From 8e37ecb7378df0a3c6d37f4aa886e8fbb692bd42 Mon Sep 17 00:00:00 2001 From: Qinyouzeng <102203523+Qinyouzeng@users.noreply.github.com> Date: Wed, 10 Apr 2024 12:04:13 +0800 Subject: [PATCH] feat: clickhouse query improved (#470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat:更改数据统计方式，源表实时计算改为统计表查询 --- Masa.Tsc.sln | 58 ++ .../Masa.Tsc.ApiGateways.Caller.csproj | 4 +- .../Extensions/DictionaryExtenistions.cs | 35 + ...ldingBlocks.StackSdks.Tsc.Contracts.csproj | 13 + .../Model/Aggregate/AggregateTypes.cs | 14 + .../Aggregate/SimpleAggregateRequestDto.cs | 25 + .../Model/BaseRequestDto.cs | 29 + .../Model/ConditionTypes.cs | 20 + .../Model/FieldConditionDto.cs | 13 + .../Model/FieldOrderDto.cs | 11 + .../Model/MappingResponseDto.cs | 11 + .../Model/Trace/TraceDatabaseResponseDto.cs | 80 +++ .../Model/Trace/TraceDtoKind.cs | 13 + .../Model/Trace/TraceExceptionResponseDto.cs | 19 + .../Model/Trace/TraceHttpResponseDto.cs | 83 +++ .../Model/Trace/TraceNetResponseDto.cs | 35 + .../Model/Trace/TraceRequestAttrDto.cs | 21 + .../Model/Trace/TraceResponseDto.cs | 111 +++ .../Service/ILogService.cs | 18 + .../Service/ITraceService.cs | 22 + .../_Imports.cs | 10 + .../ApmClickhouseInit.cs | 129 ++++ .../ApmClickhouseServiceExtensions.cs | 21 + .../Cliclhouse/ClickhouseApmService.cs | 663 ++++++++++++++++++ .../Config/Constants.cs | 40 ++ .../Interfaces/IApmService.cs | 62 ++ ...ontrib.StackSdks.Tsc.Apm.Clickhouse.csproj | 13 + .../Models/ApmEnums.cs | 10 + .../Models/Request/ApmEndpointRequestDto.cs | 9 + .../Request/ApmTraceLatencyRequestDto.cs | 19 + .../Models/Request/BaseApmRequestDto.cs | 37 + .../Models/Response/EndpointChartDto.cs | 28 + .../EndpointLatencyDistributionDto.cs | 11 + .../Models/Response/EndpointListDto.cs | 11 + .../Models/Response/ServiceListDto.cs | 66 ++ .../_Imports.cs | 20 + .../Extensions/ClickhouseInit.cs | 311 ++++++++ .../Extensions/IDbConnectionExtensitions.cs | 537 ++++++++++++++ .../MasaTscCliclhouseExtensitions.cs | 21 + .../LogService.cs | 28 + ...sa.Contrib.StackSdks.Tsc.Clickhouse.csproj | 20 + .../Model/MASAStackClickhouseConnection.cs | 46 ++ .../TraceService.cs | 39 ++ .../_Imports.cs | 21 + .../Constants/ElasticConstant.cs | 40 ++ .../Constants/LogTraceSetting.cs | 25 + .../Constants/MappingConstant.cs | 16 + .../Converters/LogResponseDtoConverter.cs | 35 + .../Converters/TraceResponseDtoConverter.cs | 35 + .../Extenistions/IElasticClientExtenstion.cs | 504 +++++++++++++ .../Extenistions/JsonElementExtensions.cs | 75 ++ .../Extenistions/ServiceExtenistion.cs | 139 ++++ .../LogService.cs | 31 + ...Contrib.StackSdks.Tsc.Elasticsearch.csproj | 20 + .../Model/ElasticseachLogResponseDto.cs | 10 + .../Model/ElasticseacherMappingResponse.cs | 17 + .../Scroll/ElasticsearchScrollRequestDto.cs | 11 + .../Scroll/ElasticsearchScrollResponseDto.cs | 9 + .../Trace/ElasticseachTraceResponseDto.cs | 13 + .../TraceService.cs | 41 ++ .../_Imports.cs | 22 + .../Enums/ResultStatuses.cs | 10 + .../Enums/ResultTypes.cs | 15 + .../Extensions/HttpClientExtensions.cs | 30 + .../Extensions/ObjectExtensions.cs | 142 ++++ .../Extensions/StringExtensions.cs | 20 + .../IMasaPrometheusClient.cs | 21 + .../Masa.Utils.Data.Prometheus.csproj | 12 + .../MasaPrometheusClient.cs | 140 ++++ .../Reponse/Exemplar/ExemplarDataModel.cs | 13 + .../Model/Reponse/Exemplar/ExemplarModel.cs | 11 + .../Exemplar/ExemplarResultResponse.cs | 9 + .../Reponse/MetaData/LabelResultResponse.cs | 9 + .../Reponse/MetaData/MetaItemValueModel.cs | 22 + .../Reponse/MetaData/MetaResultResponse.cs | 9 + .../MetaData/MetricMetaQueryRequest.cs | 17 + .../Reponse/MetaData/SeriesResultResponse.cs | 9 + .../Query/QueryResultCommonResponse.cs | 9 + .../Reponse/Query/QueryResultDataResponse.cs | 11 + .../Query/QueryResultInstantVectorResponse.cs | 11 + .../Query/QueryResultMatrixRangeResponse.cs | 11 + .../Model/Reponse/ResultBaseResponse.cs | 15 + .../Model/Request/LableValueQueryRequest.cs | 9 + .../Model/Request/MetaDataQueryRequest.cs | 13 + .../Model/Request/QueryExemplarRequest.cs | 14 + .../Model/Request/QueryRangeRequest.cs | 17 + .../Model/Request/QueryRequest.cs | 13 + .../Masa.Utils.Data.Prometheus/README.md | 75 ++ .../README.zh-CN.md | 75 ++ .../ServiceCollectionExtensions.cs | 38 + .../Masa.Utils.Data.Prometheus/_Imports.cs | 14 + .../Extensions/AddTraceLogExtenstion.cs | 2 +- .../Infrastructure/Const/ConfigConst.cs | 4 + .../AppSettingConfiguration.cs | 2 + .../Masa.Tsc.Service.Admin.csproj | 24 +- .../Services/ApmService.cs | 3 +- .../Services/TopologyService.cs | 11 +- .../Apm/ApmInputSearchComponent.razor | 70 +- .../Components/Apm/ApmLogView.razor.css | 1 - .../Components/Apm/ApmSamplePage.razor.cs | 2 +- .../Components/Apm/ApmSearchComponent.razor | 2 +- .../Apm/ApmSearchComponent.razor.cs | 57 +- .../Components/Apm/ApmTraceView.razor.css | 1 - .../Pages/Apm/Endpoint.razor.cs | 8 +- .../Pages/Apm/Service.razor | 1 + .../Pages/Apm/Service.razor.cs | 2 +- .../Apm/Services/ServiceEndpoints.razor.cs | 6 +- .../Shared/ApmComponentBase.cs | 27 +- src/Web/Masa.Tsc.Web.Admin.Server/Program.cs | 3 +- 109 files changed, 4828 insertions(+), 62 deletions(-) create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Masa.BuildingBlocks.StackSdks.Tsc.Contracts.csproj create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/AggregateTypes.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/SimpleAggregateRequestDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/BaseRequestDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/ConditionTypes.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldConditionDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldOrderDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/MappingResponseDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDtoKind.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceExceptionResponseDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceNetResponseDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceRequestAttrDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceResponseDto.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ILogService.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ITraceService.cs create mode 100644 src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/_Imports.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseInit.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IApmService.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/ApmEnums.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ClickhouseInit.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/LogService.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/TraceService.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/LogTraceSetting.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/MappingConstant.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/LogResponseDtoConverter.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/TraceResponseDtoConverter.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/JsonElementExtensions.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/LogService.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Masa.Contrib.StackSdks.Tsc.Elasticsearch.csproj create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseachLogResponseDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseacherMappingResponse.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollRequestDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollResponseDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Trace/ElasticseachTraceResponseDto.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/TraceService.cs create mode 100644 src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/_Imports.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultStatuses.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultTypes.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/HttpClientExtensions.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/ObjectExtensions.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/StringExtensions.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/IMasaPrometheusClient.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Masa.Utils.Data.Prometheus.csproj create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/MasaPrometheusClient.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarDataModel.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarModel.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarResultResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/LabelResultResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaItemValueModel.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaResultResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetricMetaQueryRequest.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/SeriesResultResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultCommonResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultDataResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultInstantVectorResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultMatrixRangeResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/ResultBaseResponse.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/LableValueQueryRequest.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/MetaDataQueryRequest.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryExemplarRequest.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRangeRequest.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRequest.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/README.md create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/README.zh-CN.md create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/ServiceCollectionExtensions.cs create mode 100644 src/Infrastructure/Masa.Utils.Data.Prometheus/_Imports.cs diff --git a/Masa.Tsc.sln b/Masa.Tsc.sln index 9e74df77..127181f6 100644 --- a/Masa.Tsc.sln +++ b/Masa.Tsc.sln @@ -41,6 +41,18 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Tsc.Service.Admin", "s EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Tsc.Web.Admin.Wasm", "src\Web\Masa.Tsc.Web.Admin.Wasm\Masa.Tsc.Web.Admin.Wasm.csproj", "{192AE8AB-9E20-44CB-895C-4F981179F92A}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Infrastructure", "Infrastructure", "{386CA149-7280-4F1F-A064-4F3B9643C082}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse", "src\Infrastructure\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse\Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj", "{2F17FF6D-31C8-4D43-9918-B31DE9E8A826}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse", "src\Infrastructure\Masa.Contrib.StackSdks.Tsc.Clickhouse\Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj", "{FCED6C6F-3EC0-47A4-84CD-0392BB663999}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Elasticsearch", "src\Infrastructure\Masa.Contrib.StackSdks.Tsc.Elasticsearch\Masa.Contrib.StackSdks.Tsc.Elasticsearch.csproj", "{860A1867-2BC7-4AAD-82E8-55CD74C0F608}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.BuildingBlocks.StackSdks.Tsc.Contracts", "src\Infrastructure\Masa.BuildingBlocks.StackSdks.Tsc.Contracts\Masa.BuildingBlocks.StackSdks.Tsc.Contracts.csproj", "{647CDA33-B497-47F7-A258-1F1831BF240A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Utils.Data.Prometheus", "src\Infrastructure\Masa.Utils.Data.Prometheus\Masa.Utils.Data.Prometheus.csproj", "{7F3CD41F-35ED-46C9-8974-D064E73B9D19}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -104,6 +116,46 @@ Global {192AE8AB-9E20-44CB-895C-4F981179F92A}.Release|Any CPU.Build.0 = Release|Any CPU {192AE8AB-9E20-44CB-895C-4F981179F92A}.Release|x86.ActiveCfg = Release|Any CPU {192AE8AB-9E20-44CB-895C-4F981179F92A}.Release|x86.Build.0 = Release|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Debug|x86.ActiveCfg = Debug|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Debug|x86.Build.0 = Debug|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Release|Any CPU.Build.0 = Release|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Release|x86.ActiveCfg = Release|Any CPU + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826}.Release|x86.Build.0 = Release|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Debug|x86.ActiveCfg = Debug|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Debug|x86.Build.0 = Debug|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Release|Any CPU.Build.0 = Release|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Release|x86.ActiveCfg = Release|Any CPU + {FCED6C6F-3EC0-47A4-84CD-0392BB663999}.Release|x86.Build.0 = Release|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Debug|Any CPU.Build.0 = Debug|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Debug|x86.ActiveCfg = Debug|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Debug|x86.Build.0 = Debug|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Release|Any CPU.ActiveCfg = Release|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Release|Any CPU.Build.0 = Release|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Release|x86.ActiveCfg = Release|Any CPU + {860A1867-2BC7-4AAD-82E8-55CD74C0F608}.Release|x86.Build.0 = Release|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Debug|x86.ActiveCfg = Debug|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Debug|x86.Build.0 = Debug|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Release|Any CPU.Build.0 = Release|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Release|x86.ActiveCfg = Release|Any CPU + {647CDA33-B497-47F7-A258-1F1831BF240A}.Release|x86.Build.0 = Release|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Debug|x86.ActiveCfg = Debug|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Debug|x86.Build.0 = Debug|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Release|Any CPU.Build.0 = Release|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Release|x86.ActiveCfg = Release|Any CPU + {7F3CD41F-35ED-46C9-8974-D064E73B9D19}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -120,6 +172,12 @@ Global {AD1AD4E4-5D65-4480-9A38-7B11FC367501} = {F3986DD3-0372-4929-B72A-09D93196541E} {C9455D57-1672-4A97-9CF2-3B23706282A6} = {4A1D2DFA-D091-4CD2-B9CB-D7AAA795ABD9} {192AE8AB-9E20-44CB-895C-4F981179F92A} = {F3986DD3-0372-4929-B72A-09D93196541E} + {386CA149-7280-4F1F-A064-4F3B9643C082} = {92F40AFA-8416-40BE-9893-D35E1924C69D} + {2F17FF6D-31C8-4D43-9918-B31DE9E8A826} = {386CA149-7280-4F1F-A064-4F3B9643C082} + {FCED6C6F-3EC0-47A4-84CD-0392BB663999} = {386CA149-7280-4F1F-A064-4F3B9643C082} + {860A1867-2BC7-4AAD-82E8-55CD74C0F608} = {386CA149-7280-4F1F-A064-4F3B9643C082} + {647CDA33-B497-47F7-A258-1F1831BF240A} = {386CA149-7280-4F1F-A064-4F3B9643C082} + {7F3CD41F-35ED-46C9-8974-D064E73B9D19} = {386CA149-7280-4F1F-A064-4F3B9643C082} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B83BA2BA-19E5-41FB-A92A-16A03DA229E4} diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Masa.Tsc.ApiGateways.Caller.csproj b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Masa.Tsc.ApiGateways.Caller.csproj index 25cf50e5..3a7742d2 100644 --- a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Masa.Tsc.ApiGateways.Caller.csproj +++ b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Masa.Tsc.ApiGateways.Caller.csproj @@ -6,8 +6,7 @@ enable - - + @@ -19,5 +18,6 @@ + diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs new file mode 100644 index 00000000..eb0fb2d3 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Extensions/DictionaryExtenistions.cs @@ -0,0 +1,35 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace System.Collections.Generic; + +public static class DictionaryExtenistions +{ + private static readonly JsonSerializerOptions _serializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + NumberHandling = JsonNumberHandling.AllowReadingFromString + }; + + public static Dictionary GroupByKeyPrefix(this Dictionary source, string prefix, Func? convertFunc = null) + { + var result = new Dictionary(); + foreach (var key in source.Keys) + { + if (!key.StartsWith(prefix)) + continue; + var value = source[key]; + var newKey = key[prefix.Length..]; + if (convertFunc != null) + value = convertFunc(source[key]); + result.Add(newKey, (T)value!); + } + return result; + } + + internal static T ConvertTo(this Dictionary dic) + { + var text = JsonSerializer.Serialize(dic, _serializerOptions); + return JsonSerializer.Deserialize(text, _serializerOptions)!; + } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Masa.BuildingBlocks.StackSdks.Tsc.Contracts.csproj b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Masa.BuildingBlocks.StackSdks.Tsc.Contracts.csproj new file mode 100644 index 00000000..bce7b7ac --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Masa.BuildingBlocks.StackSdks.Tsc.Contracts.csproj @@ -0,0 +1,13 @@ + + + + net6.0 + enable + enable + + + + + + + diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/AggregateTypes.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/AggregateTypes.cs new file mode 100644 index 00000000..232a5f5b --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/AggregateTypes.cs @@ -0,0 +1,14 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Aggregate; + +public enum AggregateTypes +{ + Count=1, + Sum, + Avg, + DistinctCount, + DateHistogram, + GroupBy +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/SimpleAggregateRequestDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/SimpleAggregateRequestDto.cs new file mode 100644 index 00000000..88931700 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/SimpleAggregateRequestDto.cs @@ -0,0 +1,25 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Aggregate; + +public class SimpleAggregateRequestDto : BaseRequestDto +{ + public string Name { get; set; } + + public string Alias { get; set; } + + public AggregateTypes Type { get; set; } + + public int MaxCount { get; set; } + + /// + /// currently support elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/search-aggregations-bucket-datehistogram-aggregation.html + /// + public string Interval { get; set; } + + /// + /// only fro type Group by, true return type is IEnumerable>,false is IEnumerable + /// + public bool AllValue { get; set; } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/BaseRequestDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/BaseRequestDto.cs new file mode 100644 index 00000000..0ca350cf --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/BaseRequestDto.cs @@ -0,0 +1,29 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; + +public class BaseRequestDto : RequestPageBase +{ + public string TraceId { get; set; } + + public string Service { get; set; } + + public string Instance { get; set; } + + public string Endpoint { get; set; } + + public string Keyword { get; set; } + + public DateTime Start { get; set; } + + public DateTime End { get; set; } + + public string RawQuery { get; set; } + + public IEnumerable Conditions { get; set; } + + public FieldOrderDto? Sort { get; set; } + + public virtual void AppendConditions() { } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/ConditionTypes.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/ConditionTypes.cs new file mode 100644 index 00000000..7bbbec1f --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/ConditionTypes.cs @@ -0,0 +1,20 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; + +public enum ConditionTypes +{ + Equal=1, + NotEqual, + Great, + Less, + GreatEqual, + LessEqual, + In, + NotIn, + Regex, + NotRegex, + Exists, + NotExists +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldConditionDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldConditionDto.cs new file mode 100644 index 00000000..1bf82de0 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldConditionDto.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; + +public class FieldConditionDto +{ + public string Name { get; set; } + + public ConditionTypes Type { get; set; } + + public object Value { get; set; } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldOrderDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldOrderDto.cs new file mode 100644 index 00000000..bdf1955d --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/FieldOrderDto.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; + +public class FieldOrderDto +{ + public string Name { get; set; } + + public bool IsDesc { get; set; } = true; +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/MappingResponseDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/MappingResponseDto.cs new file mode 100644 index 00000000..f801a14e --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/MappingResponseDto.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; + +public class MappingResponseDto +{ + public string Name { get; set; } + + public string Type { get; set; } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs new file mode 100644 index 00000000..a2fe169d --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDatabaseResponseDto.cs @@ -0,0 +1,80 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; + +public class TraceDatabaseResponseDto +{ + [JsonPropertyName("db.system")] + public virtual string System { get; set; } + + [JsonPropertyName("db.connection_string")] + public virtual string ConnectionString { get; set; } + + [JsonPropertyName("db.user")] + public virtual string User { get; set; } + + [JsonPropertyName("net.peer.ip")] + public virtual string PeerIp { get; set; } + + [JsonPropertyName("net.peer.name")] + public virtual string PeerName { get; set; } + + [JsonPropertyName("net.peer.port")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int PeerPort { get; set; } + + [JsonPropertyName("net.transport")] + public virtual string Transport { get; set; } + + [JsonPropertyName("db.jdbc.driver_classname")] + public virtual string JdbcDriverClassName { get; set; } + + [JsonPropertyName("db.mssql.instance_name")] + public virtual string MssqlInstanceName { get; set; } + + [JsonPropertyName("db.name")] + public virtual string Name { get; set; } + + [JsonPropertyName("db.statement")] + public virtual string Statement { get; set; } + + [JsonPropertyName("db.operation")] + public virtual string Operation { get; set; } + + [JsonPropertyName("db.redis.database_index")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int RedisDatabaseIndex { get; set; } + + [JsonPropertyName("db.mongodb.collection")] + public virtual string MongodbCollection { get; set; } + + [JsonPropertyName("db.sql.table")] + public virtual string SqlTable { get; set; } + + #region Cassandra + + [JsonPropertyName("db.cassandra.page_size")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int CassandraPageSize { get; set; } + + [JsonPropertyName("db.cassandra.consistency_level")] + public virtual string CassandraConsistencyLevel { get; set; } + + [JsonPropertyName("db.cassandra.table")] + public virtual string CassandraTable { get; set; } + + [JsonPropertyName("db.cassandra.idempotence")] + public virtual bool CassandraIdempotence { get; set; } + + [JsonPropertyName("db.cassandra.speculative_execution_count")] + public virtual bool CassandraSpeculativeExecutionCount { get; set; } + + [JsonPropertyName("db.cassandra.coordinator.id")] + public virtual string CassandraCoordinatorId { get; set; } + + [JsonPropertyName("db.cassandra.coordinator.dc")] + public virtual string CassandraCoordinatorDc { get; set; } + + #endregion +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDtoKind.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDtoKind.cs new file mode 100644 index 00000000..7b5d22e3 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceDtoKind.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Trace; + +public sealed class TraceDtoKind +{ + private TraceDtoKind() { } + + public const string SPAN_KIND_SERVER = nameof(SPAN_KIND_SERVER); + + public const string SPAN_KIND_CLIENT = nameof(SPAN_KIND_CLIENT); +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceExceptionResponseDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceExceptionResponseDto.cs new file mode 100644 index 00000000..58421a17 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceExceptionResponseDto.cs @@ -0,0 +1,19 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; + +public class TraceExceptionResponseDto +{ + [JsonPropertyName("exception.type")] + public virtual string Type { get; set; } + + [JsonPropertyName("exception.message")] + public virtual string Message { get; set; } + + [JsonPropertyName("exception.stacktrace")] + public virtual string StackTrace { get; set; } + + [JsonPropertyName("exception.escaped")] + public virtual bool Escaped { get; set; } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs new file mode 100644 index 00000000..cea96ace --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceHttpResponseDto.cs @@ -0,0 +1,83 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; + +public class TraceHttpResponseDto +{ + public virtual string Name { get; set; } + + public virtual int Status { get; set; } + + [JsonPropertyName("http.method")] + public virtual string Method { get; set; } + + [JsonPropertyName("http.url")] + public virtual string Url { get; set; } + + [JsonPropertyName("http.target")] + public virtual string Target { get; set; } + + [JsonPropertyName("http.host")] + public virtual string Host { get; set; } + + [JsonPropertyName("http.scheme")] + public virtual string Scheme { get; set; } + + [JsonPropertyName("http.status_code")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int StatusCode { get; set; } + + [JsonPropertyName("http.flavor")] + public virtual string Flavor { get; set; } + + [JsonPropertyName("http.user_agent")] + public virtual string UserAgent { get; set; } + + [JsonPropertyName("http.request_content_length")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int RequestContentLength { get; set; } + + [JsonPropertyName("http.request_content_length_uncompressed")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int RequestContentLengthUncompressed { get; set; } + + public Dictionary> RequestHeaders { get; set; } + + [JsonPropertyName("http.response_content_length")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int ResponseContentLength { get; set; } + + [JsonPropertyName("http.response_content_length_uncompressed")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int ResponseContentLengthUncompressed { get; set; } + + public Dictionary> ReponseHeaders { get; set; } + + [JsonPropertyName("http.retry_count")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int RetryCount { get; set; } + + [JsonPropertyName("net.peer.ip")] + public virtual string PeerIp { get; set; } + + [JsonPropertyName("net.peer.port")] + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public virtual int? PeerPort { get; set; } + + [JsonPropertyName("net.peer.name")] + public virtual string PeerName { get; set; } + + #region http server + + [JsonPropertyName("http.server_name")] + public virtual string ServerName { get; set; } + + [JsonPropertyName("http.route")] + public virtual string Route { get; set; } + + [JsonPropertyName("http.client_ip")] + public virtual string ClientIp { get; set; } + + #endregion +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceNetResponseDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceNetResponseDto.cs new file mode 100644 index 00000000..f319dc27 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceNetResponseDto.cs @@ -0,0 +1,35 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; + +public class TraceNetResponseDto +{ + public virtual string Transport { get; set; } + + public virtual string PeerIp { get; set; } + + public virtual int PeerPort { get; set; } + + public virtual string PeerName { get; set; } + + public virtual string HostIp { get; set; } + + public virtual int HostPort { get; set; } + + public virtual string HostName { get; set; } + + public virtual string HostConnectType { get; set; } + + public virtual string HostConnectSubtype { get; set; } + + public virtual string CarrierName { get; set; } + + public virtual string CarrierMCC { get; set; } + + public virtual string CarrierMNC { get; set; } + + public virtual string CarrierICC { get; set; } + + public virtual string PeerService { get; set; } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceRequestAttrDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceRequestAttrDto.cs new file mode 100644 index 00000000..10eb5c4f --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceRequestAttrDto.cs @@ -0,0 +1,21 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Trace; + +public class TraceRequestAttrDto +{ + public string Service { get; set; } + + public string Instance { get; set; } + + public string Endpoint { get; set; } + + public DateTime Start { get; set; } + + public DateTime End { get; set; } + + public string Query { get; set; } + + public int MaxCount { get; set; } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceResponseDto.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceResponseDto.cs new file mode 100644 index 00000000..fb95217f --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Trace/TraceResponseDto.cs @@ -0,0 +1,111 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; + +public class TraceResponseDto +{ + private static readonly string[] httpKeys = new string[] { "http.method" }; + private static readonly string[] databaseKeys = new string[] { "db.system" }; + private static readonly string[] exceptionKeys = new string[] { "exception.type", "exception.message" }; + + public virtual DateTime Timestamp { get; set; } + + public virtual DateTime EndTimestamp { get; set; } + + public virtual string TraceId { get; set; } + + public virtual string SpanId { get; set; } + + public virtual string ParentSpanId { get; set; } + + public virtual string Kind { get; set; } + + public virtual int TraceStatus { get; set; } + + public virtual string Name { get; set; } + + public virtual Dictionary Attributes { get; set; } + + public virtual Dictionary Resource { get; set; } + + public virtual long Duration => (long)Math.Floor((EndTimestamp - Timestamp).TotalMilliseconds); + + public virtual bool TryParseHttp(out TraceHttpResponseDto result) + { + result = default!; + if (!IsContainsAnyKey(Attributes, httpKeys)) + return false; + result = Attributes.ConvertTo(); + + result.RequestHeaders = Attributes.GroupByKeyPrefix("http.request.header.", ReadHeaderValues); + result.ReponseHeaders = Attributes.GroupByKeyPrefix("http.response.header.", ReadHeaderValues); + + result.Name = Name; + result.Status = TraceStatus; + return true; + } + + public virtual bool TryParseDatabase(out TraceDatabaseResponseDto result) + { + result = default!; + if (!IsContainsAnyKey(Attributes, databaseKeys)) + return false; + result = Attributes.ConvertTo(); + return true; + } + + public virtual bool TryParseException(out TraceExceptionResponseDto result) + { + result = default!; + if (!IsContainsAnyKey(Attributes, exceptionKeys)) + return false; + + result = Attributes.ConvertTo(); + return true; + } + + public virtual string GetDispalyName() + { + if (TryParseDatabase(out var databaseDto)) + { + return $"{(Attributes.ContainsKey("peer.service") ? Attributes["peer.service"] : "")}{databaseDto.System}{databaseDto.Name}"; + } + else if (TryParseHttp(out var traceHttpDto)) + { + return traceHttpDto.Url; + } + else if (TryParseException(out TraceExceptionResponseDto exceptionDto)) + { + return exceptionDto.Type ?? exceptionDto.Message; + } + else + return Name; + } + + private static IEnumerable ReadHeaderValues(object obj) + { + if (obj is JsonElement value) + { + if (value.ValueKind == JsonValueKind.Array) + { + return value.EnumerateArray().Select(item => item.ToString()).ToArray(); + } + else + { + return new string[] { value.ToString() }; + } + } + return new string[] { obj.ToString()! }; + } + + private static bool IsContainsAnyKey(Dictionary source, string[] keys) + { + if (source == null || !source.Any() || keys == null || !keys.Any()) + return false; + if (keys.Length == 1) + return source.ContainsKey(keys[0]); + + return keys.Any(k => source.ContainsKey(k)); + } +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ILogService.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ILogService.cs new file mode 100644 index 00000000..019f1460 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ILogService.cs @@ -0,0 +1,18 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; + +public interface ILogService +{ + Task> ListAsync(BaseRequestDto query); + + Task> GetMappingAsync(); + + /// + /// when query type: Count,Sum,Avg and DistinctCount return type is double, DateHistogram return IEnumerable> ,GroupBy return IEnumerable, AllValue is true return IEnumerable> + /// + /// + /// + Task AggregateAsync(SimpleAggregateRequestDto query); +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ITraceService.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ITraceService.cs new file mode 100644 index 00000000..d91502a9 --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ITraceService.cs @@ -0,0 +1,22 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; + +public interface ITraceService +{ + Task> GetAsync(string traceId); + + Task> ListAsync(BaseRequestDto query); + + Task> ScrollAsync(BaseRequestDto query); + + /// + /// when query type: Count,Sum,Avg and DistinctCount return type is double, DateHistogram return IEnumerable> ,GroupBy return IEnumerable + /// + /// + /// + Task AggregateAsync(SimpleAggregateRequestDto query); + + Task GetMaxDelayTraceIdAsync(BaseRequestDto query); +} diff --git a/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/_Imports.cs b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/_Imports.cs new file mode 100644 index 00000000..63ddf67f --- /dev/null +++ b/src/Infrastructure/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/_Imports.cs @@ -0,0 +1,10 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Log; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Aggregate; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; +global using Masa.Utils.Models; +global using System.Text.Json; +global using System.Text.Json.Serialization; diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseInit.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseInit.cs new file mode 100644 index 00000000..1992ac72 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseInit.cs @@ -0,0 +1,129 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +using Masa.Contrib.StackSdks.Tsc.Clickhouse.Extensions; + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; + +internal static class ApmClickhouseInit +{ + public static void Init(MasaStackClickhouseConnection connection) + { + InitErrorTable(connection); + InitAggregateTable(connection); + } + + private static void InitErrorTable(MasaStackClickhouseConnection connection) + { + var sql = new string[]{ + @$"CREATE TABLE {Constants.ErrorTable} +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `Attributes.exception.message` String CODEC(ZSTD(1)), + `Attributes.exception.type` String CODEC(ZSTD(1)), + `ServiceName` String CODEC(ZSTD(1)), + `Resource.service.namespace` String CODEC(ZSTD(1)), + `Attributes.http.target` String CODEC(ZSTD(1)), + INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_spanid SpanId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_environment `Resource.service.namespace` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_type `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_endpoint `Attributes.http.target` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_string_message `Attributes.exception.message` TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (Timestamp, + ServiceName, + `Resource.service.namespace`, + `Attributes.exception.type`, +`Attributes.http.target`) +TTL toDateTime(Timestamp) + toIntervalDay(30) +SETTINGS index_granularity = 8192, + ttl_only_drop_parts = 1; +", +$@"CREATE MATERIALIZED VIEW {Constants.ErrorTable.Replace(".",".v_")} TO {Constants.ErrorTable} +AS +SELECT +Timestamp,TraceId,SpanId, Body AS `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`, + ServiceName,ResourceAttributes['service.namespace'] AS `Resource.service.namespace`, LogAttributes['RequestPath'] AS `Attributes.http.target` +FROM {MasaStackClickhouseConnection.LogSourceTable} +WHERE mapContains(LogAttributes, 'exception.type') +"}; + ClickhouseInit.InitTable(connection, Constants.ErrorTable, sql); + } + + private static void InitAggregateTable(MasaStackClickhouseConnection connection) + { + foreach (var item in Constants.DicAggregateTable) + { + InitAggregateTable(connection, item.Key, item.Value); + } + } + + private static void InitAggregateTable(MasaStackClickhouseConnection connection, string interval, string tableName) + { + var sql = new string[] { + $@"CREATE TABLE {tableName} +( + `ServiceName` String, + `Resource.service.namespace` String, + `Attributes.http.target` String, + `Attributes.http.method` String, + `Timestamp` DateTime64(9), + `Latency` AggregateFunction(avg,Int64), + `Throughput` AggregateFunction(count,UInt8), + `Failed` AggregateFunction(sum,UInt8), + `P99` AggregateFunction(quantile(0.99),Int64), + `P95` AggregateFunction(quantile(0.95), Int64) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toYYYYMM(Timestamp) +ORDER BY ( + ServiceName, + Attributes.http.target, + Resource.service.namespace, + Timestamp + ) + TTL toDateTime(Timestamp) + toIntervalDay(30) +SETTINGS index_granularity = 8192", +$@"CREATE MATERIALIZED VIEW {tableName.Replace(".",".v_")} TO {tableName} +( + `ServiceName` String, + `Resource.service.namespace` String, + `Attributes.http.target` String, + `Attributes.http.method` String, + `Timestamp` DateTime64(9), + `Latency` AggregateFunction(avg, Float64), + `Throughput` AggregateFunction(count, UInt8), + `Failed` AggregateFunction(count, UInt8), + `P99` AggregateFunction(quantile(0.99),Int64), + `P95` AggregateFunction(quantile(0.95), Int64) +) AS +SELECT + ServiceName, + `Resource.service.namespace`, + `Attributes.http.target`, + `Attributes.http.method`, + toStartOfInterval(Timestamp,INTERVAL {interval}) AS Timestamp, + avgState(Duration) AS Latency, + countState(1) AS Throughput, + sumState(has(['400','500','501','502','503'],`Attributes.http.status_code`)) as Failed, + quantileState(0.99)(Duration) as P99, + quantileState(0.95)(Duration) as P95 +FROM {MasaStackClickhouseConnection.TraceTable} +WHERE +SpanKind='SPAN_KIND_SERVER' +GROUP BY + ServiceName, + `Resource.service.namespace`, + `Attributes.http.target`, + `Attributes.http.method`, + Timestamp" + }; + ClickhouseInit.InitTable(connection, tableName, sql); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs new file mode 100644 index 00000000..76187735 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs @@ -0,0 +1,21 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Microsoft.Extensions.DependencyInjection; + +public static class ApmClickhouseServiceExtensions +{ + internal static ILogger Logger { get; private set; } + + public static IServiceCollection AddMASAStackApmClickhouse(this IServiceCollection services, string connectionStr, string suffix = "masastack", string? logSourceTable = null, string? traceSourceTable = null) + { + services.AddMASAStackClickhouse(connectionStr, suffix, logSourceTable, traceSourceTable, con => + { + var clickhouseConnection = (MasaStackClickhouseConnection)con; + Constants.Init(clickhouseConnection.ConnectionSettings.Database, suffix); + ApmClickhouseInit.Init(clickhouseConnection); + }); + services.AddScoped(); + return services; + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs new file mode 100644 index 00000000..75acd674 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Cliclhouse/ClickhouseApmService.cs @@ -0,0 +1,663 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; + +internal class ClickhouseApmService : IApmService +{ + private readonly MasaStackClickhouseConnection _dbConnection; + private readonly ClickHouseCommand command; + private readonly ITraceService _traceService; + private readonly static object lockObj = new(); + private static Dictionary serviceOrders = new() { + {nameof(ServiceListDto.Name),SERVICE_NAME}, + {nameof(ServiceListDto.Envs),"Resource.service.namespace"}, + {nameof(ServiceListDto.Latency),"Latency"}, + {nameof(ServiceListDto.Throughput),"Throughput"}, + {nameof(ServiceListDto.Failed),"Failed"}, + }; + + private static Dictionary endpointOrders = new() { + {nameof(EndpointListDto.Name),"`Attributes.http.target`"}, + {nameof(EndpointListDto.Service),SERVICE_NAME}, + {nameof(EndpointListDto.Method),"`method`"}, + {nameof(EndpointListDto.Latency),"latency"}, + {nameof(EndpointListDto.Throughput),"throughput"}, + {nameof(EndpointListDto.Failed),"failed"}, + }; + + private static Dictionary errorOrders = new() { + {nameof(ErrorMessageDto.Type),"Type"}, + {nameof(ErrorMessageDto.Message),"Message"}, + {nameof(ErrorMessageDto.LastTime),"`time`"}, + {nameof(ErrorMessageDto.Total),"`total`"} + }; + const double MILLSECOND = 1e6; + const string SERVICE_NAME = "ServiceName"; + + private readonly ILogger _logger; + + public ClickhouseApmService(MasaStackClickhouseConnection dbConnection, ITraceService traceService, ILogger logger) + { + _traceService = traceService; + _dbConnection = dbConnection; + command = dbConnection.CreateCommand(); + if (_dbConnection.State == ConnectionState.Closed) + _dbConnection.Open(); + _logger = logger; + } + + public Task> ServicePageAsync(BaseApmRequestDto query) + { + return MetricListAsync(query, false); + } + + public Task> EndpointPageAsync(BaseApmRequestDto query) + { + return MetricListAsync(query, true); + } + + private Task> MetricListAsync(BaseApmRequestDto query, bool isEndpoint) where T : ServiceListDto, new() + { + query.IsServer = true; + query.IsMetric = true; + var (where, parameters) = AppendWhere(query); + var period = GetPeriod(query); + string groupAppend = isEndpoint ? ",`Attributes.http.target`,`Attributes.http.method`" : string.Empty; + var groupby = $"group by ServiceName{groupAppend}"; + var tableName = Constants.GetAggregateTable(period); + var countSql = $"select count(1) from(select count(1) from {tableName} where {where} {groupby})"; + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; + var orderBy = GetOrderBy(query, serviceOrders, defaultSort: SERVICE_NAME); + var sql = $@"select + ServiceName,`Resource.service.namespace1` as `Resource.service.namespace`,Latency1 as Latency, +Throughput1 as Throughput,Failed1 as Failed {groupAppend} +from( + select + ServiceName, + arrayStringConcat(groupUniqArray(`Resource.service.namespace`)) AS `Resource.service.namespace1`, + floor(sum(Latency*Throughput)/sum(Throughput)/{MILLSECOND}) as Latency1, + sum(Throughput) as Throughput1, + round(sum(Failed)*100/sum(Throughput),2) as Failed1 {groupAppend} + from( + select + ServiceName,Resource.service.namespace, + avgMerge(Latency) as Latency, + countMerge(Throughput) as Throughput, + SumMerge(Failed) as Failed {groupAppend} + from {tableName} + where {where} + group by ServiceName,`Resource.service.namespace`{groupAppend},Timestamp + ) t + {groupby} + ) {orderBy} @limit"; + SetData(sql, parameters, result, query, reader => + { + var result = new T() + { + Service = reader[0].ToString()!, + Envs = reader[1]?.ToString()?.Split(',') ?? Array.Empty(), + Latency = (long)Math.Floor(Convert.ToDouble(reader[2])), + Throughput = Math.Round(Convert.ToDouble(reader[3]), 2), + Failed = Math.Round(Convert.ToDouble(reader[4]), 2), + }; + if (result is EndpointListDto endpointListDto) + { + endpointListDto.Endpoint = reader[5].ToString()!; + endpointListDto.Method = reader[6].ToString()!; + } + return result; + }); + return Task.FromResult(result); + } + + private Task> GetEndpointAsync(BaseApmRequestDto query, string groupBy, string selectField, Func parseFn) + { + var (where, parameters) = AppendWhere(query); + var countSql = $"select count(1) from(select count(1) from {MasaStackClickhouseConnection.TraceTable} where {where} {groupBy})"; + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; + var orderBy = GetOrderBy(query, endpointOrders); + var sql = $@"select * from( select {selectField} from {MasaStackClickhouseConnection.TraceTable} where {where} {groupBy} {orderBy} @limit)"; + SetData(sql, parameters, result, query, parseFn); + return Task.FromResult(result); + } + + public Task> ChartDataAsync(BaseApmRequestDto query) + { + query.IsServer = true; + query.IsMetric = true; + var (where, parameters) = AppendWhere(query); + var period = GetPeriod(query); + bool isEndpoint = query is ApmEndpointRequestDto apmEndpointDto && string.IsNullOrEmpty(apmEndpointDto.Endpoint); + string groupAppend = isEndpoint ? ",`Attributes.http.target`,`Attributes.http.method`" : string.Empty; + var tableName = Constants.GetAggregateTable(period); + var result = new List(); + var sql = $@"select + Timestamp, + floor(Latency1/{MILLSECOND}) as Latency, + Throughput1 as Throughput, + round(Failed1*100/Throughput1,2) as Failed, + floor(P991/{MILLSECOND}) as P99, + floor(P951/{MILLSECOND}) as P95, + ServiceName{groupAppend} + from( + select + Timestamp, + avgMerge(Latency) as Latency1, + countMerge(Throughput) as Throughput1, + SumMerge(Failed) as Failed1, + quantileMerge(P99) P991, + quantileMerge(P95) P951, + ServiceName{groupAppend} + from {tableName} + where {where} + group by ServiceName{groupAppend},Timestamp + order by ServiceName{groupAppend},Timestamp + ) t"; + lock (lockObj) + { + using var reader = Query(sql, parameters); + SetChartData(result, reader); + } + GetPreviousChartData(query, sql, parameters, result); + return Task.FromResult(result.AsEnumerable()); + } + + private void GetPreviousChartData(BaseApmRequestDto query, string sql, List parameters, List result) + { + if (!query.ComparisonType.HasValue) + return; + + int day = 0; + switch (query.ComparisonType.Value) + { + case ComparisonTypes.DayBefore: + day = -1; + break; + case ComparisonTypes.WeekBefore: + day = -7; + break; + } + if (day == 0) + return; + + var paramStartTime = parameters.First(p => p.ParameterName == "startTime"); + paramStartTime.Value = ((DateTime)paramStartTime.Value!).AddDays(day); + + var paramEndTime = parameters.First(p => p.ParameterName == "endTime"); + paramEndTime.Value = ((DateTime)paramEndTime.Value!).AddDays(day); + + lock (lockObj) + { + using var readerPrevious = Query(sql, parameters); + SetChartData(result, readerPrevious, isPrevious: true); + } + } + + private static void SetChartData(List result, IDataReader reader, bool isPrevious = false) + { + if (!reader.NextResult()) + return; + ChartLineDto? current = null; + while (reader.Read()) + { + string name; + if (reader.FieldCount - 7== 0)//service + { + name = reader[6].ToString()!; + } + else + { + name = $"{reader[7]} {reader[6]}"; + } + var time = new DateTimeOffset(Convert.ToDateTime(reader[0])).ToUnixTimeSeconds(); + if (current == null || current.Name != name) + { + if (isPrevious && result.Exists(item => item.Name == name)) + { + current = result.First(item => item.Name == name); + } + else + { + current = new ChartLineDto + { + Name = name, + Previous = new List(), + Currents = new List() + }; + result.Add(current); + } + } + + ((List)(isPrevious ? current.Previous : current.Currents)).Add( + new() + { + Latency = (long)Math.Floor(Convert.ToDouble(reader[1])), + Throughput = Math.Round(Convert.ToDouble(reader[2]), 2, MidpointRounding.ToZero), + Failed = Math.Round(Convert.ToDouble(reader[3]), 2, MidpointRounding.ToZero), + P99 = Math.Round(Convert.ToDouble(reader[4]), 2, MidpointRounding.ToZero), + P95 = Math.Round(Convert.ToDouble(reader[5]), 2, MidpointRounding.ToZero), + Time = time + }); + } + } + + public Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query) + { + var (where, parameters) = AppendWhere(query); + var result = new EndpointLatencyDistributionDto(); + var p95 = Convert.ToDouble(Scalar($"select floor(quantile(0.95)(Duration/{MILLSECOND})) p95 from {MasaStackClickhouseConnection.TraceTable} where {where}", parameters)); + if (p95 is not double.NaN) + result.P95 = (long)Math.Floor(p95); + var sql = $@"select Duration/{MILLSECOND},count(1) total from {MasaStackClickhouseConnection.TraceTable} where {where} group by Duration order by Duration"; + var list = new List(); + lock (lockObj) + { + using var reader = Query(sql, parameters); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new ChartPointDto() + { + X = reader[0].ToString()!, + Y = reader[1]?.ToString()! + }; + list.Add(item); + } + } + result.Latencies = list; + return Task.FromResult(result); + } + + public Task> ErrorMessagePageAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + var (where, parameters) = AppendWhere(query); + var groupby = $"group by Type,Message{(string.IsNullOrEmpty(query.Endpoint) ? "" : ",Endpoint")}"; + var countSql = $"select count(1) from (select Attributes.exception.type as Type,Attributes.exception.message as Message,max(Timestamp) time,count(1) from {Constants.ErrorTable} where {where} {groupby})"; + PaginatedListBase result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) }; + var orderBy = GetOrderBy(query, errorOrders); + var sql = $@"select * from( select Attributes.exception.type as Type,Attributes.exception.message as Message,max(Timestamp) time,count(1) total from {Constants.ErrorTable} where {where} {groupby} {orderBy} @limit)"; + SetData(sql, parameters, result, query, reader => new ErrorMessageDto() + { + Type = reader[0]?.ToString()!, + Message = reader[1]?.ToString()!, + LastTime = Convert.ToDateTime(reader[2])!, + Total = Convert.ToInt32(reader[3]), + }); + return Task.FromResult(result); + } + + private void SetData(string sql, List parameters, PaginatedListBase result, BaseApmRequestDto query, Func parseFn) where TResult : class + { + var start = (query.Page - 1) * query.PageSize; + if (result.Total - start > 0) + { + lock (lockObj) + { + using var reader = Query(sql.Replace("@limit", $"limit {start},{query.PageSize}"), parameters); + result.Result = new(); + while (reader.NextResult()) + while (reader.Read()) + result.Result.Add(parseFn(reader)); + } + } + } + + private static (string where, List parameters) AppendWhere(TQuery query) where TQuery : BaseApmRequestDto + { + List parameters = new(); + var sql = new StringBuilder(); + sql.AppendLine(" Timestamp between @startTime and @endTime"); + parameters.Add(new ClickHouseParameter { ParameterName = "startTime", Value = MasaStackClickhouseConnection.ToTimeZone(query.Start), DbType = DbType.DateTime }); + parameters.Add(new ClickHouseParameter { ParameterName = "endTime", Value = MasaStackClickhouseConnection.ToTimeZone(query.End), DbType = DbType.DateTime }); + if (!string.IsNullOrEmpty(query.Env)) + { + sql.AppendLine(" and Resource.service.namespace=@environment"); + parameters.Add(new ClickHouseParameter { ParameterName = "environment", Value = query.Env }); + } + if (!string.IsNullOrEmpty(query.Service)) + { + sql.AppendLine(" and ServiceName=@serviceName"); + parameters.Add(new ClickHouseParameter { ParameterName = "serviceName", Value = query.Service }); + } + if (query.IsServer.HasValue && (query.IsMetric == null || !query.IsMetric.Value)) + { + sql.AppendLine(" and SpanKind=@spanKind"); + parameters.Add(new ClickHouseParameter { ParameterName = "spanKind", Value = query.IsServer.Value ? "SPAN_KIND_SERVER" : "SPAN_KIND_CLIENT" }); + } + AppendEndpoint(query as ApmEndpointRequestDto, sql, parameters); + AppendDuration(query as ApmTraceLatencyRequestDto, sql, parameters); + + if (!string.IsNullOrEmpty(query.Queries) && query.Queries.Trim().Length > 0) + { + if (!query.Queries.Trim().StartsWith("and ", StringComparison.CurrentCultureIgnoreCase)) + sql.Append(" and "); + sql.AppendLine(query.Queries); + } + + return (sql.ToString(), parameters); + } + + private static void AppendEndpoint(ApmEndpointRequestDto? traceQuery, StringBuilder sql, List parameters) + { + if (traceQuery == null || string.IsNullOrEmpty(traceQuery.Endpoint)) + return; + var name = "endpoint"; + if (traceQuery.IsLog.HasValue && traceQuery.IsLog.Value) + { + sql.AppendLine($" and Attributes.http.target like @{name}"); + parameters.Add(new ClickHouseParameter { ParameterName = name, Value = $"{traceQuery.Endpoint}%" }); + } + else + { + sql.AppendLine($" and Attributes.http.target=@{name}"); + parameters.Add(new ClickHouseParameter { ParameterName = name, Value = traceQuery.Endpoint }); + } + } + + private static void AppendDuration(ApmTraceLatencyRequestDto? query, StringBuilder sql, List parameters) + { + if (query == null || !query.LatMin.HasValue && !query.LatMax.HasValue || query.IsMetric != null && query.IsMetric.Value) return; + if (query.LatMin.HasValue && query.LatMin > 0) + { + sql.AppendLine(" and Duration >=@minDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "minDuration", Value = (long)(query.LatMin * MILLSECOND) }); + } + if (query.LatMax.HasValue && query.LatMax > 0) + { + sql.AppendLine(" and Duration <=@maxDuration"); + parameters.Add(new ClickHouseParameter { ParameterName = "maxDuration", Value = (long)(query.LatMax * MILLSECOND) }); + } + } + + public async Task> TraceLatencyDetailAsync(ApmTraceLatencyRequestDto query) + { + var queryDto = new BaseRequestDto + { + Start = query.Start, + End = query.End, + Endpoint = query.Endpoint, + Service = query.Service! + }; + var conditions = new List(); + if (!string.IsNullOrEmpty(query.Env)) + { + conditions.Add(new FieldConditionDto + { + Name = "Resource.service.namespace", + Type = ConditionTypes.Equal, + Value = query.Env + }); + } + var name = "Duration"; + if (query.LatMin.HasValue && query.LatMin.Value >= 0) + { + conditions.Add(new FieldConditionDto + { + Name = name, + Type = ConditionTypes.GreatEqual, + Value = (long)(query.LatMin.Value * MILLSECOND), + }); + } + + if (query.LatMax.HasValue && query.LatMax.Value >= 0 && ( + !query.LatMin.HasValue + || query.LatMin.HasValue && query.LatMax - query.LatMin.Value > 0)) + conditions.Add(new FieldConditionDto + { + Name = name, + Type = ConditionTypes.LessEqual, + Value = (long)(query.LatMax.Value * MILLSECOND), + }); + if (conditions.Count > 0) + queryDto.Conditions = conditions; + + return await _traceService.ListAsync(queryDto); + } + + private IDataReader Query(string sql, IEnumerable parameters) + { + try + { + command.CommandText = sql; + SetParameters(parameters); + return command.ExecuteReader(); + } + catch (Exception ex) + { + _logger.LogError(ex, "execute sql error:{Sqlraw}", sql); + throw; + } + } + + private object Scalar(string sql, IEnumerable parameters) + { + lock (lockObj) + { + command.CommandText = sql; + SetParameters(parameters); + return command.ExecuteScalar()!; + } + } + + private void SetParameters(IEnumerable parameters) + { + if (command.Parameters.Count > 0) + command.Parameters.Clear(); + if (parameters != null && parameters.Any()) + foreach (var param in parameters) + command.Parameters.Add(param); + } + + private static string? GetOrderBy(BaseApmRequestDto query, Dictionary sortFields, string? defaultSort = null, bool isDesc = false) + { + if (!string.IsNullOrEmpty(query.OrderField) && sortFields.TryGetValue(query.OrderField, out var field)) + { + if (!query.IsDesc.HasValue) + return $"order by `{field}`"; + return $"order by `{field}`{(query.IsDesc.Value ? " desc" : "")}"; + } + + if (string.IsNullOrEmpty(defaultSort)) + return null; + return $"order by `{defaultSort}`{(isDesc ? " desc" : "")}"; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _dbConnection.Close(); + _dbConnection.Dispose(); + } + } + + public Task> GetErrorChartAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + query.IsLog = true; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `time` order by `time`"; + var sql = $@"select +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +count(1) `total` +from {MasaStackClickhouseConnection.LogTable} where {where} and SeverityText='Error' and `Attributes.exception.message`!='' {groupby}"; + + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); + } + + private List getChartCountData(string sql, IEnumerable parameters, ComparisonTypes? comparisonTypes = null) + { + var result = new List(); + lock (lockObj) + { + using var currentReader = Query(sql, parameters); + SetChartCountData(result, currentReader); + } + + if (comparisonTypes.HasValue && (comparisonTypes.Value == ComparisonTypes.DayBefore || comparisonTypes.Value == ComparisonTypes.WeekBefore)) + { + var day = comparisonTypes.Value == ComparisonTypes.DayBefore ? -1 : -7; + var paramStartTime = parameters.First(p => p.ParameterName == "startTime"); + paramStartTime.Value = ((DateTime)paramStartTime.Value!).AddDays(day); + + var paramEndTime = parameters.First(p => p.ParameterName == "endTime"); + paramEndTime.Value = ((DateTime)paramEndTime.Value!).AddDays(day); + + lock (lockObj) + { + using var previousReader = Query(sql, parameters); + SetChartCountData(result, previousReader, true); + } + } + + return result; + } + + private static void SetChartCountData(List result, IDataReader reader, bool isPrevious = false) + { + if (!reader.NextResult()) + return; + ChartLineCountDto? current = null; + while (reader.Read()) + { + var name = reader[0].ToString()!; + var time = new DateTimeOffset(Convert.ToDateTime(reader[0])).ToUnixTimeSeconds(); + if (current == null || current.Name != name) + { + if (isPrevious && result.Exists(item => item.Name == name)) + { + current = result.First(item => item.Name == name); + } + else + { + current = new ChartLineCountDto + { + Name = name, + Previous = new List(), + Currents = new List() + }; + result.Add(current); + } + } + + ((List)(isPrevious ? current.Previous : current.Currents)).Add( + new() + { + Value = reader[1], + Time = time + }); + } + } + + public Task> GetEndpointChartAsync(ApmEndpointRequestDto query) + { + query.IsServer = false; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `time` order by `time`"; + var sql = $@"select +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +count(1) `total` +from {MasaStackClickhouseConnection.TraceTable} where {where} {groupby}"; + + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); + } + + public Task> GetLogChartAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + query.IsLog = true; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `time` order by `time`"; + var sql = $@"select +toStartOfInterval(`Timestamp` , INTERVAL {GetPeriod(query)} ) as `time`, +count(1) `total` +from {MasaStackClickhouseConnection.LogTable} where {where} {groupby}"; + return Task.FromResult(getChartCountData(sql, parameters, query.ComparisonType).AsEnumerable()); + } + + private static string GetPeriod(BaseApmRequestDto query) + { + var reg = new Regex(@"/d+", default, TimeSpan.FromSeconds(5)); + if (string.IsNullOrEmpty(query.Period) || !reg.IsMatch(query.Period)) + { + return GetDefaultPeriod(query.End - query.Start); + } + var unit = reg.Replace(query.Period, "").Trim().ToLower(); + var units = new List { "year", "month", "week", "day", "hour", "minute", "second" }; + var find = units.Find(s => s.StartsWith(unit)); + if (string.IsNullOrEmpty(find)) + find = "minute"; + return $"{reg.Match(query.Period).Result} {find}"; + } + + private static string GetDefaultPeriod(TimeSpan timeSpan) + { + if ((int)timeSpan.TotalHours < 1) + { + return "1 minute"; + } + + var days = (int)timeSpan.TotalDays; + if (days <= 0) + { + if ((int)timeSpan.TotalHours - 12 <= 0) + { + return "1 minute"; + } + return "30 minute"; + } + + if (days - 7 <= 0) + { + return "1 hour"; + } + + if (days - 30 <= 0) + { + return "1 day"; + } + + if (days - 365 <= 0) + { + return "1 week"; + } + + return "1 month"; + } + + public Task> GetTraceErrorsAsync(ApmEndpointRequestDto query) + { + query.IsServer = default; + query.IsLog = true; + var (where, parameters) = AppendWhere(query); + var groupby = "group by `SpanId` order by `SpanId`"; + var sql = $@"select +SpanId, +count(1) `total` +from {Constants.ErrorTable} where {where} {groupby}"; + var list = new List(); + lock (lockObj) + { + using var reader = Query(sql, parameters); + while (reader.NextResult()) + while (reader.Read()) + { + var item = new ChartPointDto() + { + X = reader[0].ToString()!, + Y = reader[1]?.ToString()! + }; + list.Add(item); + } + } + return Task.FromResult(list.AsEnumerable()); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs new file mode 100644 index 00000000..ecf4c6f2 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Config/Constants.cs @@ -0,0 +1,40 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Config; + +internal static class Constants +{ + public static string ErrorTable { get; private set; } + + private const string AggregateTable = "otel_trace_metrics@interval"; + + private static string GetAggregateTable(string interval, string suffix) + { + if (string.IsNullOrEmpty(interval)) + interval = INTERVALS[0]; + return AggregateTable.Replace("@interval", interval.Replace(" ", $"_")) + "_" + suffix; + } + + public static string GetAggregateTable(string interval) + { + return DicAggregateTable.TryGetValue(interval, out var table) ? table : default!; + } + + public static readonly int[] DefaultErrorStatus = new int[] { 400, 500, 501, 502, 503, 504, 505 }; + + public static readonly string[] INTERVALS = new string[] { "1 minute", "30 minute", "1 hour", "1 day", "1 week", "1 month" }; + + public static readonly Dictionary DicAggregateTable = new(); + + public static void Init(string database, string suffix) + { + if (!string.IsNullOrEmpty(database)) + database = $"{database}."; + ErrorTable = $"{database}otel_errors_{suffix}"; + foreach (var key in INTERVALS) + { + DicAggregateTable.Add(key, database + GetAggregateTable(key, suffix)); + } + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IApmService.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IApmService.cs new file mode 100644 index 00000000..b85f3373 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Interfaces/IApmService.cs @@ -0,0 +1,62 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; + +public interface IApmService : IDisposable +{ + /// + /// 服务列表页，服务详情页endpoints和instance公用 + /// + /// + /// + Task> ServicePageAsync(BaseApmRequestDto query); + + /// + /// trace列表 + /// + /// + /// + Task> EndpointPageAsync(BaseApmRequestDto query); + + /// + /// 可共用，service和endpoint公用 + /// + /// + /// + Task> ChartDataAsync(BaseApmRequestDto query); + + /// + /// endpoint 加载耗时分布 + /// + /// + /// + Task EndpointLatencyDistributionAsync(ApmEndpointRequestDto query); + + /// + /// tendpoint trace tree line + /// + /// + /// + Task> TraceLatencyDetailAsync(ApmTraceLatencyRequestDto query); + + /// + /// 错误列表 + /// + /// + /// + Task> ErrorMessagePageAsync(ApmEndpointRequestDto query); + + /// + /// 获取trace下的错误信息统计，按照spanId + /// + /// + /// + Task> GetTraceErrorsAsync(ApmEndpointRequestDto query); + + Task> GetErrorChartAsync(ApmEndpointRequestDto query); + + Task> GetEndpointChartAsync(ApmEndpointRequestDto query); + + Task> GetLogChartAsync(ApmEndpointRequestDto query); +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj new file mode 100644 index 00000000..9394a4a1 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.csproj @@ -0,0 +1,13 @@ + + + + net6.0 + enable + enable + + + + + + + diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/ApmEnums.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/ApmEnums.cs new file mode 100644 index 00000000..00c23676 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/ApmEnums.cs @@ -0,0 +1,10 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; + +public enum ComparisonTypes +{ + DayBefore = 1, + WeekBefore +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs new file mode 100644 index 00000000..6ec3e017 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmEndpointRequestDto.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; + +public class ApmEndpointRequestDto : BaseApmRequestDto +{ + public string Endpoint { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs new file mode 100644 index 00000000..eef88f80 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/ApmTraceLatencyRequestDto.cs @@ -0,0 +1,19 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; + +public class ApmTraceLatencyRequestDto : ApmEndpointRequestDto +{ + /// + /// unit ms + /// + public long? LatMin { get; set; } + + /// + /// unit ms + /// + public long? LatMax { get; set; } + + public new int PageSize { get; } = 1; +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs new file mode 100644 index 00000000..ed100c35 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Request/BaseApmRequestDto.cs @@ -0,0 +1,37 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; + +public class BaseApmRequestDto : RequestPageBase +{ + public string? Env { get; set; } + + public ComparisonTypes? ComparisonType { get; set; } + + public DateTime Start { get; set; } + + public DateTime End { get; set; } + + public string Period { get; set; } + + public string? Service { get; set; } + + public string? Queries { get; set; } + + public string? OrderField { get; set; } + + public bool? IsDesc { get; set; } + + public string StatusCodes { get; set; } + + internal int[] GetErrorStatusCodes() => string.IsNullOrEmpty(StatusCodes) ? Constants.DefaultErrorStatus : StatusCodes.Split(',').Select(s => Convert.ToInt32(s)).Where(num => num != 0).ToArray(); + + internal bool? IsServer { get; set; } = true; + + internal bool? IsMetric { get; set; } + + internal bool? IsTrace { get; set; } + + internal bool? IsLog { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs new file mode 100644 index 00000000..9d32bb8d --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointChartDto.cs @@ -0,0 +1,28 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class EndpointChartDto +{ + public IEnumerable P99s { get; set; } + + public IEnumerable P95s { get; set; } + + public IEnumerable Latencies { get; set; } + + public IEnumerable Throughputs { get; set; } + + public IEnumerable Fails { get; set; } +} + +public class ErrorMessageDto +{ + public string Type { get; set; } + + public string Message { get; set; } + + public DateTime LastTime { get; set; } + + public int Total { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs new file mode 100644 index 00000000..78146aa5 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointLatencyDistributionDto.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class EndpointLatencyDistributionDto +{ + public long? P95 { get; set; } + + public IEnumerable Latencies { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs new file mode 100644 index 00000000..742206f2 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/EndpointListDto.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class EndpointListDto: ServiceListDto +{ + public string Method { get; set; } + + public string Endpoint { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs new file mode 100644 index 00000000..fffcabaa --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/Models/Response/ServiceListDto.cs @@ -0,0 +1,66 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; + +public class ServiceListDto +{ + public string Service { get; set; } + + public string Name { get; set; } + + public IEnumerable Envs { get; set; } + + public long Latency { get; set; } + + public double Throughput { get; set; } + + public double Failed { get; set; } +} + +public class ChartLineDto +{ + public string Name { get; set; } + + public IEnumerable Currents { get; set; } + + public IEnumerable Previous { get; set; } +} + +public class ChartLineCountDto +{ + public string Name { get; set; } + + public IEnumerable Currents { get; set; } + + public IEnumerable Previous { get; set; } +} + +public class ChartLineCountItemDto +{ + public long Time { get; set; } + + public object Value { get; set; } +} + +public class ChartLineItemDto +{ + public long Time { get; set; } + + public long Latency { get; set; } + + public double P99 { get; set; } + + public double P95 { get; set; } + + public double Throughput { get; set; } + + public double Failed { get; set; } +} + +public class ChartPointDto +{ + public string X { get; set; } + + public string Y { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs new file mode 100644 index 00000000..ff1b5ffc --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/_Imports.cs @@ -0,0 +1,20 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using ClickHouse.Ado; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Cliclhouse; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Config; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request; +global using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response; +global using Masa.Contrib.StackSdks.Tsc.Clickhouse; +global using Masa.Utils.Models; +global using Microsoft.Extensions.Logging; +global using System.Data; +global using System.Data.Common; +global using System.Text; +global using System.Text.RegularExpressions; diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ClickhouseInit.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ClickhouseInit.cs new file mode 100644 index 00000000..1f3db6f2 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/ClickhouseInit.cs @@ -0,0 +1,311 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Clickhouse.Extensions; + +public static class ClickhouseInit +{ + private static ILogger Logger { get; set; } + + internal static MasaStackClickhouseConnection Connection { get; private set; } + + public static void Init(IServiceCollection services) + { + var serviceProvider = services.BuildServiceProvider(); + var logfactory = serviceProvider.GetService(); + Logger = logfactory?.CreateLogger("Masa.Contrib.StackSdks.Tsc.Clickhouse")!; + try + { + Connection = serviceProvider.GetRequiredService(); + if (!ExistsTable(Connection, MasaStackClickhouseConnection.TraceSourceTable)) + throw new ArgumentNullException(nameof(MasaStackClickhouseConnection.TraceSourceTable)); + if (!ExistsTable(Connection, MasaStackClickhouseConnection.LogSourceTable)) + throw new ArgumentNullException(nameof(MasaStackClickhouseConnection.LogSourceTable)); + InitLog(); + InitTrace(); + InitMappingTable(); + var timezoneStr = GetTimezone(Connection); + MasaStackClickhouseConnection.TimeZone = TZConvert.GetTimeZoneInfo(timezoneStr); + } + finally + { + Connection?.Dispose(); + } + } + + private static void InitLog() + { + string[] sql = new string[] { + @$"CREATE TABLE {MasaStackClickhouseConnection.LogTable} +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt32 CODEC(ZSTD(1)), + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` Int32 CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceSchemaUrl` String CODEC(ZSTD(1)), + `Resources` String CODEC(ZSTD(1)), + `ScopeSchemaUrl` String CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` String CODEC(ZSTD(1)), + `Scopes` String CODEC(ZSTD(1)), + `Logs` String CODEC(ZSTD(1)), + + `Resource.service.namespace` String CODEC(ZSTD(1)), + `Resource.service.version` String CODEC(ZSTD(1)), + `Resource.service.instance.id` String CODEC(ZSTD(1)), + + `Attributes.TaskId` String CODEC(ZSTD(1)), + `Attributes.exception.type` String CODEC(ZSTD(1)), + `Attributes.exception.message` String CODEC(ZSTD(1)), + `Attributes.http.target` String CODEC(ZSTD(1)), + + ResourceAttributesKeys Array(String) CODEC(ZSTD(1)), + ResourceAttributesValues Array(String) CODEC(ZSTD(1)), + LogAttributesKeys Array(String) CODEC(ZSTD(1)), + LogAttributesValues Array(String) CODEC(ZSTD(1)), + + INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_serviceinstanceid `Resource.service.instance.id` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_severitytext SeverityText TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_taskid `Attributes.TaskId` TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_log_exceptiontype `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1, + + INDEX idx_string_body Body TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1, + INDEX idx_string_exceptionmessage Attributes.exception.message TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY ( + Timestamp, + `Resource.service.namespace`, + ServiceName + ) +TTL toDateTime(Timestamp) + toIntervalDay(30) +SETTINGS index_granularity = 8192, + ttl_only_drop_parts = 1; +", + $@"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.LogTable.Replace(".",".v_")} TO {MasaStackClickhouseConnection.LogTable} +AS +SELECT +Timestamp,TraceId,SpanId,TraceFlags,SeverityText,SeverityNumber,ServiceName,Body,ResourceSchemaUrl,toJSONString(ResourceAttributes) as Resources, +ScopeSchemaUrl,ScopeName,ScopeVersion,toJSONString(ScopeAttributes) as Scopes,toJSONString(LogAttributes) as Logs, +ResourceAttributes['service.namespace'] as `Resource.service.namespace`,ResourceAttributes['service.version'] as `Resource.service.version`, +ResourceAttributes['service.instance.id'] as `Resource.service.instance.id`, +LogAttributes['TaskId'] as `Attributes.TaskId`, +LogAttributes['exception.type'] as `Attributes.exception.type`, +LogAttributes['exception.message'] as `Attributes.exception.message`, +LogAttributes['RequestPath'] as `Attributes.http.target`, +mapKeys(ResourceAttributes) as ResourceAttributesKeys,mapValues(ResourceAttributes) as ResourceAttributesValues, +mapKeys(LogAttributes) as LogAttributesKeys,mapValues(LogAttributes) as LogAttributesValues +FROM {MasaStackClickhouseConnection.LogSourceTable} +", + }; + InitTable(MasaStackClickhouseConnection.LogTable, sql); + } + + private static void InitTrace() + { + string[] sql = new string[] { + @$"CREATE TABLE {MasaStackClickhouseConnection.TraceTable} +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `ParentSpanId` String CODEC(ZSTD(1)), + `TraceState` String CODEC(ZSTD(1)), + `SpanName` LowCardinality(String) CODEC(ZSTD(1)), + `SpanKind` LowCardinality(String) CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Resources` String CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` String CODEC(ZSTD(1)), + `Spans` String CODEC(ZSTD(1)), + `Duration` Int64 CODEC(ZSTD(1)), + `StatusCode` LowCardinality(String) CODEC(ZSTD(1)), + `StatusMessage` String CODEC(ZSTD(1)), + `Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)), + `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + `Links.TraceId` Array(String) CODEC(ZSTD(1)), + `Links.SpanId` Array(String) CODEC(ZSTD(1)), + `Links.TraceState` Array(String) CODEC(ZSTD(1)), + `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + + `Resource.service.namespace` String CODEC(ZSTD(1)), + `Resource.service.version` String CODEC(ZSTD(1)), + `Resource.service.instance.id` String CODEC(ZSTD(1)), + + `Attributes.http.status_code` String CODEC(ZSTD(1)), + `Attributes.http.response_content_body` String CODEC(ZSTD(1)), + `Attributes.http.request_content_body` String CODEC(ZSTD(1)), + `Attributes.http.target` String CODEC(ZSTD(1)), + `Attributes.http.method` String CODEC(ZSTD(1)), + `Attributes.exception.type` String CODEC(ZSTD(1)), + `Attributes.exception.message` String CODEC(ZSTD(1)), + + `ResourceAttributesKeys` Array(String) CODEC(ZSTD(1)), + `ResourceAttributesValues` Array(String) CODEC(ZSTD(1)), + `SpanAttributesKeys` Array(String) CODEC(ZSTD(1)), + `SpanAttributesValues` Array(String) CODEC(ZSTD(1)), + + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_trace_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_trace_servicenamespace Resource.service.namespace TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_trace_serviceinstanceid Resource.service.instance.id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_trace_statuscode Attributes.http.status_code TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_trace_exceptiontype Attributes.exception.type TYPE bloom_filter(0.001) GRANULARITY 1, + + INDEX idx_string_requestbody Attributes.http.request_content_body TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1, + INDEX idx_string_responsebody Attributes.http.response_content_body TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1, + INDEX idx_string_exceptionmessage Attributes.exception.message TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY ( + Timestamp, + Resource.service.namespace, + ServiceName + ) +TTL toDateTime(Timestamp) + toIntervalDay(30) +SETTINGS index_granularity = 8192, + ttl_only_drop_parts = 1; +", + $@"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.TraceTable.Replace(".",".v_")} TO {MasaStackClickhouseConnection.TraceTable} +AS +SELECT + Timestamp,TraceId,SpanId,ParentSpanId,TraceState,SpanName,SpanKind,ServiceName,toJSONString(ResourceAttributes) AS Resources, + ScopeName,ScopeVersion,toJSONString(SpanAttributes) AS Spans, + Duration,StatusCode,StatusMessage,Events.Timestamp,Events.Name,Events.Attributes, + Links.TraceId,Links.SpanId,Links.TraceState,Links.Attributes, + + ResourceAttributes['service.namespace'] as `Resource.service.namespace`,ResourceAttributes['service.version'] as `Resource.service.version`, + ResourceAttributes['service.instance.id'] as `Resource.service.instance.id`, + + SpanAttributes['http.status_code'] as `Attributes.http.status_code`, + SpanAttributes['http.response_content_body'] as `Attributes.http.response_content_body`, + SpanAttributes['http.request_content_body'] as `Attributes.http.request_content_body`, + SpanAttributes['http.target'] as `Attributes.http.target`, + SpanAttributes['http.method'] as `Attributes.http.method`, + SpanAttributes['exception.type'] as `Attributes.exception.type`, + SpanAttributes['exception.message'] as `Attributes.exception.message`, + + mapKeys(ResourceAttributes) AS ResourceAttributesKeys, + mapValues(ResourceAttributes) AS ResourceAttributesValues, + mapKeys(SpanAttributes) AS SpanAttributesKeys, + mapValues(SpanAttributes) AS SpanAttributesValues +FROM {MasaStackClickhouseConnection.TraceSourceTable} +" }; + InitTable(MasaStackClickhouseConnection.TraceTable, sql); + } + + private static void InitMappingTable() + { + var mappingTable = "otel_mapping_"; + var sql = new string[]{ +$@" +CREATE TABLE {MasaStackClickhouseConnection.MappingTable} +( + `Name` Array(String), + `Type` String +) +ENGINE = MergeTree +ORDER BY Name +SETTINGS index_granularity = 8192;", +@$"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.MappingTable.Replace(mappingTable,"v_otel_traces_attribute_mapping")} to {MasaStackClickhouseConnection.MappingTable} +as +select DISTINCT arraySort(mapKeys(SpanAttributes)) as Name, 'trace_attributes' as Type +from {MasaStackClickhouseConnection.TraceSourceTable}", +$@"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.MappingTable.Replace(mappingTable,"v_otel_traces_resource_mapping")} to {MasaStackClickhouseConnection.MappingTable} +as +select DISTINCT arraySort(mapKeys(ResourceAttributes)) as Name, 'trace_resource' as Type +from {MasaStackClickhouseConnection.TraceSourceTable}", +$@"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.MappingTable.Replace(mappingTable,"v_otel_logs_attribute_mapping")} to {MasaStackClickhouseConnection.MappingTable} +as +select DISTINCT arraySort(mapKeys(LogAttributes)) as Name, 'log_attributes' as Type +from {MasaStackClickhouseConnection.LogSourceTable}", +$@"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.MappingTable.Replace(mappingTable,"v_otel_logs_resource_mapping")} to {MasaStackClickhouseConnection.MappingTable} +as +select DISTINCT arraySort(mapKeys(ResourceAttributes)) as Name, 'log_resource' as Type +from {MasaStackClickhouseConnection.LogSourceTable}", +$@"insert into {MasaStackClickhouseConnection.MappingTable} +values (['Timestamp','TraceId','SpanId','TraceFlag','SeverityText','SeverityNumber','Body'],'log_basic'), +(['Timestamp','TraceId','SpanId','ParentSpanId','TraceState','SpanKind','Duration'],'trace_basic'); +" }; + InitTable(MasaStackClickhouseConnection.MappingTable, sql); + } + + private static void InitTable(string tableName, params string[] sqls) + { + var database = Connection.ConnectionSettings.Database!; + if (!string.IsNullOrEmpty(database)) + tableName = tableName.Substring(database.Length + 1); + + if (Convert.ToInt32(Connection.ExecuteScalar($"select count() from system.tables where database ='{database}' and name in ['{tableName}']")) > 0) + return; + if (sqls == null || sqls.Length == 0) + return; + foreach (var sql in sqls) + { + ExecuteSql(Connection, sql); + } + } + + internal static bool ExistsTable(MasaStackClickhouseConnection connection, string tableName) + { + var database = connection.ConnectionSettings.Database!; + if (!string.IsNullOrEmpty(database)) + tableName = tableName.Substring(database.Length + 1); + return Convert.ToInt32(connection.ExecuteScalar($"select count() from system.tables where database ='{database}' and name in ['{tableName}']")) > 0; + } + + internal static void InitTable(MasaStackClickhouseConnection connection, string tableName, params string[] sqls) + { + if (ExistsTable(connection, tableName)) + return; + if (sqls == null || sqls.Length == 0) + return; + foreach (var sql in sqls) + { + ExecuteSql(connection, sql); + } + } + + internal static void ExecuteSql(this IDbConnection connection, string sql) + { + using var cmd = connection.CreateCommand(); + if (connection.State != ConnectionState.Open) + connection.Open(); + cmd.CommandText = sql; + try + { + cmd.ExecuteNonQuery(); + } + catch (Exception ex) + { + Logger?.LogError(ex, "Init table sql error:{RawSql}", sql); + } + } + + private static string GetTimezone(MasaStackClickhouseConnection connection) + { + using var cmd = connection.CreateCommand(); + if (connection.State != ConnectionState.Open) + connection.Open(); + var sql = "select timezone()"; + cmd.CommandText = sql; + try + { + return cmd.ExecuteScalar()?.ToString()!; + } + catch (Exception ex) + { + Logger?.LogError(ex, "ExecuteSql {RawSql} error", sql); + throw; + } + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs new file mode 100644 index 00000000..72b35023 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/IDbConnectionExtensitions.cs @@ -0,0 +1,537 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse")] +namespace System.Data.Common; + +internal static class IDbConnectionExtensitions +{ + const string ATTRIBUTE_KEY = "Attributes."; + const string RESOURCE_KEY = "Resource."; + const string TIMSTAMP_KEY = "Timestamp"; + + public static PaginatedListBase QueryTrace(this IDbConnection connection, BaseRequestDto query) + { + var (where, parameters, ors) = AppendWhere(query); + var orderBy = AppendOrderBy(query, false); + var countSql = CombineOrs($"select count() as `total` from {MasaStackClickhouseConnection.TraceTable} where {where}", ors); + var total = Convert.ToInt64(ExecuteScalar(connection, $"select sum(`total`) from {countSql}", parameters?.ToArray())); + var start = (query.Page - 1) * query.PageSize; + var result = new PaginatedListBase() { Total = total, Result = new() }; + if (total > 0 && start - total < 0) + { + var querySql = CombineOrs($"select ServiceName,{TIMSTAMP_KEY},TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}", ors, orderBy); + result.Result = Query(connection, $"select * from {querySql} as t limit {start},{query.PageSize}", parameters?.ToArray(), ConvertTraceDto); + } + return result; + } + + public static PaginatedListBase QueryLog(this IDbConnection connection, BaseRequestDto query) + { + var (where, parameters, ors) = AppendWhere(query, false); + var orderBy = AppendOrderBy(query, true); + var countSql = CombineOrs($"select count() as `total` from {MasaStackClickhouseConnection.LogTable} where {where}", ors); + var total = Convert.ToInt64(ExecuteScalar(connection, $"select sum(`total`) from {countSql}", parameters?.ToArray())); + var start = (query.Page - 1) * query.PageSize; + var result = new PaginatedListBase() { Total = total, Result = new() }; + + + if (total > 0 && start - total < 0) + { + var querySql = CombineOrs($"select {TIMSTAMP_KEY},TraceId,SpanId,TraceFlags,SeverityText,SeverityNumber,ServiceName,Body,Resources,Logs from {MasaStackClickhouseConnection.LogTable} where {where}", ors, orderBy); + result.Result = Query(connection, $"select * from {querySql} as t limit {start},{query.PageSize}", parameters?.ToArray(), ConvertLogDto); + } + return result; + } + + private static string CombineOrs(string sql, IEnumerable ors, string? orderBy = null) + { + if (ors == null || !ors.Any()) + return $"({sql} {orderBy})"; + + var text = new StringBuilder(); + foreach (var or in ors) + { + text.AppendLine($" union all {sql}{or} {orderBy}"); + } + text.Remove(0, 11).Insert(0, '(').Append(')'); + return text.ToString(); + } + + public static List GetMapping(this IDbConnection dbConnection, bool isLog) + { + var type = isLog ? "log" : "trace"; + var result = dbConnection.Query($"select DISTINCT Name from otel_mapping Array join Name where `Type`='{type}_basic' order by Name", default, ConvertToMapping); + if (result == null || result.Count == 0) + return default!; + + var attributes = dbConnection.Query($"select DISTINCT concat('{ATTRIBUTE_KEY}',Name) from otel_mapping Array join Name where `Type`='{type}_attributes' order by Name", default, ConvertToMapping); + var resources = dbConnection.Query($"select DISTINCT concat('{RESOURCE_KEY}',Name) from otel_mapping Array join Name where `Type`='resource' order by Name", default, ConvertToMapping); + if (attributes != null && attributes.Count > 0) result.AddRange(attributes); + if (resources != null && resources.Count > 0) result.AddRange(resources); + + return result; + } + + public static List GetTraceByTraceId(this IDbConnection connection, string traceId) + { + string where = $"TraceId=@TraceId"; + return Query(connection, $"select * from (select {TIMSTAMP_KEY},TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}) as t limit 1000", new IDataParameter[] { new ClickHouseParameter { ParameterName = "TraceId", Value = traceId } }, ConvertTraceDto); + } + + public static string AppendOrderBy(BaseRequestDto query, bool isLog) + { + var field = TIMSTAMP_KEY; + var isDesc = query.Sort?.IsDesc ?? true; + if (isLog && query.Sort != null && !string.IsNullOrEmpty(query.Sort.Name)) + { + field = GetName(query.Sort.Name, isLog); + isDesc = query.Sort?.IsDesc ?? false; + } + return $" order by {field}{(isDesc ? " desc" : "")}"; + } + + public static (string where, List @parameters, List ors) AppendWhere(BaseRequestDto query, bool isTrace = true) + { + var sql = new StringBuilder(); + var @paramerters = new List(); + + if (query.Start > DateTime.MinValue && query.Start < DateTime.MaxValue + && query.End > DateTime.MinValue && query.End < DateTime.MaxValue + && query.End > query.Start) + { + sql.Append($" and {TIMSTAMP_KEY} BETWEEN @Start and @End"); + @paramerters.Add(new ClickHouseParameter() { ParameterName = "Start", Value = MasaStackClickhouseConnection.ToTimeZone(query.Start), DbType = DbType.DateTime2 }); + @paramerters.Add(new ClickHouseParameter() { ParameterName = "End", Value = MasaStackClickhouseConnection.ToTimeZone(query.End), DbType = DbType.DateTime2 }); + } + if (!string.IsNullOrEmpty(query.Service)) + { + sql.Append(" and ServiceName=@ServiceName"); + @paramerters.Add(new ClickHouseParameter() { ParameterName = "ServiceName", Value = query.Service }); + } + if (!string.IsNullOrEmpty(query.Instance)) + { + sql.Append($" and `{RESOURCE_KEY}service.instance.id`=@ServiceInstanceId"); + @paramerters.Add(new ClickHouseParameter() { ParameterName = "ServiceInstanceId", Value = query.Instance }); + } + if (isTrace && !string.IsNullOrEmpty(query.Endpoint)) + { + sql.Append($" and `{ATTRIBUTE_KEY}http.target`=@HttpTarget"); + @paramerters.Add(new ClickHouseParameter() { ParameterName = "HttpTarget", Value = query.Endpoint }); + } + var ors = AppendKeyword(query.Keyword, paramerters, isTrace); + AppendConditions(query.Conditions, paramerters, sql, isTrace); + + if (!string.IsNullOrEmpty(query.RawQuery)) + sql.Append($" and ({query.RawQuery})"); + + if (sql.Length > 0) + sql.Remove(0, 4); + return (sql.ToString(), @paramerters, ors); + } + + private static List AppendKeyword(string keyword, List @paramerters, bool isTrace = true) + { + var sqls = new List(); + if (string.IsNullOrEmpty(keyword)) + return sqls; + + //status_code + if (int.TryParse(keyword, out var num) && num != 0 && num - 1000 < 0 && isTrace) + { + sqls.Add($" and `{ATTRIBUTE_KEY}http.status_code`=@HttpStatusCode"); + sqls.Add($" and `{ATTRIBUTE_KEY}http.request_content_body` like @Keyword"); + paramerters.Add(new ClickHouseParameter() { ParameterName = "HttpStatusCode", Value = num }); + paramerters.Add(new ClickHouseParameter() { ParameterName = "Keyword", Value = $"%{keyword}%" }); + return sqls; + } + + if (isTrace) + { + sqls.Add($" and `{ATTRIBUTE_KEY}http.request_content_body` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}http.response_content_body` like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}exception.message` like @Keyword"); + } + else + { + if (keyword.Equals("error", StringComparison.CurrentCultureIgnoreCase)) + sqls.Add(" and SeverityText='Error'"); + sqls.Add(" and Body like @Keyword"); + sqls.Add($" and `{ATTRIBUTE_KEY}exception.message` like @Keyword"); + } + paramerters.Add(new ClickHouseParameter() { ParameterName = "Keyword", Value = $"%{keyword}%" }); + return sqls; + } + + private static void AppendConditions(IEnumerable? conditions, List @paramerters, StringBuilder sql, bool isTrace = true) + { + if (conditions == null || !conditions.Any()) + return; + + foreach (var item in conditions) + { + var name = GetName(item.Name, !isTrace); + + if (item.Value is DateTime time) + { + item.Value = MasaStackClickhouseConnection.ToTimeZone(time); + } + if (item.Name.StartsWith(RESOURCE_KEY, StringComparison.CurrentCultureIgnoreCase)) + { + var filed = item.Name[RESOURCE_KEY.Length..]; + if (string.Equals(filed, "service.name")) + { + AppendField(item, @paramerters, sql, name, "ServiceName"); + } + else if (string.Equals(filed, "service.instance.id")) + { + AppendField(item, @paramerters, sql, name, "ServiceInstanceId"); + } + else if (string.Equals(filed, "service.namespace")) + { + AppendField(item, @paramerters, sql, name, "ServiceNameSpace"); + } + } + else if (item.Name.StartsWith(ATTRIBUTE_KEY, StringComparison.CurrentCultureIgnoreCase)) + { + var filed = item.Name[ATTRIBUTE_KEY.Length..]; + AppendField(item, @paramerters, sql, name, filed.Replace('.', '_')); + } + else + { + AppendField(item, @paramerters, sql, name, name); + } + } + } + + private static void AppendField(FieldConditionDto item, List @paramerters, StringBuilder sql, string fieldName, string paramName) + { + if (item.Value is string str && string.IsNullOrEmpty(str) || item.Value is IEnumerable collects && !collects.Any()) + return; + switch (item.Type) + { + case ConditionTypes.Equal: + { + if (@paramerters.Exists(p => p.ParameterName == paramName)) + break; + ParseWhere(sql, item.Value, paramerters, fieldName, paramName, "="); + } + break; + case ConditionTypes.NotIn: + { + ParseWhere(sql, item.Value, paramerters, fieldName, $"{paramName}s", "not in"); + } + break; + case ConditionTypes.In: + { + ParseWhere(sql, item.Value, paramerters, fieldName, $"{paramName}s", "in"); + } + break; + case ConditionTypes.LessEqual: + { + ParseWhere(sql, item.Value, paramerters, fieldName, $"lte_{paramName}", "<="); + } + break; + case ConditionTypes.GreatEqual: + { + ParseWhere(sql, item.Value, paramerters, fieldName, $"gte_{paramName}", ">="); + } + break; + case ConditionTypes.Less: + { + ParseWhere(sql, item.Value, paramerters, fieldName, $"lt_{paramName}", "<"); + } + break; + case ConditionTypes.Great: + { + ParseWhere(sql, item.Value, paramerters, fieldName, $"gt_{paramName}", ">"); + } + break; + } + } + + private static void ParseWhere(StringBuilder sql, object value, List @paramerters, string fieldName, string paramName, string compare) + { + DbType dbType = value is DateTime ? DbType.DateTime2 : DbType.AnsiString; + if (value is IEnumerable) + sql.Append($" and {fieldName} {compare} (@{paramName})"); + else + sql.Append($" and {fieldName} {compare} @{paramName}"); + @paramerters.Add(new ClickHouseParameter { ParameterName = $"{paramName}", Value = value, DbType = dbType }); + } + + public static object? ExecuteScalar(this IDbConnection dbConnection, string sql, IDataParameter[]? @parameters = null) + { + using var cmd = dbConnection.CreateCommand(); + cmd.CommandText = sql; + if (@parameters != null && @parameters.Length > 0) + foreach (var p in @parameters) + cmd.Parameters.Add(p); + OpenConnection(dbConnection); + try + { + return cmd.ExecuteScalar(); + } + catch (Exception ex) + { + MasaTscCliclhouseExtensitions.Logger?.LogError(ex, "execute sql error:{RawSql}, paramters:{Parameters}", sql, parameters); + throw; + } + } + + private static void OpenConnection(IDbConnection dbConnection) + { + if (dbConnection.State == ConnectionState.Closed) + dbConnection.Open(); + } + + public static List Query(this IDbConnection dbConnection, string sql, IDataParameter[]? @parameters, Func parse) + { + using var cmd = dbConnection.CreateCommand(); + cmd.CommandText = sql; + if (@parameters != null && @parameters.Length > 0) + foreach (var p in @parameters) + cmd.Parameters.Add(p); + OpenConnection(dbConnection); + try + { + using var reader = cmd.ExecuteReader(); + if (reader == null) + return new List(); + var list = new List(); + while (reader.NextResult()) + while (reader.Read()) + { + list.Add(parse.Invoke(reader)); + } + return list; + } + catch (Exception ex) + { + MasaTscCliclhouseExtensitions.Logger?.LogError(ex, "query sql error:{RawSql}, paramters:{Parameters}", sql, parameters); + throw; + } + } + + public static MappingResponseDto ConvertToMapping(IDataReader reader) + { + return new MappingResponseDto + { + Name = reader[0].ToString()!, + Type = "string" + }; + } + + public static TraceResponseDto ConvertTraceDto(IDataReader reader) + { + var startTime = Convert.ToDateTime(reader[TIMSTAMP_KEY]); + long ns = Convert.ToInt64(reader["Duration"]); + string resource = reader["Resources"].ToString()!, spans = reader["Spans"].ToString()!; + var result = new TraceResponseDto + { + TraceId = reader["TraceId"].ToString()!, + EndTimestamp = startTime.AddMilliseconds(ns / 1e6), + Kind = reader["SpanKind"].ToString()!, + Name = reader["SpanName"].ToString()!, + ParentSpanId = reader["ParentSpanId"].ToString()!, + SpanId = reader["SpanId"].ToString()!, + Timestamp = startTime + }; + if (!string.IsNullOrEmpty(resource)) + result.Resource = JsonSerializer.Deserialize>(resource)!; + if (!string.IsNullOrEmpty(spans)) + result.Attributes = JsonSerializer.Deserialize>(spans)!; + return result; + } + + public static LogResponseDto ConvertLogDto(IDataReader reader) + { + string resource = reader["Resources"].ToString()!, logs = reader["Logs"].ToString()!; + var result = new LogResponseDto + { + TraceId = reader["TraceId"].ToString()!, + Body = reader["Body"].ToString()!, + SeverityNumber = Convert.ToInt32(reader["SeverityNumber"]), + SeverityText = reader["SeverityText"].ToString()!, + TraceFlags = Convert.ToInt32(reader["TraceFlags"]), + SpanId = reader["SpanId"].ToString()!, + Timestamp = Convert.ToDateTime(reader[TIMSTAMP_KEY]), + }; + if (!string.IsNullOrEmpty(resource)) + result.Resource = JsonSerializer.Deserialize>(resource)!; + if (!string.IsNullOrEmpty(logs)) + result.Attributes = JsonSerializer.Deserialize>(logs)!; + return result; + } + + public static object AggregationQuery(this IDbConnection dbConnection, SimpleAggregateRequestDto requestDto, bool isLog = true) + { + var sql = new StringBuilder("select "); + var append = new StringBuilder(); + var appendWhere = new StringBuilder(); + var name = GetName(requestDto.Name, isLog); + AppendAggtype(requestDto, sql, append, name, out var isScalar); + sql.AppendFormat(" from {0} ", isLog ? MasaStackClickhouseConnection.LogTable : MasaStackClickhouseConnection.TraceTable); + var (where, @paremeters, _) = AppendWhere(requestDto, !isLog); + sql.Append($" where {appendWhere} {where}"); + sql.Append(append); + var paramArray = @paremeters?.ToArray()!; + + if (isScalar) + { + return dbConnection.ExecuteScalar(sql.ToString(), paramArray)!; + } + else + { + return AggTerm(dbConnection, sql.ToString(), paramArray, requestDto.Type, requestDto.AllValue); + } + } + + private static object AggTerm(IDbConnection dbConnection, string sql, IDataParameter[] paramArray, AggregateTypes aggregateTypes, bool isAllValue) + { + var result = dbConnection.Query(sql, paramArray, reader => + { + if (aggregateTypes == AggregateTypes.GroupBy) + { + if (isAllValue) + return KeyValuePair.Create(reader[0].ToString(), Convert.ToInt64(reader[1])); + else + return reader[0]; + } + else + { + var time = Convert.ToDateTime(reader[0]); + var timestamp = new DateTimeOffset(time).ToUnixTimeMilliseconds(); + return KeyValuePair.Create(timestamp, Convert.ToInt64(reader[1])); + } + }); + if (aggregateTypes == AggregateTypes.GroupBy) + { + if (isAllValue) + return result.Select(item => (KeyValuePair)item).ToList(); + else + return result.Select(item => item.ToString()).ToList(); + } + return result; + } + + private static void AppendAggtype(SimpleAggregateRequestDto requestDto, StringBuilder sql, StringBuilder append, string name, out bool isScalar) + { + isScalar = false; + switch (requestDto.Type) + { + case AggregateTypes.Avg: + sql.Append($"AVG({name}) as a"); + isScalar = true; + break; + case AggregateTypes.Count: + sql.Append($"Count({name}) as a"); + isScalar = true; + break; + case AggregateTypes.DistinctCount: + sql.Append($"Count(DISTINCT {name}) as a"); + isScalar = true; + break; + case AggregateTypes.Sum: + sql.Append($"SUM({name}) as a"); + isScalar = true; + break; + case AggregateTypes.GroupBy: + sql.Append($"{name} as a,Count({name}) as b"); + append.Append($" and a<>'' Group By a order by b desc"); + break; + case AggregateTypes.DateHistogram: + sql.Append($"toStartOfInterval({name}, INTERVAL {ConvertInterval(requestDto.Interval)} minute ) as `time`,count() as `count`"); + append.Append($" Group by `time` order by `time`"); + break; + } + } + + private static string GetName(string name, bool isLog) + { + if (name.Equals("@timestamp", StringComparison.CurrentCultureIgnoreCase)) + return TIMSTAMP_KEY; + + if (!isLog && name.Equals("duration", StringComparison.CurrentCultureIgnoreCase)) + return "Duration"; + + if (!isLog && name.Equals("kind", StringComparison.InvariantCultureIgnoreCase)) + return "SpanKind"; + + if (name.StartsWith(RESOURCE_KEY, StringComparison.CurrentCultureIgnoreCase)) + return GetResourceName(name); + + if (name.StartsWith(ATTRIBUTE_KEY, StringComparison.CurrentCultureIgnoreCase)) + return GetAttributeName(name, isLog); + + return name; + } + + private static string GetResourceName(string name) + { + var field = name[(RESOURCE_KEY.Length)..]; + if (field.Equals("service.name", StringComparison.CurrentCultureIgnoreCase)) + return "ServiceName"; + + if (field.Equals("service.namespace", StringComparison.CurrentCultureIgnoreCase) || field.Equals("service.instance.id", StringComparison.CurrentCultureIgnoreCase)) + return $"{RESOURCE_KEY}{field}"; + + return $"ResourceAttributesValues[indexOf(ResourceAttributesKeys,'{field}')]"; + } + + private static string GetAttributeName(string name, bool isLog) + { + var pre = isLog ? "Log" : "Span"; + var field = name[(ATTRIBUTE_KEY.Length)..]; + if (isLog && (field.Equals("exception.message", StringComparison.CurrentCultureIgnoreCase))) + return $"{ATTRIBUTE_KEY}{field}"; + + if (!isLog && (field.Equals("http.status_code", StringComparison.CurrentCultureIgnoreCase) + || field.Equals("http.request_content_body", StringComparison.CurrentCultureIgnoreCase) + || field.Equals("http.response_content_body", StringComparison.CurrentCultureIgnoreCase) + || field.Equals("exception.message", StringComparison.CurrentCultureIgnoreCase) + || field.Equals("http.target", StringComparison.CurrentCultureIgnoreCase)) + ) + return $"{ATTRIBUTE_KEY}{field}"; + + return $"{pre}AttributesValues[indexOf({pre}AttributesKeys,'{field}')]"; + } + + public static int ConvertInterval(string s) + { + var unit = Regex.Replace(s, @"\d+", "", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(5)); + int t = 1; + switch (unit) + { + case "s": + t = 1; + break; + case "m": + t = 60; + break; + case "h": + t = 3600; + break; + case "d": + t = 3600 * 24; + break; + case "w": + t = 3600 * 24 * 7; + break; + case "month": + t = 3600 * 24 * 30; + break; + } + var num = Convert.ToInt64(s.Replace(unit, "")); + num *= t; + if (num - 60 < 0) + return 1; + return (int)(num / 60); + } + + public static string GetMaxDelayTraceId(this IDbConnection dbConnection, BaseRequestDto requestDto) + { + var (where, parameters, _) = AppendWhere(requestDto); + var text = $"select * from( TraceId from {MasaStackClickhouseConnection.TraceTable} where {where} order by Duration desc) as t limit 1"; + return dbConnection.ExecuteScalar(text, parameters?.ToArray())?.ToString()!; + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs new file mode 100644 index 00000000..ae03771a --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Extensions/MasaTscCliclhouseExtensitions.cs @@ -0,0 +1,21 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +using Masa.Contrib.StackSdks.Tsc.Clickhouse.Extensions; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class MasaTscCliclhouseExtensitions +{ + internal static ILogger? Logger { get; private set; } + + public static IServiceCollection AddMASAStackClickhouse(this IServiceCollection services, string connectionStr, string suffix = "masastack", string? logSourceTable = null, string? traceSourceTable = null, Action? configer = null) + { + services.AddScoped(services => new MasaStackClickhouseConnection(connectionStr, suffix, logSourceTable, traceSourceTable)) + .AddScoped() + .AddScoped(); + ClickhouseInit.Init(services); + configer?.Invoke(services.BuildServiceProvider().GetRequiredService()!); + return services; + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/LogService.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/LogService.cs new file mode 100644 index 00000000..672a45da --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/LogService.cs @@ -0,0 +1,28 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Clickhouse; + +internal class LogService : ILogService +{ + private readonly IDbConnection _dbConnection; + public LogService(MasaStackClickhouseConnection connection) + { + _dbConnection = connection; + } + + public Task AggregateAsync(SimpleAggregateRequestDto query) + { + return Task.FromResult(_dbConnection.AggregationQuery(query)); + } + + public Task> ListAsync(BaseRequestDto query) + { + return Task.FromResult(_dbConnection.QueryLog(query)); + } + + public Task> GetMappingAsync() + { + return Task.FromResult(_dbConnection.GetMapping(true).AsEnumerable()); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj new file mode 100644 index 00000000..eaebc0b3 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs new file mode 100644 index 00000000..e53d3ad3 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/Model/MASAStackClickhouseConnection.cs @@ -0,0 +1,46 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse")] +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests")] +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Tests")] +namespace Masa.Contrib.StackSdks.Tsc.Clickhouse; + +internal sealed class MasaStackClickhouseConnection : ClickHouseConnection +{ + public static string LogSourceTable { get; private set; } + + public static string TraceSourceTable { get; private set; } + + public static string LogTable { get; private set; } + + public static string TraceTable { get; private set; } + + public static string MappingTable { get; private set; } + + public static TimeZoneInfo TimeZone { get; set; } + + public static DateTime ToTimeZone(DateTime time) + { + var newTime = time.Kind == DateTimeKind.Unspecified ? time : DateTime.SpecifyKind(time, DateTimeKind.Unspecified); + return new DateTimeOffset(newTime + TimeZone.BaseUtcOffset, TimeZone.BaseUtcOffset).DateTime; + } + + public object LockObj { get; init; } = new(); + + public MasaStackClickhouseConnection(string connection, string suffix, string? logSourceTable = null, string? traceSourceTable = null) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(suffix); + ConnectionString = connection; + logSourceTable ??= "otel_logs"; + traceSourceTable ??= "otel_traces"; + + string database = string.IsNullOrEmpty(ConnectionSettings.Database) ? default! : $"{ConnectionSettings.Database}."; + + LogTable = $"{database}{logSourceTable}_{suffix}"; + TraceTable = $"{database}{traceSourceTable}_{suffix}"; + TraceSourceTable = $"{database}{traceSourceTable}"; + LogSourceTable = $"{database}{logSourceTable}"; + MappingTable = $"{database}otel_mapping_{suffix}"; + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/TraceService.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/TraceService.cs new file mode 100644 index 00000000..8742c5d2 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/TraceService.cs @@ -0,0 +1,39 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Clickhouse; + +internal class TraceService : ITraceService +{ + private readonly IDbConnection _dbConnection; + + public TraceService(MasaStackClickhouseConnection connection) + { + _dbConnection = connection; + } + + public Task AggregateAsync(SimpleAggregateRequestDto query) + { + return Task.FromResult(_dbConnection.AggregationQuery(query, false)); + } + + public Task> GetAsync(string traceId) + { + return Task.FromResult(_dbConnection.GetTraceByTraceId(traceId).AsEnumerable()); + } + + public Task GetMaxDelayTraceIdAsync(BaseRequestDto query) + { + return Task.FromResult(_dbConnection.GetMaxDelayTraceId(query)); + } + + public Task> ListAsync(BaseRequestDto query) + { + return Task.FromResult(_dbConnection.QueryTrace(query)); + } + + public Task> ScrollAsync(BaseRequestDto query) + { + return Task.FromResult(_dbConnection.QueryTrace(query)); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs new file mode 100644 index 00000000..51f1b7ee --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Clickhouse/_Imports.cs @@ -0,0 +1,21 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using ClickHouse.Ado; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Log; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Aggregate; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; +global using Masa.Contrib.StackSdks.Tsc.Clickhouse; +global using Masa.Utils.Models; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using System.Collections; +global using System.Data; +global using System.Data.Common; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.RegularExpressions; +global using TimeZoneConverter; diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs new file mode 100644 index 00000000..4d8bf89b --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs @@ -0,0 +1,40 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Constants; + +public static class ElasticConstant +{ + internal const string LOG_CALLER_CLIENT_NAME = "masa.contrib.stacksdks.tsc.log.elasticseach.log"; + internal const string TRACE_CALLER_CLIENT_NAME = "masa.contrib.stacksdks.tsc.log.elasticseach.trace"; + internal const string DEFAULT_CALLER_CLIENT_NAME = "masa.contrib.stacksdks.tsc.log.elasticseach.all"; + + private const string TIMESTAMP = "@timestamp"; + public static string TraceId => "TraceId"; + public static string ParentId => "ParentSpanId"; + public static string SpanId => "SpanId"; + public static string ServiceName => "Resource.service.name"; + public static string ServiceInstance => "Resource.service.instance.id"; + public static string NameSpace => "Resource.service.namespace"; + public static string Endpoint => "Attributes.http.url"; + + internal static int MaxRecordCount { get; private set; } = 10000; + + public static LogTraceSetting Log { get; private set; } + + public static LogTraceSetting Trace { get; private set; } + + internal static void InitLog(string indexName, bool isIndependent = false) + { + if (string.IsNullOrEmpty(indexName)) + return; + Log = new LogTraceSetting(indexName, isIndependent, TIMESTAMP); + } + + internal static void InitTrace(string indexName, bool isIndependent = false) + { + if (string.IsNullOrEmpty(indexName)) + return; + Trace = new LogTraceSetting(indexName, isIndependent, TIMESTAMP); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/LogTraceSetting.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/LogTraceSetting.cs new file mode 100644 index 00000000..076ab002 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/LogTraceSetting.cs @@ -0,0 +1,25 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Constants; + +public class LogTraceSetting +{ + internal string IndexName { get; private set; } + + public string Timestamp { get; private set; } + + internal bool IsIndependent { get; private set; } + + internal Lazy Mappings { get; set; } + + internal LogTraceSetting(string indexName, bool isIndependent = false, string timestamp = "@timestamp") + { + if (!string.IsNullOrEmpty(IndexName) || string.IsNullOrEmpty(indexName)) + return; + + IndexName = indexName; + Timestamp = timestamp; + IsIndependent = isIndependent; + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/MappingConstant.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/MappingConstant.cs new file mode 100644 index 00000000..9e804a8c --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/MappingConstant.cs @@ -0,0 +1,16 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Constants; + +internal static class MappingConstant +{ + public const string PROPERTY = "properties"; + public const string TYPE = "type"; + public const string FIELD = "fields"; + /// + /// when field is keyword, the maximum field value length, fields beyond this length will not be indexed, but will be stored + /// + public const string MAXLENGTH = "ignore_above"; + public const string KEYWORD = "keyword"; +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/LogResponseDtoConverter.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/LogResponseDtoConverter.cs new file mode 100644 index 00000000..e9ab73e8 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/LogResponseDtoConverter.cs @@ -0,0 +1,35 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests")] +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Converters; + +internal class LogResponseDtoConverter : JsonConverter +{ + public override LogResponseDto? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (JsonDocument.TryParseValue(ref reader, out var doc)) + { + var jsonObject = doc.RootElement; + var rootText = jsonObject.GetRawText(); + var result= JsonSerializer.Deserialize(rootText); + if (result == null) + return default; + if (result.Timestamp == DateTime.MinValue || result.Timestamp == DateTime.MaxValue) + return default; + if (result.Body == null) + return default; + + result.Attributes = jsonObject.ConvertToKeyValuePairs()?.ToDictionary(kv => kv.Key, kv => kv.Value)?.GroupByKeyPrefix("Attributes.")!; + result.Resource = jsonObject.ConvertToKeyValuePairs()?.ToDictionary(kv => kv.Key, kv => kv.Value)?.GroupByKeyPrefix("Resource.")!; + + return result; + } + + throw new NotImplementedException(); + } + + public override void Write(Utf8JsonWriter writer, LogResponseDto value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, options); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/TraceResponseDtoConverter.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/TraceResponseDtoConverter.cs new file mode 100644 index 00000000..e1d6bd87 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Converters/TraceResponseDtoConverter.cs @@ -0,0 +1,35 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests")] +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Converters; + +internal class TraceResponseDtoConverter : JsonConverter +{ + public override TraceResponseDto? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (JsonDocument.TryParseValue(ref reader, out var doc)) + { + var jsonObject = doc.RootElement; + var rootText = jsonObject.GetRawText(); + var result = JsonSerializer.Deserialize(rootText); + if (result == null) + return default; + if (result.Timestamp == DateTime.MinValue || result.Timestamp == DateTime.MaxValue) + return default; + if (string.IsNullOrEmpty(result.TraceId)) + return default; + + result.Attributes = jsonObject.ConvertToKeyValuePairs().ToDictionary(kv => kv.Key, kv => kv.Value).GroupByKeyPrefix("Attributes."); + result.Resource = jsonObject.ConvertToKeyValuePairs().ToDictionary(kv => kv.Key, kv => kv.Value).GroupByKeyPrefix("Resource."); + + return result; + } + + throw new NotImplementedException(); + } + + public override void Write(Utf8JsonWriter writer, TraceResponseDto value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, options); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs new file mode 100644 index 00000000..66f11267 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs @@ -0,0 +1,504 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Nest; + +internal static class IElasticClientExtenstion +{ + public static async Task SearchAsync(this IElasticClient client, + string indexName, + TQuery query, + Func, SearchDescriptor> searchDescriptorFunc, + Action, TQuery> resultFunc) where TResult : class where TQuery : class + { + try + { + if (resultFunc is null) return; + ISearchResponse searchResponse; + if (query is ElasticsearchScrollRequestDto scrollQuery) + searchResponse = string.IsNullOrEmpty(scrollQuery.ScrollId) ? + await client.SearchAsync(s => searchDescriptorFunc(s.Index(indexName)).Scroll(scrollQuery.Scroll)) : + await client.ScrollAsync(scrollQuery.Scroll, scrollQuery.ScrollId); + else + searchResponse = await client.SearchAsync(s => searchDescriptorFunc(s.Index(indexName))); + + searchResponse.FriendlyElasticException(); + if (searchResponse.IsValid) + resultFunc.Invoke(searchResponse, query); + } + catch (Exception ex) + { + throw new UserFriendlyException($"SearchAsync execute error", ex); + } + } + + #region mapping + + public static async Task> GetMappingAsync(this ICaller caller, string indexName, CancellationToken token = default) + { + var path = $"/{indexName}/_mapping"; + var result = await caller.GetAsync(path, token); + var json = (JsonElement)result!; + foreach (var item in json.EnumerateObject()) + { + JsonElement mapping; + if (item.Value.TryGetProperty("mappings", out mapping)) + { + var mappings = GetRepProperties(mapping, default!)!; + if (mappings != null && mappings.Any()) + return mappings; + } + } + + throw new UserFriendlyException($"can't find mapping for index: {indexName}"); + } + + private static IEnumerable? GetRepProperties(JsonElement node, string? parentName = default) + { + if (node.ValueKind != JsonValueKind.Object) + return default; + + var properties = GetProperties(node); + if (properties == null || properties.Value.ValueKind != JsonValueKind.Object) + return default; + + var result = new List(); + var obj = properties.Value.EnumerateObject(); + foreach (var item in obj) + { + var type = GetType(item.Value); + var name = $"{parentName}{item.Name}"; + if (string.IsNullOrEmpty(type)) + { + var children = GetRepProperties(item.Value, $"{name}."); + if (children != null && children.Any()) + result.AddRange(children); + } + else + { + var model = new ElasticseacherMappingResponseDto + { + Name = name, + Type = type + }; + SetKeyword(item.Value, model); + result.Add(model); + } + } + return result; + } + + private static JsonElement? GetProperties(JsonElement value) + { + if (value.TryGetProperty(MappingConstant.PROPERTY, out JsonElement result)) + return result; + return null; + } + + private static string? GetType(JsonElement value) + { + if (value.TryGetProperty(MappingConstant.TYPE, out JsonElement element)) + return element.ToString(); + return default; + } + + private static void SetKeyword(JsonElement value, ElasticseacherMappingResponseDto model) + { + if (value.TryGetProperty(MappingConstant.FIELD, out JsonElement fields) && + fields.TryGetProperty(MappingConstant.KEYWORD, out JsonElement element)) + { + if (element.TryGetProperty(MappingConstant.TYPE, out JsonElement type) && type.ToString() == MappingConstant.KEYWORD) + model.IsKeyword = true; + if (element.TryGetProperty(MappingConstant.MAXLENGTH, out JsonElement maxLength)) + model.MaxLenth = maxLength.GetInt32(); + } + } + + public static void FriendlyElasticException(this ISearchResponse response) where T : class + { + if (!response.IsValid) + throw new UserFriendlyException($"elastic query error: status: {response.ServerError?.Status}, message: {response.OriginalException?.Message ?? response.ServerError?.ToString()}, DebugInformation: {response.DebugInformation}"); + } + + #endregion + + #region log + + public static async Task> SearchLogAsync(this IElasticClient client, BaseRequestDto query) + { + PaginatedListBase result = default!; + await client.SearchAsync(ElasticConstant.Log.IndexName, query, + (SearchDescriptor searchDescriptor) => searchDescriptor.AddCondition(query, true) + .AddSort(query) + .AddPageSize(query, true), + (response, q) => result = SetLogResult(response)); + return result; + } + + public static async Task AggregateLogAsync(this IElasticClient client, SimpleAggregateRequestDto query) + { + object result = default!; + await client.SearchAsync(ElasticConstant.Log.IndexName, query, + (SearchDescriptor searchDescriptor) => searchDescriptor.AddCondition(query, true) + .AddSort(query) + .AddPageSize(query, false) + .AddAggregate(query, true), + (response, q) => result = SetAggregationResult(response, q)); + return result; + } + + #endregion + + #region trace + public static async Task> SearchTraceAsync(this IElasticClient client, BaseRequestDto query) + { + PaginatedListBase result = default!; + await client.SearchAsync(ElasticConstant.Trace.IndexName, query, + (SearchDescriptor searchDescriptor) => searchDescriptor.AddCondition(query, false) + .AddSort(query, false) + .AddPageSize(query, true), + (response, q) => + { + result = q is ElasticsearchScrollRequestDto ? SetTraceScrollResult(response) : SetTraceResult(response); + }); + return result; + } + + public static async Task AggregateTraceAsync(this IElasticClient client, SimpleAggregateRequestDto query) + { + object result = default!; + await client.SearchAsync(ElasticConstant.Trace.IndexName, query, + (SearchDescriptor searchDescriptor) => searchDescriptor.AddCondition(query, false) + .AddSort(query, false) + .AddPageSize(query, false) + .AddAggregate(query, false), + (response, q) => result = SetAggregationResult(response, q)); + return result; + } + + public static async Task GetMaxDelayTraceIdAsync(this IElasticClient client, BaseRequestDto query) + { + string traceId = default!; + await client.SearchAsync(ElasticConstant.Trace.IndexName, query, + (SearchDescriptor searchDescriptor) => searchDescriptor.AddCondition(query, false) + .Sort(sort => sort.Script(script => script.Order(SortOrder.Descending).Script(s => s.Source("doc['EndTimestamp'].value.toEpochSecond()-doc['@timestamp'].value.toEpochSecond()")).Type("number"))) + .Size(1), + (response, q) => + { + var result = SetTraceResult(response); + traceId = result.Result.FirstOrDefault()?.TraceId!; + }); + + return traceId; + } + #endregion + + #region set condition + + private static SearchDescriptor AddCondition(this SearchDescriptor searchDescriptor, TQuery query, bool isLog) where TQuery : BaseRequestDto where TResult : class + { + var list = new List, QueryContainer>>(); + string timestamp = isLog ? ElasticConstant.Log.Timestamp : ElasticConstant.Trace.Timestamp; + var mappings = isLog ? ElasticConstant.Log.Mappings.Value : ElasticConstant.Trace.Mappings.Value; + + if (!string.IsNullOrEmpty(query.RawQuery)) + { + list.Add(queryContainer => queryContainer.Raw(query.RawQuery)); + } + if (query.Start > DateTime.MinValue && query.End > DateTime.MinValue && query.Start < query.End) + { + list.Add(queryContainer => queryContainer.DateRange(dateRangeQuery => dateRangeQuery.GreaterThanOrEquals(query.Start).LessThanOrEquals(query.End).Field(timestamp))); + } + if (!string.IsNullOrEmpty(query.Keyword)) + { + list.Add(queryContainer => queryContainer.QueryString(queryString => queryString.Query(query.Keyword).DefaultOperator(Operator.And))); + } + + query.AppendConditions(); + var conditions = AddFilter(query); + if (conditions != null && conditions.Any()) + { + foreach (var item in conditions) + { + var mapping = mappings.FirstOrDefault(m => string.Equals(m.Name, item.Name, StringComparison.OrdinalIgnoreCase)); + list.Add(CompareCondition(mapping, item)); + } + } + + if (list.Any()) + { + return searchDescriptor.Query(container => container.Bool(boolQuery => boolQuery.Must(list))); + } + + return searchDescriptor; + } + + private static IEnumerable AddFilter(TQuery query) where TQuery : BaseRequestDto + { + var result = query.Conditions?.ToList() ?? new(); + if (!string.IsNullOrEmpty(query.Service)) + result.Add(new FieldConditionDto + { + Name = ElasticConstant.ServiceName, + Value = query.Service, + Type = ConditionTypes.Equal + }); + if (!string.IsNullOrEmpty(query.Instance)) + result.Add(new FieldConditionDto + { + Name = ElasticConstant.ServiceInstance, + Value = query.Instance, + Type = ConditionTypes.Equal + }); + if (!string.IsNullOrEmpty(query.Endpoint)) + result.Add(new FieldConditionDto + { + Name = ElasticConstant.Endpoint, + Value = $"*{query.Endpoint}*", + Type = ConditionTypes.Regex + }); + if (!string.IsNullOrEmpty(query.TraceId)) + result.Add(new FieldConditionDto + { + Name = ElasticConstant.TraceId, + Value = query.TraceId, + Type = ConditionTypes.Equal + }); + return result; + } + + private static Func, QueryContainer> CompareCondition(ElasticseacherMappingResponseDto? mapping, FieldConditionDto query) where TResult : class + { + CreateFieldKeyword(query.Name, mapping, out var fieldName, out var keyword); + var value = query.Value; + bool isDate = value is DateTime; + + switch (query.Type) + { + case ConditionTypes.Equal: + return (container) => container.Match(f => f.Field(keyword).Query(value?.ToString())); + case ConditionTypes.NotEqual: + return (container) => !container.Match(f => f.Field(keyword).Query(value?.ToString())); + case ConditionTypes.Great: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).GreaterThan((DateTime)value)); + return (container) => container.Range(f => f.Field(keyword).GreaterThan(Convert.ToDouble(value))); + case ConditionTypes.Less: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).LessThan((DateTime)value)); + return (container) => container.Range(f => f.Field(keyword).LessThan(Convert.ToDouble(value))); + case ConditionTypes.GreatEqual: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).GreaterThanOrEquals((DateTime)value)); + return (container) => container.Range(f => f.Field(keyword).GreaterThanOrEquals(Convert.ToDouble(value))); + case ConditionTypes.LessEqual: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).LessThanOrEquals((DateTime)value)); + return (container) => container.Range(f => f.Field(keyword).LessThanOrEquals(Convert.ToDouble(value))); + case ConditionTypes.In: + return (container) => container.Terms(f => f.Field(keyword).Terms((IEnumerable)value)); + case ConditionTypes.NotIn: + return (container) => !container.Terms(f => f.Field(keyword).Terms((IEnumerable)value)); + case ConditionTypes.Exists: + return (container) => container.Exists(f => f.Field(fieldName)); + case ConditionTypes.NotExists: + return (container) => !container.Exists(f => f.Field(fieldName)); + case ConditionTypes.Regex: + return (container) => container.Wildcard(f => f.Field(fieldName).Value(value?.ToString())); + case ConditionTypes.NotRegex: + return (container) => !container.Wildcard(f => f.Field(fieldName).Value(value?.ToString())); + } + return (container) => container; + } + + #endregion + + #region set page + + private static SearchDescriptor AddPageSize(this SearchDescriptor container, TQuery query, bool hasPage) where TResult : class where TQuery : BaseRequestDto + { + if (query.Page <= 0) + query.Page = 1; + if (query.PageSize <= 0) + query.PageSize = 20; + if (query is ElasticsearchScrollRequestDto || !hasPage) + return container.Size(query.PageSize); + + var start = (query.Page - 1) * query.PageSize; + if (ElasticConstant.MaxRecordCount - start - query.PageSize <= 0) + throw new UserFriendlyException($"elastic query data max count must be less {ElasticConstant.MaxRecordCount}, please input more condition to limit"); + return container.Size(query.PageSize).From(start); + + } + #endregion + + #region set sort + + private static SearchDescriptor AddSort(this SearchDescriptor searchDescriptor, TQuery query, bool isLog = true) where TQuery : BaseRequestDto where TResult : class + { + LogTraceSetting setting = isLog ? ElasticConstant.Log : ElasticConstant.Trace; + + if (query.Sort == null) + return searchDescriptor.Sort(sort => sort.Descending(setting.Timestamp)); + + var mapping = setting.Mappings.Value.FirstOrDefault(m => string.Equals(m.Name, query.Sort.Name, StringComparison.OrdinalIgnoreCase)); + CreateFieldKeyword(query.Sort.Name, mapping, out var field, out var keyword); + + Func, SortDescriptor> sortFunc; + if (query.Sort.IsDesc) + sortFunc = (sort) => sort.Descending(keyword); + else + sortFunc = (sort) => sort.Ascending(keyword); + + return searchDescriptor.Sort(sort => sortFunc(sort)); + } + + #endregion + + #region set result + private static PaginatedListBase SetLogResult(ISearchResponse response) + { + var options = new JsonSerializerOptions(); + options.Converters.Add(new LogResponseDtoConverter()); + var text = JsonSerializer.Serialize(response.Documents); + + return new PaginatedListBase { Total = response.Total, Result = JsonSerializer.Deserialize>(text, options)! }; + } + + private static PaginatedListBase SetTraceResult(ISearchResponse response) + { + var options = new JsonSerializerOptions(); + options.Converters.Add(new TraceResponseDtoConverter()); + var text = JsonSerializer.Serialize(response.Documents); + return new PaginatedListBase { Total = response.Total, Result = JsonSerializer.Deserialize>(text, options)! }; + } + + private static ElasticsearchScrollResponseDto SetTraceScrollResult(ISearchResponse response) + { + var options = new JsonSerializerOptions(); + options.Converters.Add(new TraceResponseDtoConverter()); + var text = JsonSerializer.Serialize(response.Documents); + return new ElasticsearchScrollResponseDto { Total = response.Total, Result = JsonSerializer.Deserialize>(text, options)!, ScrollId = response.ScrollId }; + } + + #endregion + + #region aggregate + + private static SearchDescriptor AddAggregate(this SearchDescriptor searchDescriptor, TQuery aggModel, bool isLog) where TResult : class where TQuery : SimpleAggregateRequestDto + { + Func, IAggregationContainer>? aggregateFunc = null; + var mappings = isLog ? ElasticConstant.Log.Mappings.Value : ElasticConstant.Trace.Mappings.Value; + var mapping = mappings.FirstOrDefault(m => string.Equals(m.Name, aggModel.Name, StringComparison.OrdinalIgnoreCase)); + CreateFieldKeyword(aggModel.Name, mapping, out var field, out var keyword); + string aliasName = aggModel.Alias ?? field; + + switch (aggModel.Type) + { + case AggregateTypes.Count: + { + aggregateFunc = (aggContainer) => aggContainer.ValueCount(aliasName, agg => agg.Field(keyword)); + } + break; + case AggregateTypes.Sum: + { + aggregateFunc = (aggContainer) => aggContainer.Sum(aliasName, agg => agg.Field(field)); + } + break; + case AggregateTypes.Avg: + { + aggregateFunc = (aggContainer) => aggContainer.Average(aliasName, agg => agg.Field(field)); + } + break; + case AggregateTypes.DistinctCount: + { + aggregateFunc = (aggContainer) => aggContainer.Cardinality(aliasName, agg => agg.Field(keyword)); + } + break; + case AggregateTypes.DateHistogram: + { + if (mapping != null && mapping.Type != "date") + { + throw new UserFriendlyException($"Field of type [{field}] is not supported for aggregation [date_histogram]"); + } + aggregateFunc = (aggContainer) => aggContainer.DateHistogram(aliasName, agg => agg.Field(keyword).FixedInterval(new Time(aggModel.Interval))); + } + break; + case AggregateTypes.GroupBy: + { + aggregateFunc = (aggContainer) => aggContainer.Terms(aliasName, agg => agg.Field(keyword).Size(aggModel.MaxCount)); + } + break; + } + if (aggregateFunc != null) + searchDescriptor = searchDescriptor.Aggregations(aggContainer => aggregateFunc(aggContainer)); + + return searchDescriptor; + } + + private static object SetAggregationResult(ISearchResponse response, SimpleAggregateRequestDto aggModel) + { + if (response.Aggregations == null || !response.Aggregations.Any()) + return default!; + + foreach (var item in response.Aggregations.Values) + { + if (item is ValueAggregate value) + { + return GetDouble(value); + } + else if (item is BucketAggregate bucketAggregate) + { + return GetBucketValue(bucketAggregate, aggModel.Type, aggModel.AllValue); + } + } + return default!; + } + + private static double GetDouble(ValueAggregate value) + { + return value.Value ?? default; + } + + private static object GetBucketValue(BucketAggregate value, AggregateTypes type, bool isAll) + { + if (type == AggregateTypes.GroupBy) + { + if (isAll) + return value.Items.Select(it => KeyValuePair.Create(((KeyedBucket)it).Key.ToString(), ((KeyedBucket)it).DocCount)).ToList(); + else + return value.Items.Select(it => ((KeyedBucket)it).Key.ToString()).ToList(); + } + + else if (type == AggregateTypes.DateHistogram) + { + var result = new List>(); + foreach (var bucket in value.Items) + { + var dateHistogramBucket = (DateHistogramBucket)bucket; + result.Add(KeyValuePair.Create(dateHistogramBucket.Key, (dateHistogramBucket.DocCount ?? 0))); + } + return result; + } + return default!; + } + + private static void CreateFieldKeyword(string name, ElasticseacherMappingResponseDto? mapping, out string field, out string keyword) + { + if (mapping == null) + { + field = name; + keyword = name; + } + else + { + field = mapping.Name; + if (mapping.Type == "text" && mapping.IsKeyword.HasValue && mapping.IsKeyword.Value) + keyword = $"{field}.keyword"; + else + keyword = field; + } + } + + #endregion +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/JsonElementExtensions.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/JsonElementExtensions.cs new file mode 100644 index 00000000..4ec310c1 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/JsonElementExtensions.cs @@ -0,0 +1,75 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. +[assembly: InternalsVisibleTo("Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests")] +namespace System; + +internal static class JsonElementExtensions +{ + public static IEnumerable> ConvertToKeyValuePairs(this JsonElement value) + { + if (value.ValueKind != JsonValueKind.Object) + return default!; + + return GetObject(value); + } + + public static object GetValue(this JsonElement value) + { + return value.ValueKind switch + { + JsonValueKind.Object => GetObject(value), + JsonValueKind.Array => GetArray(value), + JsonValueKind.String => value.GetString()!, + JsonValueKind.Number => GetNumber(value), + JsonValueKind.True or JsonValueKind.False => value.GetBoolean(), + _ => default!, + }; + } + +#pragma warning disable S6444 + private static object GetNumber(JsonElement value) + { + var str = value.GetRawText(); + + if (Regex.IsMatch(str, @"\.")) + { + return value.GetDouble(); + } + else + { + if (!value.TryGetInt32(out int num)) + return value.GetInt64(); + return num; + } + } +#pragma warning restore S6444 + + private static IEnumerable> GetObject(JsonElement value) + { + var result = new Dictionary(); + foreach (var item in value.EnumerateObject()) + { + var v = GetValue(item.Value); + if (v == null) + continue; + result.Add(item.Name, v); + } + if (result.Any()) + return result; + return default!; + } + + public static IEnumerable GetArray(this JsonElement value) + { + var temp = value.EnumerateArray(); + if (!temp.Any()) + return default!; + var list = new List(); + foreach (var item in temp) + { + var v = GetValue(item); + list.Add(v); + } + return list; + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs new file mode 100644 index 00000000..e95ad673 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs @@ -0,0 +1,139 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +// ReSharper disable once CheckNamespace + +namespace Microsoft.Extensions.DependencyInjection; + +public static class ServiceExtenistion +{ + public static IServiceCollection AddElasticClientLog(this IServiceCollection services, string[] nodes, string indexName) + { + ElasticConstant.InitLog(indexName, true); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, nodes, ElasticConstant.LOG_CALLER_CLIENT_NAME).AddScoped(); + ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.LOG_CALLER_CLIENT_NAME, indexName); + return services; + } + + public static IServiceCollection AddElasticClientLog(this IServiceCollection services, + Action elasearchConnectionAction, Action callerAction, string indexName) + { + ElasticConstant.InitLog(indexName, true); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.LOG_CALLER_CLIENT_NAME) + .AddScoped(); + ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.LOG_CALLER_CLIENT_NAME, indexName); + return services; + } + + public static IServiceCollection AddElasticClientTrace(this IServiceCollection services, string[] nodes, string indexName) + { + ElasticConstant.InitTrace(indexName, true); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, nodes, ElasticConstant.TRACE_CALLER_CLIENT_NAME).AddScoped(); + ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.TRACE_CALLER_CLIENT_NAME, indexName); + return services; + } + + public static IServiceCollection AddElasticClientTrace(this IServiceCollection services, + Action elasearchConnectionAction, Action callerAction, string indexName) + { + ElasticConstant.InitTrace(indexName, true); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.TRACE_CALLER_CLIENT_NAME) + .AddScoped(); + ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.TRACE_CALLER_CLIENT_NAME, indexName); + return services; + } + + public static IServiceCollection AddElasticClientLogAndTrace(this IServiceCollection services, string[] nodes, string logIndexName, + string traceIndexName) + { + ElasticConstant.InitLog(logIndexName); + ElasticConstant.InitTrace(traceIndexName); + if (services.BuildServiceProvider().GetService() == null || services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, nodes, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME) + .AddScoped() + .AddScoped(); + ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, logIndexName); + ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, traceIndexName); + return services; + } + + public static IServiceCollection AddElasticClientLogAndTrace(this IServiceCollection services, + Action elasearchConnectionAction, Action callerAction, string logIndexName, + string traceIndexName) + { + ElasticConstant.InitLog(logIndexName); + ElasticConstant.InitTrace(traceIndexName); + if (services.BuildServiceProvider().GetService() == null || services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME) + .AddScoped() + .AddScoped(); + ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, logIndexName); + ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, traceIndexName); + return services; + } + + private static IServiceCollection AddElasticsearch(IServiceCollection services, string[] nodes, string name) + { + return services.AddElasticsearch(name, options => + { + options.UseNodes(nodes).UseConnectionSettings(setting => setting.EnableApiVersioningHeader(false)); + + }) + .AddCaller(name, callerBuilder => + { + callerBuilder.UseHttpClient(builder => + { + builder.BaseAddress = nodes[0]; + }).UseAuthentication(); + }); + } + + private static IServiceCollection AddElasticsearch(IServiceCollection services, + Action elasticsearchConnectionAction, + Action callerAction, string name) + { + ArgumentNullException.ThrowIfNull(callerAction); + var factory = services.BuildServiceProvider().GetService(); + var callerFactory = services.BuildServiceProvider().GetService(); + + if (factory == null || factory.Create(name) == null || callerFactory == null || callerFactory.Create(name) == null) + services.AddElasticsearch(name, elasticsearchConnectionAction) + .AddCaller(name, option => option.UseHttpClient(callerAction).UseAuthentication()); + return services; + } + + internal static IElasticClient CreateElasticClient(this IElasticClientFactory elasticsearchFactory, bool isLog) + { + if (isLog) + return elasticsearchFactory.Create(ElasticConstant.Log.IsIndependent ? ElasticConstant.LOG_CALLER_CLIENT_NAME : + ElasticConstant.DEFAULT_CALLER_CLIENT_NAME); + else + return elasticsearchFactory.Create(ElasticConstant.Trace.IsIndependent ? ElasticConstant.TRACE_CALLER_CLIENT_NAME : + ElasticConstant.DEFAULT_CALLER_CLIENT_NAME); + } + + internal static ICaller Create(this ICallerFactory callerFactory, bool isLog) + { + if (isLog) + return callerFactory.Create(ElasticConstant.Log.IsIndependent ? ElasticConstant.LOG_CALLER_CLIENT_NAME : + ElasticConstant.DEFAULT_CALLER_CLIENT_NAME); + else + return callerFactory.Create(ElasticConstant.Trace.IsIndependent ? ElasticConstant.TRACE_CALLER_CLIENT_NAME : + ElasticConstant.DEFAULT_CALLER_CLIENT_NAME); + } + + private static Lazy GetLazyMapping(IServiceCollection services, string callerName, string indexName) + { + return new Lazy(() => + { + var callerFactory = services.BuildServiceProvider().GetRequiredService(); + var caller = callerFactory.Create(callerName); + var result = caller.GetMappingAsync(indexName).Result; + return result.Select(item => (ElasticseacherMappingResponseDto)item).ToArray(); + }); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/LogService.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/LogService.cs new file mode 100644 index 00000000..5aa92137 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/LogService.cs @@ -0,0 +1,31 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch; + +internal class LogService : ILogService +{ + private readonly IElasticClient _client; + private readonly ICallerFactory _callerFactory; + + public LogService(IElasticClientFactory elasticClientFactory, ICallerFactory callerFactory) + { + _client = elasticClientFactory.CreateElasticClient(true); + _callerFactory = callerFactory; + } + + public async Task AggregateAsync(SimpleAggregateRequestDto query) + { + return await _client.AggregateLogAsync(query); + } + + public async Task> ListAsync(BaseRequestDto query) + { + return await _client.SearchLogAsync(query); + } + + public async Task> GetMappingAsync() + { + return await _callerFactory.Create(true).GetMappingAsync(ElasticConstant.Log.IndexName); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Masa.Contrib.StackSdks.Tsc.Elasticsearch.csproj b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Masa.Contrib.StackSdks.Tsc.Elasticsearch.csproj new file mode 100644 index 00000000..6d931318 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Masa.Contrib.StackSdks.Tsc.Elasticsearch.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseachLogResponseDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseachLogResponseDto.cs new file mode 100644 index 00000000..07122fd6 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseachLogResponseDto.cs @@ -0,0 +1,10 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Model; + +internal class ElasticseachLogResponseDto : LogResponseDto +{ + [JsonPropertyName("@timestamp")] + public override DateTime Timestamp { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseacherMappingResponse.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseacherMappingResponse.cs new file mode 100644 index 00000000..733b7834 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/ElasticseacherMappingResponse.cs @@ -0,0 +1,17 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Model; + +public class ElasticseacherMappingResponseDto : MappingResponseDto +{ + /// + /// if has keyword is true ,else false + /// + public bool? IsKeyword { get; set; } + + /// + /// keyword query max length + /// + public int? MaxLenth { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollRequestDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollRequestDto.cs new file mode 100644 index 00000000..a01917c8 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollRequestDto.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Model; + +public class ElasticsearchScrollRequestDto : BaseRequestDto +{ + public string Scroll { get; set; } + + public string ScrollId { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollResponseDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollResponseDto.cs new file mode 100644 index 00000000..042418a1 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Scroll/ElasticsearchScrollResponseDto.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Model; + +public class ElasticsearchScrollResponseDto: PaginatedListBase where TResult : class +{ + public string ScrollId { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Trace/ElasticseachTraceResponseDto.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Trace/ElasticseachTraceResponseDto.cs new file mode 100644 index 00000000..27e9e4b6 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Model/Trace/ElasticseachTraceResponseDto.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Model; + +internal class ElasticseachTraceResponseDto : TraceResponseDto +{ + [JsonPropertyName("@timestamp")] + public override DateTime Timestamp { get; set; } + + [JsonPropertyName("EndTimestamp")] + public override DateTime EndTimestamp { get; set; } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/TraceService.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/TraceService.cs new file mode 100644 index 00000000..c5616465 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/TraceService.cs @@ -0,0 +1,41 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch; + +internal class TraceService : ITraceService +{ + private readonly IElasticClient _client; + + public TraceService(IElasticClientFactory elasticClientFactory) + { + _client = elasticClientFactory.CreateElasticClient(false); + } + + public async Task AggregateAsync(SimpleAggregateRequestDto query) + { + return await _client.AggregateTraceAsync(query); + } + + public async Task> GetAsync(string traceId) + { + return (await _client.SearchTraceAsync(new BaseRequestDto { TraceId = traceId, Page = 1, PageSize = ElasticConstant.MaxRecordCount - 1 })).Result; + } + + public Task GetMaxDelayTraceIdAsync(BaseRequestDto query) + { + return _client.GetMaxDelayTraceIdAsync(query); + } + + public Task> ListAsync(BaseRequestDto query) + { + return _client.SearchTraceAsync(query); + } + + public Task> ScrollAsync(BaseRequestDto query) + { + if (query is not ElasticsearchScrollRequestDto) + throw new UserFriendlyException("parameter: query must is type: ElasticsearchScrollRequestDto"); + return _client.SearchTraceAsync(query); + } +} diff --git a/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/_Imports.cs b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/_Imports.cs new file mode 100644 index 00000000..07579054 --- /dev/null +++ b/src/Infrastructure/Masa.Contrib.StackSdks.Tsc.Elasticsearch/_Imports.cs @@ -0,0 +1,22 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using Masa.BuildingBlocks.Service.Caller; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Log; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Model.Aggregate; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Service; +global using Masa.BuildingBlocks.StackSdks.Tsc.Contracts.Trace; +global using Masa.Contrib.Service.Caller.HttpClient; +global using Masa.Contrib.StackSdks.Tsc.Elasticsearch; +global using Masa.Contrib.StackSdks.Tsc.Elasticsearch.Constants; +global using Masa.Contrib.StackSdks.Tsc.Elasticsearch.Converters; +global using Masa.Contrib.StackSdks.Tsc.Elasticsearch.Model; +global using Masa.Utils.Data.Elasticsearch; +global using Masa.Utils.Models; +global using Microsoft.Extensions.DependencyInjection; +global using Nest; +global using System.Runtime.CompilerServices; +global using System.Text.Json; +global using System.Text.Json.Serialization; +global using System.Text.RegularExpressions; diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultStatuses.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultStatuses.cs new file mode 100644 index 00000000..b4f33b1d --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultStatuses.cs @@ -0,0 +1,10 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Enums; + +public enum ResultStatuses +{ + Success = 1, + Error +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultTypes.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultTypes.cs new file mode 100644 index 00000000..35391ffc --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Enums/ResultTypes.cs @@ -0,0 +1,15 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Enums; + +/// +/// reference https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats +/// +public enum ResultTypes +{ + Matrix = 1, + Vector, + Scalar, + String +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/HttpClientExtensions.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/HttpClientExtensions.cs new file mode 100644 index 00000000..d170f2b8 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/HttpClientExtensions.cs @@ -0,0 +1,30 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +[assembly: InternalsVisibleTo("Masa.Utils.Data.Prometheus.Test")] +namespace System.Net.Http; + +internal static class HttpClientExtensions +{ + public static async Task GetAsync(this HttpClient client, string url, object data, ILogger logger) + { + try + { + var rep = await client.GetAsync($"{url}?{data.ToUrlParam()}"); + var str = await rep.Content.ReadAsStringAsync(); + if (rep.IsSuccessStatusCode) + { + return str; + } + else + { + return str ?? $"{{\"status\":\"error\",\"errorType\":\"unkown\",\"error\":\"{(int)rep.StatusCode}-{rep.StatusCode}\"}}"; + } + } + catch (Exception ex) + { + logger.LogError(ex, "Request Exception methd: GET, url:{url} ,data:{data}", url, data); + return $"{{\"status\":\"error\",\"errorType\":\"unkown\",\"error\":\"{ex.Message}\"}}"; + } + } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/ObjectExtensions.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/ObjectExtensions.cs new file mode 100644 index 00000000..e9253e86 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/ObjectExtensions.cs @@ -0,0 +1,142 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace System; + +public static class ObjectExtensions +{ + /// + /// Currently supported types: class, struct and types implementing the IEnumerable interface, + /// struct and class use public get properties and fields by default, + /// The IEnumerable type is directly converted to: key[]=value1&key[]=value2 + /// enum uses strings by default. If you need to use numeric values, please set isEnumString=false + /// + /// + /// + /// + /// + /// + public static string? ToUrlParam(this object? obj, bool isEnumString = true, bool isCamelCase = true, bool isUrlEncode = true) + { + return GetValue(obj, string.Empty, isEnumString, isCamelCase, isUrlEncode); + } + + private static string? GetValue(object? obj, string preStr, bool isEnumString = false, bool isCamelCase = true, bool isUrlEncode = true) + { + if (obj == null) return null; + var type = obj.GetType(); + if (type == typeof(string)) + { + var str = (string)obj; + return AppendValue(preStr, str, "=", isUrlEncode); + } + + if (type.IsValueType) + { + if (type.IsEnum) + { + var str = isEnumString ? obj.ToString() : Convert.ToInt32(obj).ToString(); + return AppendValue(preStr, str, "=", isUrlEncode); + } + + //sample value + if (type.IsPrimitive) + { + var str = obj.ToString(); + return AppendValue(preStr, str, "=", isUrlEncode); + } + + //struct + return GetObjValue(type, obj, preStr, isEnumString, isCamelCase, isUrlEncode); + } + + if (type.IsArray || type.GetInterfaces().Any(t => t.Name.IndexOf("IEnumerable") == 0)) + return GetEnumerableValue(obj, preStr, isEnumString, isCamelCase, isUrlEncode); + + if (type.IsClass) + return GetObjValue(type, obj, preStr, isEnumString, isCamelCase, isUrlEncode); + + //current type not suport + return null; + } + + private static string GetObjValue(Type type, object obj, string preStr, bool isEnumString = false, bool isCamelCase = true, bool isUrlEncode = true) + { + var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.GetProperty); + var fields = type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.GetField); + var list = new List(); + + foreach (var item in properties) + { + var str = GetMemberInfoValue(item, item.GetValue(obj), preStr, isEnumString, isCamelCase, isUrlEncode); + if (string.IsNullOrEmpty(str)) + continue; + list.Add(str); + } + + foreach (var item in fields) + { + var str = GetMemberInfoValue(item, item.GetValue(obj), preStr, isEnumString, isCamelCase, isUrlEncode); + if (string.IsNullOrEmpty(str)) + continue; + list.Add(str); + } + + if (!list.Any()) + return default!; + + list.Sort(); + return string.Join('&', list); + } + + private static string? GetMemberInfoValue(MemberInfo info, object? value, string preStr, bool isEnumString = false, bool isCamelCase = true, bool isUrlEncode = true) + { + if (value == null) + return null; + + var name = info.Name; + if (isCamelCase) + name = name.ToCamelCase(); + + return GetValue(value, AppendValue(preStr, name, ".", isUrlEncode) ?? default!, isEnumString, isCamelCase, isUrlEncode); + } + + private static string? GetEnumerableValue(object obj, string preStr, bool isEnumString = false, bool isCamelCase = true, bool isUrlEncode = true) + { + var list = new List(); + foreach (var item in (IEnumerable)obj) + { + if (item is KeyValuePair keyValue) + { + var name = keyValue.Key; + if (isCamelCase) + name = name.ToCamelCase(); + var str = GetValue(keyValue.Value, AppendValue(preStr, name, ".", isUrlEncode) ?? default!, isEnumString, isCamelCase, isUrlEncode); + if (!string.IsNullOrEmpty(str)) + list.Add(str); + } + else + { + var str = GetValue(item, $"{preStr}{(isUrlEncode ? HttpUtility.UrlEncode("[]", Encoding.UTF8) : "[]")}", isEnumString, isCamelCase, isUrlEncode); + if (!string.IsNullOrEmpty(str)) + list.Add(str); + } + } + if (!list.Any()) + return default!; + + list.Sort(); + return string.Join('&', list); + } + + private static string? AppendValue(string preStr, string? value, string splitChar, bool isUrlEncode) + { + if (string.IsNullOrEmpty(preStr) || string.IsNullOrEmpty(value)) + return value; + + if (isUrlEncode) + return $"{preStr}{splitChar}{HttpUtility.UrlEncode(value, Encoding.UTF8)}"; + else + return $"{preStr}{splitChar}{value}"; + } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/StringExtensions.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/StringExtensions.cs new file mode 100644 index 00000000..a484b7fb --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Extensions/StringExtensions.cs @@ -0,0 +1,20 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace System; + +internal static class StringExtensions +{ + public static string ToCamelCase(this string str) + { + if (string.IsNullOrEmpty(str)) + return default!; + + var span = new ReadOnlySpan(str.ToArray()); + var c = span[0]; + if (c - 'A' >= 0 && c - 'Z' <= 0) + return $"{(char)(c + 32)}{span[1..]}"; + + return str; + } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/IMasaPrometheusClient.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/IMasaPrometheusClient.cs new file mode 100644 index 00000000..656ea9cf --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/IMasaPrometheusClient.cs @@ -0,0 +1,21 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus; + +public interface IMasaPrometheusClient +{ + Task QueryAsync(QueryRequest query); + + Task QueryRangeAsync(QueryRangeRequest query); + + Task SeriesQueryAsync(MetaDataQueryRequest query); + + Task LabelsQueryAsync(MetaDataQueryRequest query); + + Task LabelValuesQueryAsync(LableValueQueryRequest query); + + Task ExemplarQueryAsync(QueryExemplarRequest query); + + Task MetricMetaQueryAsync(MetricMetaQueryRequest query); +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Masa.Utils.Data.Prometheus.csproj b/src/Infrastructure/Masa.Utils.Data.Prometheus/Masa.Utils.Data.Prometheus.csproj new file mode 100644 index 00000000..353834d8 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Masa.Utils.Data.Prometheus.csproj @@ -0,0 +1,12 @@ + + + + net6.0 + enable + enable + + + + + + diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/MasaPrometheusClient.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/MasaPrometheusClient.cs new file mode 100644 index 00000000..1dad18f5 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/MasaPrometheusClient.cs @@ -0,0 +1,140 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +[assembly: InternalsVisibleTo("Masa.Utils.Data.Prometheus.Test")] +namespace Masa.Utils.Data.Prometheus; + +internal class MasaPrometheusClient : IMasaPrometheusClient +{ + private readonly HttpClient _client; + private readonly JsonSerializerOptions _jsonSerializerOptions; + private readonly ILogger _logger; + private const string LABLES_URL = "/api/v1/labels"; + private const string QUERY_URL = "/api/v1/query"; + private const string QUERY_RANGE_URL = "/api/v1/query_range"; + private const string SERIES_URL = "/api/v1/series"; + private const string EXEMPLAR_URL = "/api/v1/query_exemplars"; + private const string LABLE_VALUE_URL = "/api/v1/label/{0}/values"; + private const string METRIC_META_URL = "/api/v1/meta"; + + public MasaPrometheusClient(HttpClient client, JsonSerializerOptions jsonSerializerOptions, ILogger logger) + { + _client = client; + _jsonSerializerOptions = jsonSerializerOptions; + _logger = logger; + } + + public async Task ExemplarQueryAsync(QueryExemplarRequest query) + { + return await QueryDataAsync(EXEMPLAR_URL, query); + } + + public async Task LabelsQueryAsync(MetaDataQueryRequest query) + { + return await QueryDataAsync(LABLES_URL, query); + } + + public async Task LabelValuesQueryAsync(LableValueQueryRequest query) + { + return await QueryDataAsync(string.Format(LABLE_VALUE_URL, query.Lable), query); + } + + public async Task QueryAsync(QueryRequest query) + { + return await QueryDataAsync(QUERY_URL, query); + } + + public async Task QueryRangeAsync(QueryRangeRequest query) + { + return await QueryDataAsync(QUERY_RANGE_URL, query); + } + + public async Task SeriesQueryAsync(MetaDataQueryRequest query) + { + return await QueryDataAsync(SERIES_URL, query); + } + + public async Task MetricMetaQueryAsync(MetricMetaQueryRequest query) + { + return await QueryDataAsync(METRIC_META_URL, query); + } + + private async Task QueryDataAsync(string url, object data) where T : ResultBaseResponse + { + var str = await _client.GetAsync(url, data, _logger); + if (string.IsNullOrEmpty(str)) + return default!; + + var baseResult = JsonSerializer.Deserialize(str, _jsonSerializerOptions); + + if (baseResult == null || baseResult.Status != ResultStatuses.Success) + { + return baseResult ?? default!; + } + + if (typeof(T) == typeof(QueryResultCommonResponse)) + { + var result = baseResult as QueryResultCommonResponse; + if (result == null || result.Data == null) + return baseResult; + switch (result.Data.ResultType) + { + case ResultTypes.Matrix: + { + var temp = JsonSerializer.Serialize(result.Data.Result, _jsonSerializerOptions); + result.Data.Result = JsonSerializer.Deserialize(temp, _jsonSerializerOptions); + if (result.Data.Result != null && result.Data.Result.Any()) + { + foreach (QueryResultMatrixRangeResponse item in result.Data.Result) + { + if (item.Values == null || !item.Values.Any()) + continue; + var array = item.Values.ToArray(); + int i = 0, max = array.Length - 1; + do + { + array[i] = ConvertJsonToObjValue(array[i]); + i++; + } + while (max - i >= 0); + item.Values = array; + } + } + return result as T ?? default!; + } + case ResultTypes.Vector: + { + var temp = JsonSerializer.Serialize(result.Data.Result, _jsonSerializerOptions); + result.Data.Result = JsonSerializer.Deserialize(temp, _jsonSerializerOptions); + if (result.Data.Result != null && result.Data.Result.Any()) + { + foreach (QueryResultInstantVectorResponse item in result.Data.Result) + { + item.Value = ConvertJsonToObjValue(item.Value); + } + } + return result as T ?? default!; + } + default: + { + if (result.Data.Result != null && result.Data.Result.Any()) + { + result.Data.Result = ConvertJsonToObjValue(result.Data.Result); + } + } + break; + } + } + + return baseResult; + } + + + private static object[] ConvertJsonToObjValue(object[]? values) + { + if (values == null || values.Length - 2 < 0) + return default!; + + return new object[] { ((JsonElement)values[0]).GetDouble(), ((JsonElement)values[1]).GetString() ?? default! }; + } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarDataModel.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarDataModel.cs new file mode 100644 index 00000000..2777b076 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarDataModel.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class ExemplarModel +{ + public IDictionary? Labels { get; set; } + + public string? Value { get; set; } + + public float TimeStamp { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarModel.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarModel.cs new file mode 100644 index 00000000..84ff5c61 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarModel.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class ExemplarDataModel +{ + public IDictionary? SeriesLabels { get; set; } + + public IEnumerable? Exemplars { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarResultResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarResultResponse.cs new file mode 100644 index 00000000..c03e8eee --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Exemplar/ExemplarResultResponse.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class ExemplarResultResponse : ResultBaseResponse +{ + public IEnumerable? Data { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/LabelResultResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/LabelResultResponse.cs new file mode 100644 index 00000000..a20a25ca --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/LabelResultResponse.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class LabelResultResponse : ResultBaseResponse +{ + public IEnumerable? Data { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaItemValueModel.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaItemValueModel.cs new file mode 100644 index 00000000..1f84aa55 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaItemValueModel.cs @@ -0,0 +1,22 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class MetaItemValueModel +{ + /// + /// metric's data type, like counter、gauge、histogram and more + /// + public string Type { get; set; } + + /// + /// metric's description ,like "Cpu usage in seconds" + /// + public string Help { get; set; } + + /// + /// metric's data unit + /// + public string Unit { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaResultResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaResultResponse.cs new file mode 100644 index 00000000..baaf230f --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetaResultResponse.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class MetaResultResponse: ResultBaseResponse +{ + public Dictionary Data { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetricMetaQueryRequest.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetricMetaQueryRequest.cs new file mode 100644 index 00000000..16deea95 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/MetricMetaQueryRequest.cs @@ -0,0 +1,17 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class MetricMetaQueryRequest +{ + /// + /// search metric name ,use full match + /// + public string Metric { get; set; } + + /// + /// default all ,if set 0, then can't return any data + /// + public int? Limit { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/SeriesResultResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/SeriesResultResponse.cs new file mode 100644 index 00000000..6422449c --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/MetaData/SeriesResultResponse.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class SeriesResultResponse : ResultBaseResponse +{ + public IEnumerable>? Data { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultCommonResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultCommonResponse.cs new file mode 100644 index 00000000..1859c35b --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultCommonResponse.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryResultCommonResponse: ResultBaseResponse +{ + public QueryResultDataResponse? Data { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultDataResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultDataResponse.cs new file mode 100644 index 00000000..209f6d57 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultDataResponse.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryResultDataResponse +{ + public ResultTypes ResultType { get; set; } + + public object[]? Result { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultInstantVectorResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultInstantVectorResponse.cs new file mode 100644 index 00000000..976986e2 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultInstantVectorResponse.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryResultInstantVectorResponse +{ + public IDictionary? Metric { get; set; } + + public object[]? Value { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultMatrixRangeResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultMatrixRangeResponse.cs new file mode 100644 index 00000000..810473d1 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/Query/QueryResultMatrixRangeResponse.cs @@ -0,0 +1,11 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryResultMatrixRangeResponse +{ + public IDictionary? Metric { get; set; } + + public IEnumerable? Values { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/ResultBaseResponse.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/ResultBaseResponse.cs new file mode 100644 index 00000000..33381e61 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Reponse/ResultBaseResponse.cs @@ -0,0 +1,15 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class ResultBaseResponse +{ + public ResultStatuses Status { get; set; } + + public string? Error { get; set; } + + public string? ErrorType { get; set; } + + public IEnumerable? Warnings { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/LableValueQueryRequest.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/LableValueQueryRequest.cs new file mode 100644 index 00000000..b1089a15 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/LableValueQueryRequest.cs @@ -0,0 +1,9 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class LableValueQueryRequest: MetaDataQueryRequest +{ + public string Lable { get; set; } = "__name__"; +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/MetaDataQueryRequest.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/MetaDataQueryRequest.cs new file mode 100644 index 00000000..00c3af56 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/MetaDataQueryRequest.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class MetaDataQueryRequest +{ + public IEnumerable? Match { get; set; } + + public string? Start { get; set; } + + public string? End { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryExemplarRequest.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryExemplarRequest.cs new file mode 100644 index 00000000..b6b9a4be --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryExemplarRequest.cs @@ -0,0 +1,14 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryExemplarRequest +{ + public string? Query { get; set; } + + public string? Start { get; set; } + + public string? End { get; set; } +} + diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRangeRequest.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRangeRequest.cs new file mode 100644 index 00000000..9d492619 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRangeRequest.cs @@ -0,0 +1,17 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryRangeRequest +{ + public string? Query { get; set; } + + public string? Start { get; set; } + + public string? End { get; set; } + + public string? Step { get; set; } + + public string? TimeOut { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRequest.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRequest.cs new file mode 100644 index 00000000..b1779587 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/Model/Request/QueryRequest.cs @@ -0,0 +1,13 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus.Model; + +public class QueryRequest +{ + public string? Query { get; set; } + + public string? Time { get; set; } + + public string? TimeOut { get; set; } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/README.md b/src/Infrastructure/Masa.Utils.Data.Prometheus/README.md new file mode 100644 index 00000000..96d06f6e --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/README.md @@ -0,0 +1,75 @@ +[中](README.zh-CN.md) | EN + +## Masa.Utils.Data.Prometheus + +[Prometheus Http Api](https://www.prometheus.io/docs/prometheus/latest/querying/api/) Client Library + +## Install: +```c# +Install-Package Masa.Utils.Data.Prometheus +``` + +Example: + +1. Inject + +```` C# +builder.Services.AddPrometheusClient("http://127.0.0.1:9090"); +```` + +2. Query Example + +```C# +public class SampleService +{ + + private IMasaPrometheusClient _client; + + public SampleService(IMasaPrometheusClient client) + { + _client=client; + } + + public async Task QueryAsync() + { + var query= new QueryRequest { + Query = "up", //metric name + Time = "2022-06-01T09:00:00.000Z" //standard time format or unix timestamp, such as: 1654045200 or 1654045200.000 + }; + var result = await _client.QueryAsync(query); + if(result.Status == ResultStatuses.Success) + { + switch(result.Data.ResultType) + { + case ResultTypes.Vector: + { + var data=result.Data.Result as QueryResultInstantVectorResponse[]; + ... + } + break; + case ResultTypes.Matrix: + { + var data=result.Data.Result as QueryResultMatrixRangeResponse[]; + ... + } + break; + default: + { + var timeSpan=(double)result.Data.Result[0]; + var value=(string)result.Data.Result[1]; + } + break; + } + } + } +} +``` + +### Current suports: + +- [query](https://www.prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) +- [query_range](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-queries) +- [series](https://www.prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers) +- [lables](https://www.prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names) +- [lable value](https://www.prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values) +- [exemplars](https://www.prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars) diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/README.zh-CN.md b/src/Infrastructure/Masa.Utils.Data.Prometheus/README.zh-CN.md new file mode 100644 index 00000000..b8c14ab1 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/README.zh-CN.md @@ -0,0 +1,75 @@ +[EN](README.md) | 中 + +## Masa.Utils.Data.Prometheus + +[Prometheus Http Api](https://www.prometheus.io/docs/prometheus/latest/querying/api/) 客户端类库 + +## 安装: +```c# +Install-Package Masa.Utils.Data.Prometheus +``` + +### 示例: + +1. 注册 + +```` C# +builder.Services.AddPrometheusClient("http://127.0.0.1:9090"); +```` + +2. 查询样例 + +```C# +public class SampleService +{ + + private IMasaPrometheusClient _client; + + public SampleService(IMasaPrometheusClient client) + { + _client=client; + } + + public async Task QueryAsync() + { + var query= new QueryRequest { + Query = "up", //metric name + Time = "2022-06-01T09:00:00.000Z" //标准时间格式或unix时间戳，如：1654045200或1654045200.000 + }; + var result = await _client.QueryAsync(query); + if(result.Status == ResultStatuses.Success) + { + switch(result.Data.ResultType) + { + case ResultTypes.Vector: + { + var data=result.Data.Result as QueryResultInstantVectorResponse[]; + ... + } + break; + case ResultTypes.Matrix: + { + var data=result.Data.Result as QueryResultMatrixRangeResponse[]; + ... + } + break; + default: + { + var timeSpan=(double)result.Data.Result[0]; + var value=(string)result.Data.Result[1]; + } + break; + } + } + } +} +``` + +### 目前只支持以下api: + +- [query](https://www.prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) +- [query_range](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-queries) +- [series](https://www.prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers) +- [lables](https://www.prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names) +- [lable value](https://www.prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values) +- [exemplars](https://www.prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars) diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/ServiceCollectionExtensions.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..6c5754da --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/ServiceCollectionExtensions.cs @@ -0,0 +1,38 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +namespace Masa.Utils.Data.Prometheus; + +public static class ServiceCollectionExtensions +{ + private const string PROMETHEUS_HTTP_CLIENT_NAME = "masa_stack_prometheus_client"; + + public static IServiceCollection AddPrometheusClient(this IServiceCollection services, string url, int timeoutSeconds = 5) + { + ArgumentNullException.ThrowIfNull(url); + + if (services.Any(service => service.GetType() == typeof(IMasaPrometheusClient))) + return services; + + var jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }; + jsonOptions.Converters.Add(new JsonStringEnumConverter()); + + if (timeoutSeconds <= 0) + timeoutSeconds = 5; + services.AddHttpClient(PROMETHEUS_HTTP_CLIENT_NAME, ops => + { + ops.BaseAddress = new Uri(url); + ops.Timeout = TimeSpan.FromSeconds(timeoutSeconds); + }); + + services.AddScoped(ServiceProvider => + { + var client = ServiceProvider.GetRequiredService().CreateClient(PROMETHEUS_HTTP_CLIENT_NAME); + return new MasaPrometheusClient(client, jsonOptions, ServiceProvider.GetRequiredService>()); + }); + return services; + } +} diff --git a/src/Infrastructure/Masa.Utils.Data.Prometheus/_Imports.cs b/src/Infrastructure/Masa.Utils.Data.Prometheus/_Imports.cs new file mode 100644 index 00000000..f5e427f8 --- /dev/null +++ b/src/Infrastructure/Masa.Utils.Data.Prometheus/_Imports.cs @@ -0,0 +1,14 @@ +// Copyright (c) MASA Stack All rights reserved. +// Licensed under the MIT License. See LICENSE.txt in the project root for license information. + +global using Masa.Utils.Data.Prometheus.Enums; +global using Masa.Utils.Data.Prometheus.Model; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using System.Collections; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Serialization; +global using System.Web; diff --git a/src/Services/Masa.Tsc.Service.Admin/Extensions/AddTraceLogExtenstion.cs b/src/Services/Masa.Tsc.Service.Admin/Extensions/AddTraceLogExtenstion.cs index f4dff7c7..2e05e4c4 100644 --- a/src/Services/Masa.Tsc.Service.Admin/Extensions/AddTraceLogExtenstion.cs +++ b/src/Services/Masa.Tsc.Service.Admin/Extensions/AddTraceLogExtenstion.cs @@ -22,7 +22,7 @@ private static void AddClickHouse() { if (ConfigConst.StorageConst.HasInit || !ConfigConst.IsClickhouse) return; if (string.IsNullOrEmpty(ConfigConst.ClikhouseConnection)) return; - _services.AddMASAStackApmClickhouse(ConfigConst.ClikhouseConnection, ConfigConst.LogIndex, ConfigConst.TraceIndex, ConfigConst.ClickHouseLogSourceTable, ConfigConst.ClickHouseTaceSourceTable); + _services.AddMASAStackApmClickhouse(ConfigConst.ClikhouseConnection, ConfigConst.ClickhouseTableSuffix, ConfigConst.ClickHouseLogSourceTable, ConfigConst.ClickHouseTaceSourceTable); ConfigConst.StorageConst.SetClickhouse(); } diff --git a/src/Services/Masa.Tsc.Service.Admin/Infrastructure/Const/ConfigConst.cs b/src/Services/Masa.Tsc.Service.Admin/Infrastructure/Const/ConfigConst.cs index 7b0deb3b..b2d9aafd 100644 --- a/src/Services/Masa.Tsc.Service.Admin/Infrastructure/Const/ConfigConst.cs +++ b/src/Services/Masa.Tsc.Service.Admin/Infrastructure/Const/ConfigConst.cs @@ -1,6 +1,8 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. +using Nest; + namespace Masa.Tsc.Service.Admin.Infrastructure.Const; internal class ConfigConst @@ -26,6 +28,8 @@ public static void SetConfiguration(AppSettingConfiguration config) public static bool IsClickhouse => Configuration.IsClickHouse; + public static string ClickhouseTableSuffix => Configuration.Clickhouse?.Suffix!; + public static string ClikhouseConnection => Configuration.Clickhouse?.Connection!; public static string ClickHouseLogSourceTable => Configuration.Clickhouse?.LogSource!; diff --git a/src/Services/Masa.Tsc.Service.Admin/Infrastructure/DccConfigurations/AppSettingConfiguration.cs b/src/Services/Masa.Tsc.Service.Admin/Infrastructure/DccConfigurations/AppSettingConfiguration.cs index f9d2b8ab..f72e162c 100644 --- a/src/Services/Masa.Tsc.Service.Admin/Infrastructure/DccConfigurations/AppSettingConfiguration.cs +++ b/src/Services/Masa.Tsc.Service.Admin/Infrastructure/DccConfigurations/AppSettingConfiguration.cs @@ -30,4 +30,6 @@ public class ClickhouseConfiguration public string LogSource { get; set; } public string TraceSource { get; set; } + + public string Suffix { get; set; } } \ No newline at end of file diff --git a/src/Services/Masa.Tsc.Service.Admin/Masa.Tsc.Service.Admin.csproj b/src/Services/Masa.Tsc.Service.Admin/Masa.Tsc.Service.Admin.csproj index 2f61da86..fa386d9c 100644 --- a/src/Services/Masa.Tsc.Service.Admin/Masa.Tsc.Service.Admin.csproj +++ b/src/Services/Masa.Tsc.Service.Admin/Masa.Tsc.Service.Admin.csproj @@ -7,9 +7,7 @@ f6cdf028-3564-47a5-a4fb-662685680853 - - - + @@ -19,12 +17,8 @@ - - - - - - + + @@ -38,8 +32,7 @@ - - + @@ -55,9 +48,12 @@ - - - + + + + + + diff --git a/src/Services/Masa.Tsc.Service.Admin/Services/ApmService.cs b/src/Services/Masa.Tsc.Service.Admin/Services/ApmService.cs index 37bad574..2eddb540 100644 --- a/src/Services/Masa.Tsc.Service.Admin/Services/ApmService.cs +++ b/src/Services/Masa.Tsc.Service.Admin/Services/ApmService.cs @@ -79,12 +79,13 @@ public async Task GetLatencyDistributions([FromS StatusCodes = string.Join(',', ConfigConst.TraceErrorStatus) }); - public async Task> GetErrors([FromServices] IApmService apmService, int page, int pageSize, string start, string end, string? env, string? service, ComparisonTypes? comparisonType, string? queries, string? orderField, bool? isDesc) + public async Task> GetErrors([FromServices] IApmService apmService, int page, int pageSize, string start, string end, string? env, string? service, string? endpoint, ComparisonTypes? comparisonType, string? queries, string? orderField, bool? isDesc) => await apmService.ErrorMessagePageAsync(new ApmEndpointRequestDto { Start = start.ParseUTCTime(), End = end.ParseUTCTime(), Env = GetEnv(env), + //Endpoint = endpoint!, Queries = queries, OrderField = orderField, IsDesc = isDesc, diff --git a/src/Services/Masa.Tsc.Service.Admin/Services/TopologyService.cs b/src/Services/Masa.Tsc.Service.Admin/Services/TopologyService.cs index 58afe301..dd955aab 100644 --- a/src/Services/Masa.Tsc.Service.Admin/Services/TopologyService.cs +++ b/src/Services/Masa.Tsc.Service.Admin/Services/TopologyService.cs @@ -12,17 +12,12 @@ public TopologyService() : base("/api/topology") public async Task StartAsync([FromServices] IEventBus eventBus, [FromQuery] DateTimeOffset? excuteTime) { - if (excuteTime == null) - excuteTime = new DateTimeOffset(DateTime.UtcNow); - var end = excuteTime.Value.ToUniversalTime().DateTime; - var start = end.AddDays(-7); - var command = new StartCommand(start, end); - await eventBus.PublishAsync(command); + await Task.CompletedTask; } - public async Task GetAsync([FromServices] IEventBus eventBus, string serviceName, int level, DateTime start, DateTime end) + public async Task GetAsync([FromServices] IEventBus eventBus, string serviceName, int level, string start, string end) { - var query = new TopologyQuery(new TopologyRequestDto { ServiceName = serviceName, Level = level, Start = start, End = end }); + var query = new TopologyQuery(new TopologyRequestDto { ServiceName = serviceName, Level = level, Start = start.ParseUTCTime(), End = end.ParseUTCTime() }); await eventBus.PublishAsync(query); return query.Result; } diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmInputSearchComponent.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmInputSearchComponent.razor index e50f9711..bcaf6526 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmInputSearchComponent.razor +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmInputSearchComponent.razor @@ -1,4 +1,72 @@ @namespace Masa.Tsc.Web.Admin.Rcl.Components.Apm @inherits ApmComponentBase - \ No newline at end of file +@* *@ + + + + + +@code { + Dictionary list = new Dictionary() + { + {0,"" }, + {1,"ServiceName" }, + {2,"Endpoint" } + + }; + + List> items = new(); + + protected override void OnInitialized() + { + base.OnInitialized(); + list[0] = Value; + items = list.ToList(); + } + + Regex reg = new Regex(@"^\S+$|(?<=(and|or)\s*)\S+"); + + string lastMatch = string.Empty; + + + private void InputChanged(string search) + { + if (reg.IsMatch(search)) + { + var str = reg.Matches(search).Last().Value; + //str.StartsWith(lastMatch) + lastMatch = str; + list[0] = search; + } + + } + + + private bool Filter(KeyValuePair _, string a, string b) + { + if (!string.IsNullOrEmpty(b) && reg.IsMatch(a)) + { + var value = reg.Matches(b).Last().Value; + + return b.StartsWith(value, StringComparison.CurrentCultureIgnoreCase); + } + return false; + } + + + + + + +} \ No newline at end of file diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmLogView.razor.css b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmLogView.razor.css index 226166fa..b44d71ae 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmLogView.razor.css +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmLogView.razor.css @@ -17,7 +17,6 @@ height: calc(100vh - 96px); border: solid 1px #ccc; background-color: white; - animation-duration: 800ms; } .text-wrap { diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSamplePage.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSamplePage.razor.cs index a032b745..6054b18c 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSamplePage.razor.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSamplePage.razor.cs @@ -36,7 +36,7 @@ private async Task OnPreAsync() private async Task OnPageAsync(int value) { - if (value < 1 || value - Total > 0) + if (value < 1 || value - Total > 0 || value - Current == 0) return; Current = value; diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor index 6dc365d2..f2d0caa3 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor @@ -80,7 +80,7 @@ OnAutoUpdate="OnTimeUpdate" ShowQuickChange ShowTimeZoneSelector - DefaultQuickRange="QuickRangeKey.Last15Minutes" + DefaultQuickRange="quickRangeKey" ShowInterval /> } diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor.cs index 9706ea13..97bc51d7 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmSearchComponent.razor.cs @@ -1,8 +1,6 @@ // Copyright (c) MASA Stack All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. -using System; - namespace Masa.Tsc.Web.Admin.Rcl.Components.Apm; public partial class ApmSearchComponent @@ -34,7 +32,6 @@ public partial class ApmSearchComponent [Inject] public IMultiEnvironmentUserContext UserContext { get; set; } - private List<(string, string)> _values = new List<(string, string)> { new("All", "All") }; private static List<(ApmComparisonTypes value, string text)> listComparisons = new() { new (ApmComparisonTypes.None, "None"), @@ -46,11 +43,16 @@ public partial class ApmSearchComponent private bool isServiceLoading = true, isEnvLoading = true; private bool isCallQuery = false; - + private QuickRangeKey quickRangeKey = QuickRangeKey.Last15Minutes; protected override async Task OnInitializedAsync() { await base.OnInitializedAsync(); + if (Search.Start > DateTime.MinValue && Search.End > Search.Start) + { + SetQuickRangeKey(Search.End - Search.Start); + } + if (!isCallQuery && Search.Start > DateTime.MinValue) { await LoadEnvironmentAsync(); @@ -60,6 +62,53 @@ protected override async Task OnInitializedAsync() } } + private void SetQuickRangeKey(TimeSpan timeSpan) + { + var minutes = (int)timeSpan.TotalMinutes; + + if (minutes - 1440 >= 0) + { + var days = minutes / 1440; + switch (days) + { + case 1: + quickRangeKey = QuickRangeKey.Last24Hours; + return; + case 2: + quickRangeKey = QuickRangeKey.Last2Days; + return; + case 7: + quickRangeKey = QuickRangeKey.Last7Days; + return; + case 30: + quickRangeKey = QuickRangeKey.Last30Days; + return; + } + } + + switch (minutes) + { + case 15: + quickRangeKey = QuickRangeKey.Last15Minutes; + return; + case 30: + quickRangeKey = QuickRangeKey.Last30Minutes; + return; + case 60: + quickRangeKey = QuickRangeKey.Last1Hour; + return; + case 180: + quickRangeKey = QuickRangeKey.Last3Hours; + return; + case 360: + quickRangeKey = QuickRangeKey.Last6Hours; + return; + case 720: + quickRangeKey = QuickRangeKey.Last12Hours; + return; + } + } + protected override void OnInitialized() { base.OnInitialized(); diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmTraceView.razor.css b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmTraceView.razor.css index cfa51dcc..c7187938 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmTraceView.razor.css +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Components/Apm/ApmTraceView.razor.css @@ -16,7 +16,6 @@ height: calc(100vh - 96px); border: solid 1px #ccc; background-color: white; - animation-duration: 800ms; } .text-wrap { diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoint.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoint.razor.cs index b301909e..cef77c55 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoint.razor.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoint.razor.cs @@ -68,7 +68,7 @@ private async Task LoadPageDataAsync() Env = Search.Environment, IsDesc = sortBy, Service = Search.Service, - Queries=Search.Text + Queries = Search.Text }; var result = await ApiCaller.ApmService.GetEndpointPageAsync(query); data.Clear(); @@ -76,8 +76,8 @@ private async Task LoadPageDataAsync() { data.AddRange(result.Result.Select(item => new ListChartData { - Name = $"{item.Method} {item.Name}", - Endpoint = item.Name, + Name = $"{item.Method} {item.Endpoint}", + Endpoint = item.Endpoint, Service = item.Service, Failed = item.Failed, Throughput = item.Throughput, @@ -97,7 +97,7 @@ private async Task LoadChartDataAsync() PageSize = defaultSize, Start = Search.Start, End = Search.End, - Endpoint=Search.Endpoint!, + Endpoint = Search.Endpoint!, Service = Search.Service, Env = Search.Environment }; diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor index 37b1a19e..592d5911 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor @@ -19,6 +19,7 @@ Items="data" TItem="ListChartData" Loading="@isTableLoading" + MustSort OnOptionsUpdate="OnTableOptionsChanged"> @if (context.Header.Value == nameof(ListChartData.Name)) diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor.cs index b2ca89ca..c4767b15 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Service.razor.cs @@ -78,7 +78,7 @@ private async Task LoadPageDataAsync() { data.AddRange(result.Result.Select(item => new ListChartData { - Name = item.Name, + Name = item.Service, Envs = string.Join(",", item.Envs), Failed = item.Failed, Throughput = item.Throughput, diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor.cs index f64b6ec5..09f0e9d9 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor.cs @@ -98,9 +98,9 @@ private async Task LoadPageDataAsync() { data.AddRange(result.Result.Select(item => new ListChartData { - Name = $"{item.Method} {item.Name}", + Name = $"{item.Method} {item.Endpoint}", Service = item.Service, - Endpoint = item.Name, + Endpoint = item.Endpoint, Failed = item.Failed, Throughput = item.Throughput, Latency = item.Latency @@ -131,7 +131,7 @@ private async Task LoadChartDataAsync() foreach (var service in data) { - var chartData = result.FirstOrDefault(s => s.Name == service.Endpoint); + var chartData = result.Find(s => s.Name == service.Endpoint); service.LatencyChartData = new(); service.ThroughputChartData = new(); service.FailedChartData = new(); diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Shared/ApmComponentBase.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Shared/ApmComponentBase.cs index 1ec4fc6a..05914147 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Rcl/Shared/ApmComponentBase.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Rcl/Shared/ApmComponentBase.cs @@ -3,7 +3,7 @@ namespace Masa.Tsc.Web.Admin.Rcl.Shared.Apm; -public partial class ApmComponentBase : BDomComponentBase, IAsyncDisposable +public partial class ApmComponentBase : BDomComponentBase { [Inject] public I18n I18n { get; set; } @@ -17,13 +17,8 @@ public partial class ApmComponentBase : BDomComponentBase, IAsyncDisposable [Inject] public NavigationManager NavigationManager { get; set; } - protected virtual SearchData Search { get; set; } - - public async ValueTask DisposeAsync() - { - //base.Dispose(true); - await Task.CompletedTask; - } + [Inject] + public SearchData Search { get; set; } public TimeZoneInfo CurrentTimeZone { get; private set; } @@ -41,12 +36,8 @@ protected override void OnAfterRender(bool firstRender) public ApmComponentBase() { - if (IsPage) - { - Search = new(); - } - } + } protected override void OnInitialized() { @@ -56,15 +47,17 @@ protected override void OnInitialized() var values = HttpUtility.ParseQueryString(uri.Query); var start = values.Get("start"); var end = values.Get("end"); - if (DateTime.TryParse(start, out var startTime) && DateTime.TryParse(end, out var endTime) && endTime > startTime) + if (DateTime.TryParse(start, out DateTime startTime) && DateTime.TryParse(end, out DateTime endTime) && endTime > startTime) { Search.Start = startTime; Search.End = endTime; } var service = values.Get("service"); var env = values.Get("env"); - Search.Environment = env; - Search.Service = service; + if (!string.IsNullOrEmpty(env)) + Search.Environment = env; + if (!string.IsNullOrEmpty(service)) + Search.Service = service; var endpoint = values.Get("endpoint"); if (!string.IsNullOrEmpty(endpoint)) @@ -99,7 +92,7 @@ protected override void OnInitialized() if (!string.IsNullOrEmpty(Search.Text) && Search.Text.Trim().StartsWith("and ")) { - Search.Text = Search.Text.Trim().Substring(4); + Search.Text = Search.Text.Trim()[4..]; } } base.OnInitialized(); diff --git a/src/Web/Masa.Tsc.Web.Admin.Server/Program.cs b/src/Web/Masa.Tsc.Web.Admin.Server/Program.cs index 4a887b72..6b08b6c9 100644 --- a/src/Web/Masa.Tsc.Web.Admin.Server/Program.cs +++ b/src/Web/Masa.Tsc.Web.Admin.Server/Program.cs @@ -8,6 +8,7 @@ builder.Services.AddRazorPages(); builder.Services.AddServerSideBlazor(); +builder.Services.AddRcl().AddScoped(); builder.Services.Configure(option => { @@ -58,8 +59,6 @@ builder.Services.AddTscHttpApiCaller(masaStackConfig.GetTscServiceDomain()).AddDccClient(redisOption); #endif -builder.Services.AddRcl().AddScoped(); - StaticWebAssetsLoader.UseStaticWebAssets(builder.Environment, builder.Configuration); //#if DEBUG