From e0fc1287f5e723f371b79c1234796d070148f5d5 Mon Sep 17 00:00:00 2001 From: Grzegorz Piwowarek Date: Fri, 24 May 2024 11:17:03 +0200 Subject: [PATCH] Don't display empty elastic responses when no calls made (#208) Disabled the Elasticsearch Response view when using a mode that never calls Elasticsearch image instead of: image --- quesma/quesma/ui/dashboard.go | 34 ++++++++-------- quesma/quesma/ui/dashboard_healthcheck.go | 6 +-- quesma/quesma/ui/data_sources.go | 2 +- quesma/quesma/ui/ingest.go | 2 +- quesma/quesma/ui/live_tail.go | 47 +++++++++++++---------- quesma/quesma/ui/live_tail_drilldown.go | 2 +- quesma/quesma/ui/management_console.go | 4 +- quesma/quesma/ui/schema.go | 4 +- 8 files changed, 54 insertions(+), 47 deletions(-) diff --git a/quesma/quesma/ui/dashboard.go b/quesma/quesma/ui/dashboard.go index 59b25e457..ef63b0b6f 100644 --- a/quesma/quesma/ui/dashboard.go +++ b/quesma/quesma/ui/dashboard.go @@ -38,23 +38,23 @@ func (qmc *QuesmaManagementConsole) generateDashboard() []byte { // One limitation is that, we don't update color of paths after initial draw. // They rarely change, so it's not a big deal for now. // Clickhouse -> Kibana - if qmc.config.ReadsFromClickhouse() { + if qmc.cfg.ReadsFromClickhouse() { status, _ := qmc.generateDashboardTrafficText(RequestStatisticKibana2Clickhouse) buffer.Html(fmt.Sprintf(``, status)) } // Elasticsearch -> Kibana - if qmc.config.ReadsFromElasticsearch() { + if qmc.cfg.ReadsFromElasticsearch() { status, _ := qmc.generateDashboardTrafficText(RequestStatisticKibana2Elasticsearch) buffer.Html(fmt.Sprintf(``, status)) } // Ingest -> Clickhouse - if qmc.config.WritesToClickhouse() { + if qmc.cfg.WritesToClickhouse() { status, _ := qmc.generateDashboardTrafficText(RequestStatisticIngest2Clickhouse) buffer.Html(fmt.Sprintf(``, status)) } // Ingest -> Elasticsearch - if qmc.config.WritesToElasticsearch() { + if qmc.cfg.WritesToElasticsearch() { status, _ := qmc.generateDashboardTrafficText(RequestStatisticIngest2Elasticsearch) buffer.Html(fmt.Sprintf(``, status)) } @@ -93,22 +93,22 @@ func (qmc *QuesmaManagementConsole) generateDashboardTrafficPanel() []byte { var buffer builder.HtmlBuffer // Clickhouse -> Kibana - if qmc.config.ReadsFromClickhouse() { + if qmc.cfg.ReadsFromClickhouse() { buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticKibana2Clickhouse, 21)) } // Elasticsearch -> Kibana - if qmc.config.ReadsFromElasticsearch() { + if qmc.cfg.ReadsFromElasticsearch() { buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticKibana2Elasticsearch, 66)) } // Ingest -> Clickhouse - if qmc.config.WritesToClickhouse() { + if qmc.cfg.WritesToClickhouse() { buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticIngest2Clickhouse, 31)) } // Ingest -> Elasticsearch - if qmc.config.WritesToElasticsearch() { + if qmc.cfg.WritesToElasticsearch() { buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticIngest2Elasticsearch, 76)) } @@ -128,22 +128,22 @@ func (qmc *QuesmaManagementConsole) generateDashboardPanel() []byte { dashboardName := "

Kibana

" storeName := "

Elasticsearch

" - if qmc.config.Elasticsearch.Url != nil && strings.Contains(qmc.config.Elasticsearch.Url.String(), "opensearch") { + if qmc.cfg.Elasticsearch.Url != nil && strings.Contains(qmc.cfg.Elasticsearch.Url.String(), "opensearch") { dashboardName = "

OpenSearch

Dashboards

" storeName = "

OpenSearch

" } clickhouseName := "

ClickHouse

" - if qmc.config.Hydrolix.Url != nil { + if qmc.cfg.Hydrolix.Url != nil { clickhouseName = "

Hydrolix

" } buffer.Html(`
`) - if qmc.config.Elasticsearch.AdminUrl != nil { - buffer.Html(``) + if qmc.cfg.Elasticsearch.AdminUrl != nil { + buffer.Html(``) } buffer.Html(dashboardName) - if qmc.config.Elasticsearch.AdminUrl != nil { + if qmc.cfg.Elasticsearch.AdminUrl != nil { buffer.Html(``) } buffer.Html(statusToDiv(qmc.checkKibana())) @@ -160,11 +160,11 @@ func (qmc *QuesmaManagementConsole) generateDashboardPanel() []byte { buffer.Html(`
`) buffer.Html(`
`) - if qmc.config.ClickHouse.AdminUrl != nil { - buffer.Html(``) + if qmc.cfg.ClickHouse.AdminUrl != nil { + buffer.Html(``) } buffer.Html(clickhouseName) - if qmc.config.ClickHouse.AdminUrl != nil { + if qmc.cfg.ClickHouse.AdminUrl != nil { buffer.Html(``) } buffer.Html(statusToDiv(qmc.checkClickhouseHealth())) @@ -198,7 +198,7 @@ func (qmc *QuesmaManagementConsole) generateDashboardPanel() []byte { duration := uint64(time.Since(qmc.startedAt).Seconds()) buffer.Html(fmt.Sprintf(`
Started: %s ago
`, secondsToTerseString(duration))) - buffer.Html(fmt.Sprintf(`
Mode: %s
`, qmc.config.Mode.String())) + buffer.Html(fmt.Sprintf(`
Mode: %s
`, qmc.cfg.Mode.String())) if h, errH := host.Info(); errH == nil { buffer.Html(fmt.Sprintf(`
Host uptime: %s
`, secondsToTerseString(h.Uptime))) diff --git a/quesma/quesma/ui/dashboard_healthcheck.go b/quesma/quesma/ui/dashboard_healthcheck.go index ccbbe01a8..fa12b5f0d 100644 --- a/quesma/quesma/ui/dashboard_healthcheck.go +++ b/quesma/quesma/ui/dashboard_healthcheck.go @@ -49,7 +49,7 @@ type healthCheckStatus struct { } func (qmc *QuesmaManagementConsole) checkClickhouseHealth() healthCheckStatus { - if !qmc.config.WritesToClickhouse() && !qmc.config.ReadsFromClickhouse() { + if !qmc.cfg.WritesToClickhouse() && !qmc.cfg.ReadsFromClickhouse() { return healthCheckStatus{"grey", "N/A (not writing)", ""} } @@ -64,12 +64,12 @@ func (qmc *QuesmaManagementConsole) checkClickhouseHealth() healthCheckStatus { } func (qmc *QuesmaManagementConsole) checkElasticsearch() healthCheckStatus { - if !qmc.config.WritesToElasticsearch() && !qmc.config.ReadsFromElasticsearch() { + if !qmc.cfg.WritesToElasticsearch() && !qmc.cfg.ReadsFromElasticsearch() { return healthCheckStatus{"grey", "N/A (not writing)", ""} } return qmc.elasticStatusCache.check(func() healthCheckStatus { - resp, err := http.Get(qmc.config.Elasticsearch.Url.String()) + resp, err := http.Get(qmc.cfg.Elasticsearch.Url.String()) if err != nil { return healthCheckStatus{"red", "Ping failed", err.Error()} } diff --git a/quesma/quesma/ui/data_sources.go b/quesma/quesma/ui/data_sources.go index 8fe4cd510..89389f754 100644 --- a/quesma/quesma/ui/data_sources.go +++ b/quesma/quesma/ui/data_sources.go @@ -36,7 +36,7 @@ func (qmc *QuesmaManagementConsole) generateDatasources() []byte { buffer.Html(`
    `) tableNames := []string{} - for tableName := range qmc.config.IndexConfig { + for tableName := range qmc.cfg.IndexConfig { tableNames = append(tableNames, tableName) } slices.Sort(tableNames) diff --git a/quesma/quesma/ui/ingest.go b/quesma/quesma/ui/ingest.go index 80321e032..511404dc5 100644 --- a/quesma/quesma/ui/ingest.go +++ b/quesma/quesma/ui/ingest.go @@ -31,7 +31,7 @@ func (qmc *QuesmaManagementConsole) generateStatistics() []byte { var buffer builder.HtmlBuffer const maxTopValues = 5 - if !qmc.config.IngestStatistics { + if !qmc.cfg.IngestStatistics { buffer.Html("

    Statistics are disabled.

    \n") buffer.Html("

     You can enable them by changing ingest_statistics setting to true.

    \n") return buffer.Bytes() diff --git a/quesma/quesma/ui/live_tail.go b/quesma/quesma/ui/live_tail.go index 108963fd7..ea4b91b08 100644 --- a/quesma/quesma/ui/live_tail.go +++ b/quesma/quesma/ui/live_tail.go @@ -3,6 +3,7 @@ package ui import ( "fmt" "mitmproxy/quesma/buildinfo" + "mitmproxy/quesma/quesma/config" "mitmproxy/quesma/quesma/ui/internal/builder" "mitmproxy/quesma/util" ) @@ -80,7 +81,7 @@ document.body.addEventListener('htmx:afterSwap', function(event) { buffer.Html(`

    Details

    `) buffer.Html(`
      `) - buffer.Html("
    • Mode: ").Text(qmc.config.Mode.String()).Html("
      • ") + buffer.Html("
    • Mode: ").Text(qmc.cfg.Mode.String()).Html("
      • ") buffer.Html(`
    `) buffer.Html("\n
") @@ -104,7 +105,7 @@ func (qmc *QuesmaManagementConsole) generateQueries() []byte { } qmc.mutex.Unlock() - queriesBytes := generateQueries(debugKeyValueSlice, true) + queriesBytes := qmc.populateQueries(debugKeyValueSlice, true) queriesStats := qmc.generateQueriesStatsPanel() unsupportedQueriesStats := qmc.generateUnsupportedQuerySidePanel() return append(queriesBytes, append(queriesStats, unsupportedQueriesStats...)...) @@ -182,7 +183,7 @@ func (qmc *QuesmaManagementConsole) generateQueriesStatsPanel() []byte { return buffer.Bytes() } -func generateQueries(debugKeyValueSlice []queryDebugInfoWithId, withLinks bool) []byte { +func (qmc *QuesmaManagementConsole) populateQueries(debugKeyValueSlice []queryDebugInfoWithId, withLinks bool) []byte { var buffer builder.HtmlBuffer buffer.Html("\n" + `
` + "\n") @@ -205,25 +206,31 @@ func generateQueries(debugKeyValueSlice []queryDebugInfoWithId, withLinks bool) buffer.Html("\n
\n") buffer.Html(`
` + "\n") - buffer.Html(`
Elasticsearch response` + "\n" + `
`) - buffer.Html(`
`) - for _, v := range debugKeyValueSlice { - if withLinks { - buffer.Html(``) - } - tookStr := fmt.Sprintf(" took %d ms", v.query.PrimaryTook.Milliseconds()) - buffer.Html("

UUID:").Text(v.id).Text(tookStr).Html("

\n") - buffer.Html(`
`)
-		if len(v.query.QueryResp) > 0 {
-			buffer.Text(string(v.query.QueryResp))
-		} else {
-			buffer.Text("(empty, request was not sent to Elasticsearch)")
-		}
-		buffer.Html("\n
") - if withLinks { - buffer.Html("\n") + // TODO revisit after modes are redone + if qmc.cfg.Mode == config.DualWriteQueryClickhouse && qmc.cfg.Elasticsearch.Call { + buffer.Html(`
Elasticsearch response` + "\n" + `
`) + buffer.Html(`
`) + for _, v := range debugKeyValueSlice { + if withLinks { + buffer.Html(``) + } + tookStr := fmt.Sprintf(" took %d ms", v.query.PrimaryTook.Milliseconds()) + buffer.Html("

UUID:").Text(v.id).Text(tookStr).Html("

\n") + buffer.Html(`
`)
+			if len(v.query.QueryResp) > 0 {
+				buffer.Text(string(v.query.QueryResp))
+			} else {
+				buffer.Text("(empty, request was not sent to Elasticsearch)")
+			}
+			buffer.Html("\n
") + if withLinks { + buffer.Html("\n") + } } + } else { + buffer.Html(`
Elasticsearch response (not applicable)` + "\n" + `
`) } + buffer.Html("\n
") buffer.Html("\n
\n") diff --git a/quesma/quesma/ui/live_tail_drilldown.go b/quesma/quesma/ui/live_tail_drilldown.go index c4e4b20d3..edb3c8ec3 100644 --- a/quesma/quesma/ui/live_tail_drilldown.go +++ b/quesma/quesma/ui/live_tail_drilldown.go @@ -289,7 +289,7 @@ func (qmc *QuesmaManagementConsole) generateReportForRequests(title string, requ buffer.Html(`
`) - buffer.Write(generateQueries(requests, true)) + buffer.Write(qmc.populateQueries(requests, true)) buffer.Html("\n
\n\n") diff --git a/quesma/quesma/ui/management_console.go b/quesma/quesma/ui/management_console.go index 3de41f598..ee236035b 100644 --- a/quesma/quesma/ui/management_console.go +++ b/quesma/quesma/ui/management_console.go @@ -78,7 +78,7 @@ type QuesmaManagementConsole struct { debugInfoMessages map[string]queryDebugInfo debugLastMessages []string responseMatcherChannel chan queryDebugInfo - config config.QuesmaConfiguration + cfg config.QuesmaConfiguration requestsStore *stats.RequestStatisticStore requestsSource chan *recordRequests startedAt time.Time @@ -98,7 +98,7 @@ func NewQuesmaManagementConsole(config config.QuesmaConfiguration, logManager *c debugInfoMessages: make(map[string]queryDebugInfo), debugLastMessages: make([]string, 0), responseMatcherChannel: make(chan queryDebugInfo, 5), - config: config, + cfg: config, requestsStore: stats.NewRequestStatisticStore(), requestsSource: make(chan *recordRequests, 100), startedAt: time.Now(), diff --git a/quesma/quesma/ui/schema.go b/quesma/quesma/ui/schema.go index 2992d4a47..2b1f8c3d5 100644 --- a/quesma/quesma/ui/schema.go +++ b/quesma/quesma/ui/schema.go @@ -97,7 +97,7 @@ func (qmc *QuesmaManagementConsole) generateSchema() []byte { columnMap[k] = c } - for _, a := range qmc.config.AliasFields(table.Name) { + for _, a := range qmc.cfg.AliasFields(table.Name) { // check for collisions if field, collide := columnMap[a.SourceFieldName]; collide { @@ -238,7 +238,7 @@ func (qmc *QuesmaManagementConsole) generateSchema() []byte { buffer.Html(``) - for _, cfg := range qmc.config.IndexConfig { + for _, cfg := range qmc.cfg.IndexConfig { buffer.Html(``) buffer.Html(``) buffer.Text(cfg.Name)