UI refactor (#39)

This is a giant change. Mostly move and rename with minor delete of unused code. Principle: - replace `html_pages` and `html_panels` with filer per each tab - reduce the amount of exposed public API using `internal` and private name - keep every non-test file below 300 lines Long term goals: - Make UI configurable (e.g. port, disable debug, limit amount of messages) - Be able to refactor so one UI can attach to multiple quesma consoles - Make modular UI, so it's easy to add new tabs, even for occasional use case More changes will follow (e.g. more packages, configurability)
QuesmaOrg · May 6, 2024 · d54389f · d54389f
1 parent 1e31736
commit d54389f
Show file tree

Hide file tree

Showing 19 changed files with 1,551 additions and 1,533 deletions.
diff --git a/quesma/quesma/ui/console_routes.go b/quesma/quesma/ui/console_routes.go
@@ -3,16 +3,18 @@ package ui
 import (
 	"embed"
 	"encoding/json"
+	"errors"
 	"github.com/gorilla/mux"
 	"mitmproxy/quesma/logger"
 	"mitmproxy/quesma/stats"
 	"net/http"
+	"runtime"
 )
 
 const (
+	uiTcpPort              = "9999"
 	managementInternalPath = "/_quesma"
 	healthPath             = managementInternalPath + "/health"
-	bypassPath             = managementInternalPath + "/bypass"
 )
 
 //go:embed asset/*
@@ -25,8 +27,6 @@ func (qmc *QuesmaManagementConsole) createRouting() *mux.Router {
 
 	router.HandleFunc(healthPath, qmc.checkHealth)
 
-	router.HandleFunc(bypassPath, bypassSwitch).Methods("POST")
-
 	router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 
 	router.HandleFunc("/", func(writer http.ResponseWriter, req *http.Request) {
@@ -56,7 +56,7 @@ func (qmc *QuesmaManagementConsole) createRouting() *mux.Router {
 	})
 
 	router.HandleFunc("/telemetry", func(writer http.ResponseWriter, req *http.Request) {
-		buf := qmc.generatePhoneHome()
+		buf := qmc.generateTelemetry()
 		_, _ = writer.Write(buf)
 	})
 
@@ -71,7 +71,7 @@ func (qmc *QuesmaManagementConsole) createRouting() *mux.Router {
 	})
 
 	router.HandleFunc("/ingest-statistics", func(writer http.ResponseWriter, req *http.Request) {
-		buf := qmc.generateStatisticsLiveTail()
+		buf := qmc.generateIngestStatistics()
 		_, _ = writer.Write(buf)
 	})
 
@@ -160,3 +160,50 @@ func (qmc *QuesmaManagementConsole) createRouting() *mux.Router {
 	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.FS(uiFs))))
 	return router
 }
+
+func (qmc *QuesmaManagementConsole) newHTTPServer() *http.Server {
+	return &http.Server{
+		Addr:    ":" + uiTcpPort,
+		Handler: qmc.createRouting(),
+	}
+}
+
+func panicRecovery(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				buf := make([]byte, 2048)
+				n := runtime.Stack(buf, false)
+				buf = buf[:n]
+
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Header().Set("Content-Type", "text/plain")
+				w.Write([]byte("Internal Server Error\n\n"))
+
+				w.Write([]byte("Stack:\n"))
+				w.Write(buf)
+				logger.Error().Msgf("recovering from err %v\n %s", err, buf)
+			}
+		}()
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func (qmc *QuesmaManagementConsole) checkHealth(writer http.ResponseWriter, _ *http.Request) {
+	health := qmc.checkElasticsearch()
+	if health.status != "red" {
+		writer.WriteHeader(200)
+		writer.Header().Set("Content-Type", "application/json")
+		_, _ = writer.Write([]byte(`{"cluster_name": "quesma"}`))
+	} else {
+		writer.WriteHeader(503)
+		_, _ = writer.Write([]byte(`Elastic search is unavailable: ` + health.message))
+	}
+}
+
+func (qmc *QuesmaManagementConsole) listenAndServe() {
+	if err := qmc.ui.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		logger.Fatal().Msgf("Error starting server: %v", err)
+	}
+}
diff --git a/quesma/quesma/ui/dashboard.go b/quesma/quesma/ui/dashboard.go
@@ -0,0 +1,228 @@
+package ui
+
+import (
+	"fmt"
+	"github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v3/host"
+	"github.com/shirou/gopsutil/v3/mem"
+	"mitmproxy/quesma/buildinfo"
+	"mitmproxy/quesma/quesma/ui/internal/buffer"
+	"mitmproxy/quesma/stats/errorstats"
+	"net/url"
+	"runtime"
+	"strings"
+	"time"
+)
+
+func (qmc *QuesmaManagementConsole) generateDashboard() []byte {
+	buffer := newBufferWithHead()
+	buffer.Write(generateTopNavigation("dashboard"))
+
+	buffer.Html(`<main id="dashboard-main">` + "\n")
+
+	// Unfortunately, we need tiny bit of javascript to pause the animation.
+	buffer.Html(`<script type="text/javascript">`)
+	buffer.Html(`var checkbox = document.getElementById("autorefresh");`)
+	buffer.Html(`var dashboard = document.getElementById("dashboard-main");`)
+	buffer.Html(`checkbox.addEventListener('change', function() {`)
+	buffer.Html(`if (this.checked) {`)
+	buffer.Html(`dashboard.classList.remove("paused");`)
+	buffer.Html(`} else {`)
+	buffer.Html(`dashboard.classList.add("paused");`)
+	buffer.Html(`}`)
+	buffer.Html(`});`)
+	buffer.Html(`</script>` + "\n")
+
+	buffer.Html(`<div id="svg-container">`)
+	buffer.Html(`<svg width="100%" height="100%" viewBox="0 0 1000 1000" preserveAspectRatio="none">` + "\n")
+	// One limitation is that, we don't update color of paths after initial draw.
+	// They rarely change, so it's not a big deal for now.
+	// Clickhouse -> Kibana
+	if qmc.config.ReadsFromClickhouse() {
+		status, _ := qmc.generateDashboardTrafficText(RequestStatisticKibana2Clickhouse)
+		buffer.Html(fmt.Sprintf(`<path d="M 0 250 L 1000 250" fill="none" stroke="%s" />`, status))
+	}
+	// Elasticsearch -> Kibana
+	if qmc.config.ReadsFromElasticsearch() {
+		status, _ := qmc.generateDashboardTrafficText(RequestStatisticKibana2Elasticsearch)
+		buffer.Html(fmt.Sprintf(`<path d="M 0 350 L 150 350 L 150 700 L 1000 700" fill="none" stroke="%s" />`, status))
+	}
+
+	// Ingest -> Clickhouse
+	if qmc.config.WritesToClickhouse() {
+		status, _ := qmc.generateDashboardTrafficText(RequestStatisticIngest2Clickhouse)
+		buffer.Html(fmt.Sprintf(`<path d="M 1000 350 L 300 350 L 300 650 L 0 650" fill="none" stroke="%s" />`, status))
+	}
+	// Ingest -> Elasticsearch
+	if qmc.config.WritesToElasticsearch() {
+		status, _ := qmc.generateDashboardTrafficText(RequestStatisticIngest2Elasticsearch)
+		buffer.Html(fmt.Sprintf(`<path d="M 1000 800 L 0 800" fill="none" stroke="%s" />`, status))
+	}
+	buffer.Html(`</svg>` + "\n")
+	buffer.Write(qmc.generateDashboardTrafficPanel())
+	buffer.Html(`</div>` + "\n")
+
+	buffer.Html(`<div id="dashboard">` + "\n")
+	buffer.Write(qmc.generateDashboardPanel())
+	buffer.Html("</div>\n")
+	buffer.Html("\n</main>\n\n")
+
+	buffer.Html("\n</body>")
+	buffer.Html("\n</html>")
+	return buffer.Bytes()
+}
+
+func (qmc *QuesmaManagementConsole) generateDashboardTrafficText(typeName string) (string, string) {
+	reqStats := qmc.requestsStore.GetRequestsStats(typeName)
+	status := "green"
+	if reqStats.ErrorRate > 0.20 {
+		status = "red"
+	}
+	return status, fmt.Sprintf("%4.1f req/s, err:%5.1f%%, p99:%3dms",
+		reqStats.RatePerMinute/60, reqStats.ErrorRate*100, reqStats.Duration99Percentile)
+}
+
+func (qmc *QuesmaManagementConsole) generateDashboardTrafficElement(typeName string, y int) string {
+	status, text := qmc.generateDashboardTrafficText(typeName)
+	return fmt.Sprintf(
+		`<div style="left: 40%%; top: %d%%" id="traffic-%s" hx-swap-oob="true" class="traffic-element %s">%s</div>`,
+		y, typeName, status, text)
+}
+
+func (qmc *QuesmaManagementConsole) generateDashboardTrafficPanel() []byte {
+	var buffer buffer.HtmlBuffer
+
+	// Clickhouse -> Kibana
+	if qmc.config.ReadsFromClickhouse() {
+		buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticKibana2Clickhouse, 21))
+	}
+
+	// Elasticsearch -> Kibana
+	if qmc.config.ReadsFromElasticsearch() {
+		buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticKibana2Elasticsearch, 66))
+	}
+
+	// Ingest -> Clickhouse
+	if qmc.config.WritesToClickhouse() {
+		buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticIngest2Clickhouse, 31))
+	}
+
+	// Ingest -> Elasticsearch
+	if qmc.config.WritesToElasticsearch() {
+		buffer.Html(qmc.generateDashboardTrafficElement(RequestStatisticIngest2Elasticsearch, 76))
+	}
+
+	return buffer.Bytes()
+}
+
+func secondsToTerseString(second uint64) string {
+	return (time.Duration(second) * time.Second).String()
+}
+
+func statusToDiv(s healthCheckStatus) string {
+	return fmt.Sprintf(`<div class="status %s" title="%s">%s</div>`, s.status, s.tooltip, s.message)
+}
+
+func (qmc *QuesmaManagementConsole) generateDashboardPanel() []byte {
+	var buffer buffer.HtmlBuffer
+
+	dashboardName := "<h3>Kibana</h3>"
+	storeName := "<h3>Elasticsearch</h3>"
+	if qmc.config.Elasticsearch.Url != nil && strings.Contains(qmc.config.Elasticsearch.Url.String(), "opensearch") {
+		dashboardName = "<h3>OpenSearch</h3><h3>Dashboards</h3>"
+		storeName = "<h3>OpenSearch</h3>"
+	}
+
+	clickhouseName := "<h3>ClickHouse</h3>"
+	if qmc.config.Hydrolix.Url != nil {
+		clickhouseName = "<h3>Hydrolix</h3>"
+	}
+
+	buffer.Html(`<div id="dashboard-kibana" class="component">`)
+	if qmc.config.Elasticsearch.AdminUrl != nil {
+		buffer.Html(fmt.Sprintf(`<a href="%s">`, qmc.config.Elasticsearch.AdminUrl.String()))
+	}
+	buffer.Html(dashboardName)
+	if qmc.config.Elasticsearch.AdminUrl != nil {
+		buffer.Html(`</a>`)
+	}
+	buffer.Html(statusToDiv(qmc.checkKibana()))
+	buffer.Html(`</div>`)
+
+	buffer.Html(`<div id="dashboard-ingest" class="component">`)
+	buffer.Html(`<h3>Ingest</h3>`)
+	buffer.Html(statusToDiv(qmc.checkIngest()))
+	buffer.Html(`</div>`)
+
+	buffer.Html(`<div id="dashboard-elasticsearch" class="component">`)
+	buffer.Html(storeName)
+	buffer.Html(statusToDiv(qmc.checkElasticsearch()))
+	buffer.Html(`</div>`)
+
+	buffer.Html(`<div id="dashboard-clickhouse" class="component">`)
+	if qmc.config.ClickHouse.AdminUrl != nil {
+		buffer.Html(fmt.Sprintf(`<a href="%s">`, qmc.config.ClickHouse.AdminUrl.String()))
+	}
+	buffer.Html(clickhouseName)
+	if qmc.config.ClickHouse.AdminUrl != nil {
+		buffer.Html(`</a>`)
+	}
+	buffer.Html(statusToDiv(qmc.checkClickhouseHealth()))
+	buffer.Html(`</div>`)
+
+	buffer.Html(`<div id="dashboard-traffic" class="component">`)
+
+	buffer.Html(`<div id="dashboard-quesma" class="component">`)
+	buffer.Html(`<h3>Quesma</h3>`)
+
+	cpuStr := ""
+	c0, err0 := cpu.Percent(0, false)
+
+	if err0 == nil {
+		cpuStr = fmt.Sprintf("Host CPU: %.1f%%", c0[0])
+	} else {
+		cpuStr = fmt.Sprintf("Host CPU: N/A (error: %s)", err0.Error())
+	}
+
+	buffer.Html(fmt.Sprintf(`<div class="status">%s</div>`, cpuStr))
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	memStr := fmt.Sprintf("Memory used: %1.f MB", float64(m.Alloc)/1024.0/1024.0)
+	if v, errV := mem.VirtualMemory(); errV == nil {
+		total := float64(v.Total) / 1024.0 / 1024.0 / 1024.0
+		memStr += fmt.Sprintf(", avail: %.1f GB", total)
+	}
+	buffer.Html(fmt.Sprintf(`<div class="status">%s</div>`, memStr))
+
+	duration := uint64(time.Since(qmc.startedAt).Seconds())
+
+	buffer.Html(fmt.Sprintf(`<div class="status">Started: %s ago</div>`, secondsToTerseString(duration)))
+	buffer.Html(fmt.Sprintf(`<div class="status">Mode: %s</div>`, qmc.config.Mode.String()))
+
+	if h, errH := host.Info(); errH == nil {
+		buffer.Html(fmt.Sprintf(`<div class="status">Host uptime: %s</div>`, secondsToTerseString(h.Uptime)))
+	}
+
+	buffer.Html("<div>Version: ")
+	buffer.Text(buildinfo.Version)
+	buffer.Html("</div>")
+
+	buffer.Html(`</div>`)
+
+	buffer.Html(`<div id="dashboard-errors" class="component">`)
+	errors := errorstats.GlobalErrorStatistics.ReturnTopErrors(5)
+	if len(errors) > 0 {
+		buffer.Html(`<h3>Top errors:</h3>`)
+		for _, e := range errors {
+			buffer.Html(fmt.Sprintf(`<div class="status">%d: <a href="/error/%s">%s</a></div>`,
+				e.Count, url.PathEscape(e.Reason), e.Reason))
+		}
+	} else {
+		buffer.Html(`<h3>No errors</h3>`)
+	}
+	buffer.Html(`</div>`)
+	buffer.Html(`</div>`)
+
+	return buffer.Bytes()
+}
diff --git a/quesma/quesma/ui/dashboard_drilldown.go b/quesma/quesma/ui/dashboard_drilldown.go
@@ -0,0 +1,32 @@
+package ui
+
+import (
+	"fmt"
+	"mitmproxy/quesma/stats/errorstats"
+)
+
+func (qmc *QuesmaManagementConsole) generateErrorForReason(reason string) []byte {
+	buffer := newBufferWithHead()
+	buffer.Write(generateTopNavigation(fmt.Sprintf("Errors with reason '%s'", reason)))
+
+	buffer.Html(`<main id="errors">`)
+	errors := errorstats.GlobalErrorStatistics.ErrorReportsForReason(reason)
+	// TODO: Make it nicer
+	for _, errorReport := range errors {
+		buffer.Html("<p>").Text(errorReport.ReportedAt.String() + " " + errorReport.DebugMessage).Html("</p>\n")
+	}
+	buffer.Html("\n</main>\n\n")
+
+	buffer.Html(`<div class="menu">`)
+	buffer.Html("\n<h2>Menu</h2>")
+
+	buffer.Html(`<form action="/">&nbsp;<input class="btn" type="submit" value="Back to dashboard" /></form>`)
+	// TODO: implement
+	// buffer.Html(`<form action="/dashboard">&nbsp;<input class="btn" type="submit" value="See requests with errors" /></form>`)
+	buffer.Html("\n</div>")
+
+	buffer.Html("\n</body>")
+	buffer.Html("\n</html>")
+
+	return buffer.Bytes()
+}