From c8f4e000fcdf60a7508f5ffbbe64f87d78a7fd49 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 13 Feb 2025 12:43:37 +0300 Subject: [PATCH 01/18] PMM-13171 Update Readme --- build/ansible/roles/grafana/files/datasources.yml | 4 ++++ docs/process/v2_to_v3_environment_variables.md | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/build/ansible/roles/grafana/files/datasources.yml b/build/ansible/roles/grafana/files/datasources.yml index f6b50d2e30..645082ccf7 100644 --- a/build/ansible/roles/grafana/files/datasources.yml +++ b/build/ansible/roles/grafana/files/datasources.yml @@ -2,6 +2,7 @@ apiVersion: 1 deleteDatasources: - name: ClickHouse orgId: 1 + datasources: - name: Metrics version: 2 @@ -14,6 +15,7 @@ datasources: httpMethod: POST keepCookies: [] timeInterval: 1s + - name: PostgreSQL version: 2 orgId: 1 @@ -35,6 +37,8 @@ datasources: orgId: 1 type: pmm-pt-summary-datasource access: proxy + +# https://github.com/grafana/clickhouse-datasource?tab=readme-ov-file#with-a-configuration-file - name: ClickHouse orgId: 1 version: 2 diff --git a/docs/process/v2_to_v3_environment_variables.md b/docs/process/v2_to_v3_environment_variables.md index 2070ddf699..88fb033a20 100644 --- a/docs/process/v2_to_v3_environment_variables.md +++ b/docs/process/v2_to_v3_environment_variables.md @@ -26,12 +26,12 @@ Below is a list of affected variables and their new names. | `PERCONA_TEST_CHECKS_PUBLIC_KEY` | | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY` | | `PERCONA_TEST_NICER_API` | | Removed in PMM v3 | | `PERCONA_TEST_PMM_CLICKHOUSE_ADDR` | `PMM_CLICKHOUSE_ADDR` | | -| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE` | | Removed in PMM v3, because of new clickhouse version. | | `PERCONA_TEST_PMM_CLICKHOUSE_DATABASE` | `PMM_CLICKHOUSE_DATABASE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE` | `PMM_CLICKHOUSE_DATASOURCE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_HOST` | `PMM_CLICKHOUSE_HOST` | | -| `PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE` | | Removed in PMM v3, because of new clickhouse version. | | `PERCONA_TEST_PMM_CLICKHOUSE_PORT` | `PMM_CLICKHOUSE_PORT` | | +| `PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE` | | Removed in PMM v3, because of new clickhouse version. | +| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE` | | Removed in PMM v3, because of new clickhouse version. | | `PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE` | `PMM_DISABLE_BUILTIN_CLICKHOUSE` | | | `PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES` | `PMM_DISABLE_BUILTIN_POSTGRES` | | | `PERCONA_TEST_INTERFACE_TO_BIND` | `PMM_INTERFACE_TO_BIND` | | @@ -53,5 +53,13 @@ Below is a list of affected variables and their new names. | `PMM_TEST_TELEMETRY_FILE` | `PMM_DEV_TELEMETRY_FILE` | | | `PERCONA_TEST_TELEMETRY_HOST` | `PMM_DEV_TELEMETRY_HOST` | | | `PERCONA_TEST_TELEMETRY_INTERVAL` | `PMM_DEV_TELEMETRY_INTERVAL` | | -| `PERCONA_TEST_TELEMETRY_RETRY_BACKOFF` | `PMM_DEV_TELEMETRY_RETRY_BACKOFF` | | +| `PERCONA_TEST_TELEMETRY_RETRY_BACKOFF` | `PMM_DEV_TELEMETRY_RETRY_BACKOFF` | | | `PERCONA_TEST_VERSION_SERVICE_URL` | | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_ADDRESS` | + + +## Changes to environment variables in v3+ + +| PMM 3 | Comments | +|--------------------------------------------|--------------------------------------------------------------| +| `PMM_CLICKHOUSE_USER` | Added in v3.1.0 | +| `PMM_CLICKHOUSE_PASSWORD` | Added in v3.1.0 | From 9878b9efdbe4260ae37df9ad90096372a5826269 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 13 Feb 2025 21:11:34 +0300 Subject: [PATCH 02/18] PMM-13171 Add credentials via env variables --- .../roles/grafana/files/datasources.yml | 8 +++++-- managed/services/supervisord/supervisord.go | 8 +++++++ managed/testdata/supervisord.d/grafana.ini | 2 ++ qan-api2/main.go | 23 +++++++++++++------ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/build/ansible/roles/grafana/files/datasources.yml b/build/ansible/roles/grafana/files/datasources.yml index 645082ccf7..a927b4f014 100644 --- a/build/ansible/roles/grafana/files/datasources.yml +++ b/build/ansible/roles/grafana/files/datasources.yml @@ -32,6 +32,7 @@ datasources: sslmode: ${PMM_POSTGRES_SSL_MODE} secureJsonData: password: ${PMM_POSTGRES_DBPASSWORD} + - name: PTSummary version: 2 orgId: 1 @@ -44,6 +45,9 @@ datasources: version: 2 type: grafana-clickhouse-datasource jsonData: + username: ${PMM_CLICKHOUSE_USERNAME} port: ${PMM_CLICKHOUSE_PORT} - server: ${PMM_CLICKHOUSE_HOST} - tlsSkipVerify: false \ No newline at end of file + host: ${PMM_CLICKHOUSE_HOST} + tlsSkipVerify: false + secureJsonData: + password: ${PMM_CLICKHOUSE_PASSWORD} \ No newline at end of file diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 20c8e792a0..0a5a078ded 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -47,6 +47,8 @@ const ( defaultClickhouseDatabase = "pmm" defaultClickhouseAddr = "127.0.0.1:9000" defaultClickhouseDataSourceAddr = "127.0.0.1:8123" + defaultClickhouseUser = "default" + defaultClickhousePassword = "" defaultVMSearchMaxQueryLen = "1MB" defaultVMSearchLatencyOffset = "5s" defaultVMSearchMaxUniqueTimeseries = "100000000" @@ -277,6 +279,8 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin clickhouseAddr := getValueFromENV("PMM_CLICKHOUSE_ADDR", defaultClickhouseAddr) clickhouseDataSourceAddr := getValueFromENV("PMM_CLICKHOUSE_DATASOURCE_ADDR", defaultClickhouseDataSourceAddr) clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2) + clickhouseUser := getValueFromENV("PMM_CLICKHOUSE_USER", defaultClickhouseUser) + clickhousePassword := getValueFromENV("PMM_CLICKHOUSE_PASSWORD", defaultClickhousePassword) vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.IsVictoriaMetricsCacheEnabled())) vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen) vmSearchLatencyOffset := getValueFromENV("VM_search_latencyOffset", defaultVMSearchLatencyOffset) @@ -308,6 +312,8 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "ClickhouseDatabase": clickhouseDatabase, "ClickhouseHost": clickhouseAddrPair[0], "ClickhousePort": clickhouseAddrPair[1], + "ClickhouseUser": clickhouseUser, + "ClickhousePassword": clickhousePassword, } s.addPostgresParams(templateParams) @@ -623,6 +629,8 @@ environment = PMM_CLICKHOUSE_DATASOURCE_ADDR="{{ .ClickhouseDataSourceAddr }}", PMM_CLICKHOUSE_HOST="{{ .ClickhouseHost }}", PMM_CLICKHOUSE_PORT="{{ .ClickhousePort }}", + PMM_CLICKHOUSE_USER="{{ .ClickhouseUser }}", + PMM_CLICKHOUSE_PASSWORD="{{ .ClickhousePassword }}", {{- if .PerconaSSODetails}} GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} diff --git a/managed/testdata/supervisord.d/grafana.ini b/managed/testdata/supervisord.d/grafana.ini index 107cdf2661..4bb20a1430 100644 --- a/managed/testdata/supervisord.d/grafana.ini +++ b/managed/testdata/supervisord.d/grafana.ini @@ -19,6 +19,8 @@ environment = PMM_CLICKHOUSE_DATASOURCE_ADDR="127.0.0.1:8123", PMM_CLICKHOUSE_HOST="127.0.0.1", PMM_CLICKHOUSE_PORT="9000", + PMM_CLICKHOUSE_USER="default", + PMM_CLICKHOUSE_PASSWORD="", user = pmm directory = /usr/share/grafana autorestart = true diff --git a/qan-api2/main.go b/qan-api2/main.go index 4e6920d021..00dc14b296 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -25,6 +25,7 @@ import ( "net" "net/http" _ "net/http/pprof" //nolint:gosec + "net/url" "os" "os/signal" "strings" @@ -62,7 +63,7 @@ import ( const ( shutdownTimeout = 3 * time.Second - defaultDsnF = "clickhouse://%s/%s" + defaultDsnF = "clickhouse://%s:%s@%s/%s" maxIdleConns = 5 maxOpenConns = 10 ) @@ -255,9 +256,11 @@ func main() { jsonBindF := kingpin.Flag("json-bind", "JSON bind address and port").Default("127.0.0.1:9922").String() debugBindF := kingpin.Flag("listen-debug-addr", "Debug server listen address").Default("127.0.0.1:9933").String() dataRetentionF := kingpin.Flag("data-retention", "QAN data Retention (in days)").Default("30").Uint() - dsnF := kingpin.Flag("dsn", "ClickHouse database DSN. Can be override with database/host/port options").Default(defaultDsnF).String() - clickHouseDatabaseF := kingpin.Flag("clickhouse-name", "Clickhouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() - clickhouseAddrF := kingpin.Flag("clickhouse-addr", "Clickhouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() + dsnF := kingpin.Flag("dsn", "ClickHouse database DSN. Can be overriden with database/host/port options").Default(defaultDsnF).String() + clickhouseDatabaseF := kingpin.Flag("clickhouse-name", "ClickHouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() + clickhouseAddrF := kingpin.Flag("clickhouse-addr", "ClickHouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() + clickhouseUserF := kingpin.Flag("clickhouse-user", "ClickHouse database user").Default("default").Envar("PMM_CLICKHOUSE_USER").String() + clickhousePasswordF := kingpin.Flag("clickhouse-password", "ClickHouse database password").Default("").Envar("PMM_CLICKHOUSE_PASSWORD").String() debugF := kingpin.Flag("debug", "Enable debug logging").Bool() traceF := kingpin.Flag("trace", "Enable trace logging (implies debug)").Bool() @@ -285,12 +288,18 @@ func main() { var dsn string if *dsnF == defaultDsnF { - dsn = fmt.Sprintf(defaultDsnF, *clickhouseAddrF, *clickHouseDatabaseF) + dsn = fmt.Sprintf(defaultDsnF, *clickhouseUserF, *clickhousePasswordF, *clickhouseAddrF, *clickhouseDatabaseF) } else { dsn = *dsnF } - l.Info("DSN: ", dsn) + u, err := url.Parse(dsn) + if err != nil { + l.Error("Failed to parse DSN: ", err) + } else { + l.Info("DSN: ", u.Redacted()) + } + db := NewDB(dsn, maxIdleConns, maxOpenConns) prom.MustRegister(sqlmetrics.NewCollector("clickhouse", "qan-api2", db.DB)) @@ -345,7 +354,7 @@ func main() { defer wg.Done() for { // Drop old partitions once in 24h. - DropOldPartition(db, *clickHouseDatabaseF, *dataRetentionF) + DropOldPartition(db, *clickhouseDatabaseF, *dataRetentionF) select { case <-ctx.Done(): return From e03e35cd6f9bc7481c7b07ec03c6a4c5ad2c615a Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 13 Feb 2025 23:56:25 +0300 Subject: [PATCH 03/18] PMM-13171 Use v1 in protobuf names --- qan-api2/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qan-api2/main.go b/qan-api2/main.go index 00dc14b296..cb7a7f258f 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -50,7 +50,7 @@ import ( "google.golang.org/grpc/reflection" "google.golang.org/protobuf/encoding/protojson" - qanpb "github.com/percona/pmm/api/qan/v1" + qanv1 "github.com/percona/pmm/api/qan/v1" "github.com/percona/pmm/qan-api2/models" aservice "github.com/percona/pmm/qan-api2/services/analytics" rservice "github.com/percona/pmm/qan-api2/services/receiver" @@ -94,8 +94,8 @@ func runGRPCServer(ctx context.Context, db *sqlx.DB, mbm *models.MetricsBucket, ) aserv := aservice.NewService(rm, mm) - qanpb.RegisterCollectorServiceServer(grpcServer, rservice.NewService(mbm)) - qanpb.RegisterQANServiceServer(grpcServer, aserv) + qanv1.RegisterCollectorServiceServer(grpcServer, rservice.NewService(mbm)) + qanv1.RegisterQANServiceServer(grpcServer, aserv) reflection.Register(grpcServer) if l.Logger.GetLevel() >= logrus.DebugLevel { @@ -155,7 +155,7 @@ func runJSONServer(ctx context.Context, grpcBindF, jsonBindF string) { type registrar func(context.Context, *grpc_gateway.ServeMux, string, []grpc.DialOption) error for _, r := range []registrar{ - qanpb.RegisterQANServiceHandlerFromEndpoint, + qanv1.RegisterQANServiceHandlerFromEndpoint, } { if err := r(ctx, proxyMux, grpcBindF, opts); err != nil { l.Panic(err) From cd382b4970903e40a3c9833d75ae0e7a52402598 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 13 Feb 2025 23:58:32 +0300 Subject: [PATCH 04/18] PMM-13171 Add empty line --- build/ansible/roles/grafana/files/datasources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ansible/roles/grafana/files/datasources.yml b/build/ansible/roles/grafana/files/datasources.yml index a927b4f014..929462d898 100644 --- a/build/ansible/roles/grafana/files/datasources.yml +++ b/build/ansible/roles/grafana/files/datasources.yml @@ -50,4 +50,4 @@ datasources: host: ${PMM_CLICKHOUSE_HOST} tlsSkipVerify: false secureJsonData: - password: ${PMM_CLICKHOUSE_PASSWORD} \ No newline at end of file + password: ${PMM_CLICKHOUSE_PASSWORD} From f10146f13bd7186744afe885f8520b2abd09d144 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 00:05:01 +0300 Subject: [PATCH 05/18] PMM-13171 Remove redundant const --- managed/services/supervisord/supervisord.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 0a5a078ded..22edab4a8e 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -48,7 +48,6 @@ const ( defaultClickhouseAddr = "127.0.0.1:9000" defaultClickhouseDataSourceAddr = "127.0.0.1:8123" defaultClickhouseUser = "default" - defaultClickhousePassword = "" defaultVMSearchMaxQueryLen = "1MB" defaultVMSearchLatencyOffset = "5s" defaultVMSearchMaxUniqueTimeseries = "100000000" @@ -280,7 +279,7 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin clickhouseDataSourceAddr := getValueFromENV("PMM_CLICKHOUSE_DATASOURCE_ADDR", defaultClickhouseDataSourceAddr) clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2) clickhouseUser := getValueFromENV("PMM_CLICKHOUSE_USER", defaultClickhouseUser) - clickhousePassword := getValueFromENV("PMM_CLICKHOUSE_PASSWORD", defaultClickhousePassword) + clickhousePassword := getValueFromENV("PMM_CLICKHOUSE_PASSWORD", "") vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.IsVictoriaMetricsCacheEnabled())) vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen) vmSearchLatencyOffset := getValueFromENV("VM_search_latencyOffset", defaultVMSearchLatencyOffset) @@ -323,7 +322,7 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin if settings.PMMPublicAddress != "" { pmmPublicAddress := settings.PMMPublicAddress if !strings.HasPrefix(pmmPublicAddress, "https://") && !strings.HasPrefix(pmmPublicAddress, "http://") { - pmmPublicAddress = fmt.Sprintf("https://%s", pmmPublicAddress) + pmmPublicAddress = "https://" + pmmPublicAddress } publicURL, err := url.Parse(pmmPublicAddress) if err != nil { From d21f9bf91f78711fb6f2b051d241897e4f860d56 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 00:11:24 +0300 Subject: [PATCH 06/18] PMM-13171 Pass CH creds to qan-api spec --- managed/services/supervisord/supervisord.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 22edab4a8e..37787767f0 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -579,6 +579,8 @@ command = environment = PMM_CLICKHOUSE_ADDR="{{ .ClickhouseAddr }}", PMM_CLICKHOUSE_DATABASE="{{ .ClickhouseDatabase }}", + PMM_CLICKHOUSE_USER="{{ .ClickhouseUser }}", + PMM_CLICKHOUSE_PASSWORD="{{ .ClickhousePassword }}", user = pmm From d7eb535b0f3d3a851be7a4ede96500bf8ec7b062 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 00:13:42 +0300 Subject: [PATCH 07/18] PMM-13171 Align parameters --- managed/services/supervisord/supervisord.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 37787767f0..14fb453ffc 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -630,8 +630,8 @@ environment = PMM_CLICKHOUSE_DATASOURCE_ADDR="{{ .ClickhouseDataSourceAddr }}", PMM_CLICKHOUSE_HOST="{{ .ClickhouseHost }}", PMM_CLICKHOUSE_PORT="{{ .ClickhousePort }}", - PMM_CLICKHOUSE_USER="{{ .ClickhouseUser }}", - PMM_CLICKHOUSE_PASSWORD="{{ .ClickhousePassword }}", + PMM_CLICKHOUSE_USER="{{ .ClickhouseUser }}", + PMM_CLICKHOUSE_PASSWORD="{{ .ClickhousePassword }}", {{- if .PerconaSSODetails}} GF_AUTH_SIGNOUT_REDIRECT_URL="https://{{ .IssuerDomain }}/login/signout?fromURI=https://{{ .PMMServerAddress }}/graph/login" {{- end}} From 4ada2cb229275ec63bed13d0996dc044061c6bd0 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 00:21:34 +0300 Subject: [PATCH 08/18] PMM-13171 Fix the spelling --- qan-api2/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qan-api2/main.go b/qan-api2/main.go index cb7a7f258f..6d982d064d 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -256,7 +256,7 @@ func main() { jsonBindF := kingpin.Flag("json-bind", "JSON bind address and port").Default("127.0.0.1:9922").String() debugBindF := kingpin.Flag("listen-debug-addr", "Debug server listen address").Default("127.0.0.1:9933").String() dataRetentionF := kingpin.Flag("data-retention", "QAN data Retention (in days)").Default("30").Uint() - dsnF := kingpin.Flag("dsn", "ClickHouse database DSN. Can be overriden with database/host/port options").Default(defaultDsnF).String() + dsnF := kingpin.Flag("dsn", "ClickHouse database DSN. Can be overridden with database/host/port options").Default(defaultDsnF).String() clickhouseDatabaseF := kingpin.Flag("clickhouse-name", "ClickHouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() clickhouseAddrF := kingpin.Flag("clickhouse-addr", "ClickHouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() clickhouseUserF := kingpin.Flag("clickhouse-user", "ClickHouse database user").Default("default").Envar("PMM_CLICKHOUSE_USER").String() From 448eaa035d16c97ffcea636acf206d81bf7f6e20 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 13:44:51 +0300 Subject: [PATCH 09/18] PMM-13171 Fix the test --- managed/testdata/supervisord.d/qan-api2.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/managed/testdata/supervisord.d/qan-api2.ini b/managed/testdata/supervisord.d/qan-api2.ini index 3422475cb7..61783d509b 100644 --- a/managed/testdata/supervisord.d/qan-api2.ini +++ b/managed/testdata/supervisord.d/qan-api2.ini @@ -8,6 +8,8 @@ command = environment = PMM_CLICKHOUSE_ADDR="127.0.0.1:9000", PMM_CLICKHOUSE_DATABASE="pmm", + PMM_CLICKHOUSE_USER="default", + PMM_CLICKHOUSE_PASSWORD="", user = pmm From 790399dbd02b0eb5e03a1abcaf304ea167dc8cee Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 17:45:01 +0300 Subject: [PATCH 10/18] PMM-13171 Add user password and change users' config --- .../ansible/roles/clickhouse/files/config.xml | 8 +- .../ansible/roles/clickhouse/files/users.xml | 110 ++++++++++++++++++ build/ansible/roles/clickhouse/tasks/main.yml | 7 +- managed/services/supervisord/supervisord.go | 3 +- managed/testdata/supervisord.d/grafana.ini | 2 +- managed/testdata/supervisord.d/qan-api2.ini | 2 +- qan-api2/main.go | 2 +- 7 files changed, 124 insertions(+), 10 deletions(-) create mode 100755 build/ansible/roles/clickhouse/files/users.xml diff --git a/build/ansible/roles/clickhouse/files/config.xml b/build/ansible/roles/clickhouse/files/config.xml index 20a08592d3..a38f51ac60 100644 --- a/build/ansible/roles/clickhouse/files/config.xml +++ b/build/ansible/roles/clickhouse/files/config.xml @@ -138,7 +138,7 @@ - 9004 + - 1 - 1 - 1 + 0 + 0 + 0 + + + + + + + + + + 1 + + + + + + + + + 7e099f39b84ea79559b3e85ea046804e63725fd1f46b37f281276aae20f86dc3 + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/build/ansible/roles/clickhouse/tasks/main.yml b/build/ansible/roles/clickhouse/tasks/main.yml index baff942535..184203de95 100644 --- a/build/ansible/roles/clickhouse/tasks/main.yml +++ b/build/ansible/roles/clickhouse/tasks/main.yml @@ -36,11 +36,14 @@ - name: Copy a customized clickhouse config copy: - src: config.xml - dest: /etc/clickhouse-server/config.xml + src: "{{ item }}" + dest: "/etc/clickhouse-server/{{ item }}" owner: pmm group: pmm mode: 0664 + loop: + - config.xml + - users.xml # We need to remove capabilities because we run PMM in an unprivileged container # But we run clickhouse as root user diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 14fb453ffc..0b6b173e06 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -48,6 +48,7 @@ const ( defaultClickhouseAddr = "127.0.0.1:9000" defaultClickhouseDataSourceAddr = "127.0.0.1:8123" defaultClickhouseUser = "default" + defaultClickhousePassword = "clickhouse" defaultVMSearchMaxQueryLen = "1MB" defaultVMSearchLatencyOffset = "5s" defaultVMSearchMaxUniqueTimeseries = "100000000" @@ -279,7 +280,7 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin clickhouseDataSourceAddr := getValueFromENV("PMM_CLICKHOUSE_DATASOURCE_ADDR", defaultClickhouseDataSourceAddr) clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2) clickhouseUser := getValueFromENV("PMM_CLICKHOUSE_USER", defaultClickhouseUser) - clickhousePassword := getValueFromENV("PMM_CLICKHOUSE_PASSWORD", "") + clickhousePassword := getValueFromENV("PMM_CLICKHOUSE_PASSWORD", defaultClickhousePassword) vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.IsVictoriaMetricsCacheEnabled())) vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen) vmSearchLatencyOffset := getValueFromENV("VM_search_latencyOffset", defaultVMSearchLatencyOffset) diff --git a/managed/testdata/supervisord.d/grafana.ini b/managed/testdata/supervisord.d/grafana.ini index 4bb20a1430..8a550f8477 100644 --- a/managed/testdata/supervisord.d/grafana.ini +++ b/managed/testdata/supervisord.d/grafana.ini @@ -20,7 +20,7 @@ environment = PMM_CLICKHOUSE_HOST="127.0.0.1", PMM_CLICKHOUSE_PORT="9000", PMM_CLICKHOUSE_USER="default", - PMM_CLICKHOUSE_PASSWORD="", + PMM_CLICKHOUSE_PASSWORD="clickhouse", user = pmm directory = /usr/share/grafana autorestart = true diff --git a/managed/testdata/supervisord.d/qan-api2.ini b/managed/testdata/supervisord.d/qan-api2.ini index 61783d509b..655912ffb9 100644 --- a/managed/testdata/supervisord.d/qan-api2.ini +++ b/managed/testdata/supervisord.d/qan-api2.ini @@ -9,7 +9,7 @@ environment = PMM_CLICKHOUSE_ADDR="127.0.0.1:9000", PMM_CLICKHOUSE_DATABASE="pmm", PMM_CLICKHOUSE_USER="default", - PMM_CLICKHOUSE_PASSWORD="", + PMM_CLICKHOUSE_PASSWORD="clickhouse", user = pmm diff --git a/qan-api2/main.go b/qan-api2/main.go index 6d982d064d..4a111dea20 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -260,7 +260,7 @@ func main() { clickhouseDatabaseF := kingpin.Flag("clickhouse-name", "ClickHouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() clickhouseAddrF := kingpin.Flag("clickhouse-addr", "ClickHouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() clickhouseUserF := kingpin.Flag("clickhouse-user", "ClickHouse database user").Default("default").Envar("PMM_CLICKHOUSE_USER").String() - clickhousePasswordF := kingpin.Flag("clickhouse-password", "ClickHouse database password").Default("").Envar("PMM_CLICKHOUSE_PASSWORD").String() + clickhousePasswordF := kingpin.Flag("clickhouse-password", "ClickHouse database user password").Default("clikchouse").Envar("PMM_CLICKHOUSE_PASSWORD").String() debugF := kingpin.Flag("debug", "Enable debug logging").Bool() traceF := kingpin.Flag("trace", "Enable trace logging (implies debug)").Bool() From 13b12d12fc5dabedc37a85e9399fc0eb4cc0d17c Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 18:07:26 +0300 Subject: [PATCH 11/18] PMM-13171 Fix the tests --- .github/workflows/managed.yml | 5 ++++- build/ansible/roles/clickhouse/files/users.xml | 0 2 files changed, 4 insertions(+), 1 deletion(-) mode change 100755 => 100644 build/ansible/roles/clickhouse/files/users.xml diff --git a/.github/workflows/managed.yml b/.github/workflows/managed.yml index dbda78d10f..dbf3720c62 100644 --- a/.github/workflows/managed.yml +++ b/.github/workflows/managed.yml @@ -70,7 +70,10 @@ jobs: run: docker image rm ${{ env.PMM_CACHE_IMAGE }} - name: Rebuild the binaries - run: docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy run-qan + run: | + # TODO: remove the line below before merging + docker cp ./build/ansible/roles/clickhouse/files/users.xml pmm-server:/etc/clickhouse-server/users.xml + docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy run-qan - name: Check the status of components continue-on-error: true diff --git a/build/ansible/roles/clickhouse/files/users.xml b/build/ansible/roles/clickhouse/files/users.xml old mode 100755 new mode 100644 From 6d12e8b5cc0f143063f0a46c827e100236a6d3c6 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 14 Feb 2025 22:37:21 +0300 Subject: [PATCH 12/18] PMM-13171 Fix the typo --- build/ansible/roles/clickhouse/tasks/main.yml | 2 +- qan-api2/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ansible/roles/clickhouse/tasks/main.yml b/build/ansible/roles/clickhouse/tasks/main.yml index 184203de95..b1674f7bc0 100644 --- a/build/ansible/roles/clickhouse/tasks/main.yml +++ b/build/ansible/roles/clickhouse/tasks/main.yml @@ -34,7 +34,7 @@ - /var/log/clickhouse-server - /run/clickhouse-server -- name: Copy a customized clickhouse config +- name: Copy customized clickhouse config files copy: src: "{{ item }}" dest: "/etc/clickhouse-server/{{ item }}" diff --git a/qan-api2/main.go b/qan-api2/main.go index 4a111dea20..0f6068f48a 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -260,7 +260,7 @@ func main() { clickhouseDatabaseF := kingpin.Flag("clickhouse-name", "ClickHouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() clickhouseAddrF := kingpin.Flag("clickhouse-addr", "ClickHouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() clickhouseUserF := kingpin.Flag("clickhouse-user", "ClickHouse database user").Default("default").Envar("PMM_CLICKHOUSE_USER").String() - clickhousePasswordF := kingpin.Flag("clickhouse-password", "ClickHouse database user password").Default("clikchouse").Envar("PMM_CLICKHOUSE_PASSWORD").String() + clickhousePasswordF := kingpin.Flag("clickhouse-password", "ClickHouse database user password").Default("clickhouse").Envar("PMM_CLICKHOUSE_PASSWORD").String() debugF := kingpin.Flag("debug", "Enable debug logging").Bool() traceF := kingpin.Flag("trace", "Enable trace logging (implies debug)").Bool() From 604f7b16de4272a96c133a691c48c87a184fe3be Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 15 Feb 2025 02:11:32 +0300 Subject: [PATCH 13/18] PMM-13171 Remove logging clickhouse password --- qan-api2/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index e97e3a90aa..022f7982ad 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -105,7 +105,7 @@ func runMigrations(dsn string) error { if err != nil { return err } - log.Println("dsn: ", dsn) + m, err := migrate.NewWithSourceInstance("iofs", d, dsn) if err != nil { return err From 6cc7ddeafd71d01581f011d0ce59154a09adfdaa Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 15 Feb 2025 02:11:52 +0300 Subject: [PATCH 14/18] PMM-13171 Add creds to clickhouse telemetry --- managed/cmd/pmm-managed/main.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index dde0379c87..b3605b8016 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -698,6 +698,8 @@ func main() { //nolint:maintidx,cyclop clickHouseDatabaseF := kingpin.Flag("clickhouse-name", "Clickhouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() clickhouseAddrF := kingpin.Flag("clickhouse-addr", "Clickhouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() + clickhouseUsernameF := kingpin.Flag("clickhouse-username", "Clickhouse database user").Default("default").Envar("PMM_CLICKHOUSE_USER").String() + clickhousePasswordF := kingpin.Flag("clickhouse-password", "Clickhouse database user password").Default("clickhouse").Envar("PMM_CLICKHOUSE_PASSWORD").String() watchtowerHostF := kingpin.Flag("watchtower-host", "Watchtower host").Default("http://watchtower:8080").Envar("PMM_WATCHTOWER_HOST").URL() @@ -750,7 +752,7 @@ func main() { //nolint:maintidx,cyclop pmmdb := ds.PmmDBSelect pmmdb.Credentials.Username = *postgresDBUsernameF pmmdb.Credentials.Password = *postgresDBPasswordF - pmmdb.DSN.Scheme = "postgres" // TODO: should be configurable + pmmdb.DSN.Scheme = "postgres" pmmdb.DSN.Host = *postgresAddrF pmmdb.DSN.DB = *postgresDBNameF q := make(url.Values) @@ -768,10 +770,15 @@ func main() { //nolint:maintidx,cyclop grafanadb.DSN.DB = "grafana" grafanadb.DSN.Params = q.Encode() - clickhouseDSN := "tcp://" + *clickhouseAddrF + "/" + *clickHouseDatabaseF + chURI := url.URL{ + Scheme: "tcp", + User: url.UserPassword(*clickhouseUsernameF, *clickhousePasswordF), + Host: *clickhouseAddrF, + Path: *clickHouseDatabaseF, + } qanDB := ds.QanDBSelect - qanDB.DSN = clickhouseDSN + qanDB.DSN = chURI.String() ds.VM.Address = *victoriaMetricsURLF @@ -918,7 +925,7 @@ func main() { //nolint:maintidx,cyclop l.Fatalf("Could not create Victoria Metrics client: %s", err) } - clickhouseClient, err := newClickhouseDB(clickhouseDSN, clickhouseMaxIdleConns, clickhouseMaxOpenConns) + clickhouseClient, err := newClickhouseDB(qanDB.DSN, clickhouseMaxIdleConns, clickhouseMaxOpenConns) if err != nil { l.Fatalf("Could not create Clickhouse client: %s", err) } From b7f9fc012ff433edc4b92a4a8c9830cd3bc257b9 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 15 Feb 2025 13:07:42 +0300 Subject: [PATCH 15/18] PMM-13171 Do not log clickhouse password --- managed/services/ha/services.go | 2 +- managed/services/telemetry/datasource_qandb_select.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/managed/services/ha/services.go b/managed/services/ha/services.go index 393044b924..2945d4e57f 100644 --- a/managed/services/ha/services.go +++ b/managed/services/ha/services.go @@ -89,7 +89,7 @@ func (s *services) StopRunningServices() { ls := service go func() { defer s.wg.Done() - s.l.Infoln("Stopping ", ls) + s.l.Infoln("Stopping", ls.ID()) ls.Stop() s.rw.Lock() defer s.rw.Unlock() diff --git a/managed/services/telemetry/datasource_qandb_select.go b/managed/services/telemetry/datasource_qandb_select.go index 94fff62d31..cc7cd399f9 100644 --- a/managed/services/telemetry/datasource_qandb_select.go +++ b/managed/services/telemetry/datasource_qandb_select.go @@ -19,6 +19,7 @@ package telemetry import ( "context" "database/sql" + "net/url" telemetryv1 "github.com/percona/saas/gen/telemetry/generic" "github.com/pkg/errors" @@ -64,7 +65,12 @@ func openQANDBConnection(dsn string, enabled bool, l *logrus.Entry) (*sql.DB, er return nil, errors.Wrap(err, "Failed to open connection to QAN DB") } if err := db.Ping(); err != nil { - l.Warnf("ClickHouse DB is not reachable [%s]: %s", dsn, err) + uri, parseErr := url.Parse(dsn) + if parseErr != nil { + l.Warnf("Failed to parse ClickHouse DSN %s", parseErr) + } else { + l.Warnf("ClickHouse DB is not reachable [%s]: %s", uri.Redacted(), err) + } } return db, nil } From 683507eef3a11769017c1dfb1f3476e83d2ab39c Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 15 Feb 2025 15:30:48 +0300 Subject: [PATCH 16/18] PMM-13171 Update docker-compose.yml --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 81196ee794..a6999bad65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,12 +34,14 @@ services: # - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s # - PMM_CLICKHOUSE_ADDR=127.0.0.1:9000 # - PMM_CLICKHOUSE_DATABASE=pmm + # - PMM_CLICKHOUSE_USER=default + # - PMM_CLICKHOUSE_PASSWORD= # - PMM_DEBUG=1 # - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/local-checks.yml # - PMM_POSTGRES_ADDR=pg # - PMM_POSTGRES_DBNAME=pmm-managed # - PMM_POSTGRES_USERNAME=pmm-managed - # - PMM_POSTGRES_DBPASSWORD=pmm-managed + # - PMM_POSTGRES_DBPASSWORD= # - PMM_POSTGRES_SSL_MODE=verify-full # - PMM_POSTGRES_SSL_CA_PATH=/tmp/certs/root.crt # - PMM_POSTGRES_SSL_KEY_PATH=/tmp/certs/pmm-managed.key From 62f1fd5589581303824068405b639b75643888dc Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 15 Feb 2025 16:10:47 +0300 Subject: [PATCH 17/18] PMM-13171 Update the documentation --- documentation/docs/install-pmm/HA.md | 32 ++++++++--------- .../baremetal/docker/preview_env_var.md | 14 ++++---- .../docs/reference/third-party/clickhouse.md | 35 +++++++++---------- 3 files changed, 39 insertions(+), 42 deletions(-) diff --git a/documentation/docs/install-pmm/HA.md b/documentation/docs/install-pmm/HA.md index ee5c8af94c..7157667bf7 100644 --- a/documentation/docs/install-pmm/HA.md +++ b/documentation/docs/install-pmm/HA.md @@ -339,13 +339,13 @@ The PMM Server orchestrates the collection, storage, and visualization of metric --ip ${PMM_ACTIVE_IP} \ -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm \ - -e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 \ - -e PERCONA_TEST_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ - -e PERCONA_TEST_POSTGRES_USERNAME=${PG_USERNAME} \ - -e PERCONA_TEST_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ + -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ + -e PMM_CLICKHOUSE_DATABASE=pmm \ + -e PMM_CLICKHOUSE_USER=default \ + -e PMM_CLICKHOUSE_PASSWORD \ + -e PMM_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ + -e PMM_POSTGRES_USERNAME=${PG_USERNAME} \ + -e PMM_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ -e GF_DATABASE_URL=postgres://${GF_USERNAME}:${GF_PASSWORD}@${PG_HOST_IP}:5432/grafana \ -e PMM_VM_URL=http://${VM_HOST_IP}:8428 \ -e PMM_TEST_HA_ENABLE=1 \ @@ -372,15 +372,15 @@ The PMM Server orchestrates the collection, storage, and visualization of metric -p 9094:9094/udp \ -p 9096:9096/udp \ -p 9097:9097 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm \ - -e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 \ - -e PERCONA_TEST_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ - -e PERCONA_TEST_POSTGRES_USERNAME=${PG_USERNAME} \ - -e PERCONA_TEST_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ + -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ + -e PMM_DISABLE_BUILTIN_POSTGRES=1 \ + -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ + -e PMM_CLICKHOUSE_DATABASE=pmm \ + -e PMM_CLICKHOUSE_USER=${CH_USER} \ + -e PMM_CLICKHOUSE_PASSWORD=${CH_PASSWORD} \ + -e POSTGRES_ADDR=${PG_HOST_IP}:5432 \ + -e POSTGRES_USERNAME=${PG_USERNAME} \ + -e POSTGRES_DBPASSWORD=${PG_PASSWORD} \ -e GF_DATABASE_URL=postgres://${GF_USERNAME}:${GF_PASSWORD}@${PG_HOST_IP}:5432/grafana \ -e PMM_VM_URL=http://${VM_HOST_IP}:8428 \ -e PMM_TEST_HA_ENABLE=1 \ diff --git a/documentation/docs/install-pmm/install-pmm-server/baremetal/docker/preview_env_var.md b/documentation/docs/install-pmm/install-pmm-server/baremetal/docker/preview_env_var.md index 887f3d9ced..78f43ff17f 100644 --- a/documentation/docs/install-pmm/install-pmm-server/baremetal/docker/preview_env_var.md +++ b/documentation/docs/install-pmm/install-pmm-server/baremetal/docker/preview_env_var.md @@ -3,10 +3,10 @@ !!! caution alert alert-warning "Warning" The `PERCONA_TEST_*` environment variables are experimental and subject to change. It is recommended that you use these variables for testing purposes only and not on production. -| Variable | Description -| -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- -| `PERCONA_TEST_SAAS_HOST` | SaaS server hostname. -| `PERCONA_TEST_PMM_CLICKHOUSE_ADDR` | Name of the host and port of the external ClickHouse database instance. -| `PERCONA_TEST_PMM_CLICKHOUSE_DATABASE` | Database name of the external ClickHouse database instance. -| `​​PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE` | The maximum number of threads in the current connection thread pool. This value cannot be bigger than max_thread_pool_size. -| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE` | The number of rows to load from tables in one block for this connection. +| Variable | Description +| ------------------------------------------------------------- | -------------------------------------------------------------------------- +| `PERCONA_TEST_SAAS_HOST` | SaaS server hostname. +| `PMM_CLICKHOUSE_ADDR` | Name of the host and port of the external ClickHouse database instance. +| `PMM_CLICKHOUSE_DATABASE` | Database name of the external ClickHouse database instance. +| `​​PMM_CLICKHOUSE_USER` | Database username. +| `PMM_CLICKHOUSE_PASSWORD` | Database username password. diff --git a/documentation/docs/reference/third-party/clickhouse.md b/documentation/docs/reference/third-party/clickhouse.md index af17139054..fb9af56283 100644 --- a/documentation/docs/reference/third-party/clickhouse.md +++ b/documentation/docs/reference/third-party/clickhouse.md @@ -4,38 +4,35 @@ You can use an external ClickHouse database instance outside the PMM Server cont ## Environment variables -PMM predefines certain flags that allow you to use ClickHouse parameters as environment variables: +PMM predefines certain flags that allow you to use ClickHouse parameters as environment variables. -!!! caution alert alert-warning "Warning" - The `PERCONA_TEST_*` environment variables are experimental and subject to change. It is recommended that you use these variables for testing purposes only and not on production. -To use ClickHouse as an external database instance, use the following environment variables: +To use ClickHouse as an external database instance, provide the following environment variables: -`PERCONA_TEST_PMM_CLICKHOUSE_ADDR` -> hostname:port -: Name of the host and port of the external ClickHouse database instance. +`PMM_CLICKHOUSE_ADDR` -> hostname:port +: Name of the host and port of the external ClickHouse database. + +`PMM_CLICKHOUSE_USER` -> username +: Username to connect to the external ClickHouse database. + +`PMM_CLICKHOUSE_PASSWORD` -> password +: User password to connect to the external ClickHouse database. **Optional environment variables** -`PERCONA_TEST_PMM_CLICKHOUSE_DATABASE` -> database name +`PMM_CLICKHOUSE_DATABASE` -> database name : Database name of the external ClickHouse database instance. -`​​PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE` -> [pool size](https://github.com/ClickHouse/ClickHouse/blob/ba26b3cf4c982f681f7f9782bba2662f97620835/programs/server/config.xml#L288) -: The maximum number of threads in the current connection thread pool. This value cannot be bigger than max_thread_pool_size. - - -`PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE` -> [max block size](https://clickhouse.com/docs/en/operations/settings/settings/#setting-max_block_size) -: The number of rows to load from tables in one block for this connection. - **Example** -To use ClickHouse as an external database instance, start the PMM docker with the specified variables for external ClickHouse: +To use ClickHouse as an external database instance, run PMM in docker or podman with the specified variables for external ClickHouse: ​​ ```sh --e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=$ADDRESS:$PORT --e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=$DB --e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=1 --e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=65000 +-e PMM_CLICKHOUSE_ADDR=$ADDRESS:$PORT +-e PMM_CLICKHOUSE_DATABASE=$DB +-e PMM_CLICKHOUSE_USER=$CH_USER +-e PMM_CLICKHOUSE_PASSWORD=$CH_PASSWORD ``` ## Troubleshooting From 155a900586f73917ea49e3663da556e1fb9a8651 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sun, 16 Feb 2025 00:45:37 +0300 Subject: [PATCH 18/18] PMM-13171 Update the HA documentation --- documentation/docs/install-pmm/HA.md | 80 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/documentation/docs/install-pmm/HA.md b/documentation/docs/install-pmm/HA.md index 7157667bf7..b572aae6e5 100644 --- a/documentation/docs/install-pmm/HA.md +++ b/documentation/docs/install-pmm/HA.md @@ -337,12 +337,12 @@ The PMM Server orchestrates the collection, storage, and visualization of metric --hostname ${PMM_ACTIVE_NODE_ID} \ --network pmm-network \ --ip ${PMM_ACTIVE_IP} \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ + -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ + -e PMM_DISABLE_BUILTIN_POSTGRES=1 \ -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ -e PMM_CLICKHOUSE_DATABASE=pmm \ - -e PMM_CLICKHOUSE_USER=default \ - -e PMM_CLICKHOUSE_PASSWORD \ + -e PMM_CLICKHOUSE_USER=${CH_USER} \ + -e PMM_CLICKHOUSE_PASSWORD=${CH_PASSWORD} \ -e PMM_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ -e PMM_POSTGRES_USERNAME=${PG_USERNAME} \ -e PMM_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ @@ -407,15 +407,15 @@ The PMM Server orchestrates the collection, storage, and visualization of metric --hostname ${PMM_PASSIVE_NODE_ID} \ --network pmm-network \ --ip ${PMM_PASSIVE_IP} \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm \ - -e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 \ - -e PERCONA_TEST_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ - -e PERCONA_TEST_POSTGRES_USERNAME=${PG_USERNAME} \ - -e PERCONA_TEST_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ + -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ + -e PMM_DISABLE_BUILTIN_POSTGRES=1 \ + -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ + -e PMM_CLICKHOUSE_DATABASE=pmm \ + -e PMM_CLICKHOUSE_USER=${CH_USER} \ + -e PMM_CLICKHOUSE_PASSWORD=${CH_PASSWORD} \ + -e POSTGRES_ADDR=${PG_HOST_IP}:5432 \ + -e POSTGRES_USERNAME=${PG_USERNAME} \ + -e POSTGRES_DBPASSWORD=${PG_PASSWORD} \ -e GF_DATABASE_URL=postgres://${GF_USERNAME}:${GF_PASSWORD}@${PG_HOST_IP}:5432/grafana \ -e PMM_VM_URL=http://${VM_HOST_IP}:8428 \ -e PMM_TEST_HA_ENABLE=1 \ @@ -442,15 +442,15 @@ The PMM Server orchestrates the collection, storage, and visualization of metric -p 9094:9094/udp \ -p 9096:9096/udp \ -p 9097:9097 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm \ - -e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 \ - -e PERCONA_TEST_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ - -e PERCONA_TEST_POSTGRES_USERNAME=${PG_USERNAME} \ - -e PERCONA_TEST_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ + -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ + -e PMM_DISABLE_BUILTIN_POSTGRES=1 \ + -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ + -e PMM_CLICKHOUSE_DATABASE=pmm \ + -e PMM_CLICKHOUSE_USER=${CH_USER} \ + -e PMM_CLICKHOUSE_PASSWORD=${CH_PASSWORD} \ + -e POSTGRES_ADDR=${PG_HOST_IP}:5432 \ + -e POSTGRES_USERNAME=${PG_USERNAME} \ + -e POSTGRES_DBPASSWORD=${PG_PASSWORD} \ -e GF_DATABASE_URL=postgres://${GF_USERNAME}:${GF_PASSWORD}@${PG_HOST_IP}:5432/grafana \ -e PMM_VM_URL=http://${VM_HOST_IP}:8428 \ -e PMM_TEST_HA_ENABLE=1 \ @@ -477,15 +477,15 @@ The PMM Server orchestrates the collection, storage, and visualization of metric --hostname ${PMM_PASSIVE2_NODE_ID} \ --network pmm-network \ --ip ${PMM_PASSIVE2_IP} \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm \ - -e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 \ - -e PERCONA_TEST_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ - -e PERCONA_TEST_POSTGRES_USERNAME=${PG_USERNAME} \ - -e PERCONA_TEST_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ + -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ + -e PMM_DISABLE_BUILTIN_POSTGRES=1 \ + -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ + -e PMM_CLICKHOUSE_DATABASE=pmm \ + -e PMM_CLICKHOUSE_USER=${CH_USER} \ + -e PMM_CLICKHOUSE_PASSWORD=${CH_PASSWORD} \ + -e POSTGRES_ADDR=${PG_HOST_IP}:5432 \ + -e POSTGRES_USERNAME=${PG_USERNAME} \ + -e POSTGRES_DBPASSWORD=${PG_PASSWORD} \ -e GF_DATABASE_URL=postgres://${GF_USERNAME}:${GF_PASSWORD}@${PG_HOST_IP}:5432/grafana \ -e PMM_VM_URL=http://${VM_HOST_IP}:8428 \ -e PMM_TEST_HA_ENABLE=1 \ @@ -512,15 +512,15 @@ The PMM Server orchestrates the collection, storage, and visualization of metric -p 9094:9094/udp \ -p 9096:9096/udp \ -p 9097:9097 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ - -e PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES=1 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm \ - -e PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 \ - -e PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 \ - -e PERCONA_TEST_POSTGRES_ADDR=${PG_HOST_IP}:5432 \ - -e PERCONA_TEST_POSTGRES_USERNAME=${PG_USERNAME} \ - -e PERCONA_TEST_POSTGRES_DBPASSWORD=${PG_PASSWORD} \ + -e PMM_DISABLE_BUILTIN_CLICKHOUSE=1 \ + -e PMM_DISABLE_BUILTIN_POSTGRES=1 \ + -e PMM_CLICKHOUSE_ADDR=${CH_HOST_IP}:9000 \ + -e PMM_CLICKHOUSE_DATABASE=pmm \ + -e PMM_CLICKHOUSE_USER=${CH_USER} \ + -e PMM_CLICKHOUSE_PASSWORD=${CH_PASSWORD} \ + -e POSTGRES_ADDR=${PG_HOST_IP}:5432 \ + -e POSTGRES_USERNAME=${PG_USERNAME} \ + -e POSTGRES_DBPASSWORD=${PG_PASSWORD} \ -e GF_DATABASE_URL=postgres://${GF_USERNAME}:${GF_PASSWORD}@${PG_HOST_IP}:5432/grafana \ -e PMM_VM_URL=http://${VM_HOST_IP}:8428 \ -e PMM_TEST_HA_ENABLE=1 \