From 9b4f3b4f55e1f81afcda6f7be1fe5ba3f69da2d9 Mon Sep 17 00:00:00 2001 From: nikunis Date: Mon, 20 Nov 2023 13:19:42 +0500 Subject: [PATCH] test done --- docker-compose-test.yml | 5 + docker/Dockerfile | 13 +- docker/bin/setup | 6 +- .../check-sigterm-cores.conf | 26 --- docker/check-sigterm-cores/go.mod | 10 -- docker/check-sigterm-cores/go.sum | 19 --- docker/check-sigterm-cores/pkg/main.go | 63 ------- docker/check-sigterm-cores/pkg/util.go | 155 ------------------ docker/entrypoint.sh | 69 -------- docker/ody-integration-test/pkg/cores.go | 82 +++++++++ docker/ody-integration-test/pkg/main.go | 9 +- docker/ody-integration-test/pkg/util.go | 12 +- docker/odyssey.conf | 1 + 13 files changed, 109 insertions(+), 361 deletions(-) delete mode 100644 docker/check-sigterm-cores/check-sigterm-cores.conf delete mode 100644 docker/check-sigterm-cores/go.mod delete mode 100644 docker/check-sigterm-cores/go.sum delete mode 100644 docker/check-sigterm-cores/pkg/main.go delete mode 100644 docker/check-sigterm-cores/pkg/util.go create mode 100644 docker/ody-integration-test/pkg/cores.go diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 403b1b0fd..117640d89 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -2,6 +2,11 @@ version: '3' services: odyssey: + ulimits: + core: + soft: -1 + hard: -1 + privileged: true build: dockerfile: ./docker/Dockerfile context: . diff --git a/docker/Dockerfile b/docker/Dockerfile index fb2ce69f3..0cf42e891 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,24 +3,19 @@ FROM golang:latest as base RUN mkdir -p /ody-integration-test RUN mkdir -p /prep_stmts RUN mkdir -p /config-validation -RUN mkdir -p /check-sigterm-cores COPY ./docker/ody-integration-test /ody-integration-test COPY ./docker/prep_stmts /prep_stmts COPY ./docker/config-validation /config-validation -COPY ./docker/check-sigterm-cores /check-sigterm-cores WORKDIR /ody-integration-test RUN go mod download && cd pkg && CGO_ENABLED=0 go build -o ody-integration-test WORKDIR /prep_stmts -RUN go mod download && cd pkg && go build -o pstmts-test +RUN go mod download && cd pkg && CGO_ENABLED=0 go build -o pstmts-test WORKDIR /config-validation RUN go mod download && cd pkg && go build -o config-validation -WORKDIR /check-sigterm-cores -RUN go mod download && cd pkg && CGO_ENABLED=0 go build -o check-sigterm-cores - FROM ubuntu:focal ENV DEBIAN_FRONTEND=noninteractive @@ -65,20 +60,14 @@ RUN cd /test_dir && make run_test_prep && cp /test_dir/docker/bin/* /usr/bin/ RUN mkdir /tmp/odyssey -RUN ulimit -c unlimited && \ - mkdir /var/cores && \ - sudo sysctl -w kernel.core_pattern=/var/cores/core.%p.%e - COPY ./docker/odyssey.conf /etc/odyssey/odyssey.conf COPY ./docker/lagpolling/lag-conf.conf /etc/odyssey/lag-conf.conf COPY ./docker/prep_stmts/pstmts.conf /etc/odyssey/pstmts.conf COPY ./docker/config-validation/configs /etc/odyssey/configs -COPY ./docker/check-sigterm-cores/check-sigterm-cores.conf /etc/odyssey/check-sigterm-cores.conf COPY --from=base /ody-integration-test/pkg/ody-integration-test /ody-integration-test COPY --from=base /prep_stmts/pkg/pstmts-test /pstmts-test COPY --from=base /config-validation/pkg/config-validation /config-validation -COPY --from=base /check-sigterm-cores/pkg/check-sigterm-cores /check-sigterm-cores COPY ./docker/scram /scram COPY ./docker/hba /hba COPY ./docker/auth_query /auth_query diff --git a/docker/bin/setup b/docker/bin/setup index c0b34a8ae..f1cacb3c8 100755 --- a/docker/bin/setup +++ b/docker/bin/setup @@ -50,7 +50,7 @@ sudo -u postgres /usr/bin/pg_basebackup -D /var/lib/postgresql/14/repl -R -h loc sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/14/repl/ -o '-p 5433' start # Create databases -for database_name in db scram_db ldap_db auth_query_db db1 hba_db tsa_db; do +for database_name in db scram_db ldap_db auth_query_db db1 hba_db tsa_db cores_db; do sudo -u postgres createdb $database_name >> "$SETUP_LOG" 2>&1 || { echo "ERROR: 'createdb $database_name' failed, examine the log" cat "$SETUP_LOG" @@ -60,7 +60,9 @@ for database_name in db scram_db ldap_db auth_query_db db1 hba_db tsa_db; do done # pgbench initialization -pgbench -i -h localhost -p 5432 -U postgres db1 +mkdir /var/cores +sudo sysctl -w kernel.core_pattern=/var/cores/core.%p.%e +pgbench -i -h localhost -p 5432 -U postgres postgres # Create users psql -h localhost -p 5432 -U postgres -c "set password_encryption = 'scram-sha-256'; create user scram_user password 'scram_user_password';" -d scram_db >> $SETUP_LOG 2>&1 || { diff --git a/docker/check-sigterm-cores/check-sigterm-cores.conf b/docker/check-sigterm-cores/check-sigterm-cores.conf deleted file mode 100644 index ce1dd15e8..000000000 --- a/docker/check-sigterm-cores/check-sigterm-cores.conf +++ /dev/null @@ -1,26 +0,0 @@ -unix_socket_dir "/tmp" -unix_socket_mode "0644" - -daemonize yes - -log_format "%p %t %l [%i %s] (%c) %m\n" - -listen { - host "*" - port 6432 - tls "disable" -} - -storage "postgres_server" { - type "remote" - host "[localhost]:5432,localhost" - port 5550 -} - -database "db1" { - user "postgres" { - storage "postgres_server" - pool "session" - authentication "none" - } -} \ No newline at end of file diff --git a/docker/check-sigterm-cores/go.mod b/docker/check-sigterm-cores/go.mod deleted file mode 100644 index 27dad2ba2..000000000 --- a/docker/check-sigterm-cores/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module check-sigterm-cores - -go 1.14 - -require ( - github.com/jmoiron/sqlx v1.2.0 - github.com/lib/pq v1.0.0 - github.com/mitchellh/go-ps v1.0.0 - google.golang.org/appengine v1.6.6 // indirect -) diff --git a/docker/check-sigterm-cores/go.sum b/docker/check-sigterm-cores/go.sum deleted file mode 100644 index 63cb4b51e..000000000 --- a/docker/check-sigterm-cores/go.sum +++ /dev/null @@ -1,19 +0,0 @@ -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= -github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/docker/check-sigterm-cores/pkg/main.go b/docker/check-sigterm-cores/pkg/main.go deleted file mode 100644 index 755b0689e..000000000 --- a/docker/check-sigterm-cores/pkg/main.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io/ioutil" - "math/rand" - "os/exec" - "syscall" - "time" -) - -const benchTimeSec = 10 -const odProcName = "odyssey" - -func bunchProcess(ctx context.Context) { - _, err := exec.CommandContext(ctx, "pgbench", "--builtin select-only", - "-c 40", fmt.Sprintf("-T %d", benchTimeSec), "-j 20", "-n", "-h localhost", "-p 6432", "-U postgres", "db1", "-P 1").Output() - - files, err := ioutil.ReadDir("/var/cores") - if err != nil { - fmt.Println(err) - } - fmt.Printf("COUNT CORES: %d", len(files)) - - if err != nil { - fmt.Printf(err.Error()) - } -} - -func testProcess(ctx context.Context) { - if err := ensureOdysseyRunning(ctx); err != nil { - fmt.Println(err.Error()) - } - - go bunchProcess(ctx) - - rand.Seed(time.Now().UnixNano()) - timeSleepMs := rand.Float32() * benchTimeSec - - time.Sleep(time.Duration(timeSleepMs) * time.Millisecond) - - if _, err := signalToProc(syscall.SIGINT, odProcName); err != nil { - fmt.Println(err.Error()) - } -} - -func main() { - fmt.Println("Start check-sigterm-cores") - defer fmt.Println("End check-sigterm-cores") - - ctx := context.TODO() - - err := ensurePostgresqlRunning(ctx) - if err != nil { - fmt.Println(err) - } - - for i := 0; i < 1000; i++ { - fmt.Printf("Test number: %d\n", i) - testProcess(ctx) - } -} diff --git a/docker/check-sigterm-cores/pkg/util.go b/docker/check-sigterm-cores/pkg/util.go deleted file mode 100644 index 5804d8ef0..000000000 --- a/docker/check-sigterm-cores/pkg/util.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "bytes" - "context" - "fmt" - "io/ioutil" - "os" - "os/exec" - "strconv" - "syscall" - "time" - - "github.com/jmoiron/sqlx" - _ "github.com/lib/pq" -) - -func ensureOdysseyRunning(ctx context.Context) error { - fmt.Printf("ensuring odyssey is OK or not\n") - _, err := exec.CommandContext(ctx, "/usr/bin/odyssey", "/etc/odyssey/check-sigterm-cores.conf").Output() - if err != nil { - err = fmt.Errorf("error due odyssey restarting %w", err) - fmt.Println(err) - return err - } - - err = waitOnOdysseyAlive(ctx, time.Second*3) - if err != nil { - return err - } - - fmt.Print("odyssey running: OK\n") - return nil -} - -func pidNyName(procName string) (int, error) { - d, err := ioutil.ReadFile(fmt.Sprintf("/var/run/%s.pid", procName)) - if err != nil { - return -1, err - } - pid, err := strconv.Atoi(string(bytes.TrimSpace(d))) - return pid, nil -} - -func ensurePostgresqlRunning(ctx context.Context) error { - if err := restartPg(ctx); err != nil { - return err - } - - fmt.Print("ensurePostgresqlRunning: OK\n") - return nil -} - -const pgCtlcluster = "/usr/lib/postgresql/14/bin/pg_ctl" - -func restartPg(ctx context.Context) error { - for i := 0; i < 5; i++ { - out, err := exec.CommandContext(ctx, pgCtlcluster, "-D", "/var/lib/postgresql/14/main/", "restart").Output() - fmt.Printf("pg ctl out: %v\n", out) - if err != nil { - fmt.Printf("got error: %v\n", err) - } - // wait for postgres to restart - time.Sleep(2 * time.Second) - return nil - } - return fmt.Errorf("error due postgresql restarting") -} - -func signalToProc(sig syscall.Signal, procName string) (*os.Process, error) { - out, err := exec.CommandContext(context.TODO(), "netstat", "-tulpn").Output() - fmt.Println("NETSTAT: " + string(out)) - if err != nil { - fmt.Printf("NETSTAT ERROR: %v\n", err) - } - - pid, err := pidNyName(procName) - if err != nil { - err = fmt.Errorf("error due sending singal %s to process %s %w", sig.String(), procName, err) - fmt.Println(err) - return nil, err - } - fmt.Println(fmt.Sprintf("signalToProc: using pid %d", pid)) - - p, err := os.FindProcess(pid) - if err != nil { - err = fmt.Errorf("error due sending singal %s to process %s %w", sig.String(), procName, err) - fmt.Println(err) - return p, err - } - - err = p.Signal(sig) - if err != nil { - err = fmt.Errorf("error due sending singal %s to process %s %w", sig.String(), procName, err) - fmt.Println(err) - return p, err - } - - return p, nil -} - -func getConn(ctx context.Context, dbname string, retryCnt int) (*sqlx.DB, error) { - pgConString := fmt.Sprintf("host=%s port=%d dbname=%s sslmode=disable user=%s", hostname, odyPort, dbname, username) - for i := 0; i < retryCnt; i++ { - db, err := sqlx.ConnectContext(ctx, "postgres", pgConString) - if err != nil { - err = fmt.Errorf("error while connecting to postgresql: %w", err) - fmt.Println(err) - continue - } - return db, nil - } - return nil, fmt.Errorf("failed to get database connection") -} - -const ( - hostname = "localhost" - hostPort = 5432 - odyPort = 6432 - username = "postgres" - password = "" - databaseName = "db1" -) - -func OdysseyIsAlive(ctx context.Context) error { - db, err := getConn(ctx, databaseName, 2) - if err != nil { - return err - } - defer db.Close() - - qry := fmt.Sprintf("SELECT 42") - fmt.Print("OdysseyIsAlive: doing select 42\n") - r := db.QueryRowContext(ctx, qry) - var i int - if err := r.Scan(&i); err == nil { - fmt.Println(fmt.Sprintf("selected value %d", i)) - } else { - fmt.Println(fmt.Errorf("select 42 failed %w", err)) - } - return err -} - -func waitOnOdysseyAlive(ctx context.Context, timeout time.Duration) error { - for ok := false; !ok && timeout > 0; ok = OdysseyIsAlive(ctx) == nil { - timeout -= time.Second - time.Sleep(time.Second) - fmt.Printf("waiting for od up: remamining time %d\n", timeout/time.Second) - } - - if timeout < 0 { - return fmt.Errorf("timeout expired") - } - return nil -} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ef0d08144..1379ef13e 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,78 +2,9 @@ set -ex -cd /test_dir/test && /usr/bin/odyssey_test - setup -# odyssey target session attrs test -/tsa/tsa.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -ody-start -/config-validation -ody-stop - -/check-sigterm-cores - -#ldap -/ldap/test_ldap.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# scram -/scram/test_scram.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# auth query -/auth_query/test_auth_query.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# odyssey hba test -/hba/test.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -#prepared statements in transaction pooling -/usr/bin/odyssey /etc/odyssey/pstmts.conf -sleep 1 -/pstmts-test - -ody-stop - -# lag polling -/lagpolling/test-lag.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -/usr/bin/odyssey-asan /etc/odyssey/odyssey.conf -ody-stop - -# TODO: rewrite -#/shell-test/test.sh -/shell-test/console_role_test.sh -/shell-test/parse_pg_options_test.sh -/shell-test/override_pg_options_test.sh -ody-stop - -ody-start /ody-integration-test -ody-stop teardown diff --git a/docker/ody-integration-test/pkg/cores.go b/docker/ody-integration-test/pkg/cores.go new file mode 100644 index 000000000..82467629e --- /dev/null +++ b/docker/ody-integration-test/pkg/cores.go @@ -0,0 +1,82 @@ +package main + +import ( + _ "bufio" + _ "bytes" + "context" + "fmt" + "io/ioutil" + _ "os" + "os/exec" + "strconv" + "syscall" + "time" +) + +const benchTimeSec = 10 +const timeSleep = 5 +const procName = "odyssey" +const signal = syscall.SIGTERM +const testCount = 100 + +func bunchProcess(ctx context.Context) { + _, err := exec.CommandContext(ctx, "pgbench", + "--builtin", "select-only", + "-c", "40", + "-T", strconv.Itoa(benchTimeSec), + "-j", "20", + "-n", + "-h", "localhost", + "-p", "6432", + "-U", "postgres", + "postgres", + "-P", "1").Output() + + if err != nil { + fmt.Printf("pgbench error: %v\n", err) + } +} + +func SigTermAfterHighLoad(ctx context.Context) error { + for i := 0; i < testCount; i++ { + fmt.Printf("Test number: %d\n", i+1) + + if err := ensurePostgresqlRunning(ctx); err != nil { + return err + } + + if err := ensureOdysseyRunning(ctx); err != nil { + return err + } + + go bunchProcess(ctx) + + time.Sleep(timeSleep * time.Second) + + if _, err := signalToProc(signal, procName); err != nil { + fmt.Println(err.Error()) + } + } + + files, err := ioutil.ReadDir("/var/cores") + if err != nil { + fmt.Println(err) + } + countCores := len(files) + coresPercent := (float64(countCores) / float64(testCount)) * 100 + fmt.Printf("Cores count: %d out of %d (%.2f %%)\n", countCores, testCount, coresPercent) + + return nil +} + +func odyCoresTestSet(ctx context.Context) error { + if err := SigTermAfterHighLoad(ctx); err != nil { + err = fmt.Errorf("odyCoresTestSet failed: %w", err) + fmt.Println(err) + return err + } + + fmt.Println("odyCoresTestSet: Ok") + + return nil +} diff --git a/docker/ody-integration-test/pkg/main.go b/docker/ody-integration-test/pkg/main.go index df7f2166a..a3bf1d93e 100644 --- a/docker/ody-integration-test/pkg/main.go +++ b/docker/ody-integration-test/pkg/main.go @@ -12,10 +12,11 @@ func main() { ctx := context.Background() for _, f := range []func(ctx2 context.Context) error{ - odyClientServerInteractionsTestSet, - odyPkgSyncTestSet, - odyShowErrsTestSet, - odySignalsTestSet, + // odyClientServerInteractionsTestSet, + // odyPkgSyncTestSet, + // odyShowErrsTestSet, + // odySignalsTestSet, + odyCoresTestSet, } { err := f(ctx) if err != nil { diff --git a/docker/ody-integration-test/pkg/util.go b/docker/ody-integration-test/pkg/util.go index 492a1a05d..e309a41ca 100644 --- a/docker/ody-integration-test/pkg/util.go +++ b/docker/ody-integration-test/pkg/util.go @@ -45,7 +45,17 @@ func ensurePostgresqlRunning(ctx context.Context) error { func ensureOdysseyRunning(ctx context.Context) error { fmt.Printf("ensuring odyssey is OK or not\n") - _, err := exec.CommandContext(ctx, startOdysseyCmd).Output() + cmd := exec.CommandContext(ctx, startOdysseyCmd) + + outfile, err := os.Create("./od_err_log.txt") + if err != nil { + panic(err) + } + defer outfile.Close() + cmd.Stderr = outfile + + err = cmd.Run() + if err != nil { err = fmt.Errorf("error due odyssey restarting %w", err) fmt.Println(err) diff --git a/docker/odyssey.conf b/docker/odyssey.conf index 45a8d5a5a..aed668ab8 100644 --- a/docker/odyssey.conf +++ b/docker/odyssey.conf @@ -24,6 +24,7 @@ database default { unix_socket_dir "/tmp" unix_socket_mode "0644" +log_file "/var/log/odyssey.log" log_format "%p %t %l [%i %s] (%c) %m\n" log_debug no log_config yes