Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: use recommended cluster settings for cockroachdb #2858

Closed
42 changes: 42 additions & 0 deletions modules/cockroachdb/cockroachdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"crypto/tls"
"crypto/x509"
"database/sql"
"encoding/pem"
"fmt"
"net"
Expand Down Expand Up @@ -90,6 +91,11 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom
return addTLS(ctx, container, o)
},
},
PostReadies: []testcontainers.ContainerHook{
func(ctx context.Context, container testcontainers.Container) error {
return runStatements(ctx, container, o)
},
},
},
},
},
Expand Down Expand Up @@ -233,6 +239,42 @@ func addTLS(ctx context.Context, container testcontainers.Container, opts option
return nil
}

// runStatements runs the configured statements against the CockroachDB container.
func runStatements(ctx context.Context, container testcontainers.Container, opts options) (err error) {
martskins marked this conversation as resolved.
Show resolved Hide resolved
if len(opts.Statements) == 0 {
return nil
}

port, err := container.MappedPort(ctx, defaultSQLPort)
if err != nil {
return fmt.Errorf("mapped port: %w", err)
}

host, err := container.Host(ctx)
if err != nil {
return fmt.Errorf("host: %w", err)
}

db, err := sql.Open("pgx/v5", connString(opts, host, port))
if err != nil {
return fmt.Errorf("sql.Open: %w", err)
}
defer func() {
cerr := db.Close()
if err == nil {
err = cerr
}
}()

for _, stmt := range opts.Statements {
if _, err = db.Exec(stmt); err != nil {
return fmt.Errorf("db.Exec: %w", err)
}
}

return nil
}

func connString(opts options, host string, port nat.Port) string {
user := url.User(opts.User)
if opts.Password != "" {
Expand Down
9 changes: 9 additions & 0 deletions modules/cockroachdb/cockroachdb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ func TestCockroach_NotRoot(t *testing.T) {
url: "postgres://test@localhost:xxxxx/defaultdb?sslmode=disable",
opts: []testcontainers.ContainerCustomizer{
cockroachdb.WithUser("test"),
// Do not run the default statements as the user used on this test is
// lacking the needed MODIFYCLUSTERSETTING privilege to run them.
cockroachdb.WithStatements(),
},
})
}
Expand All @@ -38,6 +41,9 @@ func TestCockroach_Password(t *testing.T) {
opts: []testcontainers.ContainerCustomizer{
cockroachdb.WithUser("foo"),
cockroachdb.WithPassword("bar"),
// Do not run the default statements as the user used on this test is
// lacking the needed MODIFYCLUSTERSETTING privilege to run them.
cockroachdb.WithStatements(),
},
})
}
Expand All @@ -50,6 +56,9 @@ func TestCockroach_TLS(t *testing.T) {
url: "postgres://root@localhost:xxxxx/defaultdb?sslmode=verify-full",
opts: []testcontainers.ContainerCustomizer{
cockroachdb.WithTLS(tlsCfg),
// Do not run the default statements as the user used on this test is
// lacking the needed MODIFYCLUSTERSETTING privilege to run them.
cockroachdb.WithStatements(),
},
})
}
Expand Down
59 changes: 59 additions & 0 deletions modules/cockroachdb/examples_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cockroachdb_test

import (
"context"
"database/sql"
"fmt"
"log"
"net/url"
Expand Down Expand Up @@ -50,3 +51,61 @@ func ExampleRun() {
// true
// postgres://root@localhost:xxx/defaultdb?sslmode=disable
}

func ExampleRun_withRecommendedSettings() {
ctx := context.Background()

cockroachdbContainer, err := cockroachdb.Run(ctx, "cockroachdb/cockroach:latest-v23.1", cockroachdb.WithStatements(cockroachdb.DefaultStatements...))
defer func() {
if err := testcontainers.TerminateContainer(cockroachdbContainer); err != nil {
log.Printf("failed to terminate container: %s", err)
}
}()
if err != nil {
log.Printf("failed to start container: %s", err)
return
}

state, err := cockroachdbContainer.State(ctx)
if err != nil {
log.Printf("failed to get container state: %s", err)
return
}
fmt.Println(state.Running)

addr, err := cockroachdbContainer.ConnectionString(ctx)
if err != nil {
log.Printf("failed to get connection string: %s", err)
return
}

db, err := sql.Open("pgx/v5", addr)
if err != nil {
log.Printf("failed to open connection: %s", err)
return
}
defer func() {
if err := db.Close(); err != nil {
log.Printf("failed to close connection: %s", err)
}
}()

var queueInterval string
if err := db.QueryRow("SHOW CLUSTER SETTING kv.range_merge.queue_interval").Scan(&queueInterval); err != nil {
log.Printf("failed to scan row: %s", err)
return
}
fmt.Println(queueInterval)

var statsCollectionEnabled bool
if err := db.QueryRow("SHOW CLUSTER SETTING sql.stats.automatic_collection.enabled").Scan(&statsCollectionEnabled); err != nil {
log.Printf("failed to scan row: %s", err)
return
}
fmt.Println(statsCollectionEnabled)

// Output:
// true
// 00:00:00.05
// false
}
43 changes: 34 additions & 9 deletions modules/cockroachdb/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,21 @@ package cockroachdb
import "github.com/testcontainers/testcontainers-go"

type options struct {
Database string
User string
Password string
StoreSize string
TLS *TLSConfig
Database string
User string
Password string
StoreSize string
TLS *TLSConfig
Statements []string
martskins marked this conversation as resolved.
Show resolved Hide resolved
}

func defaultOptions() options {
return options{
User: defaultUser,
Password: defaultPassword,
Database: defaultDatabase,
StoreSize: defaultStoreSize,
User: defaultUser,
Password: defaultPassword,
Database: defaultDatabase,
StoreSize: defaultStoreSize,
Statements: DefaultStatements,
}
}

Expand Down Expand Up @@ -67,3 +69,26 @@ func WithTLS(cfg *TLSConfig) Option {
o.TLS = cfg
}
}

// DefaultStatements are the settings recommended by Cockroach Labs for testing clusters.
// Note that to use these defaults the user needs to have MODIFYCLUSTERSETTING privilege.
// See https://www.cockroachlabs.com/docs/stable/local-testing for more information.
var DefaultStatements = []string{
"SET CLUSTER SETTING kv.range_merge.queue_interval = '50ms'",
"SET CLUSTER SETTING jobs.registry.interval.gc = '30s'",
"SET CLUSTER SETTING jobs.registry.interval.cancel = '180s'",
"SET CLUSTER SETTING jobs.retention_time = '15s'",
"SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false",
"SET CLUSTER SETTING kv.range_split.by_load_merge_delay = '5s'",
`ALTER RANGE default CONFIGURE ZONE USING "gc.ttlseconds" = 600`,
`ALTER DATABASE system CONFIGURE ZONE USING "gc.ttlseconds" = 600`,
}

// WithStatements sets the statements to run on the CockroachDB cluster once the container is ready.
martskins marked this conversation as resolved.
Show resolved Hide resolved
// This, in combination with DefaultStatements, can be used to configure the cluster with the settings
// recommended by Cockroach Labs.
Comment on lines +88 to +89
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion: This is somewhat confusing as this won't be combined with DefaultStatements which is how I would interpret this currently. We should clarify and ensure we clearly state the default if not configured is DefaultStatements.

Copy link
Member

@mdelapenya mdelapenya Oct 31, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we are always adding the default statements, then I think they must be private, and probably prepended to the user-provided ones.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we are always adding the default statements, then I think they must be private, and probably prepended to the user-provided ones.

I see what you mean, but given the complications with TLS and users without MODIFYCLUSTERSETTING privilege I think prepending to the user-provided statements is probably going to cause a bit of pain.

Good spot on the comment. I'll hold on that change until we've established what we do about the TLS and user privileges thing though, as it might affect this.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Private vs public for these is an interesting discussion, here's some ideas from the perspective of making the var private:

  • Pro: Others can't mess with them
  • Conn: Others can't correct if needed
  • Conn: Can't pass them into WithStatements to customise
  • Conn: Not self documenting.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One of the goals of any testcontainers module is to be simple, and maybe opinionated, helping users in not committing mistakes. So I'd look for the mechanism that creates this experience.

Whatever you find that matches that, it's fine for me 😊

func WithStatements(statements ...string) Option {
return func(o *options) {
o.Statements = statements
}
}