From b6dc9964031979543aeb65af5051f41362fb1dd2 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Wed, 1 Nov 2023 16:17:09 -0700 Subject: [PATCH 1/8] Move Sqlite schema to migrations --- backend/monoprocess/monoprocess_test.go | 5 +- .../db/migrations/000001_initial.down.sql | 4 + .../migrations/000001_initial.up.sql} | 7 +- backend/sqlite/db/schema.sql | 0 backend/sqlite/options.go | 30 ++++++++ backend/sqlite/sqlite.go | 75 ++++++++++++++----- backend/sqlite/sqlite_test.go | 8 +- bench/main.go | 4 +- go.mod | 3 +- go.sum | 8 +- samples/runner.go | 4 +- 11 files changed, 115 insertions(+), 33 deletions(-) create mode 100644 backend/sqlite/db/migrations/000001_initial.down.sql rename backend/sqlite/{schema.sql => db/migrations/000001_initial.up.sql} (87%) create mode 100644 backend/sqlite/db/schema.sql create mode 100644 backend/sqlite/options.go diff --git a/backend/monoprocess/monoprocess_test.go b/backend/monoprocess/monoprocess_test.go index 0b7f3f32..5e31f942 100644 --- a/backend/monoprocess/monoprocess_test.go +++ b/backend/monoprocess/monoprocess_test.go @@ -20,7 +20,7 @@ func Test_MonoprocessBackend(t *testing.T) { // Disable sticky workflow behavior for the test execution options = append(options, backend.WithStickyTimeout(0)) - return NewMonoprocessBackend(sqlite.NewInMemoryBackend(options...)) + return NewMonoprocessBackend(sqlite.NewInMemoryBackend(sqlite.WithBackendOptions(options...))) }, nil) } @@ -33,7 +33,7 @@ func Test_EndToEndMonoprocessBackend(t *testing.T) { // Disable sticky workflow behavior for the test execution options = append(options, backend.WithStickyTimeout(0)) - return NewMonoprocessBackend(sqlite.NewInMemoryBackend(options...)) + return NewMonoprocessBackend(sqlite.NewInMemoryBackend(sqlite.WithBackendOptions(options...))) }, nil) } @@ -43,5 +43,6 @@ func (b *monoprocessBackend) GetFutureEvents(ctx context.Context) ([]*history.Ev if testBackend, ok := b.Backend.(test.TestBackend); ok { return testBackend.GetFutureEvents(ctx) } + return nil, errors.New("not implemented") } diff --git a/backend/sqlite/db/migrations/000001_initial.down.sql b/backend/sqlite/db/migrations/000001_initial.down.sql new file mode 100644 index 00000000..928470b9 --- /dev/null +++ b/backend/sqlite/db/migrations/000001_initial.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS `instances`; +DROP TABLE IF EXISTS `pending_events`; +DROP TABLE IF EXISTS `history`; +DROP TABLE IF EXISTS `activities`; \ No newline at end of file diff --git a/backend/sqlite/schema.sql b/backend/sqlite/db/migrations/000001_initial.up.sql similarity index 87% rename from backend/sqlite/schema.sql rename to backend/sqlite/db/migrations/000001_initial.up.sql index 0c31b0e2..c8ff4256 100644 --- a/backend/sqlite/schema.sql +++ b/backend/sqlite/db/migrations/000001_initial.up.sql @@ -59,4 +59,9 @@ CREATE TABLE IF NOT EXISTS `activities` ( `visible_at` DATETIME NULL, `locked_until` DATETIME NULL, `worker` TEXT NULL -); \ No newline at end of file +); + + +CREATE INDEX IF NOT EXISTS `idx_activities_id_worker` ON `activities` (`id`, `worker`); +CREATE INDEX IF NOT EXISTS `idx_activities_locked_until` ON `activities` (`locked_until`); +CREATE INDEX IF NOT EXISTS `idx_activities_instance_id_execution_id_worker` ON `activities` (`instance_id`, `execution_id`, `worker`); diff --git a/backend/sqlite/db/schema.sql b/backend/sqlite/db/schema.sql new file mode 100644 index 00000000..e69de29b diff --git a/backend/sqlite/options.go b/backend/sqlite/options.go new file mode 100644 index 00000000..5bab53f6 --- /dev/null +++ b/backend/sqlite/options.go @@ -0,0 +1,30 @@ +package sqlite + +import ( + "github.com/cschleiden/go-workflows/backend" +) + +type options struct { + backend.Options + + // ApplyMigrations automatically applies database migrations on startup. + ApplyMigrations bool +} + +type option func(*options) + +// WithApplyMigrations automatically applies database migrations on startup. +func WithApplyMigrations(applyMigrations bool) option { + return func(o *options) { + o.ApplyMigrations = applyMigrations + } +} + +// WithBackendOptions allows to pass generic backend options. +func WithBackendOptions(opts ...backend.BackendOption) option { + return func(o *options) { + for _, opt := range opts { + opt(&o.Options) + } + } +} diff --git a/backend/sqlite/sqlite.go b/backend/sqlite/sqlite.go index a651fe40..3ad27439 100644 --- a/backend/sqlite/sqlite.go +++ b/backend/sqlite/sqlite.go @@ -3,6 +3,7 @@ package sqlite import ( "context" "database/sql" + "embed" _ "embed" "encoding/json" "errors" @@ -23,31 +24,34 @@ import ( "go.opentelemetry.io/otel/trace" _ "modernc.org/sqlite" -) - -//go:embed schema.sql -var schema string -func NewInMemoryBackend(opts ...backend.BackendOption) *sqliteBackend { - b := newSqliteBackend("file::memory:?_mode=memory", opts...) + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source/iofs" +) - b.db.SetMaxOpenConns(1) +//go:embed db/migrations/*.sql +var migrationsFS embed.FS - return b +func NewInMemoryBackend(opts ...option) *sqliteBackend { + return newSqliteBackend("file::memory:?_mode=memory", opts...) } -func NewSqliteBackend(path string, opts ...backend.BackendOption) *sqliteBackend { +func NewSqliteBackend(path string, opts ...option) *sqliteBackend { return newSqliteBackend(fmt.Sprintf("file:%v?_mutex=no&_journal=wal", path), opts...) } -func newSqliteBackend(dsn string, opts ...backend.BackendOption) *sqliteBackend { - db, err := sql.Open("sqlite", dsn) - if err != nil { - panic(err) +func newSqliteBackend(dsn string, opts ...option) *sqliteBackend { + options := &options{ + ApplyMigrations: true, + } + + for _, opt := range opts { + opt(options) } - // Initialize database - if _, err := db.Exec(schema); err != nil { + db, err := sql.Open("sqlite", dsn) + if err != nil { panic(err) } @@ -56,21 +60,56 @@ func newSqliteBackend(dsn string, opts ...backend.BackendOption) *sqliteBackend // See https://github.com/mattn/go-sqlite3/issues/274 for more context db.SetMaxOpenConns(1) - return &sqliteBackend{ + b := &sqliteBackend{ db: db, workerName: fmt.Sprintf("worker-%v", uuid.NewString()), - options: backend.ApplyOptions(opts...), + options: options, + } + + // Apply migrations + if options.ApplyMigrations { + if err := b.Migrate(); err != nil { + panic(err) + } } + + return b } type sqliteBackend struct { db *sql.DB workerName string - options backend.Options + options *options } var _ backend.Backend = (*sqliteBackend)(nil) +// Migrate applies any pending database migrations. +func (sb *sqliteBackend) Migrate() error { + dbi, err := sqlite.WithInstance(sb.db, &sqlite.Config{}) + if err != nil { + return fmt.Errorf("creating migration instance: %w", err) + } + + migrations, err := iofs.New(migrationsFS, "db/migrations") + if err != nil { + return fmt.Errorf("creating migration source: %w", err) + } + + m, err := migrate.NewWithInstance("iofs", migrations, "sqlite", dbi) + if err != nil { + return fmt.Errorf("creating migration: %w", err) + } + + if err := m.Up(); err != nil { + if !errors.Is(err, migrate.ErrNoChange) { + return fmt.Errorf("running migrations: %w", err) + } + } + + return nil +} + func (sb *sqliteBackend) Logger() *slog.Logger { return sb.options.Logger } diff --git a/backend/sqlite/sqlite_test.go b/backend/sqlite/sqlite_test.go index 767efabe..830ebfa7 100644 --- a/backend/sqlite/sqlite_test.go +++ b/backend/sqlite/sqlite_test.go @@ -14,9 +14,7 @@ func Test_SqliteBackend(t *testing.T) { test.BackendTest(t, func(options ...backend.BackendOption) test.TestBackend { // Disable sticky workflow behavior for the test execution - options = append(options, backend.WithStickyTimeout(0)) - - return NewInMemoryBackend(options...) + return NewInMemoryBackend(WithBackendOptions(append(options, backend.WithStickyTimeout(0))...)) }, nil) } @@ -27,8 +25,6 @@ func Test_EndToEndSqliteBackend(t *testing.T) { test.EndToEndBackendTest(t, func(options ...backend.BackendOption) test.TestBackend { // Disable sticky workflow behavior for the test execution - options = append(options, backend.WithStickyTimeout(0)) - - return NewInMemoryBackend(options...) + return NewInMemoryBackend(WithBackendOptions(append(options, backend.WithStickyTimeout(0))...)) }, nil) } diff --git a/bench/main.go b/bench/main.go index e739adc2..42d23064 100644 --- a/bench/main.go +++ b/bench/main.go @@ -104,12 +104,12 @@ func main() { func getBackend(b string, opt ...backend.BackendOption) backend.Backend { switch b { case "memory": - return monoprocess.NewMonoprocessBackend(sqlite.NewInMemoryBackend(opt...)) + return monoprocess.NewMonoprocessBackend(sqlite.NewInMemoryBackend(sqlite.WithBackendOptions(opt...))) case "sqlite": os.Remove("bench.sqlite") - return monoprocess.NewMonoprocessBackend(sqlite.NewSqliteBackend("bench.sqlite", opt...)) + return monoprocess.NewMonoprocessBackend(sqlite.NewSqliteBackend("bench.sqlite", sqlite.WithBackendOptions(opt...))) case "mysql": db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/?parseTime=true&interpolateParams=true", "root", "root")) diff --git a/go.mod b/go.mod index 7238b50b..0d67e8f6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/go-errors/errors v1.4.2 github.com/go-sql-driver/mysql v1.7.0 + github.com/golang-migrate/migrate/v4 v4.16.2 github.com/golangci/golangci-lint v1.54.2 github.com/google/uuid v1.3.0 github.com/jellydator/ttlcache/v3 v3.0.0 @@ -131,7 +132,7 @@ require ( github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/go.sum b/go.sum index e088d279..a6b4eae2 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA= +github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= @@ -323,8 +325,9 @@ github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Rep github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -393,6 +396,8 @@ github.com/ldez/tagliatelle v0.5.0 h1:epgfuYt9v0CG3fms0pEgIMNPuFf/LpPIfjk4kyqSio github.com/ldez/tagliatelle v0.5.0/go.mod h1:rj1HmWiL1MiKQuOONhd09iySTEkUuE/8+5jtPYz9xa4= github.com/leonklingele/grouper v1.1.1 h1:suWXRU57D4/Enn6pXR0QVqqWWrnJ9Osrz+5rjt8ivzU= github.com/leonklingele/grouper v1.1.1/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= @@ -443,6 +448,7 @@ github.com/nunnatsa/ginkgolinter v0.13.5 h1:fOsPB4CEZOPkyMqF4B9hoqOpooFWU7vWSVkC github.com/nunnatsa/ginkgolinter v0.13.5/go.mod h1:OBHy4536xtuX3102NM63XRtOyxqZOO02chsaeDWXVO8= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= diff --git a/samples/runner.go b/samples/runner.go index c41ca320..1b5b35d4 100644 --- a/samples/runner.go +++ b/samples/runner.go @@ -21,10 +21,10 @@ func GetBackend(name string, opt ...backend.BackendOption) backend.Backend { switch *b { case "memory": - return sqlite.NewInMemoryBackend(opt...) + return sqlite.NewInMemoryBackend(sqlite.WithBackendOptions(opt...)) case "sqlite": - return sqlite.NewSqliteBackend(name+".sqlite", opt...) + return sqlite.NewSqliteBackend(name+".sqlite", sqlite.WithBackendOptions(opt...)) case "mysql": return mysql.NewMysqlBackend("localhost", 3306, "root", "root", name, opt...) From eb2d2f9c9eb8969a41977757b4fd2713bc1ca797 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Wed, 1 Nov 2023 16:45:29 -0700 Subject: [PATCH 2/8] Move mysql to use migrations --- .../db/migrations/000001_initial.down.sql | 4 ++ .../migrations/000001_initial.up.sql} | 0 backend/mysql/mysql.go | 69 ++++++++++++++----- backend/mysql/mysql_test.go | 4 +- backend/mysql/options.go | 30 ++++++++ bench/main.go | 2 +- go.sum | 24 +++++++ samples/runner.go | 2 +- 8 files changed, 113 insertions(+), 22 deletions(-) create mode 100644 backend/mysql/db/migrations/000001_initial.down.sql rename backend/mysql/{schema.sql => db/migrations/000001_initial.up.sql} (100%) create mode 100644 backend/mysql/options.go diff --git a/backend/mysql/db/migrations/000001_initial.down.sql b/backend/mysql/db/migrations/000001_initial.down.sql new file mode 100644 index 00000000..928470b9 --- /dev/null +++ b/backend/mysql/db/migrations/000001_initial.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS `instances`; +DROP TABLE IF EXISTS `pending_events`; +DROP TABLE IF EXISTS `history`; +DROP TABLE IF EXISTS `activities`; \ No newline at end of file diff --git a/backend/mysql/schema.sql b/backend/mysql/db/migrations/000001_initial.up.sql similarity index 100% rename from backend/mysql/schema.sql rename to backend/mysql/db/migrations/000001_initial.up.sql diff --git a/backend/mysql/mysql.go b/backend/mysql/mysql.go index 6d6acaa9..2ccff5b0 100644 --- a/backend/mysql/mysql.go +++ b/backend/mysql/mysql.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "embed" _ "embed" "encoding/json" "errors" @@ -22,44 +23,76 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/google/uuid" "go.opentelemetry.io/otel/trace" -) -//go:embed schema.sql -var schema string + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/mysql" + "github.com/golang-migrate/migrate/v4/source/iofs" +) -func NewMysqlBackend(host string, port int, user, password, database string, opts ...backend.BackendOption) *mysqlBackend { - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true&interpolateParams=true", user, password, host, port, database) +//go:embed db/migrations/*.sql +var migrationsFS embed.FS - schemaDsn := dsn + "&multiStatements=true" - db, err := sql.Open("mysql", schemaDsn) - if err != nil { - panic(err) +func NewMysqlBackend(host string, port int, user, password, database string, opts ...option) *mysqlBackend { + options := &options{ + ApplyMigrations: true, } - if _, err := db.Exec(schema); err != nil { - panic(fmt.Errorf("initializing database: %w", err)) + for _, opt := range opts { + opt(options) } - if err := db.Close(); err != nil { - panic(err) - } + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true&interpolateParams=true", user, password, host, port, database) - db, err = sql.Open("mysql", dsn) + db, err := sql.Open("mysql", dsn) if err != nil { panic(err) } - return &mysqlBackend{ + b := &mysqlBackend{ db: db, workerName: fmt.Sprintf("worker-%v", uuid.NewString()), - options: backend.ApplyOptions(opts...), + options: options, + } + + if options.ApplyMigrations { + if err := b.Migrate(); err != nil { + panic(err) + } } + + return b } type mysqlBackend struct { db *sql.DB workerName string - options backend.Options + options *options +} + +// Migrate applies any pending database migrations. +func (sb *mysqlBackend) Migrate() error { + dbi, err := mysql.WithInstance(sb.db, &mysql.Config{}) + if err != nil { + return fmt.Errorf("creating migration instance: %w", err) + } + + migrations, err := iofs.New(migrationsFS, "db/migrations") + if err != nil { + return fmt.Errorf("creating migration source: %w", err) + } + + m, err := migrate.NewWithInstance("iofs", migrations, "mysql", dbi) + if err != nil { + return fmt.Errorf("creating migration: %w", err) + } + + if err := m.Up(); err != nil { + if !errors.Is(err, migrate.ErrNoChange) { + return fmt.Errorf("running migrations: %w", err) + } + } + + return nil } func (b *mysqlBackend) Logger() *slog.Logger { diff --git a/backend/mysql/mysql_test.go b/backend/mysql/mysql_test.go index dad367ff..2afca491 100644 --- a/backend/mysql/mysql_test.go +++ b/backend/mysql/mysql_test.go @@ -43,7 +43,7 @@ func Test_MysqlBackend(t *testing.T) { options = append(options, backend.WithStickyTimeout(0)) - return NewMysqlBackend("localhost", 3306, testUser, testPassword, dbName, options...) + return NewMysqlBackend("localhost", 3306, testUser, testPassword, dbName, WithBackendOptions(options...)) }, func(b test.TestBackend) { db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/?parseTime=true&interpolateParams=true", testUser, testPassword)) if err != nil { @@ -84,7 +84,7 @@ func TestMySqlBackendE2E(t *testing.T) { options = append(options, backend.WithStickyTimeout(0)) - return NewMysqlBackend("localhost", 3306, testUser, testPassword, dbName, options...) + return NewMysqlBackend("localhost", 3306, testUser, testPassword, dbName, WithBackendOptions(options...)) }, func(b test.TestBackend) { db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/?parseTime=true&interpolateParams=true", testUser, testPassword)) if err != nil { diff --git a/backend/mysql/options.go b/backend/mysql/options.go new file mode 100644 index 00000000..c0fdc902 --- /dev/null +++ b/backend/mysql/options.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "github.com/cschleiden/go-workflows/backend" +) + +type options struct { + backend.Options + + // ApplyMigrations automatically applies database migrations on startup. + ApplyMigrations bool +} + +type option func(*options) + +// WithApplyMigrations automatically applies database migrations on startup. +func WithApplyMigrations(applyMigrations bool) option { + return func(o *options) { + o.ApplyMigrations = applyMigrations + } +} + +// WithBackendOptions allows to pass generic backend options. +func WithBackendOptions(opts ...backend.BackendOption) option { + return func(o *options) { + for _, opt := range opts { + opt(&o.Options) + } + } +} diff --git a/bench/main.go b/bench/main.go index 42d23064..3fe80432 100644 --- a/bench/main.go +++ b/bench/main.go @@ -129,7 +129,7 @@ func getBackend(b string, opt ...backend.BackendOption) backend.Backend { panic(err) } - return monoprocess.NewMonoprocessBackend(mysql.NewMysqlBackend("localhost", 3306, "root", "root", "bench", opt...)) + return monoprocess.NewMonoprocessBackend(mysql.NewMysqlBackend("localhost", 3306, "root", "root", "bench", mysql.WithBackendOptions(opt...))) case "redis": rclient := redisv8.NewUniversalClient(&redisv8.UniversalOptions{ diff --git a/go.sum b/go.sum index a6b4eae2..b6491e97 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/Antonboom/errname v0.1.12 h1:oh9ak2zUtsLp5oaEd/erjB4GPu9w19NyoIskZClD github.com/Antonboom/errname v0.1.12/go.mod h1:bK7todrzvlaZoQagP1orKzWXv59X/x0W0Io2XT1Ssro= github.com/Antonboom/nilnil v0.1.7 h1:ofgL+BA7vlA1K2wNQOsHzLJ2Pw5B5DpWRLdDAVvvTow= github.com/Antonboom/nilnil v0.1.7/go.mod h1:TP+ScQWVEq0eSIxqU8CbdT5DFWoHp0MbP+KMUO1BKYQ= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -58,6 +60,8 @@ github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 h1:3ZBs7LAezy8gh0uECsA6C github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0/go.mod h1:rZLTje5A9kFBe0pzhpe2TdhRniBF++PRHQuRpR8esVc= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard/v2 v2.1.0 h1:aQl70G173h/GZYhWf36aE5H0KaujXfVMnn/f1kSDVYY= github.com/OpenPeeDeeP/depguard/v2 v2.1.0/go.mod h1:PUBgk35fX4i7JDmwzlJwJ+GMe6NfO1723wmJMgPThNQ= @@ -139,6 +143,16 @@ github.com/denis-tingaikin/go-header v0.4.3 h1:tEaZKAlqql6SKCY++utLmkPLd6K8IBM20 github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw= +github.com/dhui/dktest v0.3.16/go.mod h1:gYaA3LRmM8Z4vJl2MA0THIigJoZrwOansEOsp+kqxp0= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= +github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -214,6 +228,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA= github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -429,6 +445,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -436,6 +454,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA= github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= @@ -453,6 +473,10 @@ github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= diff --git a/samples/runner.go b/samples/runner.go index 1b5b35d4..352309c5 100644 --- a/samples/runner.go +++ b/samples/runner.go @@ -27,7 +27,7 @@ func GetBackend(name string, opt ...backend.BackendOption) backend.Backend { return sqlite.NewSqliteBackend(name+".sqlite", sqlite.WithBackendOptions(opt...)) case "mysql": - return mysql.NewMysqlBackend("localhost", 3306, "root", "root", name, opt...) + return mysql.NewMysqlBackend("localhost", 3306, "root", "root", name, mysql.WithBackendOptions(opt...)) case "redis": rclient := redisv9.NewUniversalClient(&redisv9.UniversalOptions{ From 8058b350145e9ea5045a09464807aa5061380246 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Wed, 1 Nov 2023 21:45:07 -0700 Subject: [PATCH 3/8] Use multi statement db connection --- backend/mysql/mysql.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/mysql/mysql.go b/backend/mysql/mysql.go index 2ccff5b0..7db2a9dc 100644 --- a/backend/mysql/mysql.go +++ b/backend/mysql/mysql.go @@ -49,6 +49,7 @@ func NewMysqlBackend(host string, port int, user, password, database string, opt } b := &mysqlBackend{ + dsn: dsn, db: db, workerName: fmt.Sprintf("worker-%v", uuid.NewString()), options: options, @@ -64,6 +65,7 @@ func NewMysqlBackend(host string, port int, user, password, database string, opt } type mysqlBackend struct { + dsn string db *sql.DB workerName string options *options @@ -71,7 +73,13 @@ type mysqlBackend struct { // Migrate applies any pending database migrations. func (sb *mysqlBackend) Migrate() error { - dbi, err := mysql.WithInstance(sb.db, &mysql.Config{}) + schemaDsn := sb.dsn + "&multiStatements=true" + db, err := sql.Open("mysql", schemaDsn) + if err != nil { + return fmt.Errorf("opening schema database: %w", err) + } + + dbi, err := mysql.WithInstance(db, &mysql.Config{}) if err != nil { return fmt.Errorf("creating migration instance: %w", err) } From 8891bc2e68546ded1983b0e0c0e101cde13e8e50 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Wed, 1 Nov 2023 22:11:57 -0700 Subject: [PATCH 4/8] Correctly set default options --- backend/mysql/mysql.go | 1 + backend/sqlite/sqlite.go | 1 + 2 files changed, 2 insertions(+) diff --git a/backend/mysql/mysql.go b/backend/mysql/mysql.go index 7db2a9dc..2d51f0cd 100644 --- a/backend/mysql/mysql.go +++ b/backend/mysql/mysql.go @@ -34,6 +34,7 @@ var migrationsFS embed.FS func NewMysqlBackend(host string, port int, user, password, database string, opts ...option) *mysqlBackend { options := &options{ + Options: backend.ApplyOptions(), ApplyMigrations: true, } diff --git a/backend/sqlite/sqlite.go b/backend/sqlite/sqlite.go index 3ad27439..1088ee74 100644 --- a/backend/sqlite/sqlite.go +++ b/backend/sqlite/sqlite.go @@ -43,6 +43,7 @@ func NewSqliteBackend(path string, opts ...option) *sqliteBackend { func newSqliteBackend(dsn string, opts ...option) *sqliteBackend { options := &options{ + Options: backend.ApplyOptions(), ApplyMigrations: true, } From a8770fa19a69d98b7d22011aae927f2a9855d581 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Wed, 1 Nov 2023 22:19:58 -0700 Subject: [PATCH 5/8] Close migration db connection after applying migrations --- backend/mysql/mysql.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/mysql/mysql.go b/backend/mysql/mysql.go index 2d51f0cd..e9c8ab99 100644 --- a/backend/mysql/mysql.go +++ b/backend/mysql/mysql.go @@ -101,6 +101,10 @@ func (sb *mysqlBackend) Migrate() error { } } + if err := db.Close(); err != nil { + return fmt.Errorf("closing schema database: %w", err) + } + return nil } From e8729b74abdc099037ef1a0fa1beace33789bde4 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Wed, 1 Nov 2023 22:31:35 -0700 Subject: [PATCH 6/8] Close DB connection for mysql tests --- backend/mysql/mysql_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/mysql/mysql_test.go b/backend/mysql/mysql_test.go index 2afca491..1495e9a5 100644 --- a/backend/mysql/mysql_test.go +++ b/backend/mysql/mysql_test.go @@ -45,6 +45,10 @@ func Test_MysqlBackend(t *testing.T) { return NewMysqlBackend("localhost", 3306, testUser, testPassword, dbName, WithBackendOptions(options...)) }, func(b test.TestBackend) { + if err := b.(*mysqlBackend).db.Close(); err != nil { + panic(err) + } + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/?parseTime=true&interpolateParams=true", testUser, testPassword)) if err != nil { panic(err) @@ -86,6 +90,10 @@ func TestMySqlBackendE2E(t *testing.T) { return NewMysqlBackend("localhost", 3306, testUser, testPassword, dbName, WithBackendOptions(options...)) }, func(b test.TestBackend) { + if err := b.(*mysqlBackend).db.Close(); err != nil { + panic(err) + } + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/?parseTime=true&interpolateParams=true", testUser, testPassword)) if err != nil { panic(err) From 8ac191ff2d2fbee058c8dbc2229f52c114d13772 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Thu, 2 Nov 2023 09:10:32 -0700 Subject: [PATCH 7/8] Add documentation for migration behavior --- README.md | 6 +++++- backend/mysql/README.md | 8 ++++++++ backend/sqlite/README.md | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 backend/mysql/README.md create mode 100644 backend/sqlite/README.md diff --git a/README.md b/README.md index 057c0c5c..ae55a6a6 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ For all backends, for now the initial schema is applied upon first usage. In the #### Sqlite -The Sqlite backend implementation supports two different modes, in-memory and on-disk. +The Sqlite backend implementation supports two different modes, in-memory and on-disk: * In-memory: ```go @@ -136,12 +136,16 @@ The Sqlite backend implementation supports two different modes, in-memory and on b := sqlite.NewSqliteBackend("simple.sqlite") ``` +By default the schema is automatically created/migrations are automatically applied. Use `WithApplyMigrations(false)` to disable this behavior. + #### MySql ```go b := mysql.NewMysqlBackend("localhost", 3306, "root", "SqlPassw0rd", "simple") ``` +By default the schema is automatically created/migrations are automatically applied. Use `WithApplyMigrations(false)` to disable this behavior. + #### Redis ```go diff --git a/backend/mysql/README.md b/backend/mysql/README.md new file mode 100644 index 00000000..d7e67897 --- /dev/null +++ b/backend/mysql/README.md @@ -0,0 +1,8 @@ +# Sqlite backend + +## Adding a migration + +1. Install [golang-migrate/migrate](https://www.github.com/golang-migrate/migrate) +1. ```bash + migrate create -ext sql -dir ./db/migrations -seq + ``` \ No newline at end of file diff --git a/backend/sqlite/README.md b/backend/sqlite/README.md new file mode 100644 index 00000000..d7e67897 --- /dev/null +++ b/backend/sqlite/README.md @@ -0,0 +1,8 @@ +# Sqlite backend + +## Adding a migration + +1. Install [golang-migrate/migrate](https://www.github.com/golang-migrate/migrate) +1. ```bash + migrate create -ext sql -dir ./db/migrations -seq + ``` \ No newline at end of file From c6961f716fa437797cc2ab615a50ef88cd11f441 Mon Sep 17 00:00:00 2001 From: Christopher Schleiden Date: Thu, 2 Nov 2023 09:32:16 -0700 Subject: [PATCH 8/8] Remove outdated schema file --- backend/sqlite/db/schema.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/sqlite/db/schema.sql diff --git a/backend/sqlite/db/schema.sql b/backend/sqlite/db/schema.sql deleted file mode 100644 index e69de29b..00000000