Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build dynamic statements #102

Open
wants to merge 62 commits into
base: support-sqlite
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
9e1a1d1
Introduce database#QueryBuilder
jrauh01 Nov 28, 2024
3367c4a
Add database#QueryBuilder to database#DB
jrauh01 Nov 28, 2024
cf99627
Introduce database#InsertStatement
jrauh01 Nov 28, 2024
4392b4e
Add and implement QueryBuilder.InsertStatement()
jrauh01 Nov 28, 2024
fef3b29
Add DB.BuildInsertStatement()
jrauh01 Nov 28, 2024
5fcfaaf
Introduce database#SelectStatement
jrauh01 Nov 28, 2024
ce4ed84
Add and implement QueryBuilder.SelectStatement()
jrauh01 Nov 28, 2024
02f7ad7
Add DB.BuildSelectStatement()
jrauh01 Nov 28, 2024
24508e8
Introduce database#InsertSelectStatement
jrauh01 Nov 28, 2024
c52293f
Add and implement QueryBuilder.InsertSelectStatement()
jrauh01 Nov 28, 2024
47eef5a
Add DB.BuildInsertSelectStatement()
jrauh01 Nov 28, 2024
9e19371
Move statements to own files
jrauh01 Nov 28, 2024
677a309
Introduce database#DeleteStatement
jrauh01 Nov 28, 2024
527c0ed
Add and implement QueryBuilder.DeleteStatement()
jrauh01 Nov 28, 2024
12181bd
Add BD.BuildDeleteStatement()
jrauh01 Nov 28, 2024
9d8da97
Add and implement QueryBuilder.DeleteAllStatement()
jrauh01 Nov 28, 2024
d667389
Add DB.BuildDeleteAllStatement()
jrauh01 Nov 28, 2024
f28a170
Add and implement QueryBuilder.InsertIgnoreStatement()
jrauh01 Nov 28, 2024
7ea0a5d
Add DB.BuildInsertIgnoreStatement()
jrauh01 Nov 28, 2024
dd82336
Introduce database#UpdateStatement
jrauh01 Nov 28, 2024
f8d02ea
Add and implement QueryBuilder.UpdateStatement()
jrauh01 Nov 28, 2024
268ea18
Add DB.BuildUpdateStatement()
jrauh01 Nov 28, 2024
78fb60c
Add and implement QueryBuilder.UpdateAllStatement()
jrauh01 Nov 28, 2024
42b3b1c
Add DB.BuildUpdateAllStatement()
jrauh01 Nov 28, 2024
9f1ffb8
Add and implement QueryBuilder.UpsertStatement()
jrauh01 Nov 29, 2024
fd30ad1
Add DB.BuildUpsertStatement()
jrauh01 Nov 29, 2024
8608c13
Rename QueryBuilder.driver to 'dbDriver'
jrauh01 Nov 29, 2024
efd981e
Adjust unsupported driver error message
jrauh01 Nov 29, 2024
56f4131
Make statements generic
jrauh01 Nov 29, 2024
313100f
Remove generics from statements
jrauh01 Dec 2, 2024
b851787
Introduce database#UpsertStatement
jrauh01 Dec 2, 2024
0efb511
Use UpsertStatement in QueryBuilder.UpsertStatement()
jrauh01 Dec 2, 2024
0efd7d7
Use UpsertStatement in DB.BuildUpsertStatement()
jrauh01 Dec 2, 2024
8670f66
Rename constructor funcs for statements
jrauh01 Dec 2, 2024
4d781fb
Add not retryable error
jrauh01 Dec 3, 2024
cf19615
Return number of placeholders for upsert
jrauh01 Dec 3, 2024
5cc9299
wip: streamed statements
jrauh01 Dec 3, 2024
cbee0a6
Use fmt.Errorf() instead of errors.New()
jrauh01 Dec 3, 2024
9f86a7d
Add functional options for insert
jrauh01 Dec 3, 2024
b52cc3d
Add functional options for update
jrauh01 Dec 3, 2024
c854604
Add docs for statements
jrauh01 Dec 4, 2024
2d83c36
Add custom errors for query builder
jrauh01 Dec 4, 2024
6fea5b1
Sort columns for unit testing
jrauh01 Dec 4, 2024
ead7c97
Fix bug: cache is changed via reference
jrauh01 Dec 4, 2024
8f65e96
Introduce new errors
jrauh01 Dec 4, 2024
b0e3835
Make InsertSelect a standalone statement
jrauh01 Dec 4, 2024
5e8d7f3
Build SET clause from columns
jrauh01 Dec 4, 2024
af28f81
Add support for sqlite
jrauh01 Dec 9, 2024
7111124
Use own error type for returning error
jrauh01 Dec 9, 2024
9233a43
Add unit tests for query builder
jrauh01 Dec 9, 2024
6165fc5
Create pgsql constraint without function
jrauh01 Dec 9, 2024
a1092af
Make MockEntity implement Entity directly
jrauh01 Dec 10, 2024
4264d81
Add docs example for upsert streamed
jrauh01 Dec 10, 2024
df13c53
Move unit test data to database package
jrauh01 Dec 10, 2024
17a77cf
wip: unit tests for upsert streamed
jrauh01 Dec 11, 2024
25f80f1
Revise the functional options for statements
jrauh01 Dec 11, 2024
dbe6f5c
Add testutils for database
jrauh01 Dec 11, 2024
86e878c
Add query builder to DB
jrauh01 Dec 11, 2024
70e89a1
Append callbacks instead overwriting
jrauh01 Dec 11, 2024
8baec90
Add extra function for prefilling test database
jrauh01 Dec 11, 2024
242c1e9
Use one line if for error handling
jrauh01 Dec 12, 2024
f2b199d
wip: more upsert examples
jrauh01 Dec 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions database/contracts.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ type IDer interface {
// EntityFactoryFunc knows how to create an Entity.
type EntityFactoryFunc func() Entity

type EntityConstraint[T any] interface {
Entity
*T
}

// Upserter implements the Upsert method,
// which returns a part of the object for ON DUPLICATE KEY UPDATE.
type Upserter interface {
Expand Down
6 changes: 6 additions & 0 deletions database/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type DB struct {
Options *Options

addr string
queryBuilder QueryBuilder
columnMap ColumnMap
logger *logging.Logger
tableSemaphores map[string]*semaphore.Weighted
Expand Down Expand Up @@ -256,6 +257,7 @@ func NewDbFromConfig(c *Config, logger *logging.Logger, connectorCallbacks Retry
return &DB{
DB: db,
Options: &c.Options,
queryBuilder: NewQueryBuilder(db.DriverName()),
columnMap: NewColumnMap(db.Mapper),
addr: addr,
logger: logger,
Expand Down Expand Up @@ -893,3 +895,7 @@ func (db *DB) Log(ctx context.Context, query string, counter *com.Counter) perio
db.logger.Debugf("Finished executing %q with %d rows in %s", query, counter.Total(), tick.Elapsed)
}))
}

func (db *DB) QueryBuilder() QueryBuilder {
return db.queryBuilder
}
213 changes: 213 additions & 0 deletions database/delete.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
package database

import (
"context"
"fmt"
"github.com/icinga/icinga-go-library/backoff"
"github.com/icinga/icinga-go-library/com"
"github.com/icinga/icinga-go-library/retry"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
"reflect"
"time"
)

// DeleteStatement is the interface for building DELETE statements.
type DeleteStatement interface {
// From sets the table name for the DELETE statement.
// Overrides the table name provided by the entity.
From(table string) DeleteStatement

// SetWhere sets the where clause for the DELETE statement.
SetWhere(where string) DeleteStatement

// Entity returns the entity associated with the DELETE statement.
Entity() Entity

// Table returns the table name for the DELETE statement.
Table() string

Where() string
}

// NewDeleteStatement returns a new deleteStatement for the given entity.
func NewDeleteStatement(entity Entity) DeleteStatement {
return &deleteStatement{
entity: entity,
}
}

// deleteStatement is the default implementation of the DeleteStatement interface.
type deleteStatement struct {
entity Entity
table string
where string
}

func (d *deleteStatement) From(table string) DeleteStatement {
d.table = table

return d
}

func (d *deleteStatement) SetWhere(where string) DeleteStatement {
d.where = where

return d
}

func (d *deleteStatement) Entity() Entity {
return d.entity
}

func (d *deleteStatement) Table() string {
return d.table
}

func (d *deleteStatement) Where() string {
return d.where
}

// DeleteOption is a functional option for DeleteStreamed().
type DeleteOption func(opts *deleteOptions)

// WithDeleteStatement sets the DELETE statement to be used for deleting entities.
func WithDeleteStatement(stmt DeleteStatement) DeleteOption {
return func(opts *deleteOptions) {
opts.stmt = stmt
}
}

// WithOnDelete sets the callbacks for a successful DELETE operation.
func WithOnDelete(onDelete ...OnSuccess[any]) DeleteOption {
return func(opts *deleteOptions) {
opts.onDelete = append(opts.onDelete, onDelete...)
}
}

// deleteOptions stores the options for DeleteStreamed.
type deleteOptions struct {
stmt DeleteStatement
onDelete []OnSuccess[any]
}

// DeleteStreamed deletes entities from the given channel from the database.
func DeleteStreamed(
ctx context.Context,
db *DB,
entityType Entity,
entities <-chan any,
options ...DeleteOption,
) error {
opts := &deleteOptions{}
for _, option := range options {
option(opts)
}

first, forward, err := com.CopyFirst(ctx, entities)
if err != nil {
return errors.Wrap(err, "can't copy first entity")
}

sem := db.GetSemaphoreForTable(TableName(entityType))

var stmt string

if opts.stmt != nil {
stmt, err = db.QueryBuilder().DeleteStatement(opts.stmt)
if err != nil {
return err
}
} else {
stmt, err = db.QueryBuilder().DeleteStatement(NewDeleteStatement(entityType))
if err != nil {
return err
}
}

switch reflect.TypeOf(first).Kind() {
case reflect.Struct, reflect.Map:
return namedBulkExec(ctx, db, stmt, db.Options.MaxPlaceholdersPerStatement, sem, forward, com.NeverSplit[any], opts.onDelete...)
default:
return bulkExec(ctx, db, stmt, db.Options.MaxPlaceholdersPerStatement, sem, forward, opts.onDelete...)
}
}

func bulkExec(
ctx context.Context, db *DB, query string, count int, sem *semaphore.Weighted, arg <-chan any, onSuccess ...OnSuccess[any],
) error {
var counter com.Counter
defer db.Log(ctx, query, &counter).Stop()

g, ctx := errgroup.WithContext(ctx)
// Use context from group.
bulk := com.Bulk(ctx, arg, count, com.NeverSplit[any])

g.Go(func() error {
g, ctx := errgroup.WithContext(ctx)

for b := range bulk {
if err := sem.Acquire(ctx, 1); err != nil {
return errors.Wrap(err, "can't acquire semaphore")
}

g.Go(func(b []any) func() error {
return func() error {
defer sem.Release(1)

return retry.WithBackoff(
ctx,
func(context.Context) error {
var valCollection []any

for _, v := range b {
val := reflect.ValueOf(v)
if val.Kind() == reflect.Slice {
for i := 0; i < val.Len(); i++ {
valCollection = append(valCollection, val.Index(i).Interface())
}
} else {
valCollection = append(valCollection, val.Interface())
}
}

stmt, args, err := sqlx.In(query, valCollection)
if err != nil {
return fmt.Errorf(
"%w: %w",
retry.ErrNotRetryable,
errors.Wrapf(err, "can't build placeholders for %q", query),
)
}

stmt = db.Rebind(stmt)
_, err = db.ExecContext(ctx, stmt, args...)
if err != nil {
return CantPerformQuery(err, query)
}

counter.Add(uint64(len(b)))

for _, onSuccess := range onSuccess {
if err := onSuccess(ctx, b); err != nil {
return err
}
}

return nil
},
retry.Retryable,
backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
db.GetDefaultRetrySettings(),
)
}
}(b))
}

return g.Wait()
})

return g.Wait()
}
Loading
Loading