From db808f136b6452f5b7b2caa1748054006c3adc8c Mon Sep 17 00:00:00 2001 From: Yosyp Buchma Date: Mon, 18 Sep 2023 17:35:11 +0300 Subject: [PATCH 1/5] Faster MySQL GetTableColumnsMetaData query --- generator/mysql/query_set.go | 49 +++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/generator/mysql/query_set.go b/generator/mysql/query_set.go index 9eb92577..442b74be 100644 --- a/generator/mysql/query_set.go +++ b/generator/mysql/query_set.go @@ -39,27 +39,36 @@ ORDER BY table_name; func (m mySqlQuerySet) GetTableColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) { query := ` -SELECT COLUMN_NAME AS "column.Name", - IS_NULLABLE = "YES" AS "column.IsNullable", - columns.COLUMN_COMMENT as "column.Comment", - (EXISTS( - SELECT 1 +SELECT + col.COLUMN_NAME AS "column.Name", + col.IS_NULLABLE = "YES" AS "column.IsNullable", + col.COLUMN_COMMENT AS "column.Comment", + pk.IsPrimaryKey AS "column.IsPrimaryKey", + IF (col.COLUMN_TYPE = 'tinyint(1)', + 'boolean', + IF (col.DATA_TYPE = 'enum', + CONCAT(col.TABLE_NAME, '_', col.COLUMN_NAME), + col.DATA_TYPE) + ) AS "dataType.Name", + IF (col.DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind", + col.COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned" +FROM + information_schema.columns AS col +LEFT JOIN ( + SELECT k.column_name, 1 AS IsPrimaryKey FROM information_schema.table_constraints t - JOIN information_schema.key_column_usage k USING(constraint_name,table_schema,table_name) - WHERE table_schema = ? AND table_name = ? AND t.constraint_type='PRIMARY KEY' AND k.column_name = columns.column_name - )) AS "column.IsPrimaryKey", - IF (COLUMN_TYPE = 'tinyint(1)', - 'boolean', - IF (DATA_TYPE='enum', - CONCAT(TABLE_NAME, '_', COLUMN_NAME), - DATA_TYPE) - ) AS "dataType.Name", - IF (DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind", - COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned" -FROM information_schema.columns -WHERE table_schema = ? AND table_name = ? -ORDER BY ordinal_position; -` + JOIN information_schema.key_column_usage k USING(constraint_name, table_schema, table_name) + WHERE t.table_schema = ? + AND t.table_name = ? + AND t.constraint_type = 'PRIMARY KEY' +) AS pk ON col.COLUMN_NAME = pk.column_name +WHERE + col.table_schema = ? + AND col.table_name = ? +ORDER BY + col.ordinal_position; + ` + var columns []metadata.Column _, err := qrm.Query(context.Background(), db, query, []interface{}{schemaName, tableName, schemaName, tableName}, &columns) if err != nil { From 98dfce2ae587ebd9b2a2064f457af464e19b9ded Mon Sep 17 00:00:00 2001 From: Yosyp Buchma Date: Mon, 18 Sep 2023 17:35:58 +0300 Subject: [PATCH 2/5] Concurrent GetTableColumnsMetaData for MySQL --- generator/mysql/query_set.go | 39 ++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/generator/mysql/query_set.go b/generator/mysql/query_set.go index 442b74be..95d6a93b 100644 --- a/generator/mysql/query_set.go +++ b/generator/mysql/query_set.go @@ -4,7 +4,9 @@ import ( "context" "database/sql" "fmt" + "runtime" "strings" + "sync" "github.com/go-jet/jet/v2/generator/metadata" "github.com/go-jet/jet/v2/qrm" @@ -27,11 +29,40 @@ ORDER BY table_name; return nil, fmt.Errorf("failed to query %s metadata result: %w", tableType, err) } + tblChan := make(chan int, len(tables)) + errChan := make(chan error, 1) + + wg := sync.WaitGroup{} + for i := 0; i < runtime.NumCPU(); i++ { + wg.Add(1) + go func() { + defer wg.Done() + var err1 error + for tblIdx := range tblChan { + tables[tblIdx].Columns, err1 = m.GetTableColumnsMetaData(db, schemaName, tables[tblIdx].Name) + if err1 != nil { + select { + case errChan <- fmt.Errorf("failed to get '%s' table columns metadata: %w", tables[tblIdx].Name, err1): + return + default: + } + return + } + } + }() + } + for i := range tables { - tables[i].Columns, err = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name) - if err != nil { - return nil, fmt.Errorf("failed to get '%s' table columns metadata: %w", tables[i].Name, err) - } + tblChan <- i + } + + close(tblChan) + wg.Wait() + + select { + case err = <-errChan: + return nil, err + default: } return tables, nil From ffabf8b26e90b94fb019988d06dbca26d6aceece Mon Sep 17 00:00:00 2001 From: Yosyp Buchma Date: Tue, 19 Sep 2023 20:56:21 +0300 Subject: [PATCH 3/5] coalesce pk.IsPrimaryKey --- generator/mysql/query_set.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/mysql/query_set.go b/generator/mysql/query_set.go index 95d6a93b..f8863572 100644 --- a/generator/mysql/query_set.go +++ b/generator/mysql/query_set.go @@ -74,7 +74,7 @@ SELECT col.COLUMN_NAME AS "column.Name", col.IS_NULLABLE = "YES" AS "column.IsNullable", col.COLUMN_COMMENT AS "column.Comment", - pk.IsPrimaryKey AS "column.IsPrimaryKey", + COALESCE(pk.IsPrimaryKey, 0) AS "column.IsPrimaryKey", IF (col.COLUMN_TYPE = 'tinyint(1)', 'boolean', IF (col.DATA_TYPE = 'enum', From f472becd892d3cfca593ecca62aa92389e6f3ebb Mon Sep 17 00:00:00 2001 From: Yosyp Buchma Date: Tue, 19 Sep 2023 20:56:54 +0300 Subject: [PATCH 4/5] simplified concurrent querying --- generator/mysql/query_set.go | 49 ++++++++++-------------------------- go.mod | 1 + go.sum | 2 ++ 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/generator/mysql/query_set.go b/generator/mysql/query_set.go index f8863572..b58ff82c 100644 --- a/generator/mysql/query_set.go +++ b/generator/mysql/query_set.go @@ -4,12 +4,11 @@ import ( "context" "database/sql" "fmt" - "runtime" "strings" - "sync" "github.com/go-jet/jet/v2/generator/metadata" "github.com/go-jet/jet/v2/qrm" + "golang.org/x/sync/errgroup" ) // mySqlQuerySet is dialect query set for MySQL @@ -29,43 +28,21 @@ ORDER BY table_name; return nil, fmt.Errorf("failed to query %s metadata result: %w", tableType, err) } - tblChan := make(chan int, len(tables)) - errChan := make(chan error, 1) - - wg := sync.WaitGroup{} - for i := 0; i < runtime.NumCPU(); i++ { - wg.Add(1) - go func() { - defer wg.Done() - var err1 error - for tblIdx := range tblChan { - tables[tblIdx].Columns, err1 = m.GetTableColumnsMetaData(db, schemaName, tables[tblIdx].Name) - if err1 != nil { - select { - case errChan <- fmt.Errorf("failed to get '%s' table columns metadata: %w", tables[tblIdx].Name, err1): - return - default: - } - return - } - } - }() - } - - for i := range tables { - tblChan <- i - } + const maxConns = 32 + db.SetMaxOpenConns(maxConns) + db.SetMaxIdleConns(maxConns) - close(tblChan) - wg.Wait() - - select { - case err = <-errChan: - return nil, err - default: + wg := errgroup.Group{} + for i := 0; i < len(tables); i++ { + i := i + wg.Go(func() (err1 error) { + tables[i].Columns, err1 = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name) + return err1 + }) } - return tables, nil + err = wg.Wait() + return tables, err } func (m mySqlQuerySet) GetTableColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) { diff --git a/go.mod b/go.mod index 685bebb3..d0a810a8 100644 --- a/go.mod +++ b/go.mod @@ -18,5 +18,6 @@ require ( github.com/shopspring/decimal v1.3.1 github.com/stretchr/testify v1.8.2 github.com/volatiletech/null/v8 v8.1.2 + golang.org/x/sync v0.3.0 gopkg.in/guregu/null.v4 v4.0.0 ) diff --git a/go.sum b/go.sum index 2c11a02b..4b06a0a9 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From dd8d043cb88440207670da9604777e994a78e830 Mon Sep 17 00:00:00 2001 From: Yosyp Buchma Date: Wed, 20 Sep 2023 13:13:50 +0300 Subject: [PATCH 5/5] moved mysql connection pool config to openConection func --- generator/mysql/mysql_generator.go | 5 +++++ generator/mysql/query_set.go | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/generator/mysql/mysql_generator.go b/generator/mysql/mysql_generator.go index 635bcbc2..7495bec4 100644 --- a/generator/mysql/mysql_generator.go +++ b/generator/mysql/mysql_generator.go @@ -12,6 +12,8 @@ import ( mysqldr "github.com/go-sql-driver/mysql" ) +const mysqlMaxConns = 10 + // DBConnection contains MySQL connection details type DBConnection struct { Host string @@ -83,6 +85,9 @@ func openConnection(connectionString string) (*sql.DB, error) { return nil, fmt.Errorf("failed to open mysql connection: %w", err) } + db.SetMaxOpenConns(mysqlMaxConns) + db.SetMaxIdleConns(mysqlMaxConns) + err = db.Ping() if err != nil { return nil, fmt.Errorf("failed to ping database: %w", err) diff --git a/generator/mysql/query_set.go b/generator/mysql/query_set.go index b58ff82c..80406bdb 100644 --- a/generator/mysql/query_set.go +++ b/generator/mysql/query_set.go @@ -28,10 +28,6 @@ ORDER BY table_name; return nil, fmt.Errorf("failed to query %s metadata result: %w", tableType, err) } - const maxConns = 32 - db.SetMaxOpenConns(maxConns) - db.SetMaxIdleConns(maxConns) - wg := errgroup.Group{} for i := 0; i < len(tables); i++ { i := i