Skip to content

Commit

Permalink
Merge pull request #274 from josephbuchma/faster-mysql-db-info-querying
Browse files Browse the repository at this point in the history
10x mysql db info querying speedup
  • Loading branch information
go-jet authored Sep 20, 2023
2 parents e2fe799 + dd8d043 commit 6a13530
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 26 deletions.
5 changes: 5 additions & 0 deletions generator/mysql/mysql_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
mysqldr "github.com/go-sql-driver/mysql"
)

const mysqlMaxConns = 10

// DBConnection contains MySQL connection details
type DBConnection struct {
Host string
Expand Down Expand Up @@ -83,6 +85,9 @@ func openConnection(connectionString string) (*sql.DB, error) {
return nil, fmt.Errorf("failed to open mysql connection: %w", err)
}

db.SetMaxOpenConns(mysqlMaxConns)
db.SetMaxIdleConns(mysqlMaxConns)

err = db.Ping()
if err != nil {
return nil, fmt.Errorf("failed to ping database: %w", err)
Expand Down
65 changes: 39 additions & 26 deletions generator/mysql/query_set.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/go-jet/jet/v2/generator/metadata"
"github.com/go-jet/jet/v2/qrm"
"golang.org/x/sync/errgroup"
)

// mySqlQuerySet is dialect query set for MySQL
Expand All @@ -27,39 +28,51 @@ ORDER BY table_name;
return nil, fmt.Errorf("failed to query %s metadata result: %w", tableType, err)
}

for i := range tables {
tables[i].Columns, err = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name)
if err != nil {
return nil, fmt.Errorf("failed to get '%s' table columns metadata: %w", tables[i].Name, err)
}
wg := errgroup.Group{}
for i := 0; i < len(tables); i++ {
i := i
wg.Go(func() (err1 error) {
tables[i].Columns, err1 = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name)
return err1
})
}

return tables, nil
err = wg.Wait()
return tables, err
}

func (m mySqlQuerySet) GetTableColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) {
query := `
SELECT COLUMN_NAME AS "column.Name",
IS_NULLABLE = "YES" AS "column.IsNullable",
columns.COLUMN_COMMENT as "column.Comment",
(EXISTS(
SELECT 1
SELECT
col.COLUMN_NAME AS "column.Name",
col.IS_NULLABLE = "YES" AS "column.IsNullable",
col.COLUMN_COMMENT AS "column.Comment",
COALESCE(pk.IsPrimaryKey, 0) AS "column.IsPrimaryKey",
IF (col.COLUMN_TYPE = 'tinyint(1)',
'boolean',
IF (col.DATA_TYPE = 'enum',
CONCAT(col.TABLE_NAME, '_', col.COLUMN_NAME),
col.DATA_TYPE)
) AS "dataType.Name",
IF (col.DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind",
col.COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned"
FROM
information_schema.columns AS col
LEFT JOIN (
SELECT k.column_name, 1 AS IsPrimaryKey
FROM information_schema.table_constraints t
JOIN information_schema.key_column_usage k USING(constraint_name,table_schema,table_name)
WHERE table_schema = ? AND table_name = ? AND t.constraint_type='PRIMARY KEY' AND k.column_name = columns.column_name
)) AS "column.IsPrimaryKey",
IF (COLUMN_TYPE = 'tinyint(1)',
'boolean',
IF (DATA_TYPE='enum',
CONCAT(TABLE_NAME, '_', COLUMN_NAME),
DATA_TYPE)
) AS "dataType.Name",
IF (DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind",
COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned"
FROM information_schema.columns
WHERE table_schema = ? AND table_name = ?
ORDER BY ordinal_position;
`
JOIN information_schema.key_column_usage k USING(constraint_name, table_schema, table_name)
WHERE t.table_schema = ?
AND t.table_name = ?
AND t.constraint_type = 'PRIMARY KEY'
) AS pk ON col.COLUMN_NAME = pk.column_name
WHERE
col.table_schema = ?
AND col.table_name = ?
ORDER BY
col.ordinal_position;
`

var columns []metadata.Column
_, err := qrm.Query(context.Background(), db, query, []interface{}{schemaName, tableName, schemaName, tableName}, &columns)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@ require (
github.com/shopspring/decimal v1.3.1
github.com/stretchr/testify v1.8.2
github.com/volatiletech/null/v8 v8.1.2
golang.org/x/sync v0.3.0
gopkg.in/guregu/null.v4 v4.0.0
)
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down

0 comments on commit 6a13530

Please sign in to comment.