Gleipnir/jet
4
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge pull request #274 from josephbuchma/faster-mysql-db-info-querying

Browse source 
10x mysql db info querying speedup
This commit is contained in:
go-jet 2023-09-20 19:16:47 +02:00 • committed by GitHub
commit 6a13530ec1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 47 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

5
generator/mysql/mysql_generator.go
View file

@ -12,6 +12,8 @@ import (
mysqldr "github.com/go-sql-driver/mysql"
)
const mysqlMaxConns = 10
// DBConnection contains MySQL connection details
type DBConnection struct {
Host string
@ -83,6 +85,9 @@ func openConnection(connectionString string) (*sql.DB, error) {
return nil, fmt.Errorf("failed to open mysql connection: %w", err)
}
db.SetMaxOpenConns(mysqlMaxConns)
db.SetMaxIdleConns(mysqlMaxConns)
err = db.Ping()
if err != nil {
return nil, fmt.Errorf("failed to ping database: %w", err)

63
generator/mysql/query_set.go
View file

@ -8,6 +8,7 @@ import (
"github.com/go-jet/jet/v2/generator/metadata"
"github.com/go-jet/jet/v2/qrm"
"golang.org/x/sync/errgroup"
)
// mySqlQuerySet is dialect query set for MySQL
@ -27,39 +28,51 @@ ORDER BY table_name;
return nil, fmt.Errorf("failed to query %s metadata result: %w", tableType, err)
}
for i := range tables {
tables[i].Columns, err = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name)
if err != nil {
return nil, fmt.Errorf("failed to get '%s' table columns metadata: %w", tables[i].Name, err)
}
wg := errgroup.Group{}
for i := 0; i < len(tables); i++ {
i := i
wg.Go(func() (err1 error) {
tables[i].Columns, err1 = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name)
return err1
})
}
return tables, nil
err = wg.Wait()
return tables, err
}
func (m mySqlQuerySet) GetTableColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) {
query := `
SELECT COLUMN_NAME AS "column.Name",
IS_NULLABLE = "YES" AS "column.IsNullable",
columns.COLUMN_COMMENT as "column.Comment",
(EXISTS(
SELECT 1
FROM information_schema.table_constraints t
JOIN information_schema.key_column_usage k USING(constraint_name,table_schema,table_name)
WHERE table_schema = ? AND table_name = ? AND t.constraint_type='PRIMARY KEY' AND k.column_name = columns.column_name
)) AS "column.IsPrimaryKey",
IF (COLUMN_TYPE = 'tinyint(1)',
SELECT
col.COLUMN_NAME AS "column.Name",
col.IS_NULLABLE = "YES" AS "column.IsNullable",
col.COLUMN_COMMENT AS "column.Comment",
COALESCE(pk.IsPrimaryKey, 0) AS "column.IsPrimaryKey",
IF (col.COLUMN_TYPE = 'tinyint(1)',
'boolean',
IF (DATA_TYPE='enum',
CONCAT(TABLE_NAME, '_', COLUMN_NAME),
DATA_TYPE)
IF (col.DATA_TYPE = 'enum',
CONCAT(col.TABLE_NAME, '_', col.COLUMN_NAME),
col.DATA_TYPE)
) AS "dataType.Name",
IF (DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind",
COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned"
FROM information_schema.columns
WHERE table_schema = ? AND table_name = ?
ORDER BY ordinal_position;
`
IF (col.DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind",
col.COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned"
FROM
information_schema.columns AS col
LEFT JOIN (
SELECT k.column_name, 1 AS IsPrimaryKey
FROM information_schema.table_constraints t
JOIN information_schema.key_column_usage k USING(constraint_name, table_schema, table_name)
WHERE t.table_schema = ?
AND t.table_name = ?
AND t.constraint_type = 'PRIMARY KEY'
) AS pk ON col.COLUMN_NAME = pk.column_name
WHERE
col.table_schema = ?
AND col.table_name = ?
ORDER BY
col.ordinal_position;
`
var columns []metadata.Column
_, err := qrm.Query(context.Background(), db, query, []interface{}{schemaName, tableName, schemaName, tableName}, &columns)
if err != nil {

1
go.mod
View file

@ -18,5 +18,6 @@ require (
github.com/shopspring/decimal v1.3.1
github.com/stretchr/testify v1.8.2
github.com/volatiletech/null/v8 v8.1.2
golang.org/x/sync v0.3.0
gopkg.in/guregu/null.v4 v4.0.0
)

2
go.sum
View file

@ -182,6 +182,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=