Merge pull request #274 from josephbuchma/faster-mysql-db-info-querying

10x mysql db info querying speedup

generator/mysql/mysql_generator.go

@@ -12,6 +12,8 @@ import (
 	mysqldr "github.com/go-sql-driver/mysql"
 )
 
+const mysqlMaxConns = 10
+
 // DBConnection contains MySQL connection details
 type DBConnection struct {
 	Host     string
@@ -83,6 +85,9 @@ func openConnection(connectionString string) (*sql.DB, error) {
 		return nil, fmt.Errorf("failed to open mysql connection: %w", err)
 	}
 
+	db.SetMaxOpenConns(mysqlMaxConns)
+	db.SetMaxIdleConns(mysqlMaxConns)
+
 	err = db.Ping()
 	if err != nil {
 		return nil, fmt.Errorf("failed to ping database: %w", err)

generator/mysql/query_set.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-jet/jet/v2/generator/metadata"
 	"github.com/go-jet/jet/v2/qrm"
+	"golang.org/x/sync/errgroup"
 )
 
 // mySqlQuerySet is dialect query set for MySQL
@@ -27,39 +28,51 @@ ORDER BY table_name;
 		return nil, fmt.Errorf("failed to query %s metadata result: %w", tableType, err)
 	}
 
-	for i := range tables {
+	wg := errgroup.Group{}
-		tables[i].Columns, err = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name)
+	for i := 0; i < len(tables); i++ {
-		if err != nil {
+		i := i
-			return nil, fmt.Errorf("failed to get '%s' table columns metadata: %w", tables[i].Name, err)
+		wg.Go(func() (err1 error) {
-		}
+			tables[i].Columns, err1 = m.GetTableColumnsMetaData(db, schemaName, tables[i].Name)
+			return err1
+		})
 	}
 
-	return tables, nil
+	err = wg.Wait()
+	return tables, err
 }
 
 func (m mySqlQuerySet) GetTableColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) {
 	query := `
-SELECT COLUMN_NAME AS "column.Name", 
+SELECT
-	IS_NULLABLE = "YES" AS "column.IsNullable",
+		col.COLUMN_NAME AS "column.Name",
-	columns.COLUMN_COMMENT as "column.Comment",
+		col.IS_NULLABLE = "YES" AS "column.IsNullable",
-	(EXISTS(
+		col.COLUMN_COMMENT AS "column.Comment",
-		SELECT 1
+		COALESCE(pk.IsPrimaryKey, 0) AS "column.IsPrimaryKey",
+		IF (col.COLUMN_TYPE = 'tinyint(1)',
+				'boolean',
+				IF (col.DATA_TYPE = 'enum',
+						CONCAT(col.TABLE_NAME, '_', col.COLUMN_NAME),
+						col.DATA_TYPE)
+		) AS "dataType.Name",
+		IF (col.DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind",
+		col.COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned"
+FROM
+		information_schema.columns AS col
+LEFT JOIN (
+		SELECT k.column_name, 1 AS IsPrimaryKey
 		FROM information_schema.table_constraints t
-			JOIN information_schema.key_column_usage k USING(constraint_name,table_schema,table_name)
+		JOIN information_schema.key_column_usage k USING(constraint_name, table_schema, table_name)
-		WHERE table_schema = ? AND table_name = ? AND t.constraint_type='PRIMARY KEY' AND k.column_name = columns.column_name
+		WHERE t.table_schema =  ?
-	)) AS "column.IsPrimaryKey",
+			AND t.table_name = ?
-	IF (COLUMN_TYPE = 'tinyint(1)', 
+			AND t.constraint_type = 'PRIMARY KEY'
-			'boolean', 
+) AS pk ON col.COLUMN_NAME = pk.column_name
-			IF (DATA_TYPE='enum', 
+WHERE
-					CONCAT(TABLE_NAME, '_', COLUMN_NAME), 
+		col.table_schema = ?
-					DATA_TYPE)
+		AND col.table_name = ?
-	) AS "dataType.Name", 
+ORDER BY
-	IF (DATA_TYPE = 'enum', 'enum', 'base') AS "dataType.Kind", 
+		col.ordinal_position;
-	COLUMN_TYPE LIKE '%unsigned%' AS "dataType.IsUnsigned"
+	`
-FROM information_schema.columns
+
-WHERE table_schema = ? AND table_name = ?
-ORDER BY ordinal_position;
-`
 	var columns []metadata.Column
 	_, err := qrm.Query(context.Background(), db, query, []interface{}{schemaName, tableName, schemaName, tableName}, &columns)
 	if err != nil {

go.mod

@@ -18,5 +18,6 @@ require (
 	github.com/shopspring/decimal v1.3.1
 	github.com/stretchr/testify v1.8.2
 	github.com/volatiletech/null/v8 v8.1.2
+	golang.org/x/sync v0.3.0
 	gopkg.in/guregu/null.v4 v4.0.0
 )

go.sum

@@ -182,6 +182,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=