Add support for running integration tests with dockerized test databases.

.gitignore

@@ -19,4 +19,5 @@
 gen
 .gentestdata
 .tests/testdata/
-.gen
+.gen
+.docker

tests/Makefile

@@ -0,0 +1,50 @@
+
+
+setup: checkout-testdata docker-compose-up
+
+# checkout-testdata will checkout testdata from separate repository into git submodule.
+checkout-testdata:
+	git submodule init
+	git submodule update
+	cd ./testdata && git fetch && git checkout master && git pull
+
+# docker-compose-up will download docker image for each of the databases listed in docker-compose.yaml file, and then it will initialize
+# database with testdata retrieved in previous step.
+# On the first run this action might take couple of minutes. Docker temp data are stored in .docker directory.
+docker-compose-up:
+	docker-compose up
+
+
+# jet-gen will call generator on each of the test databases to generate sql builder and model files need to run the tests.
+jet-gen: install-jet-gen jet-gen-pg jet-gen-mysql jet-gen-mariadb jet-gen-sqlite
+
+install-jet-gen:
+	go build -o ${GOPATH}/bin/jet ../cmd/jet/
+
+jet-gen-pg:
+	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=dvds  -path=./.gentestdata/
+	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook  -path=./.gentestdata/
+	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook2  -path=./.gentestdata/
+	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=test_sample  -path=./.gentestdata/
+
+jet-gen-mysql:
+	jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/dvds" -path=./.gentestdata/mysql
+	jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/dvds2" -path=./.gentestdata/mysql
+	jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/test_sample" -path=./.gentestdata/mysql
+
+jet-gen-mariadb:
+	jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/dvds" -path=./.gentestdata/mysql
+	jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/dvds2" -path=./.gentestdata/mysql
+	jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/test_sample" -path=./.gentestdata/mysql
+
+jet-gen-sqlite:
+	jet -source=sqlite -dsn="./testdata/init/sqlite/chinook.db" -schema=dvds -path=./.gentestdata/sqlite/chinook
+	jet -source=sqlite -dsn="./testdata/init/sqlite/sakila.db" -schema=dvds -path=./.gentestdata/sqlite/sakila
+	jet -source=sqlite -dsn="./testdata/init/sqlite/test_sample.db" -schema=dvds -path=./.gentestdata/sqlite/test_sample
+
+
+# docker-compose-cleanup will stop and remove test containers, volumes, and images.
+# If temp .docker/ folder is still not empty, use 'sudo rm -rf .docker' to remove it.
+cleanup:
+	docker-compose down --volumes
+	$(info INFO: If the ./.docker folder is still not empty, use 'sudo rm -rf .docker/' to remove it.)

tests/Readme.md

@@ -0,0 +1,30 @@
+
+# Integration tests
+
+This folder contains integration tests intended to test jet generator, statements and query result mapping with a running database.
+
+## How to run tests?
+
+Before we can run tests, we need to set up and initialize test databases.
+To simplify the process there is a Makefile with a list of helper commands.
+```shell
+# We first need to checkout testdata from separate repository into git submodule,
+# then download docker image for each of the databases listed in docker-compose.yaml file, and 
+# finally run and initialize databases with downloaded test data.
+# Note that on the first run this command might take a couple of minutes.
+make setup
+
+# When databases are ready, we can generate sql builder and model types needed to compile the tests.
+make jet-gen
+```
+
+Then we can run the tests the usual way:
+```shell
+go test -v ./...
+```
+
+To removes test containers, volumes, and images:
+```shell
+make cleanup
+# If temp ./.docker folder is still not empty, use 'sudo rm -rf .docker' to remove it.
+```

tests/dbconfig/dbconfig.go

@@ -8,7 +8,7 @@ import (
 // Postgres test database connection parameters
 const (
 	PgHost     = "localhost"
-	PgPort     = 5432
+	PgPort     = 50901
 	PgUser     = "jet"
 	PgPassword = "jet"
 	PgDBName   = "jetdb"
@@ -20,13 +20,24 @@ var PostgresConnectString = fmt.Sprintf("host=%s port=%d user=%s password=%s dbn
 // MySQL test database connection parameters
 const (
 	MySqLHost     = "localhost"
-	MySQLPort     = 3306
+	MySQLPort     = 50902
 	MySQLUser     = "jet"
 	MySQLPassword = "jet"
+
+	MariaDBHost     = "localhost"
+	MariaDBPort     = 50903
+	MariaDBUser     = "jet"
+	MariaDBPassword = "jet"
 )
 
-// MySQLConnectionString is MySQL driver connection string to test database
+// MySQLConnectionString is MySQL connection string for test database
-var MySQLConnectionString = fmt.Sprintf("%s:%s@tcp(%s:%d)/", MySQLUser, MySQLPassword, MySqLHost, MySQLPort)
+func MySQLConnectionString(isMariaDB bool, dbName string) string {
+	if isMariaDB {
+		return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", MariaDBUser, MariaDBPassword, MariaDBHost, MariaDBPort, dbName)
+	}
+
+	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", MySQLUser, MySQLPassword, MySqLHost, MySQLPort, dbName)
+}
 
 // sqllite
 var (

tests/docker-compose.yaml

@@ -0,0 +1,42 @@
+version: '3'
+services:
+  postgres:
+    image: postgres:14.1
+    restart: always
+    environment:
+      - POSTGRES_USER=jet
+      - POSTGRES_PASSWORD=jet
+      - POSTGRES_DB=jetdb
+    ports:
+      - '50901:5432'
+    volumes:
+      - ./testdata/init/postgres:/docker-entrypoint-initdb.d
+      - ./.docker/postgres-data:/var/lib/postgresql/data
+
+  mysql:
+    image: mysql:8.0.27
+    command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1']
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: jet
+      MYSQL_USER: jet
+      MYSQL_PASSWORD: jet
+    ports:
+      - '50902:3306'
+    volumes:
+      - ./testdata/init/mysql:/docker-entrypoint-initdb.d
+      - ./.docker/mysql-data:/var/lib/mysql
+
+  mariadb:
+    image: mariadb:10.3.32
+    command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1']
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: jet
+      MYSQL_USER: jet
+      MYSQL_PASSWORD: jet
+    ports:
+      - '50903:3306'
+    volumes:
+      - ./testdata/init/mysql:/docker-entrypoint-initdb.d
+      - ./.docker/mariadb-data:/var/lib/mysql

tests/init/Readme.md

@@ -0,0 +1,6 @@
+
+The `init` command can be used to initialize test databases on the local host machine, if needed.
+Update [dbconfig](../dbconfig/dbconfig.go) with your local database parameters.
+
+The recommended way to initialize test databases is by a docker container. 
+See tests [Readme.md](../Readme.md).

tests/init/init.go

@@ -71,8 +71,13 @@ func initMySQLDB() {
 	}
 
 	for _, dbName := range mySQLDBs {
-		cmdLine := fmt.Sprintf("mysql -h 127.0.0.1 -u %s -p%s %s < %s",
+		cmdLine := fmt.Sprintf("mysql -h %s -P%d -u %s -p%s %s < %s",
-			dbconfig.MySQLUser, dbconfig.MySQLPassword, dbName, "./testdata/init/mysql/"+dbName+".sql")
+			dbconfig.MySqLHost,
+			dbconfig.MySQLPort,
+			dbconfig.MySQLUser,
+			dbconfig.MySQLPassword,
+			dbName,
+			"./testdata/init/mysql/"+dbName+".sql")
 
 		fmt.Println(cmdLine)
 

tests/mysql/alltypes_test.go

@@ -31,10 +31,6 @@ func TestAllTypes(t *testing.T) {
 
 	require.Equal(t, len(dest), 2)
 
-	if sourceIsMariaDB() { // MariaDB saves current timestamp in a case of NULL value insert
-		return
-	}
-
 	//testutils.PrintJson(dest)
 	testutils.AssertJSON(t, dest, allTypesJson)
 }
@@ -49,10 +45,6 @@ func TestAllTypesViewSelect(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, len(dest), 2)
 
-	if sourceIsMariaDB() { // MariaDB saves current timestamp in a case of NULL value insert
-		return
-	}
-
 	testutils.AssertJSON(t, dest, allTypesJson)
 }
 

tests/mysql/generator_template_test.go

@@ -25,18 +25,30 @@ var defaultViewSQLBuilderFilePath = path.Join(tempTestDir, "dvds/view")
 var defaultEnumSQLBuilderFilePath = path.Join(tempTestDir, "dvds/enum")
 var defaultActorSQLBuilderFilePath = path.Join(tempTestDir, "dvds/table", "actor.go")
 
-var dbConnection = mysql2.DBConnection{
+func dbConnection(dbName string) mysql2.DBConnection {
-	Host:     dbconfig.MySqLHost,
+	if sourceIsMariaDB() {
-	Port:     dbconfig.MySQLPort,
+		return mysql2.DBConnection{
-	User:     dbconfig.MySQLUser,
+			Host:     dbconfig.MariaDBHost,
-	Password: dbconfig.MySQLPassword,
+			Port:     dbconfig.MariaDBPort,
-	DBName:   "dvds",
+			User:     dbconfig.MariaDBUser,
+			Password: dbconfig.MariaDBPassword,
+			DBName:   dbName,
+		}
+	}
+
+	return mysql2.DBConnection{
+		Host:     dbconfig.MySqLHost,
+		Port:     dbconfig.MySQLPort,
+		User:     dbconfig.MySQLUser,
+		Password: dbconfig.MySQLPassword,
+		DBName:   dbName,
+	}
 }
 
 func TestGeneratorTemplate_Schema_ChangePath(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).UsePath("new/schema/path")
@@ -54,7 +66,7 @@ func TestGeneratorTemplate_Schema_ChangePath(t *testing.T) {
 func TestGeneratorTemplate_Model_SkipGeneration(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -75,7 +87,7 @@ func TestGeneratorTemplate_Model_SkipGeneration(t *testing.T) {
 func TestGeneratorTemplate_SQLBuilder_SkipGeneration(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -98,7 +110,7 @@ func TestGeneratorTemplate_Model_ChangePath(t *testing.T) {
 
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -116,7 +128,7 @@ func TestGeneratorTemplate_SQLBuilder_ChangePath(t *testing.T) {
 
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -137,7 +149,7 @@ func TestGeneratorTemplate_SQLBuilder_ChangePath(t *testing.T) {
 func TestGeneratorTemplate_Model_RenameFilesAndTypes(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -175,7 +187,7 @@ func TestGeneratorTemplate_Model_RenameFilesAndTypes(t *testing.T) {
 func TestGeneratorTemplate_Model_SkipTableAndEnum(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -203,7 +215,7 @@ func TestGeneratorTemplate_Model_SkipTableAndEnum(t *testing.T) {
 func TestGeneratorTemplate_SQLBuilder_SkipTableAndEnum(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -236,7 +248,7 @@ func TestGeneratorTemplate_SQLBuilder_SkipTableAndEnum(t *testing.T) {
 func TestGeneratorTemplate_SQLBuilder_ChangeTypeAndFileName(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -277,7 +289,7 @@ func TestGeneratorTemplate_Model_AddTags(t *testing.T) {
 
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -318,7 +330,7 @@ func TestGeneratorTemplate_Model_AddTags(t *testing.T) {
 func TestGeneratorTemplate_Model_ChangeFieldTypes(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).
@@ -361,7 +373,7 @@ func TestGeneratorTemplate_Model_ChangeFieldTypes(t *testing.T) {
 func TestGeneratorTemplate_SQLBuilder_ChangeColumnTypes(t *testing.T) {
 	err := mysql2.Generate(
 		tempTestDir,
-		dbConnection,
+		dbConnection("dvds"),
 		template.Default(postgres2.Dialect).
 			UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
 				return template.DefaultSchema(schemaMetaData).

tests/mysql/generator_test.go

@@ -1,10 +1,10 @@
 package mysql
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"strconv"
 	"testing"
 
 	"github.com/go-jet/jet/v2/generator/mysql"
@@ -19,13 +19,7 @@ const genTestDir3 = "./.gentestdata3/mysql"
 func TestGenerator(t *testing.T) {
 
 	for i := 0; i < 3; i++ {
-		err := mysql.Generate(genTestDir3, mysql.DBConnection{
+		err := mysql.Generate(genTestDir3, dbConnection("dvds"))
-			Host:     dbconfig.MySqLHost,
-			Port:     dbconfig.MySQLPort,
-			User:     dbconfig.MySQLUser,
-			Password: dbconfig.MySQLPassword,
-			DBName:   "dvds",
-		})
 
 		require.NoError(t, err)
 
@@ -33,17 +27,11 @@ func TestGenerator(t *testing.T) {
 	}
 
 	for i := 0; i < 3; i++ {
-		dsn := fmt.Sprintf("%[1]s:%[2]s@tcp(%[3]s:%[4]d)/%[5]s",
+		dsn := dbconfig.MySQLConnectionString(sourceIsMariaDB(), "dvds")
-			dbconfig.MySQLUser,
+
-			dbconfig.MySQLPassword,
-			dbconfig.MySqLHost,
-			dbconfig.MySQLPort,
-			"dvds",
-		)
 		err := mysql.GenerateDSN(dsn, genTestDir3)
 
 		require.NoError(t, err)
-
 		assertGeneratedFiles(t)
 	}
 
@@ -55,8 +43,27 @@ func TestCmdGenerator(t *testing.T) {
 	err := os.RemoveAll(genTestDir3)
 	require.NoError(t, err)
 
-	cmd := exec.Command("jet", "-source=MySQL", "-dbname=dvds", "-host=localhost", "-port=3306",
+	var cmd *exec.Cmd
-		"-user=jet", "-password=jet", "-path="+genTestDir3)
+
+	if sourceIsMariaDB() {
+		cmd = exec.Command("jet",
+			"-source=MariaDB",
+			"-dbname=dvds",
+			"-host="+dbconfig.MariaDBHost,
+			"-port="+strconv.Itoa(dbconfig.MariaDBPort),
+			"-user="+dbconfig.MariaDBUser,
+			"-password="+dbconfig.MariaDBPassword,
+			"-path="+genTestDir3)
+	} else {
+		cmd = exec.Command("jet",
+			"-source=MySQL",
+			"-dbname=dvds",
+			"-host="+dbconfig.MySqLHost,
+			"-port="+strconv.Itoa(dbconfig.MySQLPort),
+			"-user="+dbconfig.MySQLUser,
+			"-password="+dbconfig.MySQLPassword,
+			"-path="+genTestDir3)
+	}
 
 	cmd.Stderr = os.Stderr
 	cmd.Stdout = os.Stdout
@@ -70,13 +77,7 @@ func TestCmdGenerator(t *testing.T) {
 	require.NoError(t, err)
 
 	// check that generation via DSN works
-	dsn := fmt.Sprintf("mysql://%[1]s:%[2]s@tcp(%[3]s:%[4]d)/%[5]s",
+	dsn := "mysql://" + dbconfig.MySQLConnectionString(sourceIsMariaDB(), "dvds")
-		dbconfig.MySQLUser,
-		dbconfig.MySQLPassword,
-		dbconfig.MySqLHost,
-		dbconfig.MySQLPort,
-		"dvds",
-	)
 	cmd = exec.Command("jet", "-dsn="+dsn, "-path="+genTestDir3)
 
 	cmd.Stderr = os.Stderr

tests/mysql/main_test.go

@@ -36,7 +36,7 @@ func TestMain(m *testing.M) {
 	defer profile.Start().Stop()
 
 	var err error
-	db, err = sql.Open("mysql", dbconfig.MySQLConnectionString)
+	db, err = sql.Open("mysql", dbconfig.MySQLConnectionString(sourceIsMariaDB(), ""))
 	if err != nil {
 		panic("Failed to connect to test db" + err.Error())
 	}

tests/mysql/select_test.go

@@ -389,8 +389,6 @@ LIMIT ?;
 			).
 			LIMIT(1000)
 
-		//fmt.Println(query.Sql())
-
 		testutils.AssertStatementSql(t, query, expectedSQL, int64(1000))
 
 		var dest []struct {
@@ -414,12 +412,7 @@ LIMIT ?;
 		err := query.Query(db, &dest)
 
 		require.NoError(t, err)
-		//require.Equal(t, len(dest), 1)
-		//require.Equal(t, len(dest[0].Films), 10)
-		//require.Equal(t, len(dest[0].Films[0].Actors), 10)
-
 		//testutils.SaveJsonFile(dest, "./mysql/testdata/lang_film_actor_inventory_rental.json")
-
 		testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/lang_film_actor_inventory_rental.json")
 	}
 }

tests/testdata

@@ -1 +1 @@
-Subproject commit 946bc1e5d3e162154eade8b79ff915e4c4986efd
+Subproject commit f8329498fd778157a1aac8d471d09ff39cdd5de1