From 972fc1d9bf28ef0a91aa47454073ae141492e875 Mon Sep 17 00:00:00 2001 From: go-jet Date: Fri, 17 Dec 2021 16:59:43 +0100 Subject: [PATCH] Add support for running integration tests with dockerized test databases. --- .gitignore | 3 +- tests/Makefile | 50 +++++++++++++++++++++++++ tests/Readme.md | 30 +++++++++++++++ tests/dbconfig/dbconfig.go | 19 ++++++++-- tests/docker-compose.yaml | 42 +++++++++++++++++++++ tests/init/Readme.md | 6 +++ tests/init/init.go | 9 ++++- tests/mysql/alltypes_test.go | 8 ---- tests/mysql/generator_template_test.go | 48 +++++++++++++++--------- tests/mysql/generator_test.go | 51 +++++++++++++------------- tests/mysql/main_test.go | 2 +- tests/mysql/select_test.go | 7 ---- tests/testdata | 2 +- 13 files changed, 210 insertions(+), 67 deletions(-) create mode 100644 tests/Makefile create mode 100644 tests/Readme.md create mode 100644 tests/docker-compose.yaml create mode 100644 tests/init/Readme.md diff --git a/.gitignore b/.gitignore index 153be12..4909287 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ gen .gentestdata .tests/testdata/ -.gen \ No newline at end of file +.gen +.docker \ No newline at end of file diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..c0d31bd --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,50 @@ + + +setup: checkout-testdata docker-compose-up + +# checkout-testdata will checkout testdata from separate repository into git submodule. +checkout-testdata: + git submodule init + git submodule update + cd ./testdata && git fetch && git checkout master && git pull + +# docker-compose-up will download docker image for each of the databases listed in docker-compose.yaml file, and then it will initialize +# database with testdata retrieved in previous step. +# On the first run this action might take couple of minutes. Docker temp data are stored in .docker directory. +docker-compose-up: + docker-compose up + + +# jet-gen will call generator on each of the test databases to generate sql builder and model files need to run the tests. +jet-gen: install-jet-gen jet-gen-pg jet-gen-mysql jet-gen-mariadb jet-gen-sqlite + +install-jet-gen: + go build -o ${GOPATH}/bin/jet ../cmd/jet/ + +jet-gen-pg: + jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=dvds -path=./.gentestdata/ + jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook -path=./.gentestdata/ + jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook2 -path=./.gentestdata/ + jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=test_sample -path=./.gentestdata/ + +jet-gen-mysql: + jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/dvds" -path=./.gentestdata/mysql + jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/dvds2" -path=./.gentestdata/mysql + jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/test_sample" -path=./.gentestdata/mysql + +jet-gen-mariadb: + jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/dvds" -path=./.gentestdata/mysql + jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/dvds2" -path=./.gentestdata/mysql + jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/test_sample" -path=./.gentestdata/mysql + +jet-gen-sqlite: + jet -source=sqlite -dsn="./testdata/init/sqlite/chinook.db" -schema=dvds -path=./.gentestdata/sqlite/chinook + jet -source=sqlite -dsn="./testdata/init/sqlite/sakila.db" -schema=dvds -path=./.gentestdata/sqlite/sakila + jet -source=sqlite -dsn="./testdata/init/sqlite/test_sample.db" -schema=dvds -path=./.gentestdata/sqlite/test_sample + + +# docker-compose-cleanup will stop and remove test containers, volumes, and images. +# If temp .docker/ folder is still not empty, use 'sudo rm -rf .docker' to remove it. +cleanup: + docker-compose down --volumes + $(info INFO: If the ./.docker folder is still not empty, use 'sudo rm -rf .docker/' to remove it.) diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 0000000..40bc456 --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,30 @@ + +# Integration tests + +This folder contains integration tests intended to test jet generator, statements and query result mapping with a running database. + +## How to run tests? + +Before we can run tests, we need to set up and initialize test databases. +To simplify the process there is a Makefile with a list of helper commands. +```shell +# We first need to checkout testdata from separate repository into git submodule, +# then download docker image for each of the databases listed in docker-compose.yaml file, and +# finally run and initialize databases with downloaded test data. +# Note that on the first run this command might take a couple of minutes. +make setup + +# When databases are ready, we can generate sql builder and model types needed to compile the tests. +make jet-gen +``` + +Then we can run the tests the usual way: +```shell +go test -v ./... +``` + +To removes test containers, volumes, and images: +```shell +make cleanup +# If temp ./.docker folder is still not empty, use 'sudo rm -rf .docker' to remove it. +``` \ No newline at end of file diff --git a/tests/dbconfig/dbconfig.go b/tests/dbconfig/dbconfig.go index ef89c1b..b09ab80 100644 --- a/tests/dbconfig/dbconfig.go +++ b/tests/dbconfig/dbconfig.go @@ -8,7 +8,7 @@ import ( // Postgres test database connection parameters const ( PgHost = "localhost" - PgPort = 5432 + PgPort = 50901 PgUser = "jet" PgPassword = "jet" PgDBName = "jetdb" @@ -20,13 +20,24 @@ var PostgresConnectString = fmt.Sprintf("host=%s port=%d user=%s password=%s dbn // MySQL test database connection parameters const ( MySqLHost = "localhost" - MySQLPort = 3306 + MySQLPort = 50902 MySQLUser = "jet" MySQLPassword = "jet" + + MariaDBHost = "localhost" + MariaDBPort = 50903 + MariaDBUser = "jet" + MariaDBPassword = "jet" ) -// MySQLConnectionString is MySQL driver connection string to test database -var MySQLConnectionString = fmt.Sprintf("%s:%s@tcp(%s:%d)/", MySQLUser, MySQLPassword, MySqLHost, MySQLPort) +// MySQLConnectionString is MySQL connection string for test database +func MySQLConnectionString(isMariaDB bool, dbName string) string { + if isMariaDB { + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", MariaDBUser, MariaDBPassword, MariaDBHost, MariaDBPort, dbName) + } + + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", MySQLUser, MySQLPassword, MySqLHost, MySQLPort, dbName) +} // sqllite var ( diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml new file mode 100644 index 0000000..fc24385 --- /dev/null +++ b/tests/docker-compose.yaml @@ -0,0 +1,42 @@ +version: '3' +services: + postgres: + image: postgres:14.1 + restart: always + environment: + - POSTGRES_USER=jet + - POSTGRES_PASSWORD=jet + - POSTGRES_DB=jetdb + ports: + - '50901:5432' + volumes: + - ./testdata/init/postgres:/docker-entrypoint-initdb.d + - ./.docker/postgres-data:/var/lib/postgresql/data + + mysql: + image: mysql:8.0.27 + command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1'] + restart: always + environment: + MYSQL_ROOT_PASSWORD: jet + MYSQL_USER: jet + MYSQL_PASSWORD: jet + ports: + - '50902:3306' + volumes: + - ./testdata/init/mysql:/docker-entrypoint-initdb.d + - ./.docker/mysql-data:/var/lib/mysql + + mariadb: + image: mariadb:10.3.32 + command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1'] + restart: always + environment: + MYSQL_ROOT_PASSWORD: jet + MYSQL_USER: jet + MYSQL_PASSWORD: jet + ports: + - '50903:3306' + volumes: + - ./testdata/init/mysql:/docker-entrypoint-initdb.d + - ./.docker/mariadb-data:/var/lib/mysql diff --git a/tests/init/Readme.md b/tests/init/Readme.md new file mode 100644 index 0000000..65e61d9 --- /dev/null +++ b/tests/init/Readme.md @@ -0,0 +1,6 @@ + +The `init` command can be used to initialize test databases on the local host machine, if needed. +Update [dbconfig](../dbconfig/dbconfig.go) with your local database parameters. + +The recommended way to initialize test databases is by a docker container. +See tests [Readme.md](../Readme.md). \ No newline at end of file diff --git a/tests/init/init.go b/tests/init/init.go index aa04fb5..eb49fb7 100644 --- a/tests/init/init.go +++ b/tests/init/init.go @@ -71,8 +71,13 @@ func initMySQLDB() { } for _, dbName := range mySQLDBs { - cmdLine := fmt.Sprintf("mysql -h 127.0.0.1 -u %s -p%s %s < %s", - dbconfig.MySQLUser, dbconfig.MySQLPassword, dbName, "./testdata/init/mysql/"+dbName+".sql") + cmdLine := fmt.Sprintf("mysql -h %s -P%d -u %s -p%s %s < %s", + dbconfig.MySqLHost, + dbconfig.MySQLPort, + dbconfig.MySQLUser, + dbconfig.MySQLPassword, + dbName, + "./testdata/init/mysql/"+dbName+".sql") fmt.Println(cmdLine) diff --git a/tests/mysql/alltypes_test.go b/tests/mysql/alltypes_test.go index 2132d7a..4ae7f5b 100644 --- a/tests/mysql/alltypes_test.go +++ b/tests/mysql/alltypes_test.go @@ -31,10 +31,6 @@ func TestAllTypes(t *testing.T) { require.Equal(t, len(dest), 2) - if sourceIsMariaDB() { // MariaDB saves current timestamp in a case of NULL value insert - return - } - //testutils.PrintJson(dest) testutils.AssertJSON(t, dest, allTypesJson) } @@ -49,10 +45,6 @@ func TestAllTypesViewSelect(t *testing.T) { require.NoError(t, err) require.Equal(t, len(dest), 2) - if sourceIsMariaDB() { // MariaDB saves current timestamp in a case of NULL value insert - return - } - testutils.AssertJSON(t, dest, allTypesJson) } diff --git a/tests/mysql/generator_template_test.go b/tests/mysql/generator_template_test.go index e915e0f..6af636b 100644 --- a/tests/mysql/generator_template_test.go +++ b/tests/mysql/generator_template_test.go @@ -25,18 +25,30 @@ var defaultViewSQLBuilderFilePath = path.Join(tempTestDir, "dvds/view") var defaultEnumSQLBuilderFilePath = path.Join(tempTestDir, "dvds/enum") var defaultActorSQLBuilderFilePath = path.Join(tempTestDir, "dvds/table", "actor.go") -var dbConnection = mysql2.DBConnection{ - Host: dbconfig.MySqLHost, - Port: dbconfig.MySQLPort, - User: dbconfig.MySQLUser, - Password: dbconfig.MySQLPassword, - DBName: "dvds", +func dbConnection(dbName string) mysql2.DBConnection { + if sourceIsMariaDB() { + return mysql2.DBConnection{ + Host: dbconfig.MariaDBHost, + Port: dbconfig.MariaDBPort, + User: dbconfig.MariaDBUser, + Password: dbconfig.MariaDBPassword, + DBName: dbName, + } + } + + return mysql2.DBConnection{ + Host: dbconfig.MySqLHost, + Port: dbconfig.MySQLPort, + User: dbconfig.MySQLUser, + Password: dbconfig.MySQLPassword, + DBName: dbName, + } } func TestGeneratorTemplate_Schema_ChangePath(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData).UsePath("new/schema/path") @@ -54,7 +66,7 @@ func TestGeneratorTemplate_Schema_ChangePath(t *testing.T) { func TestGeneratorTemplate_Model_SkipGeneration(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -75,7 +87,7 @@ func TestGeneratorTemplate_Model_SkipGeneration(t *testing.T) { func TestGeneratorTemplate_SQLBuilder_SkipGeneration(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -98,7 +110,7 @@ func TestGeneratorTemplate_Model_ChangePath(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -116,7 +128,7 @@ func TestGeneratorTemplate_SQLBuilder_ChangePath(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -137,7 +149,7 @@ func TestGeneratorTemplate_SQLBuilder_ChangePath(t *testing.T) { func TestGeneratorTemplate_Model_RenameFilesAndTypes(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -175,7 +187,7 @@ func TestGeneratorTemplate_Model_RenameFilesAndTypes(t *testing.T) { func TestGeneratorTemplate_Model_SkipTableAndEnum(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -203,7 +215,7 @@ func TestGeneratorTemplate_Model_SkipTableAndEnum(t *testing.T) { func TestGeneratorTemplate_SQLBuilder_SkipTableAndEnum(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -236,7 +248,7 @@ func TestGeneratorTemplate_SQLBuilder_SkipTableAndEnum(t *testing.T) { func TestGeneratorTemplate_SQLBuilder_ChangeTypeAndFileName(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -277,7 +289,7 @@ func TestGeneratorTemplate_Model_AddTags(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -318,7 +330,7 @@ func TestGeneratorTemplate_Model_AddTags(t *testing.T) { func TestGeneratorTemplate_Model_ChangeFieldTypes(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). @@ -361,7 +373,7 @@ func TestGeneratorTemplate_Model_ChangeFieldTypes(t *testing.T) { func TestGeneratorTemplate_SQLBuilder_ChangeColumnTypes(t *testing.T) { err := mysql2.Generate( tempTestDir, - dbConnection, + dbConnection("dvds"), template.Default(postgres2.Dialect). UseSchema(func(schemaMetaData metadata.Schema) template.Schema { return template.DefaultSchema(schemaMetaData). diff --git a/tests/mysql/generator_test.go b/tests/mysql/generator_test.go index 033f699..61f364b 100644 --- a/tests/mysql/generator_test.go +++ b/tests/mysql/generator_test.go @@ -1,10 +1,10 @@ package mysql import ( - "fmt" "io/ioutil" "os" "os/exec" + "strconv" "testing" "github.com/go-jet/jet/v2/generator/mysql" @@ -19,13 +19,7 @@ const genTestDir3 = "./.gentestdata3/mysql" func TestGenerator(t *testing.T) { for i := 0; i < 3; i++ { - err := mysql.Generate(genTestDir3, mysql.DBConnection{ - Host: dbconfig.MySqLHost, - Port: dbconfig.MySQLPort, - User: dbconfig.MySQLUser, - Password: dbconfig.MySQLPassword, - DBName: "dvds", - }) + err := mysql.Generate(genTestDir3, dbConnection("dvds")) require.NoError(t, err) @@ -33,17 +27,11 @@ func TestGenerator(t *testing.T) { } for i := 0; i < 3; i++ { - dsn := fmt.Sprintf("%[1]s:%[2]s@tcp(%[3]s:%[4]d)/%[5]s", - dbconfig.MySQLUser, - dbconfig.MySQLPassword, - dbconfig.MySqLHost, - dbconfig.MySQLPort, - "dvds", - ) + dsn := dbconfig.MySQLConnectionString(sourceIsMariaDB(), "dvds") + err := mysql.GenerateDSN(dsn, genTestDir3) require.NoError(t, err) - assertGeneratedFiles(t) } @@ -55,8 +43,27 @@ func TestCmdGenerator(t *testing.T) { err := os.RemoveAll(genTestDir3) require.NoError(t, err) - cmd := exec.Command("jet", "-source=MySQL", "-dbname=dvds", "-host=localhost", "-port=3306", - "-user=jet", "-password=jet", "-path="+genTestDir3) + var cmd *exec.Cmd + + if sourceIsMariaDB() { + cmd = exec.Command("jet", + "-source=MariaDB", + "-dbname=dvds", + "-host="+dbconfig.MariaDBHost, + "-port="+strconv.Itoa(dbconfig.MariaDBPort), + "-user="+dbconfig.MariaDBUser, + "-password="+dbconfig.MariaDBPassword, + "-path="+genTestDir3) + } else { + cmd = exec.Command("jet", + "-source=MySQL", + "-dbname=dvds", + "-host="+dbconfig.MySqLHost, + "-port="+strconv.Itoa(dbconfig.MySQLPort), + "-user="+dbconfig.MySQLUser, + "-password="+dbconfig.MySQLPassword, + "-path="+genTestDir3) + } cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout @@ -70,13 +77,7 @@ func TestCmdGenerator(t *testing.T) { require.NoError(t, err) // check that generation via DSN works - dsn := fmt.Sprintf("mysql://%[1]s:%[2]s@tcp(%[3]s:%[4]d)/%[5]s", - dbconfig.MySQLUser, - dbconfig.MySQLPassword, - dbconfig.MySqLHost, - dbconfig.MySQLPort, - "dvds", - ) + dsn := "mysql://" + dbconfig.MySQLConnectionString(sourceIsMariaDB(), "dvds") cmd = exec.Command("jet", "-dsn="+dsn, "-path="+genTestDir3) cmd.Stderr = os.Stderr diff --git a/tests/mysql/main_test.go b/tests/mysql/main_test.go index 7f2e801..6a35cb8 100644 --- a/tests/mysql/main_test.go +++ b/tests/mysql/main_test.go @@ -36,7 +36,7 @@ func TestMain(m *testing.M) { defer profile.Start().Stop() var err error - db, err = sql.Open("mysql", dbconfig.MySQLConnectionString) + db, err = sql.Open("mysql", dbconfig.MySQLConnectionString(sourceIsMariaDB(), "")) if err != nil { panic("Failed to connect to test db" + err.Error()) } diff --git a/tests/mysql/select_test.go b/tests/mysql/select_test.go index 5a88acf..a1dc3fc 100644 --- a/tests/mysql/select_test.go +++ b/tests/mysql/select_test.go @@ -389,8 +389,6 @@ LIMIT ?; ). LIMIT(1000) - //fmt.Println(query.Sql()) - testutils.AssertStatementSql(t, query, expectedSQL, int64(1000)) var dest []struct { @@ -414,12 +412,7 @@ LIMIT ?; err := query.Query(db, &dest) require.NoError(t, err) - //require.Equal(t, len(dest), 1) - //require.Equal(t, len(dest[0].Films), 10) - //require.Equal(t, len(dest[0].Films[0].Actors), 10) - //testutils.SaveJsonFile(dest, "./mysql/testdata/lang_film_actor_inventory_rental.json") - testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/lang_film_actor_inventory_rental.json") } } diff --git a/tests/testdata b/tests/testdata index 946bc1e..f832949 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 946bc1e5d3e162154eade8b79ff915e4c4986efd +Subproject commit f8329498fd778157a1aac8d471d09ff39cdd5de1