Gleipnir/jet
4
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Add support for running integration tests with dockerized test databases.

Browse source
This commit is contained in:
go-jet 2021-12-17 16:59:43 +01:00
parent 4d5abc85c6
commit 972fc1d9bf
13 changed files with 210 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -20,3 +20,4 @@ gen
.gentestdata
.tests/testdata/
.gen
.docker

50
tests/Makefile Normal file
View file

@ -0,0 +1,50 @@
setup: checkout-testdata docker-compose-up
# checkout-testdata will checkout testdata from separate repository into git submodule.
checkout-testdata:
git submodule init
git submodule update
cd ./testdata && git fetch && git checkout master && git pull
# docker-compose-up will download docker image for each of the databases listed in docker-compose.yaml file, and then it will initialize
# database with testdata retrieved in previous step.
# On the first run this action might take couple of minutes. Docker temp data are stored in .docker directory.
docker-compose-up:
docker-compose up
# jet-gen will call generator on each of the test databases to generate sql builder and model files need to run the tests.
jet-gen: install-jet-gen jet-gen-pg jet-gen-mysql jet-gen-mariadb jet-gen-sqlite
install-jet-gen:
go build -o ${GOPATH}/bin/jet ../cmd/jet/
jet-gen-pg:
jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=dvds -path=./.gentestdata/
jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook -path=./.gentestdata/
jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook2 -path=./.gentestdata/
jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=test_sample -path=./.gentestdata/
jet-gen-mysql:
jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/dvds" -path=./.gentestdata/mysql
jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/dvds2" -path=./.gentestdata/mysql
jet -source=mysql -dsn="jet:jet@tcp(localhost:50902)/test_sample" -path=./.gentestdata/mysql
jet-gen-mariadb:
jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/dvds" -path=./.gentestdata/mysql
jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/dvds2" -path=./.gentestdata/mysql
jet -source=mariadb -dsn="jet:jet@tcp(localhost:50903)/test_sample" -path=./.gentestdata/mysql
jet-gen-sqlite:
jet -source=sqlite -dsn="./testdata/init/sqlite/chinook.db" -schema=dvds -path=./.gentestdata/sqlite/chinook
jet -source=sqlite -dsn="./testdata/init/sqlite/sakila.db" -schema=dvds -path=./.gentestdata/sqlite/sakila
jet -source=sqlite -dsn="./testdata/init/sqlite/test_sample.db" -schema=dvds -path=./.gentestdata/sqlite/test_sample
# docker-compose-cleanup will stop and remove test containers, volumes, and images.
# If temp .docker/ folder is still not empty, use 'sudo rm -rf .docker' to remove it.
cleanup:
docker-compose down --volumes
$(info INFO: If the ./.docker folder is still not empty, use 'sudo rm -rf .docker/' to remove it.)

30
tests/Readme.md Normal file
View file

@ -0,0 +1,30 @@
# Integration tests
This folder contains integration tests intended to test jet generator, statements and query result mapping with a running database.
## How to run tests?
Before we can run tests, we need to set up and initialize test databases.
To simplify the process there is a Makefile with a list of helper commands.
```shell
# We first need to checkout testdata from separate repository into git submodule,
# then download docker image for each of the databases listed in docker-compose.yaml file, and
# finally run and initialize databases with downloaded test data.
# Note that on the first run this command might take a couple of minutes.
make setup
# When databases are ready, we can generate sql builder and model types needed to compile the tests.
make jet-gen
```
Then we can run the tests the usual way:
```shell
go test -v ./...
```
To removes test containers, volumes, and images:
```shell
make cleanup
# If temp ./.docker folder is still not empty, use 'sudo rm -rf .docker' to remove it.
```

19
tests/dbconfig/dbconfig.go
View file

@ -8,7 +8,7 @@ import (
// Postgres test database connection parameters
const (
PgHost = "localhost"
PgPort = 5432
PgPort = 50901
PgUser = "jet"
PgPassword = "jet"
PgDBName = "jetdb"
@ -20,13 +20,24 @@ var PostgresConnectString = fmt.Sprintf("host=%s port=%d user=%s password=%s dbn
// MySQL test database connection parameters
const (
MySqLHost = "localhost"
MySQLPort = 3306
MySQLPort = 50902
MySQLUser = "jet"
MySQLPassword = "jet"
MariaDBHost = "localhost"
MariaDBPort = 50903
MariaDBUser = "jet"
MariaDBPassword = "jet"
)
// MySQLConnectionString is MySQL driver connection string to test database
var MySQLConnectionString = fmt.Sprintf("%s:%s@tcp(%s:%d)/", MySQLUser, MySQLPassword, MySqLHost, MySQLPort)
// MySQLConnectionString is MySQL connection string for test database
func MySQLConnectionString(isMariaDB bool, dbName string) string {
if isMariaDB {
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", MariaDBUser, MariaDBPassword, MariaDBHost, MariaDBPort, dbName)
}
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", MySQLUser, MySQLPassword, MySqLHost, MySQLPort, dbName)
}
// sqllite
var (

42
tests/docker-compose.yaml Normal file
View file

@ -0,0 +1,42 @@
version: '3'
services:
postgres:
image: postgres:14.1
restart: always
environment:
- POSTGRES_USER=jet
- POSTGRES_PASSWORD=jet
- POSTGRES_DB=jetdb
ports:
- '50901:5432'
volumes:
- ./testdata/init/postgres:/docker-entrypoint-initdb.d
- ./.docker/postgres-data:/var/lib/postgresql/data
mysql:
image: mysql:8.0.27
command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1']
restart: always
environment:
MYSQL_ROOT_PASSWORD: jet
MYSQL_USER: jet
MYSQL_PASSWORD: jet
ports:
- '50902:3306'
volumes:
- ./testdata/init/mysql:/docker-entrypoint-initdb.d
- ./.docker/mysql-data:/var/lib/mysql
mariadb:
image: mariadb:10.3.32
command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1']
restart: always
environment:
MYSQL_ROOT_PASSWORD: jet
MYSQL_USER: jet
MYSQL_PASSWORD: jet
ports:
- '50903:3306'
volumes:
- ./testdata/init/mysql:/docker-entrypoint-initdb.d
- ./.docker/mariadb-data:/var/lib/mysql

6
tests/init/Readme.md Normal file
View file

@ -0,0 +1,6 @@
The `init` command can be used to initialize test databases on the local host machine, if needed.
Update [dbconfig](../dbconfig/dbconfig.go) with your local database parameters.
The recommended way to initialize test databases is by a docker container.
See tests [Readme.md](../Readme.md).

9
tests/init/init.go
View file

@ -71,8 +71,13 @@ func initMySQLDB() {
}
for _, dbName := range mySQLDBs {
cmdLine := fmt.Sprintf("mysql -h 127.0.0.1 -u %s -p%s %s < %s",
dbconfig.MySQLUser, dbconfig.MySQLPassword, dbName, "./testdata/init/mysql/"+dbName+".sql")
cmdLine := fmt.Sprintf("mysql -h %s -P%d -u %s -p%s %s < %s",
dbconfig.MySqLHost,
dbconfig.MySQLPort,
dbconfig.MySQLUser,
dbconfig.MySQLPassword,
dbName,
"./testdata/init/mysql/"+dbName+".sql")
fmt.Println(cmdLine)

8
tests/mysql/alltypes_test.go
View file

@ -31,10 +31,6 @@ func TestAllTypes(t *testing.T) {
require.Equal(t, len(dest), 2)
if sourceIsMariaDB() { // MariaDB saves current timestamp in a case of NULL value insert
return
}
//testutils.PrintJson(dest)
testutils.AssertJSON(t, dest, allTypesJson)
}
@ -49,10 +45,6 @@ func TestAllTypesViewSelect(t *testing.T) {
require.NoError(t, err)
require.Equal(t, len(dest), 2)
if sourceIsMariaDB() { // MariaDB saves current timestamp in a case of NULL value insert
return
}
testutils.AssertJSON(t, dest, allTypesJson)
}

40
tests/mysql/generator_template_test.go
View file

@ -25,18 +25,30 @@ var defaultViewSQLBuilderFilePath = path.Join(tempTestDir, "dvds/view")
var defaultEnumSQLBuilderFilePath = path.Join(tempTestDir, "dvds/enum")
var defaultActorSQLBuilderFilePath = path.Join(tempTestDir, "dvds/table", "actor.go")
var dbConnection = mysql2.DBConnection{
func dbConnection(dbName string) mysql2.DBConnection {
if sourceIsMariaDB() {
return mysql2.DBConnection{
Host: dbconfig.MariaDBHost,
Port: dbconfig.MariaDBPort,
User: dbconfig.MariaDBUser,
Password: dbconfig.MariaDBPassword,
DBName: dbName,
}
}
return mysql2.DBConnection{
Host: dbconfig.MySqLHost,
Port: dbconfig.MySQLPort,
User: dbconfig.MySQLUser,
Password: dbconfig.MySQLPassword,
DBName: "dvds",
DBName: dbName,
}
}
func TestGeneratorTemplate_Schema_ChangePath(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).UsePath("new/schema/path")
@ -54,7 +66,7 @@ func TestGeneratorTemplate_Schema_ChangePath(t *testing.T) {
func TestGeneratorTemplate_Model_SkipGeneration(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -75,7 +87,7 @@ func TestGeneratorTemplate_Model_SkipGeneration(t *testing.T) {
func TestGeneratorTemplate_SQLBuilder_SkipGeneration(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -98,7 +110,7 @@ func TestGeneratorTemplate_Model_ChangePath(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -116,7 +128,7 @@ func TestGeneratorTemplate_SQLBuilder_ChangePath(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -137,7 +149,7 @@ func TestGeneratorTemplate_SQLBuilder_ChangePath(t *testing.T) {
func TestGeneratorTemplate_Model_RenameFilesAndTypes(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -175,7 +187,7 @@ func TestGeneratorTemplate_Model_RenameFilesAndTypes(t *testing.T) {
func TestGeneratorTemplate_Model_SkipTableAndEnum(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -203,7 +215,7 @@ func TestGeneratorTemplate_Model_SkipTableAndEnum(t *testing.T) {
func TestGeneratorTemplate_SQLBuilder_SkipTableAndEnum(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -236,7 +248,7 @@ func TestGeneratorTemplate_SQLBuilder_SkipTableAndEnum(t *testing.T) {
func TestGeneratorTemplate_SQLBuilder_ChangeTypeAndFileName(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -277,7 +289,7 @@ func TestGeneratorTemplate_Model_AddTags(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -318,7 +330,7 @@ func TestGeneratorTemplate_Model_AddTags(t *testing.T) {
func TestGeneratorTemplate_Model_ChangeFieldTypes(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).
@ -361,7 +373,7 @@ func TestGeneratorTemplate_Model_ChangeFieldTypes(t *testing.T) {
func TestGeneratorTemplate_SQLBuilder_ChangeColumnTypes(t *testing.T) {
err := mysql2.Generate(
tempTestDir,
dbConnection,
dbConnection("dvds"),
template.Default(postgres2.Dialect).
UseSchema(func(schemaMetaData metadata.Schema) template.Schema {
return template.DefaultSchema(schemaMetaData).

51
tests/mysql/generator_test.go
View file

@ -1,10 +1,10 @@
package mysql
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"strconv"
"testing"
"github.com/go-jet/jet/v2/generator/mysql"
@ -19,13 +19,7 @@ const genTestDir3 = "./.gentestdata3/mysql"
func TestGenerator(t *testing.T) {
for i := 0; i < 3; i++ {
err := mysql.Generate(genTestDir3, mysql.DBConnection{
Host: dbconfig.MySqLHost,
Port: dbconfig.MySQLPort,
User: dbconfig.MySQLUser,
Password: dbconfig.MySQLPassword,
DBName: "dvds",
})
err := mysql.Generate(genTestDir3, dbConnection("dvds"))
require.NoError(t, err)
@ -33,17 +27,11 @@ func TestGenerator(t *testing.T) {
}
for i := 0; i < 3; i++ {
dsn := fmt.Sprintf("%[1]s:%[2]s@tcp(%[3]s:%[4]d)/%[5]s",
dbconfig.MySQLUser,
dbconfig.MySQLPassword,
dbconfig.MySqLHost,
dbconfig.MySQLPort,
"dvds",
)
dsn := dbconfig.MySQLConnectionString(sourceIsMariaDB(), "dvds")
err := mysql.GenerateDSN(dsn, genTestDir3)
require.NoError(t, err)
assertGeneratedFiles(t)
}
@ -55,8 +43,27 @@ func TestCmdGenerator(t *testing.T) {
err := os.RemoveAll(genTestDir3)
require.NoError(t, err)
cmd := exec.Command("jet", "-source=MySQL", "-dbname=dvds", "-host=localhost", "-port=3306",
"-user=jet", "-password=jet", "-path="+genTestDir3)
var cmd *exec.Cmd
if sourceIsMariaDB() {
cmd = exec.Command("jet",
"-source=MariaDB",
"-dbname=dvds",
"-host="+dbconfig.MariaDBHost,
"-port="+strconv.Itoa(dbconfig.MariaDBPort),
"-user="+dbconfig.MariaDBUser,
"-password="+dbconfig.MariaDBPassword,
"-path="+genTestDir3)
} else {
cmd = exec.Command("jet",
"-source=MySQL",
"-dbname=dvds",
"-host="+dbconfig.MySqLHost,
"-port="+strconv.Itoa(dbconfig.MySQLPort),
"-user="+dbconfig.MySQLUser,
"-password="+dbconfig.MySQLPassword,
"-path="+genTestDir3)
}
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
@ -70,13 +77,7 @@ func TestCmdGenerator(t *testing.T) {
require.NoError(t, err)
// check that generation via DSN works
dsn := fmt.Sprintf("mysql://%[1]s:%[2]s@tcp(%[3]s:%[4]d)/%[5]s",
dbconfig.MySQLUser,
dbconfig.MySQLPassword,
dbconfig.MySqLHost,
dbconfig.MySQLPort,
"dvds",
)
dsn := "mysql://" + dbconfig.MySQLConnectionString(sourceIsMariaDB(), "dvds")
cmd = exec.Command("jet", "-dsn="+dsn, "-path="+genTestDir3)
cmd.Stderr = os.Stderr

2
tests/mysql/main_test.go
View file

@ -36,7 +36,7 @@ func TestMain(m *testing.M) {
defer profile.Start().Stop()
var err error
db, err = sql.Open("mysql", dbconfig.MySQLConnectionString)
db, err = sql.Open("mysql", dbconfig.MySQLConnectionString(sourceIsMariaDB(), ""))
if err != nil {
panic("Failed to connect to test db" + err.Error())
}

7
tests/mysql/select_test.go
View file

@ -389,8 +389,6 @@ LIMIT ?;
).
LIMIT(1000)
//fmt.Println(query.Sql())
testutils.AssertStatementSql(t, query, expectedSQL, int64(1000))
var dest []struct {
@ -414,12 +412,7 @@ LIMIT ?;
err := query.Query(db, &dest)
require.NoError(t, err)
//require.Equal(t, len(dest), 1)
//require.Equal(t, len(dest[0].Films), 10)
//require.Equal(t, len(dest[0].Films[0].Actors), 10)
//testutils.SaveJsonFile(dest, "./mysql/testdata/lang_film_actor_inventory_rental.json")
testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/lang_film_actor_inventory_rental.json")
}
}

2
tests/testdata

@ -1 +1 @@
Subproject commit 946bc1e5d3e162154eade8b79ff915e4c4986efd
Subproject commit f8329498fd778157a1aac8d471d09ff39cdd5de1