From b92af7ca6ebd86eaa1d16a2206078dae68c9f68f Mon Sep 17 00:00:00 2001 From: go-jet Date: Sat, 18 Dec 2021 13:55:40 +0100 Subject: [PATCH] Update circle.ci --- .circleci/config.yml | 170 ++++++++++------------ .gitignore | 3 +- tests/Makefile | 20 ++- tests/Readme.md | 5 +- tests/dbconfig/dbconfig.go | 4 +- tests/docker-compose.yaml | 3 - tests/init/init.go | 38 +++-- tests/postgres/generator_template_test.go | 2 +- tests/postgres/generator_test.go | 10 +- 9 files changed, 128 insertions(+), 127 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 108ecfd..1571843 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,28 +1,37 @@ # Golang CircleCI 2.0 configuration file # # Check https://circleci.com/docs/2.0/language-go/ for more details -version: 2 +version: 2.1 +orbs: + codecov: codecov/codecov@3.1.1 jobs: - build-postgres-and-mysql: + build_and_tests: docker: # specify the version - - image: circleci/golang:1.13 - - - image: circleci/postgres:10.8-alpine - environment: # environment variables for primary container + - image: circleci/golang:1.16 + - image: circleci/postgres:12 + environment: POSTGRES_USER: jet POSTGRES_PASSWORD: jet POSTGRES_DB: jetdb + PGPORT: 50901 - - image: circleci/mysql:8.0.16 - command: [--default-authentication-plugin=mysql_native_password] + - image: circleci/mysql:8.0.27 + command: [ --default-authentication-plugin=mysql_native_password ] environment: MYSQL_ROOT_PASSWORD: jet MYSQL_DATABASE: dvds MYSQL_USER: jet MYSQL_PASSWORD: jet + MYSQL_TCP_PORT: 50902 - working_directory: /go/src/github.com/go-jet/jet + - image: circleci/mariadb:10.3 + command: [ '--default-authentication-plugin=mysql_native_password', '--port=50903' ] + environment: + MYSQL_ROOT_PASSWORD: jet + MYSQL_DATABASE: dvds + MYSQL_USER: jet + MYSQL_PASSWORD: jet environment: # environment variables for the build itself TEST_RESULTS: /tmp/test-results # path to where test results will be saved @@ -32,25 +41,22 @@ jobs: - run: name: Submodule init - command: | - git submodule init - git submodule update - cd ./tests/testdata && git fetch && git checkout master + command: cd tests && make checkout-testdata + - restore_cache: # restores saved cache if no changes are detected since last run + keys: + - go-mod-v4-{{ checksum "go.sum" }} - run: - name: Install dependencies - command: | - cd /go/src/github.com/go-jet/jet - go get github.com/jstemmer/go-junit-report - go build -o /home/circleci/.local/bin/jet ./cmd/jet/ + name: Install jet generator + command: cd tests && make install-jet-gen - run: name: Waiting for Postgres to be ready command: | for i in `seq 1 10`; do - nc -z localhost 5432 && echo Success && exit 0 + nc -z localhost 50901 && echo Success && exit 0 echo -n . sleep 1 done @@ -61,39 +67,71 @@ jobs: command: | for i in `seq 1 10`; do - nc -z 127.0.0.1 3306 && echo Success && exit 0 + nc -z 127.0.0.1 50902 && echo Success && exit 0 echo -n . sleep 1 done echo Failed waiting for MySQL && exit 1 + + - run: + name: Waiting for MariaDB to be ready + command: | + for i in `seq 1 10`; + do + nc -z 127.0.0.1 50903 && echo Success && exit 0 + echo -n . + sleep 1 + done + echo Failed waiting for MySQL && exit 1 + - run: name: Install MySQL CLI; command: | sudo apt-get --allow-releaseinfo-change update && sudo apt-get install default-mysql-client - run: - name: Create MySQL user and databases + name: Create MySQL/MariaDB user and test databases command: | - mysql -h 127.0.0.1 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';" - mysql -h 127.0.0.1 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';" - mysql -h 127.0.0.1 -u jet -pjet -e "create database test_sample" - mysql -h 127.0.0.1 -u jet -pjet -e "create database dvds2" + mysql -h 127.0.0.1 -P 50902 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';" + mysql -h 127.0.0.1 -P 50902 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';" + mysql -h 127.0.0.1 -P 50902 -u jet -pjet -e "create database test_sample" + mysql -h 127.0.0.1 -P 50902 -u jet -pjet -e "create database dvds2" + + mysql -h 127.0.0.1 -P 50903 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';" + mysql -h 127.0.0.1 -P 50903 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';" + mysql -h 127.0.0.1 -P 50903 -u jet -pjet -e "create database test_sample" + mysql -h 127.0.0.1 -P 50903 -u jet -pjet -e "create database dvds2" - run: - name: Init Postgres database - command: | - cd tests - go run ./init/init.go -testsuite all - cd .. + name: Init databases + command: | + cd tests + go run ./init/init.go -testsuite all + # to create test results report + - run: + name: Install go-junit-report + command: go install github.com/jstemmer/go-junit-report@latest - run: mkdir -p $TEST_RESULTS - # this will run all tests and exclude test files from code coverage report - - run: MY_SQL_SOURCE=MySQL go test -v ./... -covermode=atomic -coverpkg=github.com/go-jet/jet/postgres/...,github.com/go-jet/jet/mysql/...,github.com/go-jet/jet/sqlite/...,github.com/go-jet/jet/qrm/...,github.com/go-jet/jet/generator/...,github.com/go-jet/jet/internal/... -coverprofile=cover.out 2>&1 | go-junit-report > $TEST_RESULTS/results.xml - - run: - name: Upload code coverage - command: bash <(curl -s https://codecov.io/bash) + # this will run all tests and exclude test files from code coverage report + - run: | + go test -v ./... \ + -covermode=atomic \ + -coverpkg=github.com/go-jet/jet/v2/postgres/...,github.com/go-jet/jet/v2/mysql/...,github.com/go-jet/jet/v2/sqlite/...,github.com/go-jet/jet/v2/qrm/...,github.com/go-jet/jet/v2/generator/...,github.com/go-jet/jet/v2/internal/... \ + -coverprofile=cover.out 2>&1 | go-junit-report > $TEST_RESULTS/results.xml + + # run mariaDB tests. No need to collect coverage, because coverage is already included with mysql tests + - run: MY_SQL_SOURCE=MariaDB go test -v ./tests/mysql/ + + - save_cache: + key: go-mod-v4-{{ checksum "go.sum" }} + paths: + - "/go/pkg/mod" + + - codecov/upload: + file: cover.out - store_artifacts: # Upload test summary for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ path: /tmp/test-results @@ -101,69 +139,9 @@ jobs: - store_test_results: # Upload test results for display in Test Summary: https://circleci.com/docs/2.0/collect-test-data/ path: /tmp/test-results - build-mariadb: - docker: - # specify the version - - image: circleci/golang:1.13 - - - image: circleci/mariadb:10.3 - command: [--default-authentication-plugin=mysql_native_password] - environment: - MYSQL_ROOT_PASSWORD: jet - MYSQL_DATABASE: dvds - MYSQL_USER: jet - MYSQL_PASSWORD: jet - - working_directory: /go/src/github.com/go-jet/jet - - environment: # environment variables for the build itself - TEST_RESULTS: /tmp/test-results # path to where test results will be saved - - steps: - - checkout - - - run: - name: Submodule init - command: | - git submodule init - git submodule update - cd ./tests/testdata && git fetch && git checkout master - - - run: - name: Install dependencies - command: | - cd /go/src/github.com/go-jet/jet - go get github.com/jstemmer/go-junit-report - go build -o /home/circleci/.local/bin/jet ./cmd/jet/ - - - run: - name: Install MySQL CLI; - command: | - sudo apt-get --allow-releaseinfo-change update && sudo apt-get install default-mysql-client - - - run: - name: Init MariaDB database - command: | - mysql -h 127.0.0.1 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';" - mysql -h 127.0.0.1 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';" - mysql -h 127.0.0.1 -u jet -pjet -e "create database test_sample" - mysql -h 127.0.0.1 -u jet -pjet -e "create database dvds2" - - - run: - name: Init MariaDB database - command: | - cd tests - go run ./init/init.go -testsuite MariaDB - cd .. - - - run: - name: Run MariaDB tests - command: | - MY_SQL_SOURCE=MariaDB go test -v ./tests/mysql/ workflows: version: 2 build_and_test: jobs: - - build-postgres-and-mysql - - build-mariadb + - build_and_tests \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4909287..4d7da45 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ gen .gentestdata .tests/testdata/ .gen -.docker \ No newline at end of file +.docker +.env \ No newline at end of file diff --git a/tests/Makefile b/tests/Makefile index c0d31bd..632c3d2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -14,14 +14,28 @@ checkout-testdata: docker-compose-up: docker-compose up +init-all: + go run ./init/init.go -testsuite all + +init-postgres: + go run ./init/init.go -testsuite postgres + +init-mysql: + go run ./init/init.go -testsuite mysql + +init-mariadb: + go run ./init/init.go -testsuite mariadb + +init-sqlite: + go run ./init/init.go -testsuite sqlite # jet-gen will call generator on each of the test databases to generate sql builder and model files need to run the tests. -jet-gen: install-jet-gen jet-gen-pg jet-gen-mysql jet-gen-mariadb jet-gen-sqlite +jet-gen-all: install-jet-gen jet-gen-postgres jet-gen-mysql jet-gen-mariadb jet-gen-sqlite install-jet-gen: go build -o ${GOPATH}/bin/jet ../cmd/jet/ -jet-gen-pg: +jet-gen-postgres: jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=dvds -path=./.gentestdata/ jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook -path=./.gentestdata/ jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook2 -path=./.gentestdata/ @@ -44,7 +58,5 @@ jet-gen-sqlite: # docker-compose-cleanup will stop and remove test containers, volumes, and images. -# If temp .docker/ folder is still not empty, use 'sudo rm -rf .docker' to remove it. cleanup: docker-compose down --volumes - $(info INFO: If the ./.docker folder is still not empty, use 'sudo rm -rf .docker/' to remove it.) diff --git a/tests/Readme.md b/tests/Readme.md index 40bc456..097e5f3 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -14,8 +14,8 @@ To simplify the process there is a Makefile with a list of helper commands. # Note that on the first run this command might take a couple of minutes. make setup -# When databases are ready, we can generate sql builder and model types needed to compile the tests. -make jet-gen +# When databases are ready, we can generate sql builder and model types for each of the test databases +make jet-gen-all ``` Then we can run the tests the usual way: @@ -26,5 +26,4 @@ go test -v ./... To removes test containers, volumes, and images: ```shell make cleanup -# If temp ./.docker folder is still not empty, use 'sudo rm -rf .docker' to remove it. ``` \ No newline at end of file diff --git a/tests/dbconfig/dbconfig.go b/tests/dbconfig/dbconfig.go index b09ab80..bbf73f9 100644 --- a/tests/dbconfig/dbconfig.go +++ b/tests/dbconfig/dbconfig.go @@ -19,12 +19,12 @@ var PostgresConnectString = fmt.Sprintf("host=%s port=%d user=%s password=%s dbn // MySQL test database connection parameters const ( - MySqLHost = "localhost" + MySqLHost = "127.0.0.1" MySQLPort = 50902 MySQLUser = "jet" MySQLPassword = "jet" - MariaDBHost = "localhost" + MariaDBHost = "127.0.0.1" MariaDBPort = 50903 MariaDBUser = "jet" MariaDBPassword = "jet" diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index fc24385..2e913f1 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -11,7 +11,6 @@ services: - '50901:5432' volumes: - ./testdata/init/postgres:/docker-entrypoint-initdb.d - - ./.docker/postgres-data:/var/lib/postgresql/data mysql: image: mysql:8.0.27 @@ -25,7 +24,6 @@ services: - '50902:3306' volumes: - ./testdata/init/mysql:/docker-entrypoint-initdb.d - - ./.docker/mysql-data:/var/lib/mysql mariadb: image: mariadb:10.3.32 @@ -39,4 +37,3 @@ services: - '50903:3306' volumes: - ./testdata/init/mysql:/docker-entrypoint-initdb.d - - ./.docker/mariadb-data:/var/lib/mysql diff --git a/tests/init/init.go b/tests/init/init.go index eb49fb7..c1c842a 100644 --- a/tests/init/init.go +++ b/tests/init/init.go @@ -4,6 +4,7 @@ import ( "database/sql" "flag" "fmt" + "github.com/go-jet/jet/v2/generator/mysql" "github.com/go-jet/jet/v2/generator/sqlite" "github.com/go-jet/jet/v2/tests/internal/utils/repo" "io/ioutil" @@ -11,7 +12,6 @@ import ( "os/exec" "strings" - "github.com/go-jet/jet/v2/generator/mysql" "github.com/go-jet/jet/v2/generator/postgres" "github.com/go-jet/jet/v2/internal/utils/throw" "github.com/go-jet/jet/v2/tests/dbconfig" @@ -39,7 +39,7 @@ func main() { } if testSuite == "mysql" || testSuite == "mariadb" { - initMySQLDB() + initMySQLDB(testSuite == "mariadb") return } @@ -48,8 +48,9 @@ func main() { return } - initMySQLDB() initPostgresDB() + initMySQLDB(false) + initMySQLDB(true) initSQLiteDB() } @@ -62,7 +63,7 @@ func initSQLiteDB() { throw.OnError(err) } -func initMySQLDB() { +func initMySQLDB(isMariaDB bool) { mySQLDBs := []string{ "dvds", @@ -71,12 +72,19 @@ func initMySQLDB() { } for _, dbName := range mySQLDBs { - cmdLine := fmt.Sprintf("mysql -h %s -P%d -u %s -p%s %s < %s", - dbconfig.MySqLHost, - dbconfig.MySQLPort, - dbconfig.MySQLUser, - dbconfig.MySQLPassword, - dbName, + host := dbconfig.MySqLHost + port := dbconfig.MySQLPort + user := dbconfig.MySQLUser + pass := dbconfig.MySQLPassword + + if isMariaDB { + host = dbconfig.MariaDBHost + port = dbconfig.MariaDBPort + user = dbconfig.MariaDBUser + pass = dbconfig.MariaDBPassword + } + + cmdLine := fmt.Sprintf("mysql -h %s -P %d -u %s -p%s %s < %s", host, port, user, pass, dbName, "./testdata/init/mysql/"+dbName+".sql") fmt.Println(cmdLine) @@ -90,10 +98,10 @@ func initMySQLDB() { throw.OnError(err) err = mysql.Generate("./.gentestdata/mysql", mysql.DBConnection{ - Host: dbconfig.MySqLHost, - Port: dbconfig.MySQLPort, - User: dbconfig.MySQLUser, - Password: dbconfig.MySQLPassword, + Host: host, + Port: port, + User: user, + Password: pass, DBName: dbName, }) @@ -104,7 +112,7 @@ func initMySQLDB() { func initPostgresDB() { db, err := sql.Open("postgres", dbconfig.PostgresConnectString) if err != nil { - panic("Failed to connect to test db") + panic("Failed to connect to test db: " + err.Error()) } defer func() { err := db.Close() diff --git a/tests/postgres/generator_template_test.go b/tests/postgres/generator_template_test.go index 85dd01e..e8f59c7 100644 --- a/tests/postgres/generator_template_test.go +++ b/tests/postgres/generator_template_test.go @@ -27,7 +27,7 @@ var defaultActorSQLBuilderFilePath = path.Join(tempTestDir, "jetdb/dvds/table", var dbConnection = postgres.DBConnection{ Host: dbconfig.PgHost, - Port: 5432, + Port: dbconfig.PgPort, User: dbconfig.PgUser, Password: dbconfig.PgPassword, DBName: dbconfig.PgDBName, diff --git a/tests/postgres/generator_test.go b/tests/postgres/generator_test.go index 77c8aee..fe159aa 100644 --- a/tests/postgres/generator_test.go +++ b/tests/postgres/generator_test.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "reflect" + "strconv" "testing" "github.com/go-jet/jet/v2/generator/postgres" @@ -52,8 +53,13 @@ func TestCmdGenerator(t *testing.T) { err := os.RemoveAll(genTestDir2) require.NoError(t, err) - cmd := exec.Command("jet", "-source=PostgreSQL", "-dbname=jetdb", "-host=localhost", "-port=5432", - "-user=jet", "-password=jet", "-schema=dvds", "-path="+genTestDir2) + cmd := exec.Command("jet", "-source=PostgreSQL", "-dbname=jetdb", "-host=localhost", + "-port="+strconv.Itoa(dbconfig.PgPort), + "-user=jet", + "-password=jet", + "-schema=dvds", + "-path="+genTestDir2) + cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout