Update circle.ci

.circleci/config.yml

@@ -1,28 +1,37 @@
 # Golang CircleCI 2.0 configuration file
 #
 # Check https://circleci.com/docs/2.0/language-go/ for more details
-version: 2
+version: 2.1
+orbs:
+  codecov: codecov/codecov@3.1.1
 jobs:
-  build-postgres-and-mysql:
+  build_and_tests:
     docker:
       # specify the version
-      - image: circleci/golang:1.13
+      - image: circleci/golang:1.16
-
+      - image: circleci/postgres:12
-      - image: circleci/postgres:10.8-alpine
+        environment:
-        environment: # environment variables for primary container
           POSTGRES_USER: jet
           POSTGRES_PASSWORD: jet
           POSTGRES_DB: jetdb
+          PGPORT: 50901
 
-      - image: circleci/mysql:8.0.16
+      - image: circleci/mysql:8.0.27
-        command: [--default-authentication-plugin=mysql_native_password]
+        command: [ --default-authentication-plugin=mysql_native_password ]
         environment:
           MYSQL_ROOT_PASSWORD: jet
           MYSQL_DATABASE: dvds
           MYSQL_USER: jet
           MYSQL_PASSWORD: jet
+          MYSQL_TCP_PORT: 50902
 
-    working_directory: /go/src/github.com/go-jet/jet
+      - image: circleci/mariadb:10.3
+        command: [ '--default-authentication-plugin=mysql_native_password', '--port=50903' ]
+        environment:
+          MYSQL_ROOT_PASSWORD: jet
+          MYSQL_DATABASE: dvds
+          MYSQL_USER: jet
+          MYSQL_PASSWORD: jet
 
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -32,25 +41,22 @@ jobs:
 
       - run:
           name: Submodule init
-          command: |
+          command: cd tests && make checkout-testdata
-            git submodule init
-            git submodule update
-            cd ./tests/testdata && git fetch && git checkout master
 
+      - restore_cache: # restores saved cache if no changes are detected since last run
+          keys:
+            - go-mod-v4-{{ checksum "go.sum" }}
 
       - run:
-          name: Install dependencies
+          name: Install jet generator
-          command: |
+          command: cd tests && make install-jet-gen
-            cd /go/src/github.com/go-jet/jet
-            go get github.com/jstemmer/go-junit-report
-            go build -o /home/circleci/.local/bin/jet ./cmd/jet/
 
       - run:
           name: Waiting for Postgres to be ready
           command: |
             for i in `seq 1 10`;
             do
-              nc -z localhost 5432 && echo Success && exit 0
+              nc -z localhost 50901 && echo Success && exit 0
               echo -n .
               sleep 1
             done
@@ -61,39 +67,71 @@ jobs:
           command: |
             for i in `seq 1 10`;
             do
-              nc -z 127.0.0.1 3306 && echo Success && exit 0
+              nc -z 127.0.0.1 50902 && echo Success && exit 0
               echo -n .
               sleep 1
             done
             echo Failed waiting for MySQL && exit 1
+
+      - run:
+          name: Waiting for MariaDB to be ready
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z 127.0.0.1 50903 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for MySQL && exit 1      
+
       - run:
           name: Install MySQL CLI;
           command: |
             sudo apt-get --allow-releaseinfo-change update && sudo apt-get install default-mysql-client
 
       - run:
-          name: Create MySQL user and databases
+          name: Create MySQL/MariaDB user and test databases
           command: |
-            mysql -h 127.0.0.1 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';"
+            mysql -h 127.0.0.1 -P 50902 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';"
-            mysql -h 127.0.0.1 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';"
+            mysql -h 127.0.0.1 -P 50902 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';"
-            mysql -h 127.0.0.1 -u jet -pjet -e "create database test_sample"
+            mysql -h 127.0.0.1 -P 50902 -u jet -pjet -e "create database test_sample"
-            mysql -h 127.0.0.1 -u jet -pjet -e "create database dvds2"
+            mysql -h 127.0.0.1 -P 50902 -u jet -pjet -e "create database dvds2"
+            
+            mysql -h 127.0.0.1 -P 50903 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';"
+            mysql -h 127.0.0.1 -P 50903 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';"
+            mysql -h 127.0.0.1 -P 50903 -u jet -pjet -e "create database test_sample"
+            mysql -h 127.0.0.1 -P 50903 -u jet -pjet -e "create database dvds2"
 
       - run:
-         name: Init Postgres database
+          name: Init databases
           command: |
             cd tests
             go run ./init/init.go -testsuite all
-           cd ..
 
+      # to create test results report
+      - run:
+          name: Install go-junit-report
+          command: go install github.com/jstemmer/go-junit-report@latest
 
       - run: mkdir -p $TEST_RESULTS
-      # this will run all tests and exclude test files from code coverage report
-      - run: MY_SQL_SOURCE=MySQL go test -v ./... -covermode=atomic -coverpkg=github.com/go-jet/jet/postgres/...,github.com/go-jet/jet/mysql/...,github.com/go-jet/jet/sqlite/...,github.com/go-jet/jet/qrm/...,github.com/go-jet/jet/generator/...,github.com/go-jet/jet/internal/... -coverprofile=cover.out 2>&1 | go-junit-report > $TEST_RESULTS/results.xml
 
-      - run:
+      # this will run all tests and exclude test files from code coverage report
-          name: Upload code coverage
+      - run:  |
-          command: bash <(curl -s https://codecov.io/bash)
+          go test -v ./... \
+            -covermode=atomic \
+            -coverpkg=github.com/go-jet/jet/v2/postgres/...,github.com/go-jet/jet/v2/mysql/...,github.com/go-jet/jet/v2/sqlite/...,github.com/go-jet/jet/v2/qrm/...,github.com/go-jet/jet/v2/generator/...,github.com/go-jet/jet/v2/internal/... \
+            -coverprofile=cover.out 2>&1 | go-junit-report > $TEST_RESULTS/results.xml
+
+      # run mariaDB tests. No need to collect coverage, because coverage is already included with mysql tests
+      - run: MY_SQL_SOURCE=MariaDB go test -v  ./tests/mysql/
+
+      - save_cache:
+          key: go-mod-v4-{{ checksum "go.sum" }}
+          paths:
+            - "/go/pkg/mod"
+
+      - codecov/upload:
+          file: cover.out
 
       - store_artifacts: # Upload test summary for display in Artifacts: https://circleci.com/docs/2.0/artifacts/
           path: /tmp/test-results
@@ -101,69 +139,9 @@ jobs:
 
       - store_test_results: # Upload test results for display in Test Summary: https://circleci.com/docs/2.0/collect-test-data/
           path: /tmp/test-results
-  build-mariadb:
-    docker:
-      # specify the version
-      - image: circleci/golang:1.13
-
-      - image: circleci/mariadb:10.3
-        command: [--default-authentication-plugin=mysql_native_password]
-        environment:
-          MYSQL_ROOT_PASSWORD: jet
-          MYSQL_DATABASE: dvds
-          MYSQL_USER: jet
-          MYSQL_PASSWORD: jet
-
-    working_directory: /go/src/github.com/go-jet/jet
-
-    environment: # environment variables for the build itself
-      TEST_RESULTS: /tmp/test-results # path to where test results will be saved
-
-    steps:
-      - checkout
-
-      - run:
-          name: Submodule init
-          command: |
-            git submodule init
-            git submodule update
-            cd ./tests/testdata && git fetch && git checkout master
-
-      - run:
-          name: Install dependencies
-          command: |
-            cd /go/src/github.com/go-jet/jet
-            go get github.com/jstemmer/go-junit-report
-            go build -o /home/circleci/.local/bin/jet ./cmd/jet/
-
-      - run:
-          name: Install MySQL CLI;
-          command: |
-            sudo apt-get --allow-releaseinfo-change update && sudo apt-get install default-mysql-client
-
-      - run:
-          name: Init MariaDB database
-          command: |
-            mysql -h 127.0.0.1 -u root -pjet -e "grant all privileges on *.* to 'jet'@'%';"
-            mysql -h 127.0.0.1 -u root -pjet -e "set global sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION';"
-            mysql -h 127.0.0.1 -u jet -pjet -e "create database test_sample"
-            mysql -h 127.0.0.1 -u jet -pjet -e "create database dvds2"
-
-      - run:
-          name: Init MariaDB database
-          command: |
-            cd tests
-            go run ./init/init.go -testsuite MariaDB
-            cd ..
-
-      - run:
-          name: Run MariaDB tests
-          command: |
-            MY_SQL_SOURCE=MariaDB go test -v  ./tests/mysql/
 
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - build-postgres-and-mysql
+      - build_and_tests
-      - build-mariadb

.gitignore

@@ -21,3 +21,4 @@ gen
 .tests/testdata/
 .gen
 .docker
+.env

tests/Makefile

@@ -14,14 +14,28 @@ checkout-testdata:
 docker-compose-up:
 	docker-compose up
 
+init-all:
+	go run ./init/init.go -testsuite all
+
+init-postgres:
+	go run ./init/init.go -testsuite postgres
+
+init-mysql:
+	go run ./init/init.go -testsuite mysql
+
+init-mariadb:
+	go run ./init/init.go -testsuite mariadb
+
+init-sqlite:
+	go run ./init/init.go -testsuite sqlite
 
 # jet-gen will call generator on each of the test databases to generate sql builder and model files need to run the tests.
-jet-gen: install-jet-gen jet-gen-pg jet-gen-mysql jet-gen-mariadb jet-gen-sqlite
+jet-gen-all: install-jet-gen jet-gen-postgres jet-gen-mysql jet-gen-mariadb jet-gen-sqlite
 
 install-jet-gen:
 	go build -o ${GOPATH}/bin/jet ../cmd/jet/
 
-jet-gen-pg:
+jet-gen-postgres:
 	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=dvds  -path=./.gentestdata/
 	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook  -path=./.gentestdata/
 	jet -dsn=postgres://jet:jet@localhost:50901/jetdb?sslmode=disable -schema=chinook2  -path=./.gentestdata/
@@ -44,7 +58,5 @@ jet-gen-sqlite:
 
 
 # docker-compose-cleanup will stop and remove test containers, volumes, and images.
-# If temp .docker/ folder is still not empty, use 'sudo rm -rf .docker' to remove it.
 cleanup:
 	docker-compose down --volumes
-	$(info INFO: If the ./.docker folder is still not empty, use 'sudo rm -rf .docker/' to remove it.)

tests/Readme.md

@@ -14,8 +14,8 @@ To simplify the process there is a Makefile with a list of helper commands.
 # Note that on the first run this command might take a couple of minutes.
 make setup
 
-# When databases are ready, we can generate sql builder and model types needed to compile the tests.
+# When databases are ready, we can generate sql builder and model types for each of the test databases
-make jet-gen
+make jet-gen-all
 ```
 
 Then we can run the tests the usual way:
@@ -26,5 +26,4 @@ go test -v ./...
 To removes test containers, volumes, and images:
 ```shell
 make cleanup
-# If temp ./.docker folder is still not empty, use 'sudo rm -rf .docker' to remove it.
 ```

tests/dbconfig/dbconfig.go

@@ -19,12 +19,12 @@ var PostgresConnectString = fmt.Sprintf("host=%s port=%d user=%s password=%s dbn
 
 // MySQL test database connection parameters
 const (
-	MySqLHost     = "localhost"
+	MySqLHost     = "127.0.0.1"
 	MySQLPort     = 50902
 	MySQLUser     = "jet"
 	MySQLPassword = "jet"
 
-	MariaDBHost     = "localhost"
+	MariaDBHost     = "127.0.0.1"
 	MariaDBPort     = 50903
 	MariaDBUser     = "jet"
 	MariaDBPassword = "jet"

tests/docker-compose.yaml

@@ -11,7 +11,6 @@ services:
       - '50901:5432'
     volumes:
       - ./testdata/init/postgres:/docker-entrypoint-initdb.d
-      - ./.docker/postgres-data:/var/lib/postgresql/data
 
   mysql:
     image: mysql:8.0.27
@@ -25,7 +24,6 @@ services:
       - '50902:3306'
     volumes:
       - ./testdata/init/mysql:/docker-entrypoint-initdb.d
-      - ./.docker/mysql-data:/var/lib/mysql
 
   mariadb:
     image: mariadb:10.3.32
@@ -39,4 +37,3 @@ services:
       - '50903:3306'
     volumes:
       - ./testdata/init/mysql:/docker-entrypoint-initdb.d
-      - ./.docker/mariadb-data:/var/lib/mysql

tests/init/init.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"flag"
 	"fmt"
+	"github.com/go-jet/jet/v2/generator/mysql"
 	"github.com/go-jet/jet/v2/generator/sqlite"
 	"github.com/go-jet/jet/v2/tests/internal/utils/repo"
 	"io/ioutil"
@@ -11,7 +12,6 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/go-jet/jet/v2/generator/mysql"
 	"github.com/go-jet/jet/v2/generator/postgres"
 	"github.com/go-jet/jet/v2/internal/utils/throw"
 	"github.com/go-jet/jet/v2/tests/dbconfig"
@@ -39,7 +39,7 @@ func main() {
 	}
 
 	if testSuite == "mysql" || testSuite == "mariadb" {
-		initMySQLDB()
+		initMySQLDB(testSuite == "mariadb")
 		return
 	}
 
@@ -48,8 +48,9 @@ func main() {
 		return
 	}
 
-	initMySQLDB()
 	initPostgresDB()
+	initMySQLDB(false)
+	initMySQLDB(true)
 	initSQLiteDB()
 }
 
@@ -62,7 +63,7 @@ func initSQLiteDB() {
 	throw.OnError(err)
 }
 
-func initMySQLDB() {
+func initMySQLDB(isMariaDB bool) {
 
 	mySQLDBs := []string{
 		"dvds",
@@ -71,12 +72,19 @@ func initMySQLDB() {
 	}
 
 	for _, dbName := range mySQLDBs {
-		cmdLine := fmt.Sprintf("mysql -h %s -P%d -u %s -p%s %s < %s",
+		host := dbconfig.MySqLHost
-			dbconfig.MySqLHost,
+		port := dbconfig.MySQLPort
-			dbconfig.MySQLPort,
+		user := dbconfig.MySQLUser
-			dbconfig.MySQLUser,
+		pass := dbconfig.MySQLPassword
-			dbconfig.MySQLPassword,
+
-			dbName,
+		if isMariaDB {
+			host = dbconfig.MariaDBHost
+			port = dbconfig.MariaDBPort
+			user = dbconfig.MariaDBUser
+			pass = dbconfig.MariaDBPassword
+		}
+
+		cmdLine := fmt.Sprintf("mysql -h %s -P %d -u %s -p%s %s < %s", host, port, user, pass, dbName,
 			"./testdata/init/mysql/"+dbName+".sql")
 
 		fmt.Println(cmdLine)
@@ -90,10 +98,10 @@ func initMySQLDB() {
 		throw.OnError(err)
 
 		err = mysql.Generate("./.gentestdata/mysql", mysql.DBConnection{
-			Host:     dbconfig.MySqLHost,
+			Host:     host,
-			Port:     dbconfig.MySQLPort,
+			Port:     port,
-			User:     dbconfig.MySQLUser,
+			User:     user,
-			Password: dbconfig.MySQLPassword,
+			Password: pass,
 			DBName:   dbName,
 		})
 
@@ -104,7 +112,7 @@ func initMySQLDB() {
 func initPostgresDB() {
 	db, err := sql.Open("postgres", dbconfig.PostgresConnectString)
 	if err != nil {
-		panic("Failed to connect to test db")
+		panic("Failed to connect to test db: " + err.Error())
 	}
 	defer func() {
 		err := db.Close()

tests/postgres/generator_template_test.go

@@ -27,7 +27,7 @@ var defaultActorSQLBuilderFilePath = path.Join(tempTestDir, "jetdb/dvds/table",
 
 var dbConnection = postgres.DBConnection{
 	Host:       dbconfig.PgHost,
-	Port:       5432,
+	Port:       dbconfig.PgPort,
 	User:       dbconfig.PgUser,
 	Password:   dbconfig.PgPassword,
 	DBName:     dbconfig.PgDBName,

tests/postgres/generator_test.go

@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"testing"
 
 	"github.com/go-jet/jet/v2/generator/postgres"
@@ -52,8 +53,13 @@ func TestCmdGenerator(t *testing.T) {
 	err := os.RemoveAll(genTestDir2)
 	require.NoError(t, err)
 
-	cmd := exec.Command("jet", "-source=PostgreSQL", "-dbname=jetdb", "-host=localhost", "-port=5432",
+	cmd := exec.Command("jet", "-source=PostgreSQL", "-dbname=jetdb", "-host=localhost",
-		"-user=jet", "-password=jet", "-schema=dvds", "-path="+genTestDir2)
+		"-port="+strconv.Itoa(dbconfig.PgPort),
+		"-user=jet",
+		"-password=jet",
+		"-schema=dvds",
+		"-path="+genTestDir2)
+
 	cmd.Stderr = os.Stderr
 	cmd.Stdout = os.Stdout