From 7f48e9fb67070e4c029ae6b1f2a87d97e2d96cdb Mon Sep 17 00:00:00 2001 From: go-jet Date: Thu, 1 Feb 2024 15:20:49 +0100 Subject: [PATCH] Add support for materialized views. --- generator/metadata/column_meta_data.go | 1 + generator/postgres/query_set.go | 83 +++++++++++++--------- generator/template/sql_builder_template.go | 15 ++-- tests/docker-compose.yaml | 4 +- tests/postgres/alltypes_test.go | 51 ++++++++++--- tests/postgres/generator_test.go | 22 ++++-- tests/postgres/sample_test.go | 12 +++- tests/testdata | 2 +- 8 files changed, 127 insertions(+), 63 deletions(-) diff --git a/generator/metadata/column_meta_data.go b/generator/metadata/column_meta_data.go index 55533f4..679f6c1 100644 --- a/generator/metadata/column_meta_data.go +++ b/generator/metadata/column_meta_data.go @@ -33,6 +33,7 @@ const ( EnumType DataTypeKind = "enum" UserDefinedType DataTypeKind = "user-defined" ArrayType DataTypeKind = "array" + RangeType DataTypeKind = "range" ) // DataType contains information about column data type diff --git a/generator/postgres/query_set.go b/generator/postgres/query_set.go index 856173d..2d8835b 100644 --- a/generator/postgres/query_set.go +++ b/generator/postgres/query_set.go @@ -26,8 +26,25 @@ ORDER BY table_name; return nil, fmt.Errorf("failed to query %s metadata: %w", tableType, err) } + // add materialized views separately, because materialized views are not part of standard information schema + if tableType == metadata.ViewTable { + matViewQuery := ` + select matviewname as "table.name" + from pg_matviews + where schemaname = $1; + ` + var matViews []metadata.Table + + _, err := qrm.Query(context.Background(), db, matViewQuery, []interface{}{schemaName}, &matViews) + if err != nil { + return nil, fmt.Errorf("failed to query materialized view metadata: %w", err) + } + + tables = append(tables, matViews...) + } + for i := range tables { - tables[i].Columns, err = p.GetTableColumnsMetaData(db, schemaName, tables[i].Name) + tables[i].Columns, err = getColumnsMetaData(db, schemaName, tables[i].Name) if err != nil { return nil, fmt.Errorf("failed to query %s columns metadata: %w", tableType, err) } @@ -36,39 +53,39 @@ ORDER BY table_name; return tables, nil } -func (p postgresQuerySet) GetTableColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) { +func getColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]metadata.Column, error) { query := ` -WITH primaryKeys AS ( - SELECT column_name - FROM information_schema.key_column_usage AS c - LEFT JOIN information_schema.table_constraints AS t - ON t.constraint_name = c.constraint_name AND - c.table_schema = t.table_schema AND - c.table_name = t.table_name - WHERE t.table_schema = $1 AND t.table_name = $2 AND t.constraint_type = 'PRIMARY KEY' -) -SELECT column_name as "column.Name", - is_nullable = 'YES' as "column.isNullable", - is_generated = 'ALWAYS' or is_generated = 'YES' as "column.isGenerated", - (EXISTS(SELECT 1 from primaryKeys as pk where pk.column_name = columns.column_name)) as "column.IsPrimaryKey", - dataType.kind as "dataType.Kind", - (case dataType.Kind when 'base' then data_type else LTRIM(udt_name, '_') end) as "dataType.Name", - FALSE as "dataType.isUnsigned" -FROM information_schema.columns, - LATERAL (select (case data_type - when 'ARRAY' then 'array' - when 'USER-DEFINED' then - case (select t.typtype - from pg_type as t - join pg_namespace as p on p.oid = t.typnamespace - where t.typname = columns.udt_name and p.nspname = $1) - when 'e' then 'enum' - else 'user-defined' - end - else 'base' - end) as Kind) as dataType -where table_schema = $1 and table_name = $2 -order by ordinal_position; +select + attr.attname as "column.Name", + exists( + select 1 + from pg_catalog.pg_index indx + where attr.attrelid = indx.indrelid and attr.attnum = any(indx.indkey) and indx.indisprimary + ) as "column.IsPrimaryKey", + not attr.attnotnull as "column.isNullable", + attr.attgenerated = 's' as "column.isGenerated", + (case tp.typtype + when 'b' then 'base' + when 'd' then 'base' + when 'e' then 'enum' + when 'r' then 'range' + end) as "dataType.Kind", + (case when tp.typtype = 'd' then (select pg_type.typname from pg_catalog.pg_type where pg_type.oid = tp.typbasetype) + when tp.typcategory = 'A' then pg_catalog.format_type(attr.atttypid, attr.atttypmod) + else tp.typname + end) as "dataType.Name", + false as "dataType.isUnsigned" +from pg_catalog.pg_attribute as attr + join pg_catalog.pg_class as cls on cls.oid = attr.attrelid + join pg_catalog.pg_namespace as ns on ns.oid = cls.relnamespace + join pg_catalog.pg_type as tp on tp.oid = attr.atttypid +where + ns.nspname = $1 and + cls.relname = $2 and + not attr.attisdropped and + attr.attnum > 0 +order by + attr.attnum; ` var columns []metadata.Column _, err := qrm.Query(context.Background(), db, query, []interface{}{schemaName, tableName}, &columns) diff --git a/generator/template/sql_builder_template.go b/generator/template/sql_builder_template.go index c9ba7c4..869b8e5 100644 --- a/generator/template/sql_builder_template.go +++ b/generator/template/sql_builder_template.go @@ -142,14 +142,15 @@ func DefaultTableSQLBuilderColumn(columnMetaData metadata.Column) TableSQLBuilde // getSqlBuilderColumnType returns type of jet sql builder column func getSqlBuilderColumnType(columnMetaData metadata.Column) string { - if columnMetaData.DataType.Kind != metadata.BaseType { + if columnMetaData.DataType.Kind != metadata.BaseType && + columnMetaData.DataType.Kind != metadata.RangeType { return "String" } switch strings.ToLower(columnMetaData.DataType.Name) { - case "boolean": + case "boolean", "bool": return "Bool" - case "smallint", "integer", "bigint", + case "smallint", "integer", "bigint", "int2", "int4", "int8", "tinyint", "mediumint", "int", "year": //MySQL return "Integer" case "date": @@ -157,21 +158,21 @@ func getSqlBuilderColumnType(columnMetaData metadata.Column) string { case "timestamp without time zone", "timestamp", "datetime": //MySQL: return "Timestamp" - case "timestamp with time zone": + case "timestamp with time zone", "timestamptz": return "Timestampz" case "time without time zone", "time": //MySQL return "Time" - case "time with time zone": + case "time with time zone", "timetz": return "Timez" case "interval": return "Interval" case "user-defined", "enum", "text", "character", "character varying", "bytea", "uuid", "tsvector", "bit", "bit varying", "money", "json", "jsonb", "xml", "point", "line", "ARRAY", - "char", "varchar", "nvarchar", "binary", "varbinary", + "char", "varchar", "nvarchar", "binary", "varbinary", "bpchar", "varbit", "tinyblob", "blob", "mediumblob", "longblob", "tinytext", "mediumtext", "longtext": // MySQL return "String" - case "real", "numeric", "decimal", "double precision", "float", + case "real", "numeric", "decimal", "double precision", "float", "float4", "float8", "double": // MySQL return "Float" default: diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 9c562fb..9b3af50 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -39,14 +39,14 @@ services: - ./testdata/init/mysql:/docker-entrypoint-initdb.d cockroach: - image: cockroachdb/cockroach-unstable:v22.1.0-beta.4 + image: cockroachdb/cockroach-unstable:v23.1.0-rc.2 environment: - COCKROACH_USER=jet - COCKROACH_PASSWORD=jet - COCKROACH_DATABASE=jetdb ports: - "26257:26257" - command: start-single-node --insecure + command: start-single-node --accept-sql-without-tls # volumes: # - ./testdata/init/cockroach:/docker-entrypoint-initdb.d diff --git a/tests/postgres/alltypes_test.go b/tests/postgres/alltypes_test.go index 8a11191..d41feee 100644 --- a/tests/postgres/alltypes_test.go +++ b/tests/postgres/alltypes_test.go @@ -17,31 +17,52 @@ import ( "github.com/go-jet/jet/v2/tests/testdata/results/common" ) +var AllTypesAllColumns = AllTypes.AllColumns. + Except(IntegerColumn("rowid")) // cockroachDB: exclude rowid column + func TestAllTypesSelect(t *testing.T) { var dest []model.AllTypes - err := AllTypes.SELECT( - AllTypesAllColumns, - ).LIMIT(2). + err := AllTypes.SELECT(AllTypesAllColumns). + LIMIT(2). Query(db, &dest) - require.NoError(t, err) + require.NoError(t, err) testutils.AssertDeepEqual(t, dest[0], allTypesRow0) testutils.AssertDeepEqual(t, dest[1], allTypesRow1) } func TestAllTypesViewSelect(t *testing.T) { type AllTypesView model.AllTypes - var dest []AllTypesView - err := view.AllTypesView.SELECT(view.AllTypesView.AllColumns).Query(db, &dest) - require.NoError(t, err) + err := SELECT(view.AllTypesView.AllColumns). + FROM(view.AllTypesView). + Query(db, &dest) + require.NoError(t, err) testutils.AssertDeepEqual(t, dest[0], AllTypesView(allTypesRow0)) testutils.AssertDeepEqual(t, dest[1], AllTypesView(allTypesRow1)) } +func TestMaterializedViewAllTypes(t *testing.T) { + stmt := SELECT( + view.AllTypesMaterializedView.AllColumns. + Except(IntegerColumn("rowid")), // cockroachDB: exclude rowid column + ).FROM( + view.AllTypesMaterializedView, + ) + + type AllTypesMaterializedView model.AllTypes + var dest []AllTypesMaterializedView + + err := stmt.Query(db, &dest) + + require.NoError(t, err) + testutils.AssertDeepEqual(t, dest[0], AllTypesMaterializedView(allTypesRow0)) + testutils.AssertDeepEqual(t, dest[1], AllTypesMaterializedView(allTypesRow1)) +} + func TestAllTypesInsertModel(t *testing.T) { skipForPgxDriver(t) // pgx driver bug ERROR: date/time field value out of range: "0000-01-01 12:05:06Z" (SQLSTATE 22008) @@ -64,8 +85,6 @@ func TestAllTypesInsertModel(t *testing.T) { }) } -var AllTypesAllColumns = AllTypes.AllColumns.Except(IntegerColumn("rowid")) - func TestAllTypesInsertQuery(t *testing.T) { query := AllTypes.INSERT(AllTypesAllColumns). QUERY( @@ -230,7 +249,9 @@ SELECT "allTypesSubQuery"."all_types.small_int_ptr" AS "all_types.small_int_ptr" "allTypesSubQuery"."all_types.text_array" AS "all_types.text_array", "allTypesSubQuery"."all_types.jsonb_array" AS "all_types.jsonb_array", "allTypesSubQuery"."all_types.text_multi_dim_array_ptr" AS "all_types.text_multi_dim_array_ptr", - "allTypesSubQuery"."all_types.text_multi_dim_array" AS "all_types.text_multi_dim_array" + "allTypesSubQuery"."all_types.text_multi_dim_array" AS "all_types.text_multi_dim_array", + "allTypesSubQuery"."all_types.mood_ptr" AS "all_types.mood_ptr", + "allTypesSubQuery"."all_types.mood" AS "all_types.mood" FROM ( SELECT all_types.small_int_ptr AS "all_types.small_int_ptr", all_types.small_int AS "all_types.small_int", @@ -292,7 +313,9 @@ FROM ( all_types.text_array AS "all_types.text_array", all_types.jsonb_array AS "all_types.jsonb_array", all_types.text_multi_dim_array_ptr AS "all_types.text_multi_dim_array_ptr", - all_types.text_multi_dim_array AS "all_types.text_multi_dim_array" + all_types.text_multi_dim_array AS "all_types.text_multi_dim_array", + all_types.mood_ptr AS "all_types.mood_ptr", + all_types.mood AS "all_types.mood" FROM test_sample.all_types ) AS "allTypesSubQuery" LIMIT 2; @@ -1279,6 +1302,8 @@ RETURNING all_types.json AS "all_types.json"; }) } +var moodSad = model.Mood_Sad + var allTypesRow0 = model.AllTypes{ SmallIntPtr: testutils.Int16Ptr(14), SmallInt: 14, @@ -1343,6 +1368,8 @@ var allTypesRow0 = model.AllTypes{ JsonbArray: `{"{\"a\": 1, \"b\": 2}","{\"a\": 3, \"b\": 4}"}`, TextMultiDimArrayPtr: testutils.StringPtr("{{meeting,lunch},{training,presentation}}"), TextMultiDimArray: "{{meeting,lunch},{training,presentation}}", + MoodPtr: &moodSad, + Mood: model.Mood_Happy, } var allTypesRow1 = model.AllTypes{ @@ -1409,6 +1436,8 @@ var allTypesRow1 = model.AllTypes{ JsonbArray: `{"{\"a\": 1, \"b\": 2}","{\"a\": 3, \"b\": 4}"}`, TextMultiDimArrayPtr: nil, TextMultiDimArray: "{{meeting,lunch},{training,presentation}}", + MoodPtr: nil, + Mood: model.Mood_Ok, } func TestAliasedDuplicateSliceSubType(t *testing.T) { diff --git a/tests/postgres/generator_test.go b/tests/postgres/generator_test.go index a0257e7..5aaaa31 100644 --- a/tests/postgres/generator_test.go +++ b/tests/postgres/generator_test.go @@ -548,11 +548,12 @@ func UseSchema(schema string) { ` func TestGeneratedAllTypesSQLBuilderFiles(t *testing.T) { - skipForCockroachDB(t) + skipForCockroachDB(t) // because of rowid column enumDir := filepath.Join(testRoot, "/.gentestdata/jetdb/test_sample/enum/") modelDir := filepath.Join(testRoot, "/.gentestdata/jetdb/test_sample/model/") tableDir := filepath.Join(testRoot, "/.gentestdata/jetdb/test_sample/table/") + viewDir := filepath.Join(testRoot, "/.gentestdata/jetdb/test_sample/view/") testutils.AssertFileNamesEqual(t, enumDir, "mood.go", "level.go") testutils.AssertFileContent(t, enumDir+"/mood.go", moodEnumContent) @@ -560,15 +561,16 @@ func TestGeneratedAllTypesSQLBuilderFiles(t *testing.T) { testutils.AssertFileNamesEqual(t, modelDir, "all_types.go", "all_types_view.go", "employee.go", "link.go", "mood.go", "person.go", "person_phone.go", "weird_names_table.go", "level.go", "user.go", "floats.go", "people.go", - "components.go", "vulnerabilities.go") - + "components.go", "vulnerabilities.go", "all_types_materialized_view.go") testutils.AssertFileContent(t, modelDir+"/all_types.go", allTypesModelContent) testutils.AssertFileNamesEqual(t, tableDir, "all_types.go", "employee.go", "link.go", "person.go", "person_phone.go", "weird_names_table.go", "user.go", "floats.go", "people.go", "table_use_schema.go", "components.go", "vulnerabilities.go") - testutils.AssertFileContent(t, tableDir+"/all_types.go", allTypesTableContent) + + testutils.AssertFileNamesEqual(t, viewDir, "all_types_materialized_view.go", "all_types_view.go", + "view_use_schema.go") } var moodEnumContent = ` @@ -698,6 +700,8 @@ type AllTypes struct { JsonbArray string TextMultiDimArrayPtr *string TextMultiDimArray string + MoodPtr *Mood + Mood Mood } ` @@ -782,6 +786,8 @@ type allTypesTable struct { JsonbArray postgres.ColumnString TextMultiDimArrayPtr postgres.ColumnString TextMultiDimArray postgres.ColumnString + MoodPtr postgres.ColumnString + Mood postgres.ColumnString AllColumns postgres.ColumnList MutableColumns postgres.ColumnList @@ -883,8 +889,10 @@ func newAllTypesTableImpl(schemaName, tableName, alias string) allTypesTable { JsonbArrayColumn = postgres.StringColumn("jsonb_array") TextMultiDimArrayPtrColumn = postgres.StringColumn("text_multi_dim_array_ptr") TextMultiDimArrayColumn = postgres.StringColumn("text_multi_dim_array") - allColumns = postgres.ColumnList{SmallIntPtrColumn, SmallIntColumn, IntegerPtrColumn, IntegerColumn, BigIntPtrColumn, BigIntColumn, DecimalPtrColumn, DecimalColumn, NumericPtrColumn, NumericColumn, RealPtrColumn, RealColumn, DoublePrecisionPtrColumn, DoublePrecisionColumn, SmallserialColumn, SerialColumn, BigserialColumn, VarCharPtrColumn, VarCharColumn, CharPtrColumn, CharColumn, TextPtrColumn, TextColumn, ByteaPtrColumn, ByteaColumn, TimestampzPtrColumn, TimestampzColumn, TimestampPtrColumn, TimestampColumn, DatePtrColumn, DateColumn, TimezPtrColumn, TimezColumn, TimePtrColumn, TimeColumn, IntervalPtrColumn, IntervalColumn, BooleanPtrColumn, BooleanColumn, PointPtrColumn, BitPtrColumn, BitColumn, BitVaryingPtrColumn, BitVaryingColumn, TsvectorPtrColumn, TsvectorColumn, UUIDPtrColumn, UUIDColumn, XMLPtrColumn, XMLColumn, JSONPtrColumn, JSONColumn, JsonbPtrColumn, JsonbColumn, IntegerArrayPtrColumn, IntegerArrayColumn, TextArrayPtrColumn, TextArrayColumn, JsonbArrayColumn, TextMultiDimArrayPtrColumn, TextMultiDimArrayColumn} - mutableColumns = postgres.ColumnList{SmallIntPtrColumn, SmallIntColumn, IntegerPtrColumn, IntegerColumn, BigIntPtrColumn, BigIntColumn, DecimalPtrColumn, DecimalColumn, NumericPtrColumn, NumericColumn, RealPtrColumn, RealColumn, DoublePrecisionPtrColumn, DoublePrecisionColumn, SmallserialColumn, SerialColumn, BigserialColumn, VarCharPtrColumn, VarCharColumn, CharPtrColumn, CharColumn, TextPtrColumn, TextColumn, ByteaPtrColumn, ByteaColumn, TimestampzPtrColumn, TimestampzColumn, TimestampPtrColumn, TimestampColumn, DatePtrColumn, DateColumn, TimezPtrColumn, TimezColumn, TimePtrColumn, TimeColumn, IntervalPtrColumn, IntervalColumn, BooleanPtrColumn, BooleanColumn, PointPtrColumn, BitPtrColumn, BitColumn, BitVaryingPtrColumn, BitVaryingColumn, TsvectorPtrColumn, TsvectorColumn, UUIDPtrColumn, UUIDColumn, XMLPtrColumn, XMLColumn, JSONPtrColumn, JSONColumn, JsonbPtrColumn, JsonbColumn, IntegerArrayPtrColumn, IntegerArrayColumn, TextArrayPtrColumn, TextArrayColumn, JsonbArrayColumn, TextMultiDimArrayPtrColumn, TextMultiDimArrayColumn} + MoodPtrColumn = postgres.StringColumn("mood_ptr") + MoodColumn = postgres.StringColumn("mood") + allColumns = postgres.ColumnList{SmallIntPtrColumn, SmallIntColumn, IntegerPtrColumn, IntegerColumn, BigIntPtrColumn, BigIntColumn, DecimalPtrColumn, DecimalColumn, NumericPtrColumn, NumericColumn, RealPtrColumn, RealColumn, DoublePrecisionPtrColumn, DoublePrecisionColumn, SmallserialColumn, SerialColumn, BigserialColumn, VarCharPtrColumn, VarCharColumn, CharPtrColumn, CharColumn, TextPtrColumn, TextColumn, ByteaPtrColumn, ByteaColumn, TimestampzPtrColumn, TimestampzColumn, TimestampPtrColumn, TimestampColumn, DatePtrColumn, DateColumn, TimezPtrColumn, TimezColumn, TimePtrColumn, TimeColumn, IntervalPtrColumn, IntervalColumn, BooleanPtrColumn, BooleanColumn, PointPtrColumn, BitPtrColumn, BitColumn, BitVaryingPtrColumn, BitVaryingColumn, TsvectorPtrColumn, TsvectorColumn, UUIDPtrColumn, UUIDColumn, XMLPtrColumn, XMLColumn, JSONPtrColumn, JSONColumn, JsonbPtrColumn, JsonbColumn, IntegerArrayPtrColumn, IntegerArrayColumn, TextArrayPtrColumn, TextArrayColumn, JsonbArrayColumn, TextMultiDimArrayPtrColumn, TextMultiDimArrayColumn, MoodPtrColumn, MoodColumn} + mutableColumns = postgres.ColumnList{SmallIntPtrColumn, SmallIntColumn, IntegerPtrColumn, IntegerColumn, BigIntPtrColumn, BigIntColumn, DecimalPtrColumn, DecimalColumn, NumericPtrColumn, NumericColumn, RealPtrColumn, RealColumn, DoublePrecisionPtrColumn, DoublePrecisionColumn, SmallserialColumn, SerialColumn, BigserialColumn, VarCharPtrColumn, VarCharColumn, CharPtrColumn, CharColumn, TextPtrColumn, TextColumn, ByteaPtrColumn, ByteaColumn, TimestampzPtrColumn, TimestampzColumn, TimestampPtrColumn, TimestampColumn, DatePtrColumn, DateColumn, TimezPtrColumn, TimezColumn, TimePtrColumn, TimeColumn, IntervalPtrColumn, IntervalColumn, BooleanPtrColumn, BooleanColumn, PointPtrColumn, BitPtrColumn, BitColumn, BitVaryingPtrColumn, BitVaryingColumn, TsvectorPtrColumn, TsvectorColumn, UUIDPtrColumn, UUIDColumn, XMLPtrColumn, XMLColumn, JSONPtrColumn, JSONColumn, JsonbPtrColumn, JsonbColumn, IntegerArrayPtrColumn, IntegerArrayColumn, TextArrayPtrColumn, TextArrayColumn, JsonbArrayColumn, TextMultiDimArrayPtrColumn, TextMultiDimArrayColumn, MoodPtrColumn, MoodColumn} ) return allTypesTable{ @@ -952,6 +960,8 @@ func newAllTypesTableImpl(schemaName, tableName, alias string) allTypesTable { JsonbArray: JsonbArrayColumn, TextMultiDimArrayPtr: TextMultiDimArrayPtrColumn, TextMultiDimArray: TextMultiDimArrayColumn, + MoodPtr: MoodPtrColumn, + Mood: MoodColumn, AllColumns: allColumns, MutableColumns: mutableColumns, diff --git a/tests/postgres/sample_test.go b/tests/postgres/sample_test.go index 8bc8105..1df72dd 100644 --- a/tests/postgres/sample_test.go +++ b/tests/postgres/sample_test.go @@ -120,9 +120,15 @@ RETURNING floats.decimal_ptr AS "floats.decimal_ptr", } func TestUUIDComplex(t *testing.T) { - query := Person.INNER_JOIN(PersonPhone, PersonPhone.PersonID.EQ(Person.PersonID)). - SELECT(Person.AllColumns, PersonPhone.AllColumns). - ORDER_BY(Person.PersonID.ASC(), PersonPhone.PhoneID.ASC()) + query := SELECT( + Person.AllColumns, + PersonPhone.AllColumns, + ).FROM( + Person.INNER_JOIN(PersonPhone, PersonPhone.PersonID.EQ(Person.PersonID)), + ).ORDER_BY( + Person.PersonID.ASC(), + PersonPhone.PhoneID.ASC(), + ) t.Run("slice of structs", func(t *testing.T) { diff --git a/tests/testdata b/tests/testdata index 3398b97..b2f98e8 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 3398b9735b9d097d2ee0c282976726affc6b96f0 +Subproject commit b2f98e8297c34e86e02ada4226c125de53f64a6d