From 929109622e7a3a6fa6125048cee531da14d71e2d Mon Sep 17 00:00:00 2001 From: Volker Lieber <42102008+VolkerLieber@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:34:34 +0200 Subject: [PATCH 1/6] Include postgres comments in output #391 --- generator/postgres/query_set.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/postgres/query_set.go b/generator/postgres/query_set.go index abb21ba..d549cf1 100644 --- a/generator/postgres/query_set.go +++ b/generator/postgres/query_set.go @@ -57,6 +57,7 @@ func getColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]meta query := ` select attr.attname as "column.Name", + dsc.description as "column.Comment", exists( select 1 from pg_catalog.pg_index indx @@ -81,6 +82,7 @@ from pg_catalog.pg_attribute as attr join pg_catalog.pg_class as cls on cls.oid = attr.attrelid join pg_catalog.pg_namespace as ns on ns.oid = cls.relnamespace join pg_catalog.pg_type as tp on tp.oid = attr.atttypid + left join pg_catalog.pg_description as dsc on dsc.objoid = attr.attrelid and dsc.objsubid = attr.attnum where ns.nspname = $1 and cls.relname = $2 and From ff82eb5df7f1e971a1d32956a9bbf23f2d976884 Mon Sep 17 00:00:00 2001 From: Volker Lieber <42102008+VolkerLieber@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:41:27 +0200 Subject: [PATCH 2/6] Implemented postgres table and enum comment generation --- generator/metadata/enum_meta_data.go | 17 +++++++++++++++-- generator/metadata/table_meta_data.go | 13 +++++++++++++ generator/postgres/query_set.go | 2 +- generator/template/file_templates.go | 4 ++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/generator/metadata/enum_meta_data.go b/generator/metadata/enum_meta_data.go index 7aea3d6..733150b 100644 --- a/generator/metadata/enum_meta_data.go +++ b/generator/metadata/enum_meta_data.go @@ -1,7 +1,20 @@ package metadata +import "regexp" + // Enum metadata struct type Enum struct { - Name string `sql:"primary_key"` - Values []string + Name string `sql:"primary_key"` + Comment string + Values []string +} + +// GoLangComment returns enum comment without ascii control characters +func (e Enum) GoLangComment() string { + if e.Comment == "" { + return "" + } + + // remove ascii control characters from string + return regexp.MustCompile(`[[:cntrl:]]+`).ReplaceAllString(e.Comment, "") } diff --git a/generator/metadata/table_meta_data.go b/generator/metadata/table_meta_data.go index df9514e..95fd3c4 100644 --- a/generator/metadata/table_meta_data.go +++ b/generator/metadata/table_meta_data.go @@ -1,8 +1,11 @@ package metadata +import "regexp" + // Table metadata struct type Table struct { Name string `sql:"primary_key"` + Comment string Columns []Column } @@ -20,3 +23,13 @@ func (t Table) MutableColumns() []Column { return ret } + +// GoLangComment returns table comment without ascii control characters +func (t Table) GoLangComment() string { + if t.Comment == "" { + return "" + } + + // remove ascii control characters from string + return regexp.MustCompile(`[[:cntrl:]]+`).ReplaceAllString(t.Comment, "") +} diff --git a/generator/postgres/query_set.go b/generator/postgres/query_set.go index d549cf1..50129d1 100644 --- a/generator/postgres/query_set.go +++ b/generator/postgres/query_set.go @@ -14,7 +14,7 @@ type postgresQuerySet struct{} func (p postgresQuerySet) GetTablesMetaData(db *sql.DB, schemaName string, tableType metadata.TableType) ([]metadata.Table, error) { query := ` -SELECT table_name as "table.name" +SELECT table_name as "table.name", obj_description((table_schema||'.'||quote_ident(table_name))::regclass) as "table.comment" FROM information_schema.tables WHERE table_schema = $1 and table_type = $2 ORDER BY table_name; diff --git a/generator/template/file_templates.go b/generator/template/file_templates.go index f3aa505..45104db 100644 --- a/generator/template/file_templates.go +++ b/generator/template/file_templates.go @@ -26,6 +26,7 @@ import ( var {{tableTemplate.InstanceName}} = new{{tableTemplate.TypeName}}("{{schemaName}}", "{{.Name}}", "{{tableTemplate.DefaultAlias}}") +{{if .Comment }} // {{.GoLangComment}} {{end}} type {{structImplName}} struct { {{dialect.PackageName}}.Table @@ -119,6 +120,7 @@ import ( {{end}} {{$modelTableTemplate := tableTemplate}} +{{if .Comment }} // {{.GoLangComment}} {{end}} type {{$modelTableTemplate.TypeName}} struct { {{- range .Columns}} {{- $field := structField .}} @@ -132,6 +134,7 @@ var enumSQLBuilderTemplate = `package {{package}} import "github.com/go-jet/jet/v2/{{dialect.PackageName}}" +{{if .Comment }} // {{.GoLangComment}} {{end}} var {{enumTemplate.InstanceName}} = &struct { {{- range $index, $value := .Values}} {{enumValueName $value}} {{dialect.PackageName}}.StringExpression @@ -148,6 +151,7 @@ var enumModelTemplate = `package {{package}} import "errors" +{{if .Comment }} // {{.GoLangComment}} {{end}} type {{$enumTemplate.TypeName}} string const ( From 0f21699a1f66480f2fa3093b18e736cf01bff7f8 Mon Sep 17 00:00:00 2001 From: Volker Lieber <42102008+VolkerLieber@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:42:27 +0200 Subject: [PATCH 3/6] Implemented postgres enum comment generation --- generator/postgres/query_set.go | 1 + 1 file changed, 1 insertion(+) diff --git a/generator/postgres/query_set.go b/generator/postgres/query_set.go index 50129d1..4474c56 100644 --- a/generator/postgres/query_set.go +++ b/generator/postgres/query_set.go @@ -103,6 +103,7 @@ order by func (p postgresQuerySet) GetEnumsMetaData(db *sql.DB, schemaName string) ([]metadata.Enum, error) { query := ` SELECT t.typname as "enum.name", + obj_description(t.oid) as "enum.comment", e.enumlabel as "values" FROM pg_catalog.pg_type t JOIN pg_catalog.pg_enum e on t.oid = e.enumtypid From 30e02dc9c07bbc0bf89f36d27f9c00b9b84b8123 Mon Sep 17 00:00:00 2001 From: Volker Lieber <42102008+VolkerLieber@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:45:06 +0200 Subject: [PATCH 4/6] Implemented postgres generator comment tests https://github.com/go-jet/jet-test-data/pull/6 --- tests/postgres/generator_test.go | 111 +++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/tests/postgres/generator_test.go b/tests/postgres/generator_test.go index fe1407f..87a7eee 100644 --- a/tests/postgres/generator_test.go +++ b/tests/postgres/generator_test.go @@ -604,6 +604,7 @@ func TestGeneratedAllTypesSQLBuilderFiles(t *testing.T) { "mood.go", "person.go", "person_phone.go", "weird_names_table.go", "level.go", "user.go", "floats.go", "people.go", "components.go", "vulnerabilities.go", "all_types_materialized_view.go", "sample_ranges.go") testutils.AssertFileContent(t, modelDir+"/all_types.go", allTypesModelContent) + testutils.AssertFileContent(t, modelDir+"/link.go", linkModelContent) testutils.AssertFileNamesEqual(t, tableDir, "all_types.go", "employee.go", "link.go", "person.go", "person_phone.go", "weird_names_table.go", "user.go", "floats.go", "people.go", "table_use_schema.go", @@ -611,6 +612,8 @@ func TestGeneratedAllTypesSQLBuilderFiles(t *testing.T) { testutils.AssertFileContent(t, tableDir+"/all_types.go", allTypesTableContent) testutils.AssertFileContent(t, tableDir+"/sample_ranges.go", sampleRangeTableContent) + testutils.AssertFileContent(t, tableDir+"/link.go", linkTableContent) + testutils.AssertFileNamesEqual(t, viewDir, "all_types_materialized_view.go", "all_types_view.go", "view_use_schema.go") } @@ -650,6 +653,7 @@ package enum import "github.com/go-jet/jet/v2/postgres" +// Level enum var Level = &struct { Level1 postgres.StringExpression Level2 postgres.StringExpression @@ -747,6 +751,25 @@ type AllTypes struct { } ` +var linkModelContent = ` +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +// Link table +type Link struct { + ID int64 ` + "`sql:\"primary_key\"`" + ` // this is link id + URL string // link url + Name string // Unicode characters comment ₲鬼佬℧⇄↻ + Description *string // '"Z\%_ +} +` + var allTypesTableContent = ` // // Code generated by go-jet DO NOT EDIT. @@ -1103,3 +1126,91 @@ func newSampleRangesTableImpl(schemaName, tableName, alias string) sampleRangesT } } ` + +var linkTableContent = ` +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var Link = newLinkTable("test_sample", "link", "") + +// Link table +type linkTable struct { + postgres.Table + + // Columns + ID postgres.ColumnInteger // this is link id + URL postgres.ColumnString // link url + Name postgres.ColumnString // Unicode characters comment ₲鬼佬℧⇄↻ + Description postgres.ColumnString // '"Z\%_ + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type LinkTable struct { + linkTable + + EXCLUDED linkTable +} + +// AS creates new LinkTable with assigned alias +func (a LinkTable) AS(alias string) *LinkTable { + return newLinkTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new LinkTable with assigned schema name +func (a LinkTable) FromSchema(schemaName string) *LinkTable { + return newLinkTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new LinkTable with assigned table prefix +func (a LinkTable) WithPrefix(prefix string) *LinkTable { + return newLinkTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new LinkTable with assigned table suffix +func (a LinkTable) WithSuffix(suffix string) *LinkTable { + return newLinkTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newLinkTable(schemaName, tableName, alias string) *LinkTable { + return &LinkTable{ + linkTable: newLinkTableImpl(schemaName, tableName, alias), + EXCLUDED: newLinkTableImpl("", "excluded", ""), + } +} + +func newLinkTableImpl(schemaName, tableName, alias string) linkTable { + var ( + IDColumn = postgres.IntegerColumn("id") + URLColumn = postgres.StringColumn("url") + NameColumn = postgres.StringColumn("name") + DescriptionColumn = postgres.StringColumn("description") + allColumns = postgres.ColumnList{IDColumn, URLColumn, NameColumn, DescriptionColumn} + mutableColumns = postgres.ColumnList{URLColumn, NameColumn, DescriptionColumn} + ) + + return linkTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + URL: URLColumn, + Name: NameColumn, + Description: DescriptionColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} +` From 64884e496971236fe98fa10e9b0857ae15b127bf Mon Sep 17 00:00:00 2001 From: Volker Lieber <42102008+VolkerLieber@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:17:21 +0200 Subject: [PATCH 5/6] Extracted golang comment format function --- generator/metadata/column_meta_data.go | 14 ------------- generator/metadata/enum_meta_data.go | 12 ------------ generator/metadata/table_meta_data.go | 12 ------------ generator/template/file_templates.go | 12 ++++++------ generator/template/format.go | 13 +++++++++++++ generator/template/format_test.go | 27 ++++++++++++++++++++++++++ generator/template/process.go | 4 ++++ 7 files changed, 50 insertions(+), 44 deletions(-) create mode 100644 generator/template/format.go create mode 100644 generator/template/format_test.go diff --git a/generator/metadata/column_meta_data.go b/generator/metadata/column_meta_data.go index 1502719..ecd61e2 100644 --- a/generator/metadata/column_meta_data.go +++ b/generator/metadata/column_meta_data.go @@ -1,9 +1,5 @@ package metadata -import ( - "regexp" -) - // Column struct type Column struct { Name string `sql:"primary_key"` @@ -15,16 +11,6 @@ type Column struct { Comment string } -// GoLangComment returns column comment without ascii control characters -func (c Column) GoLangComment() string { - if c.Comment == "" { - return "" - } - - // remove ascii control characters from string - return regexp.MustCompile(`[[:cntrl:]]+`).ReplaceAllString(c.Comment, "") -} - // DataTypeKind is database type kind(base, enum, user-defined, array) type DataTypeKind string diff --git a/generator/metadata/enum_meta_data.go b/generator/metadata/enum_meta_data.go index 733150b..8cce596 100644 --- a/generator/metadata/enum_meta_data.go +++ b/generator/metadata/enum_meta_data.go @@ -1,20 +1,8 @@ package metadata -import "regexp" - // Enum metadata struct type Enum struct { Name string `sql:"primary_key"` Comment string Values []string } - -// GoLangComment returns enum comment without ascii control characters -func (e Enum) GoLangComment() string { - if e.Comment == "" { - return "" - } - - // remove ascii control characters from string - return regexp.MustCompile(`[[:cntrl:]]+`).ReplaceAllString(e.Comment, "") -} diff --git a/generator/metadata/table_meta_data.go b/generator/metadata/table_meta_data.go index 95fd3c4..1d56bc0 100644 --- a/generator/metadata/table_meta_data.go +++ b/generator/metadata/table_meta_data.go @@ -1,7 +1,5 @@ package metadata -import "regexp" - // Table metadata struct type Table struct { Name string `sql:"primary_key"` @@ -23,13 +21,3 @@ func (t Table) MutableColumns() []Column { return ret } - -// GoLangComment returns table comment without ascii control characters -func (t Table) GoLangComment() string { - if t.Comment == "" { - return "" - } - - // remove ascii control characters from string - return regexp.MustCompile(`[[:cntrl:]]+`).ReplaceAllString(t.Comment, "") -} diff --git a/generator/template/file_templates.go b/generator/template/file_templates.go index 45104db..1538031 100644 --- a/generator/template/file_templates.go +++ b/generator/template/file_templates.go @@ -26,14 +26,14 @@ import ( var {{tableTemplate.InstanceName}} = new{{tableTemplate.TypeName}}("{{schemaName}}", "{{.Name}}", "{{tableTemplate.DefaultAlias}}") -{{if .Comment }} // {{.GoLangComment}} {{end}} +{{golangComment .Comment}} type {{structImplName}} struct { {{dialect.PackageName}}.Table // Columns {{- range $i, $c := .Columns}} {{- $field := columnField $c}} - {{$field.Name}} {{dialect.PackageName}}.Column{{$field.Type}} {{- if $c.Comment }} // {{$c.GoLangComment}} {{end}} + {{$field.Name}} {{dialect.PackageName}}.Column{{$field.Type}} {{golangComment .Comment}} {{- end}} AllColumns {{dialect.PackageName}}.ColumnList @@ -120,11 +120,11 @@ import ( {{end}} {{$modelTableTemplate := tableTemplate}} -{{if .Comment }} // {{.GoLangComment}} {{end}} +{{golangComment .Comment}} type {{$modelTableTemplate.TypeName}} struct { {{- range .Columns}} {{- $field := structField .}} - {{$field.Name}} {{$field.Type.Name}} ` + "{{$field.TagsString}}" + ` {{- if .Comment }} // {{.GoLangComment}} {{end}} + {{$field.Name}} {{$field.Type.Name}} ` + "{{$field.TagsString}}" + ` {{golangComment .Comment}} {{- end}} } @@ -134,7 +134,7 @@ var enumSQLBuilderTemplate = `package {{package}} import "github.com/go-jet/jet/v2/{{dialect.PackageName}}" -{{if .Comment }} // {{.GoLangComment}} {{end}} +{{golangComment .Comment}} var {{enumTemplate.InstanceName}} = &struct { {{- range $index, $value := .Values}} {{enumValueName $value}} {{dialect.PackageName}}.StringExpression @@ -151,7 +151,7 @@ var enumModelTemplate = `package {{package}} import "errors" -{{if .Comment }} // {{.GoLangComment}} {{end}} +{{golangComment .Comment}} type {{$enumTemplate.TypeName}} string const ( diff --git a/generator/template/format.go b/generator/template/format.go new file mode 100644 index 0000000..bd88fb1 --- /dev/null +++ b/generator/template/format.go @@ -0,0 +1,13 @@ +package template + +import "regexp" + +// Returns the provided string as golang comment without ascii control characters +func formatGolangComment(comment string) string { + if len(comment) == 0 { + return "" + } + + // Format as colang comment and remove ascii control characters from string + return "// " + regexp.MustCompile(`[[:cntrl:]]+`).ReplaceAllString(comment, "") +} diff --git a/generator/template/format_test.go b/generator/template/format_test.go new file mode 100644 index 0000000..b43b61d --- /dev/null +++ b/generator/template/format_test.go @@ -0,0 +1,27 @@ +package template + +import "testing" + +func Test_formatGolangComment(t *testing.T) { + type args struct { + comment string + } + tests := []struct { + name string + args args + want string + }{ + {name: "Empty string", args: args{comment: ""}, want: ""}, + {name: "Non-empty string", args: args{comment: "This is a comment"}, want: "// This is a comment"}, + {name: "String with control characters", args: args{comment: "This is a comment with control characters \x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f and text after"}, want: "// This is a comment with control characters and text after"}, + {name: "String with escape characters", args: args{comment: "This is a comment with escape characters \n\r\t and text after"}, want: "// This is a comment with escape characters and text after"}, + {name: "String with unicode characters", args: args{comment: "This is a comment with unicode characters ₲鬼佬℧⇄↻ and text after"}, want: "// This is a comment with unicode characters ₲鬼佬℧⇄↻ and text after"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatGolangComment(tt.args.comment); got != tt.want { + t.Errorf("formatGoLangComment() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/generator/template/process.go b/generator/template/process.go index 3f3798a..5abef87 100644 --- a/generator/template/process.go +++ b/generator/template/process.go @@ -140,6 +140,7 @@ func processEnumSQLBuilder(dirPath string, dialect jet.Dialect, enumsMetaData [] "enumValueName": func(enumValue string) string { return enumTemplate.ValueName(enumValue) }, + "golangComment": formatGolangComment, }) if err != nil { return fmt.Errorf("failed to generete enum type %s: %w", enumTemplate.FileName, err) @@ -215,6 +216,7 @@ func processTableSQLBuilder(fileTypes, dirPath string, "insertedRowAlias": func() string { return insertedRowAlias(dialect) }, + "golangComment": formatGolangComment, }) if err != nil { return fmt.Errorf("failed to generate table sql builder type %s: %w", tableSQLBuilder.TypeName, err) @@ -307,6 +309,7 @@ func processTableModels(fileTypes, modelDirPath string, tablesMetaData []metadat "structField": func(columnMetaData metadata.Column) TableModelField { return tableTemplate.Field(columnMetaData) }, + "golangComment": formatGolangComment, }) if err != nil { return fmt.Errorf("failed to generate model type '%s': %w", tableMetaData.Name, err) @@ -347,6 +350,7 @@ func processEnumModels(modelDir string, enumsMetaData []metadata.Enum, modelTemp "valueName": func(value string) string { return enumTemplate.ValueName(value) }, + "golangComment": formatGolangComment, }) if err != nil { From b5f04ffea8c66718ee104fe1ee1ee6d8ef07aa0b Mon Sep 17 00:00:00 2001 From: Volker Lieber <42102008+VolkerLieber@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:33:23 +0200 Subject: [PATCH 6/6] Improved postgres table comment generation --- generator/postgres/query_set.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/generator/postgres/query_set.go b/generator/postgres/query_set.go index 4474c56..fc4135a 100644 --- a/generator/postgres/query_set.go +++ b/generator/postgres/query_set.go @@ -14,7 +14,7 @@ type postgresQuerySet struct{} func (p postgresQuerySet) GetTablesMetaData(db *sql.DB, schemaName string, tableType metadata.TableType) ([]metadata.Table, error) { query := ` -SELECT table_name as "table.name", obj_description((table_schema||'.'||quote_ident(table_name))::regclass) as "table.comment" +SELECT table_name as "table.name", obj_description((quote_ident(table_schema)||'.'||quote_ident(table_name))::regclass) as "table.comment" FROM information_schema.tables WHERE table_schema = $1 and table_type = $2 ORDER BY table_name; @@ -57,7 +57,7 @@ func getColumnsMetaData(db *sql.DB, schemaName string, tableName string) ([]meta query := ` select attr.attname as "column.Name", - dsc.description as "column.Comment", + col_description(attr.attrelid, attr.attnum) as "column.Comment", exists( select 1 from pg_catalog.pg_index indx @@ -82,7 +82,6 @@ from pg_catalog.pg_attribute as attr join pg_catalog.pg_class as cls on cls.oid = attr.attrelid join pg_catalog.pg_namespace as ns on ns.oid = cls.relnamespace join pg_catalog.pg_type as tp on tp.oid = attr.atttypid - left join pg_catalog.pg_description as dsc on dsc.objoid = attr.attrelid and dsc.objsubid = attr.attnum where ns.nspname = $1 and cls.relname = $2 and