From 7b16e432ff9dc714f795cf462f11a6e5da1916ab Mon Sep 17 00:00:00 2001 From: go-jet Date: Fri, 21 Feb 2025 19:55:01 +0100 Subject: [PATCH 01/18] Add support for SELECT_JSON statements. --- .circleci/config.yml | 4 +- README.md | 2 +- cmd/jet/version.go | 2 +- internal/3rdparty/snaker/snaker.go | 13 +- internal/3rdparty/snaker/snaker_test.go | 2 + internal/jet/alias.go | 5 + internal/jet/clause.go | 8 +- internal/jet/column.go | 16 +- internal/jet/column_list.go | 6 + internal/jet/expression.go | 9 +- internal/jet/projection.go | 12 + internal/jet/raw_statement.go | 4 +- internal/jet/serializer.go | 22 +- internal/jet/sql_builder.go | 4 + internal/jet/statement.go | 97 ++- internal/jet/utils.go | 17 + internal/jet/with_statement.go | 4 +- internal/testutils/test_utils.go | 35 +- internal/utils/datetime/duration.go | 38 +- internal/utils/min/min.go | 9 - mysql/functions.go | 5 + mysql/select_json.go | 79 +++ mysql/select_statement.go | 6 +- mysql/table.go | 2 +- postgres/select_json.go | 131 ++++ postgres/select_statement.go | 22 +- postgres/set_statement.go | 16 +- postgres/table.go | 2 +- qrm/internal/null_types.go | 46 +- qrm/qrm.go | 173 ++++- qrm/scan_context.go | 28 +- qrm/utill.go | 20 +- tests/docker-compose.yaml | 2 +- tests/mysql/alltypes_test.go | 50 +- tests/mysql/bench_test.go | 138 ++++ tests/mysql/select_json_test.go | 437 ++++++++++++ tests/mysql/select_test.go | 84 ++- tests/postgres/alltypes_test.go | 100 ++- tests/postgres/chinook_db_test.go | 156 +++- tests/postgres/main_test.go | 12 +- tests/postgres/northwind_test.go | 169 ++++- tests/postgres/sample_test.go | 57 +- tests/postgres/scan_test.go | 2 +- tests/postgres/select_json_test.go | 908 ++++++++++++++++++++++++ tests/postgres/select_test.go | 83 ++- tests/testdata | 2 +- 46 files changed, 2732 insertions(+), 307 deletions(-) delete mode 100644 internal/utils/min/min.go create mode 100644 mysql/select_json.go create mode 100644 postgres/select_json.go create mode 100644 tests/mysql/bench_test.go create mode 100644 tests/mysql/select_json_test.go create mode 100644 tests/postgres/select_json_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index a36737b..f74334f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: build_and_tests: docker: # specify the version - - image: cimg/go:1.22.8 + - image: cimg/go:1.23.6 # Please keep the version in sync with test/docker-compose.yaml - image: cimg/postgres:14.10 @@ -29,7 +29,7 @@ jobs: MYSQL_TCP_PORT: 50902 # Please keep the version in sync with test/docker-compose.yaml - - image: circleci/mariadb:10.3 + - image: circleci/mariadb:11.7 command: [ '--default-authentication-plugin=mysql_native_password', '--port=50903' ] environment: MYSQL_ROOT_PASSWORD: jet diff --git a/README.md b/README.md index 3d5f57d..70e29e9 100644 --- a/README.md +++ b/README.md @@ -579,5 +579,5 @@ To run the tests, additional dependencies are required: ## License -Copyright 2019-2024 Goran Bjelanovic +Copyright 2019-2025 Goran Bjelanovic Licensed under the Apache License, Version 2.0. diff --git a/cmd/jet/version.go b/cmd/jet/version.go index 3300008..f5371e4 100644 --- a/cmd/jet/version.go +++ b/cmd/jet/version.go @@ -1,3 +1,3 @@ package main -const version = "v2.11.1" +const version = "v2.12.0" diff --git a/internal/3rdparty/snaker/snaker.go b/internal/3rdparty/snaker/snaker.go index 32a19e6..4177e1c 100644 --- a/internal/3rdparty/snaker/snaker.go +++ b/internal/3rdparty/snaker/snaker.go @@ -40,14 +40,23 @@ func snakeToCamel(s string, upperCase bool) string { if upperCase || i > 0 { result += camelizeWord(word, len(words) > 1) - } else { - result += word + } else { // lowerCase and i == 0 + result += toLowerFirstLetter(word) } } return result } +func toLowerFirstLetter(s string) string { + if s == "" { + return s + } + runes := []rune(s) + runes[0] = unicode.ToLower(runes[0]) + return string(runes) +} + func camelizeWord(word string, force bool) string { runes := []rune(word) diff --git a/internal/3rdparty/snaker/snaker_test.go b/internal/3rdparty/snaker/snaker_test.go index f828a91..9cc5ef7 100644 --- a/internal/3rdparty/snaker/snaker_test.go +++ b/internal/3rdparty/snaker/snaker_test.go @@ -8,6 +8,8 @@ import ( func TestSnakeToCamel(t *testing.T) { require.Equal(t, SnakeToCamel(""), "") require.Equal(t, SnakeToCamel("potato_"), "Potato") + require.Equal(t, SnakeToCamel("potato_", false), "potato") + require.Equal(t, SnakeToCamel("Potato_", false), "potato") require.Equal(t, SnakeToCamel("this_has_to_be_uppercased"), "ThisHasToBeUppercased") require.Equal(t, SnakeToCamel("this_is_an_id"), "ThisIsAnID") require.Equal(t, SnakeToCamel("this_is_an_identifier"), "ThisIsAnIdentifier") diff --git a/internal/jet/alias.go b/internal/jet/alias.go index 8693b13..5374031 100644 --- a/internal/jet/alias.go +++ b/internal/jet/alias.go @@ -30,3 +30,8 @@ func (a *alias) serializeForProjection(statement StatementType, out *SQLBuilder) out.WriteString("AS") out.WriteAlias(a.alias) } + +func (a *alias) serializeForJsonObj(statement StatementType, out *SQLBuilder) { + out.WriteJsonObjKey(a.alias) + a.expression.serialize(statement, out) +} diff --git a/internal/jet/clause.go b/internal/jet/clause.go index 533d223..68d3fee 100644 --- a/internal/jet/clause.go +++ b/internal/jet/clause.go @@ -52,6 +52,10 @@ func (s *ClauseSelect) Projections() ProjectionList { // Serialize serializes clause into SQLBuilder func (s *ClauseSelect) Serialize(statementType StatementType, out *SQLBuilder, options ...SerializeOption) { + if len(s.ProjectionList) == 0 { + panic("jet: SELECT clause has to have at least one projection") + } + out.NewLine() out.WriteString("SELECT") s.OptimizerHints.Serialize(statementType, out, options...) @@ -66,10 +70,6 @@ func (s *ClauseSelect) Serialize(statementType StatementType, out *SQLBuilder, o out.WriteByte(')') } - if len(s.ProjectionList) == 0 { - panic("jet: SELECT clause has to have at least one projection") - } - out.WriteProjections(statementType, s.ProjectionList) } diff --git a/internal/jet/column.go b/internal/jet/column.go index 2b1b930..702518a 100644 --- a/internal/jet/column.go +++ b/internal/jet/column.go @@ -2,6 +2,10 @@ package jet +import ( + "github.com/go-jet/jet/v2/internal/3rdparty/snaker" +) + // Column is common column interface for all types of columns. type Column interface { Name() string @@ -97,7 +101,17 @@ func (c ColumnExpressionImpl) serializeForProjection(statement StatementType, ou c.serialize(statement, out) out.WriteString("AS") - out.WriteAlias(c.defaultAlias()) + + if statement.IsSelectJSON() { + out.WriteAlias(snaker.SnakeToCamel(c.name, false)) + } else { + out.WriteAlias(c.defaultAlias()) + } +} + +func (c ColumnExpressionImpl) serializeForJsonObj(statement StatementType, out *SQLBuilder) { + out.WriteJsonObjKey(snaker.SnakeToCamel(c.name, false)) + c.serialize(statement, out) } func (c ColumnExpressionImpl) serialize(statement StatementType, out *SQLBuilder, options ...SerializeOption) { diff --git a/internal/jet/column_list.go b/internal/jet/column_list.go index a07b9ba..18acd81 100644 --- a/internal/jet/column_list.go +++ b/internal/jet/column_list.go @@ -78,6 +78,12 @@ func (cl ColumnList) serializeForProjection(statement StatementType, out *SQLBui SerializeProjectionList(statement, projections, out) } +func (cl ColumnList) serializeForJsonObj(statement StatementType, out *SQLBuilder) { + projections := ColumnListToProjectionList(cl) + + SerializeProjectionListJsonObj(statement, projections, out) +} + // dummy column interface implementation // Name is placeholder for ColumnList to implement Column interface diff --git a/internal/jet/expression.go b/internal/jet/expression.go index 9999803..5c152ce 100644 --- a/internal/jet/expression.go +++ b/internal/jet/expression.go @@ -2,7 +2,7 @@ package jet import "fmt" -// Expression is common interface for all expressions. +// Expression is a common interface for all expressions. // Can be Bool, Int, Float, String, Date, Time, Timez, Timestamp or Timestampz expressions. type Expression interface { Serializer @@ -89,9 +89,16 @@ func (e *ExpressionInterfaceImpl) serializeForGroupBy(statement StatementType, o } func (e *ExpressionInterfaceImpl) serializeForProjection(statement StatementType, out *SQLBuilder) { + if statement.IsSelectJSON() { + panic("jet: expression need to be aliased when used as SELECT JSON projection.") + } e.Parent.serialize(statement, out, NoWrap) } +func (e *ExpressionInterfaceImpl) serializeForJsonObj(statement StatementType, out *SQLBuilder) { + panic("jet: expression need to be aliased when used as SELECT JSON projection.") +} + func (e *ExpressionInterfaceImpl) serializeForOrderBy(statement StatementType, out *SQLBuilder) { e.Parent.serialize(statement, out, NoWrap) } diff --git a/internal/jet/projection.go b/internal/jet/projection.go index 3b2ccd8..647c1ba 100644 --- a/internal/jet/projection.go +++ b/internal/jet/projection.go @@ -3,6 +3,7 @@ package jet // Projection is interface for all projection types. Types that can be part of, for instance SELECT clause. type Projection interface { serializeForProjection(statement StatementType, out *SQLBuilder) + serializeForJsonObj(statement StatementType, out *SQLBuilder) fromImpl(subQuery SelectTable) Projection } @@ -28,6 +29,10 @@ func (pl ProjectionList) serializeForProjection(statement StatementType, out *SQ SerializeProjectionList(statement, pl, out) } +func (pl ProjectionList) serializeForJsonObj(statement StatementType, out *SQLBuilder) { + SerializeProjectionListJsonObj(statement, pl, out) +} + // As will create new projection list where each column is wrapped with a new table alias. // tableAlias should be in the form 'name' or 'name.*', or it can be an empty string, which will remove existing table alias. // For instance: If projection list has a column 'Artist.Name', and tableAlias is 'Musician.*', returned projection list will @@ -79,3 +84,10 @@ func (pl ProjectionList) Except(toExclude ...Column) ProjectionList { return ret } + +// JsonProjectionList redefines []Projection so projections can be serialized as json object key/values +type JsonProjectionList []Projection + +func (j JsonProjectionList) serialize(statement StatementType, out *SQLBuilder, options ...SerializeOption) { + SerializeProjectionListJsonObj(statement, j, out) +} diff --git a/internal/jet/raw_statement.go b/internal/jet/raw_statement.go index 99fb8eb..427b3c8 100644 --- a/internal/jet/raw_statement.go +++ b/internal/jet/raw_statement.go @@ -1,7 +1,7 @@ package jet type rawStatementImpl struct { - serializerStatementInterfaceImpl + statementInterfaceImpl RawQuery string NamedArguments map[string]interface{} @@ -10,7 +10,7 @@ type rawStatementImpl struct { // RawStatement creates new sql statements from raw query and optional map of named arguments func RawStatement(dialect Dialect, rawQuery string, namedArgument ...map[string]interface{}) SerializerStatement { newRawStatement := rawStatementImpl{ - serializerStatementInterfaceImpl: serializerStatementInterfaceImpl{ + statementInterfaceImpl: statementInterfaceImpl{ dialect: dialect, statementType: "", parent: nil, diff --git a/internal/jet/serializer.go b/internal/jet/serializer.go index 9d36de4..9d457a9 100644 --- a/internal/jet/serializer.go +++ b/internal/jet/serializer.go @@ -22,16 +22,22 @@ func (s SerializeOption) WithFallTrough(options []SerializeOption) []SerializeOp // StatementType is type of the SQL statement type StatementType string +func (s StatementType) IsSelectJSON() bool { + return s == SelectJsonObjStatementType || s == SelectJsonArrStatementType +} + // Statement types const ( - SelectStatementType StatementType = "SELECT" - InsertStatementType StatementType = "INSERT" - UpdateStatementType StatementType = "UPDATE" - DeleteStatementType StatementType = "DELETE" - SetStatementType StatementType = "SET" - LockStatementType StatementType = "LOCK" - UnLockStatementType StatementType = "UNLOCK" - WithStatementType StatementType = "WITH" + SelectStatementType StatementType = "SELECT" + SelectJsonObjStatementType StatementType = "SELECT_JSON_OBJ" + SelectJsonArrStatementType StatementType = "SELECT_JSON_ARR" + InsertStatementType StatementType = "INSERT" + UpdateStatementType StatementType = "UPDATE" + DeleteStatementType StatementType = "DELETE" + SetStatementType StatementType = "SET" + LockStatementType StatementType = "LOCK" + UnLockStatementType StatementType = "UNLOCK" + WithStatementType StatementType = "WITH" ) // Serializer interface diff --git a/internal/jet/sql_builder.go b/internal/jet/sql_builder.go index 46f47ad..87a5814 100644 --- a/internal/jet/sql_builder.go +++ b/internal/jet/sql_builder.go @@ -99,6 +99,10 @@ func (s *SQLBuilder) WriteString(str string) { s.write([]byte(str)) } +func (s *SQLBuilder) WriteJsonObjKey(key string) { + s.WriteString(fmt.Sprintf(`'%s', `, key)) +} + // WriteIdentifier adds identifier to output SQL func (s *SQLBuilder) WriteIdentifier(name string, alwaysQuote ...bool) { if s.shouldQuote(name, alwaysQuote...) { diff --git a/internal/jet/statement.go b/internal/jet/statement.go index 2ca229d..20e4665 100644 --- a/internal/jet/statement.go +++ b/internal/jet/statement.go @@ -7,25 +7,49 @@ import ( "time" ) -// Statement is common interface for all statements(SELECT, INSERT, UPDATE, DELETE, LOCK) +// Statement is a common interface for all SQL statements, including SELECT, SELECT_JSON_ARR, SELECT_JSON_OBJ, INSERT, +// UPDATE, DELETE, and LOCK. type Statement interface { - // Sql returns parametrized sql query with list of arguments. + // Sql returns a parameterized SQL query along with its list of arguments. Sql() (query string, args []interface{}) - // DebugSql returns debug query where every parametrized placeholder is replaced with its argument string representation. - // Do not use it in production. Use it only for debug purposes. + + // DebugSql returns a debug-friendly SQL query where all parameterized placeholders + // are replaced with their respective argument string representations. + // + // Warning: This method should only be used for debugging purposes. + // Do not use it in production, as it may lead to security risks such as SQL injection. DebugSql() (query string) - // Query executes statement over database connection/transaction db and stores row results in destination. - // Destination can be either pointer to struct or pointer to a slice. - // If destination is pointer to struct and query result set is empty, method returns qrm.ErrNoRows. + + // Query executes statement on the provided database connection or transaction (db), + // storing the retrieved row results in the given destination. + // Destination must be a pointer to either a struct or a slice. + // If the destination is a pointer to a struct and the query returns no rows, Query returns qrm.ErrNoRows. Query(db qrm.Queryable, destination interface{}) error - // QueryContext executes statement with a context over database connection/transaction db and stores row result in destination. - // Destination can be either pointer to struct or pointer to a slice. - // If destination is pointer to struct and query result set is empty, method returns qrm.ErrNoRows. + + // QueryContext executes statement with a context over database connection/transaction db, + // storing the retrieved row results in the given destination. + // Destination must be a pointer to either a struct or a slice. + // If the destination is a pointer to a struct and the query returns no rows, Query returns qrm.ErrNoRows. QueryContext(ctx context.Context, db qrm.Queryable, destination interface{}) error + + // QueryJSON executes the given statement within the provided context on the database connection/transaction (db) + // and unmarshals the JSON result into the destination. + // If the statement is created as SELECT_JSON_ARR, the destination must be a pointer to a slice of structs or a + // pointer to []map[string]any. + // If the statement is created as SELECT_JSON_OBJ, the destination must be a pointer to a struct or a pointer to + // map[string]any. + // QueryJSON can also be used by other SQL statements that generate JSON on the server. The only requirement is + // that the query must return exactly one row with a single column; otherwise, an error is returned. + // If the destination is a pointer to a struct (or []map[string]any) and the query result set is empty, the method + // returns qrm.ErrNoRows. + QueryJSON(ctx context.Context, db qrm.Queryable, destination interface{}) error + // Exec executes statement over db connection/transaction without returning any rows. Exec(db qrm.Executable) (sql.Result, error) + // ExecContext executes statement with context over db connection/transaction without returning any rows. ExecContext(ctx context.Context, db qrm.Executable) (sql.Result, error) + // Rows executes statements over db connection/transaction and returns rows Rows(ctx context.Context, db qrm.Queryable) (*Rows, error) } @@ -60,14 +84,14 @@ type SerializerHasProjections interface { HasProjections } -// serializerStatementInterfaceImpl struct -type serializerStatementInterfaceImpl struct { +// statementInterfaceImpl struct +type statementInterfaceImpl struct { dialect Dialect statementType StatementType parent SerializerStatement } -func (s *serializerStatementInterfaceImpl) Sql() (query string, args []interface{}) { +func (s *statementInterfaceImpl) Sql() (query string, args []interface{}) { queryData := &SQLBuilder{Dialect: s.dialect} @@ -77,7 +101,7 @@ func (s *serializerStatementInterfaceImpl) Sql() (query string, args []interface return } -func (s *serializerStatementInterfaceImpl) DebugSql() (query string) { +func (s *statementInterfaceImpl) DebugSql() (query string) { sqlBuilder := &SQLBuilder{Dialect: s.dialect, Debug: true} s.parent.serialize(s.statementType, sqlBuilder, NoWrap) @@ -86,11 +110,29 @@ func (s *serializerStatementInterfaceImpl) DebugSql() (query string) { return } -func (s *serializerStatementInterfaceImpl) Query(db qrm.Queryable, destination interface{}) error { +func (s *statementInterfaceImpl) Query(db qrm.Queryable, destination interface{}) error { return s.QueryContext(context.Background(), db, destination) } -func (s *serializerStatementInterfaceImpl) QueryContext(ctx context.Context, db qrm.Queryable, destination interface{}) error { +func (s *statementInterfaceImpl) QueryContext(ctx context.Context, db qrm.Queryable, destination interface{}) error { + return s.query(ctx, func(query string, args []interface{}) (int64, error) { + return qrm.Query(ctx, db, query, args, destination) + }) +} + +func (s *statementInterfaceImpl) QueryJSON(ctx context.Context, db qrm.Queryable, destination interface{}) error { + return s.query(ctx, func(query string, args []interface{}) (int64, error) { + if s.statementType == SelectJsonObjStatementType { + return qrm.QueryJsonObj(ctx, db, query, args, destination) + } + return qrm.QueryJsonArr(ctx, db, query, args, destination) + }) +} + +func (s *statementInterfaceImpl) query( + ctx context.Context, + queryFunc func(query string, args []interface{}) (int64, error), +) error { query, args := s.Sql() callLogger(ctx, s) @@ -99,7 +141,7 @@ func (s *serializerStatementInterfaceImpl) QueryContext(ctx context.Context, db var err error duration := duration(func() { - rowsProcessed, err = qrm.Query(ctx, db, query, args, destination) + rowsProcessed, err = queryFunc(query, args) }) callQueryLoggerFunc(ctx, QueryInfo{ @@ -112,11 +154,11 @@ func (s *serializerStatementInterfaceImpl) QueryContext(ctx context.Context, db return err } -func (s *serializerStatementInterfaceImpl) Exec(db qrm.Executable) (res sql.Result, err error) { +func (s *statementInterfaceImpl) Exec(db qrm.Executable) (res sql.Result, err error) { return s.ExecContext(context.Background(), db) } -func (s *serializerStatementInterfaceImpl) ExecContext(ctx context.Context, db qrm.Executable) (res sql.Result, err error) { +func (s *statementInterfaceImpl) ExecContext(ctx context.Context, db qrm.Executable) (res sql.Result, err error) { query, args := s.Sql() callLogger(ctx, s) @@ -141,7 +183,7 @@ func (s *serializerStatementInterfaceImpl) ExecContext(ctx context.Context, db q return res, err } -func (s *serializerStatementInterfaceImpl) Rows(ctx context.Context, db qrm.Queryable) (*Rows, error) { +func (s *statementInterfaceImpl) Rows(ctx context.Context, db qrm.Queryable) (*Rows, error) { query, args := s.Sql() callLogger(ctx, s) @@ -191,11 +233,15 @@ type ExpressionStatement interface { } // NewExpressionStatementImpl creates new expression statement -func NewExpressionStatementImpl(Dialect Dialect, statementType StatementType, parent ExpressionStatement, clauses ...Clause) ExpressionStatement { +func NewExpressionStatementImpl(Dialect Dialect, + statementType StatementType, + parent ExpressionStatement, + clauses ...Clause) ExpressionStatement { + return &expressionStatementImpl{ ExpressionInterfaceImpl{Parent: parent}, statementImpl{ - serializerStatementInterfaceImpl: serializerStatementInterfaceImpl{ + statementInterfaceImpl: statementInterfaceImpl{ parent: parent, dialect: Dialect, statementType: statementType, @@ -211,13 +257,16 @@ type expressionStatementImpl struct { } func (s *expressionStatementImpl) serializeForProjection(statement StatementType, out *SQLBuilder) { + if statement.IsSelectJSON() { + panic("jet: SELECT JSON statements need to be aliased when used as a projection.") + } s.serialize(statement, out) } // NewStatementImpl creates new statementImpl func NewStatementImpl(Dialect Dialect, statementType StatementType, parent SerializerStatement, clauses ...Clause) SerializerStatement { return &statementImpl{ - serializerStatementInterfaceImpl: serializerStatementInterfaceImpl{ + statementInterfaceImpl: statementInterfaceImpl{ parent: parent, dialect: Dialect, statementType: statementType, @@ -227,7 +276,7 @@ func NewStatementImpl(Dialect Dialect, statementType StatementType, parent Seria } type statementImpl struct { - serializerStatementInterfaceImpl + statementInterfaceImpl Clauses []Clause } diff --git a/internal/jet/utils.go b/internal/jet/utils.go index 466f2a5..5278605 100644 --- a/internal/jet/utils.go +++ b/internal/jet/utils.go @@ -58,6 +58,23 @@ func SerializeProjectionList(statement StatementType, projections []Projection, } } +// SerializeProjectionListJsonObj serializes a list of projections for JSON object +func SerializeProjectionListJsonObj(statement StatementType, projections []Projection, out *SQLBuilder) { + + for i, p := range projections { + if i > 0 { + out.WriteString(",") + out.NewLine() + } + + if p == nil { + panic("jet: Projection is nil") + } + + p.serializeForJsonObj(statement, out) + } +} + // SerializeColumnNames func func SerializeColumnNames(columns []Column, out *SQLBuilder) { for i, col := range columns { diff --git a/internal/jet/with_statement.go b/internal/jet/with_statement.go index 03330e6..d507072 100644 --- a/internal/jet/with_statement.go +++ b/internal/jet/with_statement.go @@ -7,7 +7,7 @@ func WITH(dialect Dialect, recursive bool, cte ...*CommonTableExpression) func(s newWithImpl := &withImpl{ recursive: recursive, ctes: cte, - serializerStatementInterfaceImpl: serializerStatementInterfaceImpl{ + statementInterfaceImpl: statementInterfaceImpl{ dialect: dialect, statementType: WithStatementType, }, @@ -25,7 +25,7 @@ func WITH(dialect Dialect, recursive bool, cte ...*CommonTableExpression) func(s } type withImpl struct { - serializerStatementInterfaceImpl + statementInterfaceImpl recursive bool ctes []*CommonTableExpression primaryStatement SerializerStatement diff --git a/internal/testutils/test_utils.go b/internal/testutils/test_utils.go index 817de32..2e2fc44 100644 --- a/internal/testutils/test_utils.go +++ b/internal/testutils/test_utils.go @@ -115,6 +115,16 @@ func AssertJSON(t *testing.T, data interface{}, expectedJSON string) { require.Equal(t, dataJson, expectedJSON) } +// AssertJsonEqual checks if actual and expected json representation are the same +func AssertJsonEqual(t require.TestingT, actual, expected interface{}, option ...cmp.Option) { + actualJsonData, err := json.MarshalIndent(actual, "", "\t") + require.NoError(t, err) + expectedJsonData, err := json.MarshalIndent(expected, "", "\t") + require.NoError(t, err) + + require.Equal(t, actualJsonData, expectedJsonData) +} + // SaveJSONFile saves v as json at testRelativePath // nolint:unused func SaveJSONFile(v interface{}, testRelativePath string) { @@ -127,7 +137,10 @@ func SaveJSONFile(v interface{}, testRelativePath string) { } // AssertJSONFile check if data json representation is the same as json at testRelativePath -func AssertJSONFile(t *testing.T, data interface{}, testRelativePath string) { +func AssertJSONFile(t require.TestingT, data interface{}, testRelativePath string) { + if _, ok := t.(*testing.B); ok { + return // skip assert for benchmarks + } filePath := getFullPath(testRelativePath) fileJSONData, err := os.ReadFile(filePath) // #nosec G304 @@ -145,7 +158,11 @@ func AssertJSONFile(t *testing.T, data interface{}, testRelativePath string) { } // AssertStatementSql check if statement Sql() is the same as expectedQuery and expectedArgs -func AssertStatementSql(t *testing.T, query jet.PrintableStatement, expectedQuery string, expectedArgs ...interface{}) { +func AssertStatementSql(t require.TestingT, query jet.PrintableStatement, expectedQuery string, expectedArgs ...interface{}) { + if _, ok := t.(*testing.B); ok { + return // skip assert for benchmarks + } + queryStr, args := query.Sql() assertQueryString(t, queryStr, expectedQuery) @@ -255,6 +272,16 @@ func AssertQueryPanicErr(t *testing.T, stmt jet.Statement, db qrm.DB, dest inter _ = stmt.Query(db, dest) } +// AssertQueryJsonPanicErr check if statement QueryJSON execution panics with error errString +func AssertQueryJsonPanicErr(t *testing.T, stmt jet.Statement, db qrm.DB, dest interface{}, errString string) { + defer func() { + r := recover() + require.Equal(t, r, errString) + }() + + _ = stmt.QueryJSON(context.Background(), db, dest) +} + // AssertFileContent check if file content at filePath contains expectedContent text. func AssertFileContent(t *testing.T, filePath string, expectedContent string) { enumFileData, err := os.ReadFile(filePath) // #nosec G304 @@ -283,14 +310,14 @@ func AssertFileNamesEqual(t *testing.T, dirPath string, fileNames ...string) { } // AssertDeepEqual checks if actual and expected objects are deeply equal. -func AssertDeepEqual(t *testing.T, actual, expected interface{}, option ...cmp.Option) { +func AssertDeepEqual(t require.TestingT, actual, expected interface{}, option ...cmp.Option) { if !assert.True(t, cmp.Equal(actual, expected, option...)) { printDiff(actual, expected, option...) t.FailNow() } } -func assertQueryString(t *testing.T, actual, expected string) { +func assertQueryString(t require.TestingT, actual, expected string) { if !assert.Equal(t, actual, expected) { printDiff(actual, expected) t.FailNow() diff --git a/internal/utils/datetime/duration.go b/internal/utils/datetime/duration.go index 11cc57f..a702623 100644 --- a/internal/utils/datetime/duration.go +++ b/internal/utils/datetime/duration.go @@ -1,6 +1,9 @@ package datetime -import "time" +import ( + //"github.com/go-jet/jet/v2/internal/utils/min" + "time" +) // ExtractTimeComponents extracts number of days, hours, minutes, seconds, microseconds from duration func ExtractTimeComponents(duration time.Duration) (days, hours, minutes, seconds, microseconds int64) { @@ -20,3 +23,36 @@ func ExtractTimeComponents(duration time.Duration) (days, hours, minutes, second return } + +// TryParseAsTime attempts to parse the provided value as a time using one of the given formats. +// +// The function iterates over the provided formats and tries to parse the value into a time.Time object. +// It returns the parsed time and a boolean indicating whether the parsing was successful. +func TryParseAsTime(value interface{}, formats []string) (time.Time, bool) { + + var timeStr string + + switch v := value.(type) { + case string: + timeStr = v + case []byte: + timeStr = string(v) + case int64: + return time.Unix(v, 0), true // sqlite + default: + return time.Time{}, false + } + + for _, format := range formats { + formatLen := min(len(format), len(timeStr)) + t, err := time.Parse(format[:formatLen], timeStr) + + if err != nil { + continue + } + + return t, true + } + + return time.Time{}, false +} diff --git a/internal/utils/min/min.go b/internal/utils/min/min.go deleted file mode 100644 index 0e92146..0000000 --- a/internal/utils/min/min.go +++ /dev/null @@ -1,9 +0,0 @@ -package min - -// Int returns minimum of two int values -func Int(a, b int) int { - if a < b { - return a - } - return b -} diff --git a/mysql/functions.go b/mysql/functions.go index ceec7ab..3eb16da 100644 --- a/mysql/functions.go +++ b/mysql/functions.go @@ -148,6 +148,11 @@ var NTH_VALUE = jet.NTH_VALUE //--------------------- String functions ------------------// +// HEX function in MySQL takes an input and returns its equivalent hexadecimal representation +func HEX(expression Expression) StringExpression { + return StringExp(Func("HEX", expression)) +} + // BIT_LENGTH returns number of bits in string expression var BIT_LENGTH = jet.BIT_LENGTH diff --git a/mysql/select_json.go b/mysql/select_json.go new file mode 100644 index 0000000..990e4ea --- /dev/null +++ b/mysql/select_json.go @@ -0,0 +1,79 @@ +package mysql + +import ( + "github.com/go-jet/jet/v2/internal/jet" +) + +// SelectJsonStatement is an interface for MySQL statements that generate JSON on the server. +type SelectJsonStatement interface { + Statement + jet.Serializer + + AS(alias string) Projection + + FROM(table ReadableTable) SelectJsonStatement + WHERE(condition BoolExpression) SelectJsonStatement + ORDER_BY(orderByClauses ...OrderByClause) SelectJsonStatement + LIMIT(limit int64) SelectJsonStatement + OFFSET(offset int64) SelectJsonStatement +} + +// SELECT_JSON_ARR creates a new SelectJsonStatement with a list of projections. +func SELECT_JSON_ARR(projections ...Projection) SelectJsonStatement { + return newSelectStatementJson(projections, jet.SelectJsonArrStatementType) +} + +// SELECT_JSON_OBJ creates a new SelectJsonStatement with a list of projections. +func SELECT_JSON_OBJ(projections ...Projection) SelectJsonStatement { + return newSelectStatementJson(projections, jet.SelectJsonObjStatementType) +} + +type selectJsonStatement struct { + *selectStatementImpl +} + +func newSelectStatementJson(projections []Projection, statementType jet.StatementType) SelectJsonStatement { + newSelect := &selectJsonStatement{ + selectStatementImpl: newSelectStatement(statementType, nil, nil), + } + + newSelect.Select.ProjectionList = ProjectionList{constructJsonFunc(projections, statementType).AS("json")} + + return newSelect +} + +func constructJsonFunc(projections []Projection, statementType jet.StatementType) Expression { + jsonObj := Func("JSON_OBJECT", CustomExpression(jet.JsonProjectionList(projections))) + + if statementType == jet.SelectJsonArrStatementType { + return Func("JSON_ARRAYAGG", jsonObj) + } + + return jsonObj +} + +func (s *selectJsonStatement) FROM(table ReadableTable) SelectJsonStatement { + s.From.Tables = []jet.Serializer{table} + + return s +} + +func (s *selectJsonStatement) WHERE(condition BoolExpression) SelectJsonStatement { + s.Where.Condition = condition + return s +} + +func (s *selectJsonStatement) ORDER_BY(orderByClauses ...OrderByClause) SelectJsonStatement { + s.OrderBy.List = orderByClauses + return s +} + +func (s *selectJsonStatement) LIMIT(limit int64) SelectJsonStatement { + s.Limit.Count = limit + return s +} + +func (s *selectJsonStatement) OFFSET(offset int64) SelectJsonStatement { + s.Offset.Count = Int(offset) + return s +} diff --git a/mysql/select_statement.go b/mysql/select_statement.go index aaeff9a..915fe93 100644 --- a/mysql/select_statement.go +++ b/mysql/select_statement.go @@ -62,12 +62,12 @@ type SelectStatement interface { // SELECT creates new SelectStatement with list of projections func SELECT(projection Projection, projections ...Projection) SelectStatement { - return newSelectStatement(nil, append([]Projection{projection}, projections...)) + return newSelectStatement(jet.SelectStatementType, nil, append([]Projection{projection}, projections...)) } -func newSelectStatement(table ReadableTable, projections []Projection) SelectStatement { +func newSelectStatement(stmtType jet.StatementType, table ReadableTable, projections []Projection) *selectStatementImpl { newSelect := &selectStatementImpl{} - newSelect.ExpressionStatement = jet.NewExpressionStatementImpl(Dialect, jet.SelectStatementType, newSelect, + newSelect.ExpressionStatement = jet.NewExpressionStatementImpl(Dialect, stmtType, newSelect, &newSelect.Select, &newSelect.From, &newSelect.Where, diff --git a/mysql/table.go b/mysql/table.go index 0ae7ee8..ebc67e2 100644 --- a/mysql/table.go +++ b/mysql/table.go @@ -50,7 +50,7 @@ type readableTableInterfaceImpl struct { // Generates a select query on the current tableName. func (r readableTableInterfaceImpl) SELECT(projection1 Projection, projections ...Projection) SelectStatement { - return newSelectStatement(r.parent, append([]Projection{projection1}, projections...)) + return newSelectStatement(jet.SelectStatementType, r.parent, append([]Projection{projection1}, projections...)) } // Creates a inner join tableName Expression using onCondition. diff --git a/postgres/select_json.go b/postgres/select_json.go new file mode 100644 index 0000000..5d23277 --- /dev/null +++ b/postgres/select_json.go @@ -0,0 +1,131 @@ +package postgres + +import ( + "github.com/go-jet/jet/v2/internal/jet" + "strings" +) + +// SELECT_JSON_ARR creates a new SelectJsonStatement with a list of projections. +func SELECT_JSON_ARR(projections ...Projection) SelectStatement { + return newSelectStatementJson(projections, jet.SelectJsonArrStatementType) +} + +// SELECT_JSON_OBJ creates a new SelectJsonStatement with a list of projections. +func SELECT_JSON_OBJ(projections ...Projection) SelectStatement { + return newSelectStatementJson(projections, jet.SelectJsonObjStatementType) +} + +type selectJsonStatement struct { + *selectStatementImpl + + subQuery *selectStatementImpl + statementType jet.StatementType +} + +func (s *selectJsonStatement) AS(alias string) Projection { + s.setSubQueryAlias(strings.ToLower(alias) + "_") + + return s.selectStatementImpl.AS(alias) +} + +func (s *selectJsonStatement) FROM(table ...ReadableTable) SelectStatement { + s.subQuery.From.Tables = readableTablesToSerializerList(table) + + return s +} + +func (s *selectJsonStatement) DISTINCT(on ...jet.ColumnExpression) SelectStatement { + s.subQuery.Select.Distinct = true + s.subQuery.Select.DistinctOnColumns = on + return s +} + +func (s *selectJsonStatement) WHERE(condition BoolExpression) SelectStatement { + s.subQuery.Where.Condition = condition + return s +} + +func (s *selectJsonStatement) GROUP_BY(groupByClauses ...GroupByClause) SelectStatement { + s.subQuery.GroupBy.List = groupByClauses + return s +} + +func (s *selectJsonStatement) HAVING(boolExpression BoolExpression) SelectStatement { + s.subQuery.Having.Condition = boolExpression + return s +} + +func (s *selectJsonStatement) WINDOW(name string) windowExpand { + s.subQuery.Window.Definitions = append(s.subQuery.Window.Definitions, jet.WindowDefinition{Name: name}) + return windowExpand{ + selectStatement: s.subQuery, + rootStmt: s, + } +} + +func (s *selectJsonStatement) ORDER_BY(orderByClauses ...OrderByClause) SelectStatement { + s.subQuery.OrderBy.List = orderByClauses + return s +} + +func (s *selectJsonStatement) LIMIT(limit int64) SelectStatement { + s.subQuery.Limit.Count = limit + return s +} + +func (s *selectJsonStatement) OFFSET(offset int64) SelectStatement { + s.subQuery.Offset.Count = Int(offset) + return s +} + +func (s *selectJsonStatement) OFFSET_e(offset IntegerExpression) SelectStatement { + s.subQuery.Offset.Count = offset + return s +} + +func (s *selectJsonStatement) FETCH_FIRST(count IntegerExpression) fetchExpand { + s.subQuery.Fetch.Count = count + + return fetchExpand{ + selectStatement: s.subQuery, + rootStmt: s, + } +} + +func (s *selectJsonStatement) FOR(lock RowLock) SelectStatement { + s.subQuery.For.Lock = lock + return s +} + +func newSelectStatementJson(projections []Projection, statementType jet.StatementType) SelectStatement { + newSelectJson := &selectJsonStatement{ + selectStatementImpl: newSelectStatement(statementType, nil, nil), + subQuery: newSelectStatement(statementType, nil, projections), + statementType: statementType, + } + + newSelectJson.setOperatorsImpl.stmtRoot = newSelectJson + + newSelectJson.setSubQueryAlias("") + + return newSelectJson +} + +func (s *selectJsonStatement) setSubQueryAlias(alias string) { + subQueryAlias := alias + "records" + jsonAlias := alias + "json" + + s.Select.ProjectionList = ProjectionList{constructJsonFunc(s.statementType, subQueryAlias).AS(jsonAlias)} + + s.From.Tables = []jet.Serializer{newSelectTable(s.subQuery, subQueryAlias, nil)} +} + +func constructJsonFunc(statementType jet.StatementType, subQueryAlias string) Expression { + rowToJson := Func("row_to_json", CustomExpression(Token(subQueryAlias))) + + if statementType == jet.SelectJsonArrStatementType { + return Func("json_agg", rowToJson) + } + + return rowToJson +} diff --git a/postgres/select_statement.go b/postgres/select_statement.go index 2a48fc5..f2427af 100644 --- a/postgres/select_statement.go +++ b/postgres/select_statement.go @@ -70,12 +70,12 @@ type SelectStatement interface { // SELECT creates new SelectStatement with list of projections func SELECT(projection Projection, projections ...Projection) SelectStatement { - return newSelectStatement(nil, append([]Projection{projection}, projections...)) + return newSelectStatement(jet.SelectStatementType, nil, append([]Projection{projection}, projections...)) } -func newSelectStatement(table ReadableTable, projections []Projection) SelectStatement { +func newSelectStatement(stmtType jet.StatementType, table ReadableTable, projections []Projection) *selectStatementImpl { newSelect := &selectStatementImpl{} - newSelect.ExpressionStatement = jet.NewExpressionStatementImpl(Dialect, jet.SelectStatementType, newSelect, + newSelect.ExpressionStatement = jet.NewExpressionStatementImpl(Dialect, stmtType, newSelect, &newSelect.Select, &newSelect.From, &newSelect.Where, @@ -94,7 +94,7 @@ func newSelectStatement(table ReadableTable, projections []Projection) SelectSta } newSelect.Limit.Count = -1 - newSelect.setOperatorsImpl.parent = newSelect + newSelect.setOperatorsImpl.stmtRoot = newSelect return newSelect } @@ -144,7 +144,10 @@ func (s *selectStatementImpl) HAVING(boolExpression BoolExpression) SelectStatem func (s *selectStatementImpl) WINDOW(name string) windowExpand { s.Window.Definitions = append(s.Window.Definitions, jet.WindowDefinition{Name: name}) - return windowExpand{selectStatement: s} + return windowExpand{ + selectStatement: s, + rootStmt: s, + } } func (s *selectStatementImpl) ORDER_BY(orderByClauses ...OrderByClause) SelectStatement { @@ -172,6 +175,7 @@ func (s *selectStatementImpl) FETCH_FIRST(count IntegerExpression) fetchExpand { return fetchExpand{ selectStatement: s, + rootStmt: s, } } @@ -188,6 +192,7 @@ func (s *selectStatementImpl) AsTable(alias string) SelectTable { type windowExpand struct { selectStatement *selectStatementImpl + rootStmt SelectStatement } func (w windowExpand) AS(window ...jet.Window) SelectStatement { @@ -196,7 +201,7 @@ func (w windowExpand) AS(window ...jet.Window) SelectStatement { } windowsDefinition := w.selectStatement.Window.Definitions windowsDefinition[len(windowsDefinition)-1].Window = window[0] - return w.selectStatement + return w.rootStmt } func toJetFrameOffset(offset int64) jet.Serializer { @@ -216,16 +221,17 @@ func readableTablesToSerializerList(tables []ReadableTable) []jet.Serializer { type fetchExpand struct { selectStatement *selectStatementImpl + rootStmt SelectStatement } func (f fetchExpand) ROWS_ONLY() SelectStatement { f.selectStatement.Fetch.WithTies = false - return f.selectStatement + return f.rootStmt } func (f fetchExpand) ROWS_WITH_TIES() SelectStatement { f.selectStatement.Fetch.WithTies = true - return f.selectStatement + return f.rootStmt } diff --git a/postgres/set_statement.go b/postgres/set_statement.go index 0dee00d..5f553cb 100644 --- a/postgres/set_statement.go +++ b/postgres/set_statement.go @@ -65,31 +65,31 @@ type setOperators interface { } type setOperatorsImpl struct { - parent setOperators + stmtRoot setOperators } func (s *setOperatorsImpl) UNION(rhs SelectStatement) setStatement { - return UNION(s.parent, rhs) + return UNION(s.stmtRoot, rhs) } func (s *setOperatorsImpl) UNION_ALL(rhs SelectStatement) setStatement { - return UNION_ALL(s.parent, rhs) + return UNION_ALL(s.stmtRoot, rhs) } func (s *setOperatorsImpl) INTERSECT(rhs SelectStatement) setStatement { - return INTERSECT(s.parent, rhs) + return INTERSECT(s.stmtRoot, rhs) } func (s *setOperatorsImpl) INTERSECT_ALL(rhs SelectStatement) setStatement { - return INTERSECT_ALL(s.parent, rhs) + return INTERSECT_ALL(s.stmtRoot, rhs) } func (s *setOperatorsImpl) EXCEPT(rhs SelectStatement) setStatement { - return EXCEPT(s.parent, rhs) + return EXCEPT(s.stmtRoot, rhs) } func (s *setOperatorsImpl) EXCEPT_ALL(rhs SelectStatement) setStatement { - return EXCEPT_ALL(s.parent, rhs) + return EXCEPT_ALL(s.stmtRoot, rhs) } type setStatementImpl struct { @@ -110,7 +110,7 @@ func newSetStatementImpl(operator string, all bool, selects []jet.SerializerStat newSetStatement.setOperator.Selects = selects newSetStatement.setOperator.Limit.Count = -1 - newSetStatement.setOperatorsImpl.parent = newSetStatement + newSetStatement.setOperatorsImpl.stmtRoot = newSetStatement return newSetStatement } diff --git a/postgres/table.go b/postgres/table.go index f90c114..aa54213 100644 --- a/postgres/table.go +++ b/postgres/table.go @@ -55,7 +55,7 @@ type readableTableInterfaceImpl struct { // Generates a select query on the current tableName. func (r readableTableInterfaceImpl) SELECT(projection1 Projection, projections ...Projection) SelectStatement { - return newSelectStatement(r.parent, append([]Projection{projection1}, projections...)) + return newSelectStatement(jet.SelectStatementType, r.parent, append([]Projection{projection1}, projections...)) } // Creates a inner join tableName Expression using onCondition. diff --git a/qrm/internal/null_types.go b/qrm/internal/null_types.go index 85d9c68..6f7a270 100644 --- a/qrm/internal/null_types.go +++ b/qrm/internal/null_types.go @@ -4,10 +4,9 @@ import ( "database/sql" "database/sql/driver" "fmt" - "github.com/go-jet/jet/v2/internal/utils/min" + "github.com/go-jet/jet/v2/internal/utils/datetime" "reflect" "strconv" - "time" ) var ( @@ -64,7 +63,12 @@ func (nt *NullTime) Scan(value interface{}) error { // Some of the drivers (pgx, mysql) are not parsing all of the time formats(date, time with time zone,...) and are just forwarding string value. // At this point we try to parse those values using some of the predefined formats - nt.Time, nt.Valid = tryParseAsTime(value) + nt.Time, nt.Valid = datetime.TryParseAsTime(value, []string{ + "2006-01-02 15:04:05-07:00", // sqlite + "2006-01-02 15:04:05.999999", // go-sql-driver/mysql + "15:04:05-07", // pgx + "15:04:05.999999", // pgx + }) if !nt.Valid { return fmt.Errorf("can't scan time.Time from %q", value) @@ -73,42 +77,6 @@ func (nt *NullTime) Scan(value interface{}) error { return nil } -var formats = []string{ - "2006-01-02 15:04:05-07:00", // sqlite - "2006-01-02 15:04:05.999999", // go-sql-driver/mysql - "15:04:05-07", // pgx - "15:04:05.999999", // pgx -} - -func tryParseAsTime(value interface{}) (time.Time, bool) { - - var timeStr string - - switch v := value.(type) { - case string: - timeStr = v - case []byte: - timeStr = string(v) - case int64: - return time.Unix(v, 0), true // sqlite - default: - return time.Time{}, false - } - - for _, format := range formats { - formatLen := min.Int(len(format), len(timeStr)) - t, err := time.Parse(format[:formatLen], timeStr) - - if err != nil { - continue - } - - return t, true - } - - return time.Time{}, false -} - // NullUInt64 struct type NullUInt64 struct { UInt64 uint64 diff --git a/qrm/qrm.go b/qrm/qrm.go index edd9387..f1ef6ad 100644 --- a/qrm/qrm.go +++ b/qrm/qrm.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/go-jet/jet/v2/internal/3rdparty/json" "github.com/go-jet/jet/v2/internal/utils/must" "reflect" ) @@ -12,10 +13,130 @@ import ( // ErrNoRows is returned by Query when query result set is empty var ErrNoRows = errors.New("qrm: no rows in result set") -// Query executes Query Result Mapping (QRM) of `query` with list of parametrized arguments `arg` over database connection `db` -// using context `ctx` into destination `destPtr`. -// Destination can be either pointer to struct or pointer to slice of structs. -// If destination is pointer to struct and query result set is empty, method returns qrm.ErrNoRows. +// QueryJsonObj executes a SQL query that returns a JSON object, unmarshals the result into the provided destination, +// and returns the number of rows processed. +// +// The query must return exactly one row with a single column; otherwise, an error is returned. +// +// Parameters: +// +// ctx - The context for managing query execution (timeouts, cancellations). +// db - The database connection or transaction that implements the Queryable interface. +// query - The SQL query string to be executed. +// args - A slice of arguments to be used with the query. +// destPtr - A pointer to the variable where the unmarshaled JSON result will be stored. +// The destination should be a pointer to a struct or map[string]any. +// +// Returns: +// +// rowsProcessed - The number of rows processed by the query execution. +// err - An error if query execution or unmarshaling fails. +func QueryJsonObj(ctx context.Context, db Queryable, query string, args []interface{}, destPtr interface{}) (rowsProcessed int64, err error) { + must.BeInitializedPtr(destPtr, "jet: destination is nil") + must.BeTypeKind(destPtr, reflect.Ptr, jsonDestObjErr) + destType := reflect.TypeOf(destPtr).Elem() + must.BeTrue(destType.Kind() == reflect.Struct || destType.Kind() == reflect.Map, jsonDestObjErr) + + return queryJson(ctx, db, query, args, destPtr) +} + +// QueryJsonArr executes a SQL query that returns a JSON array, unmarshals the result into the provided destination, +// and returns the number of rows processed. +// +// The query must return exactly one row with a single column; otherwise, an error is returned. +// +// Parameters: +// +// ctx - The context for managing query execution (timeouts, cancellations). +// db - The database connection or transaction that implements the Queryable interface. +// query - The SQL query string to be executed. +// args - A slice of arguments to be used with the query. +// destPtr - A pointer to the variable where the unmarshaled JSON array will be stored. +// The destination should be a pointer to a slice of structs or []map[string]any. +// +// Returns: +// +// rowsProcessed - The number of rows processed by the query execution. +// err - An error if query execution or unmarshaling fails. +func QueryJsonArr(ctx context.Context, db Queryable, query string, args []interface{}, destPtr interface{}) (rowsProcessed int64, err error) { + must.BeInitializedPtr(destPtr, "jet: destination is nil") + must.BeTypeKind(destPtr, reflect.Ptr, jsonDestArrErr) + destType := reflect.TypeOf(destPtr).Elem() + must.BeTrue(destType.Kind() == reflect.Slice, jsonDestArrErr) + + return queryJson(ctx, db, query, args, destPtr) +} + +var jsonDestObjErr = "jet: destination has to be a pointer to struct or pointer to map[string]any" +var jsonDestArrErr = "jet: destination has to be a pointer to slice of struct or pointer to []map[string]any" + +func queryJson(ctx context.Context, db Queryable, query string, args []interface{}, destPtr interface{}) (rowsProcessed int64, err error) { + must.BeInitializedPtr(db, "jet: db is nil") + + var rows *sql.Rows + rows, err = db.QueryContext(ctx, query, args...) + + if err != nil { + return 0, err + } + + defer rows.Close() + + if !rows.Next() { + err = rows.Err() + if err != nil { + return 0, err + } + return 0, ErrNoRows + } + + var jsonData []byte + err = rows.Scan(&jsonData) + + if err != nil { + return 1, err + } + + if jsonData == nil { + return 1, nil + } + + err = json.Unmarshal(jsonData, &destPtr) + + if err != nil { + return 1, err + } + + if rows.Next() { + return 1, fmt.Errorf("jet: query returned more then one row") + } + + err = rows.Close() + if err != nil { + return 1, err + } + + return 1, nil +} + +// Query executes a Query Result Mapping (QRM) of the provided SQL `query` with a list of parameterized arguments `args` +// over the database connection `db` using the provided context `ctx` and stores the result in the destination `destPtr`. +// +// The destination must be a pointer to either a struct or a slice of structs +// If the destination is a pointer to a struct and no rows are returned, the method returns qrm.ErrNoRows. +// +// Parameters: +// +// ctx - The context for managing query execution (timeouts, cancellations). +// db - The database connection or transaction implementing the Queryable interface. +// query - The SQL query string to be executed. +// args - A slice of arguments to be used with the query. +// destPtr - A pointer to the variable where the query result will be stored. This can be a pointer to a struct or a slice of structs. +// +// Returns: +// +// rowsProcessed - The number of rows processed by the query execution. +// err - An error if query execution or result mapping fails, or if no rows are found when a struct is expected. func Query(ctx context.Context, db Queryable, query string, args []interface{}, destPtr interface{}) (rowsProcessed int64, err error) { must.BeInitializedPtr(db, "jet: db is nil") @@ -185,7 +306,7 @@ func mapRowToSlice( func mapRowToBaseTypeSlice(scanContext *ScanContext, slicePtrValue reflect.Value, field *reflect.StructField) (updated bool, err error) { index := 0 if field != nil { - typeName, columnName := getTypeAndFieldName("", *field) + typeName, columnName, _ := getTypeAndFieldName("", *field) if index = scanContext.typeToColumnIndex(typeName, columnName); index < 0 { return } @@ -233,9 +354,11 @@ func mapRowToStruct( continue } - fieldMap := typeInf.fieldMappings[i] + fieldMappingInfo := typeInf.fieldMappings[i] - if fieldMap.complexType { + switch fieldMappingInfo.Type { + + case complexType: var changed bool changed, err = mapRowToDestinationValue(scanContext, concat(groupKey, ":", field.Name), fieldValue, &field) @@ -246,13 +369,12 @@ func mapRowToStruct( if changed { updated = true } - - } else { - if mapOnlySlices || fieldMap.rowIndex == -1 { + default: + if mapOnlySlices || fieldMappingInfo.rowIndex == -1 { continue } - scannedValue := scanContext.rowElemValue(fieldMap.rowIndex) + scannedValue := scanContext.rowElemValue(fieldMappingInfo.rowIndex) if !scannedValue.IsValid() { setZeroValue(fieldValue) // scannedValue is nil, destination should be set to zero value @@ -261,7 +383,8 @@ func mapRowToStruct( updated = true - if fieldMap.implementsScanner { + switch fieldMappingInfo.Type { + case implementsScanner: initializeValueIfNilPtr(fieldValue) fieldScanner := getScanner(fieldValue) @@ -270,14 +393,27 @@ func mapRowToStruct( err := fieldScanner.Scan(value) if err != nil { - return updated, fmt.Errorf(`can't scan %T(%q) to '%s %s': %w`, value, value, field.Name, field.Type.String(), err) + return updated, qrmAssignError(scannedValue, field, err) } - } else { + case jsonUnmarshal: + value, ok := scannedValue.Interface().([]byte) + + if !ok { + return updated, qrmAssignError(scannedValue, field, fmt.Errorf("value not convertable to []byte")) + } + + fieldInterface := fieldValue.Addr().Interface() + + err := json.Unmarshal(value, fieldInterface) + + if err != nil { + return updated, qrmAssignError(scannedValue, field, err) + } + default: // simple type err := assign(scannedValue, fieldValue) if err != nil { - return updated, fmt.Errorf(`can't assign %T(%q) to '%s %s': %w`, scannedValue.Interface(), scannedValue.Interface(), - field.Name, field.Type.String(), err) + return updated, qrmAssignError(scannedValue, field, err) } } } @@ -286,6 +422,11 @@ func mapRowToStruct( return } +func qrmAssignError(scannedValue reflect.Value, field reflect.StructField, err error) error { + return fmt.Errorf(`can't assign %T(%q) to '%s %s': %w`, scannedValue.Interface(), scannedValue.Interface(), + field.Name, field.Type.String(), err) +} + func mapRowToDestinationValue( scanContext *ScanContext, groupKey string, diff --git a/qrm/scan_context.go b/qrm/scan_context.go index 7a3c538..e28c4f1 100644 --- a/qrm/scan_context.go +++ b/qrm/scan_context.go @@ -75,10 +75,18 @@ type typeInfo struct { fieldMappings []fieldMapping } +type fieldMappingType int + +const ( + simpleType fieldMappingType = iota + complexType // slice and struct are complex types supported + implementsScanner + jsonUnmarshal +) + type fieldMapping struct { - complexType bool // slice and struct are complex types - rowIndex int // index in ScanContext.row - implementsScanner bool + rowIndex int // index in ScanContext.row + Type fieldMappingType } func (s *ScanContext) getTypeInfo(structType reflect.Type, parentField *reflect.StructField) typeInfo { @@ -100,17 +108,21 @@ func (s *ScanContext) getTypeInfo(structType reflect.Type, parentField *reflect. for i := 0; i < structType.NumField(); i++ { field := structType.Field(i) - newTypeName, fieldName := getTypeAndFieldName(typeName, field) + newTypeName, fieldName, jsonUnmarshaler := getTypeAndFieldName(typeName, field) columnIndex := s.typeToColumnIndex(newTypeName, fieldName) fieldMap := fieldMapping{ rowIndex: columnIndex, } - if implementsScannerType(field.Type) { - fieldMap.implementsScanner = true + if jsonUnmarshaler { + fieldMap.Type = jsonUnmarshal + } else if implementsScannerType(field.Type) { + fieldMap.Type = implementsScanner } else if !isSimpleModelType(field.Type) { - fieldMap.complexType = true + fieldMap.Type = complexType + } else { + fieldMap.Type = simpleType } newTypeInfo.fieldMappings = append(newTypeInfo.fieldMappings, fieldMap) @@ -188,7 +200,7 @@ func (s *ScanContext) getGroupKeyInfo( fieldType := indirectType(field.Type) if isPrimaryKey(field, primaryKeyOverwrites) { - newTypeName, fieldName := getTypeAndFieldName(typeName, field) + newTypeName, fieldName, _ := getTypeAndFieldName(typeName, field) pkIndex := s.typeToColumnIndex(newTypeName, fieldName) diff --git a/qrm/utill.go b/qrm/utill.go index b43ee29..1775fda 100644 --- a/qrm/utill.go +++ b/qrm/utill.go @@ -107,20 +107,26 @@ func getTypeName(structType reflect.Type, parentField *reflect.StructField) stri return toCommonIdentifier(aliasParts[0]) } -func getTypeAndFieldName(structType string, field reflect.StructField) (string, string) { +func getTypeAndFieldName(structType string, field reflect.StructField) (string, string, bool) { aliasTag := field.Tag.Get("alias") - if aliasTag == "" { - return structType, field.Name + if aliasTag != "" { + aliasParts := strings.Split(aliasTag, ".") + + if len(aliasParts) == 1 { + return structType, toCommonIdentifier(aliasParts[0]), false + } + + return toCommonIdentifier(aliasParts[0]), toCommonIdentifier(aliasParts[1]), false } - aliasParts := strings.Split(aliasTag, ".") + jsonColumnTag := field.Tag.Get("json_column") - if len(aliasParts) == 1 { - return structType, toCommonIdentifier(aliasParts[0]) + if jsonColumnTag != "" { + return "", toCommonIdentifier(jsonColumnTag), true } - return toCommonIdentifier(aliasParts[0]), toCommonIdentifier(aliasParts[1]) + return structType, field.Name, false } var replacer = strings.NewReplacer(" ", "", "-", "", "_", "") diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index bcbbb25..3bef6b7 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -26,7 +26,7 @@ services: - ./testdata/init/mysql:/docker-entrypoint-initdb.d mariadb: - image: mariadb:10.3 + image: mariadb:11.7 command: ['--default-authentication-plugin=mysql_native_password', '--log_bin_trust_function_creators=1'] restart: always environment: diff --git a/tests/mysql/alltypes_test.go b/tests/mysql/alltypes_test.go index 61bc7f2..2440339 100644 --- a/tests/mysql/alltypes_test.go +++ b/tests/mysql/alltypes_test.go @@ -23,19 +23,63 @@ func TestAllTypes(t *testing.T) { var dest []model.AllTypes - err := AllTypes. - SELECT(AllTypes.AllColumns). + err := SELECT(AllTypes.AllColumns). + FROM(AllTypes). LIMIT(2). Query(db, &dest) require.NoError(t, err) - require.Equal(t, len(dest), 2) //testutils.PrintJson(dest) testutils.AssertJSON(t, dest, allTypesJson) } +func TestAllTypesJSON(t *testing.T) { + + stmt := SELECT_JSON_ARR( + AllTypes.AllColumns.Except( + AllTypes.JSON, + AllTypes.JSONPtr, + AllTypes.Bit, + AllTypes.BitPtr, + AllTypes.Blob, + AllTypes.BlobPtr, + AllTypes.Binary, + AllTypes.BinaryPtr, + AllTypes.VarBinary, + AllTypes.VarBinaryPtr, + ), + CAST(AllTypes.JSON).AS_CHAR().AS("Json"), + CAST(AllTypes.JSONPtr).AS_CHAR().AS("JsonPtr"), + CAST(AllTypes.Bit).AS_CHAR().AS("Bit"), + CAST(AllTypes.BitPtr).AS_CHAR().AS("BitPtr"), + + // TODO: remove when binary string is implemented + CONCAT(String("\\x"), HEX(AllTypes.Blob)).AS("Blob"), + CONCAT(String("\\x"), HEX(AllTypes.BlobPtr)).AS("BlobPtr"), + + CONCAT(String("\\x"), HEX(AllTypes.Binary)).AS("Binary"), + CONCAT(String("\\x"), HEX(AllTypes.BinaryPtr)).AS("BinaryPtr"), + + CONCAT(String("\\x"), HEX(AllTypes.VarBinary)).AS("VarBinary"), + CONCAT(String("\\x"), HEX(AllTypes.VarBinaryPtr)).AS("VarBinaryPtr"), + ).FROM(AllTypes) + + var dest []model.AllTypes + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + // fix float rounding lost before comparison + dest[0].Float = 3.33 + dest[0].FloatPtr = ptr.Of(3.33) + dest[1].Float = 3.33 + + //fmt.Println(allTypesJson) + testutils.AssertJSON(t, dest, allTypesJson) +} + func TestAllTypesViewSelect(t *testing.T) { type AllTypesView model.AllTypes diff --git a/tests/mysql/bench_test.go b/tests/mysql/bench_test.go new file mode 100644 index 0000000..4595ea9 --- /dev/null +++ b/tests/mysql/bench_test.go @@ -0,0 +1,138 @@ +//go:build bench +// +build bench + +package mysql + +import ( + "github.com/go-jet/jet/v2/internal/testutils" + . "github.com/go-jet/jet/v2/mysql" + "github.com/go-jet/jet/v2/tests/.gentestdata/mysql/dvds/model" + . "github.com/go-jet/jet/v2/tests/.gentestdata/mysql/dvds/table" + "github.com/stretchr/testify/require" + "testing" +) + +type allInfo []struct { + model.Actor + + Films []struct { + model.Film + + Language model.Language + Categories []model.Category + + Inventories []struct { + model.Inventory + + Rentals []struct { + model.Rental + + Customer model.Customer + } + } + } +} + +func BenchmarkTestDVDsJoinEverything(b *testing.B) { + for i := 0; i < b.N; i++ { + testDVDsJoinEverything(b) + } +} + +func TestDVDsJoinEverything(t *testing.T) { + testDVDsJoinEverything(t) +} + +func testDVDsJoinEverything(t require.TestingT) { + stmt := SELECT( + Actor.AllColumns, + Film.AllColumns, + Language.AllColumns, + Category.AllColumns, + Inventory.AllColumns, + Rental.AllColumns, + Customer.AllColumns, + ).FROM( + Actor. + LEFT_JOIN(FilmActor, Actor.ActorID.EQ(FilmActor.ActorID)). + LEFT_JOIN(Film, Film.FilmID.EQ(FilmActor.FilmID)). + LEFT_JOIN(Language, Language.LanguageID.EQ(Film.LanguageID)). + LEFT_JOIN(FilmCategory, FilmCategory.FilmID.EQ(Film.FilmID)). + LEFT_JOIN(Category, Category.CategoryID.EQ(FilmCategory.CategoryID)). + LEFT_JOIN(Inventory, Inventory.FilmID.EQ(Film.FilmID)). + LEFT_JOIN(Rental, Rental.InventoryID.EQ(Inventory.InventoryID)). + LEFT_JOIN(Customer, Customer.CustomerID.EQ(Rental.CustomerID)), + ).ORDER_BY( + Actor.ActorID.ASC(), + Film.FilmID.ASC(), + Category.CategoryID.ASC(), + Inventory.InventoryID.ASC(), + Rental.RentalID.ASC(), + ) + + var dest allInfo + + err := stmt.Query(db, &dest) + require.NoError(t, err) + + //testutils.SaveJSONFile(dest, "./testdata/results/mysql/dvds_join_everything.json") + testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/dvds_join_everything.json") +} + +func BenchmarkTestDVDsJoinEverythingJSON(b *testing.B) { + for i := 0; i < b.N; i++ { + testDVDsJoinEverythingJSON(b) + } +} + +func TestDVDsJoinEverythingJSON(t *testing.T) { + testDVDsJoinEverythingJSON(t) +} + +func testDVDsJoinEverythingJSON(t require.TestingT) { + stmt := SELECT_JSON_ARR( + Actor.ActorID, Actor.FirstName, Actor.LastName, Actor.LastUpdate, + + SELECT_JSON_ARR( + Film.AllColumns, + + SELECT_JSON_OBJ(Language.AllColumns). + FROM(Language). + WHERE(Language.LanguageID.EQ(Film.LanguageID)).AS("Language"), + + SELECT_JSON_ARR(Category.AllColumns). + FROM(Category.INNER_JOIN(FilmCategory, FilmCategory.CategoryID.EQ(Category.CategoryID))). + WHERE(FilmCategory.FilmID.EQ(Film.FilmID)).AS("Categories"), + + SELECT_JSON_ARR( + Inventory.AllColumns, + + SELECT_JSON_ARR( + Rental.AllColumns, + + SELECT_JSON_OBJ(Customer.AllColumns). + FROM(Customer). + WHERE(Customer.CustomerID.EQ(Rental.CustomerID)).AS("Customer"), + ).FROM(Rental). + WHERE(Rental.InventoryID.EQ(Inventory.InventoryID)). + ORDER_BY(Rental.RentalID).AS("Rentals"), + ).FROM(Inventory). + WHERE(Inventory.FilmID.EQ(Film.FilmID)). + ORDER_BY(Inventory.InventoryID).AS("Inventories"), + ).FROM(Film. + INNER_JOIN(FilmActor, FilmActor.FilmID.EQ(Film.FilmID)), + ).WHERE(FilmActor.ActorID.EQ(Actor.ActorID)). + ORDER_BY(Film.FilmID.ASC()).AS("Films"), + ).FROM(Actor). + ORDER_BY(Actor.ActorID.ASC()) + + //fmt.Println(stmt.DebugSql()) + + var dest allInfo + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + //testutils.SaveJSONFile(dest, "./testdata/results/mysql/dvds_join_everything2.json") + testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/dvds_join_everything.json") +} diff --git a/tests/mysql/select_json_test.go b/tests/mysql/select_json_test.go new file mode 100644 index 0000000..2d0ccef --- /dev/null +++ b/tests/mysql/select_json_test.go @@ -0,0 +1,437 @@ +package mysql + +import ( + "context" + "fmt" + "github.com/go-jet/jet/v2/qrm" + "strings" + "testing" + + "github.com/go-jet/jet/v2/internal/testutils" + . "github.com/go-jet/jet/v2/mysql" + "github.com/go-jet/jet/v2/tests/.gentestdata/mysql/dvds/model" + . "github.com/go-jet/jet/v2/tests/.gentestdata/mysql/dvds/table" + + "github.com/stretchr/testify/require" +) + +var ctx = context.Background() + +func TestSelectJsonObj(t *testing.T) { + stmt := SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.ActorID.EQ(Int(2))) + + testutils.AssertStatementSql(t, stmt, ` +SELECT JSON_OBJECT('actorID', actor.actor_id, + 'firstName', actor.first_name, + 'lastName', actor.last_name, + 'lastUpdate', actor.last_update) AS "json" +FROM dvds.actor +WHERE actor.actor_id = ?; +`, int64(2)) + + var dest model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.Nil(t, err) + + testutils.AssertDeepEqual(t, dest, actor2) + requireLogged(t, stmt) + requireQueryLogged(t, stmt, 1) +} + +func TestSelectJsonObj_NestedObj(t *testing.T) { + stmt := SELECT_JSON_OBJ( + Actor.AllColumns, + + SELECT_JSON_OBJ(Film.AllColumns). + FROM(FilmActor.INNER_JOIN(Film, Film.FilmID.EQ(FilmActor.FilmID))). + WHERE(Actor.ActorID.EQ(FilmActor.ActorID)). + ORDER_BY(Film.Length.DESC()). + LIMIT(1).AS("LongestFilm"), + ).FROM( + Actor, + ).WHERE( + Actor.ActorID.EQ(Int(2)), + ) + + testutils.AssertStatementSql(t, stmt, ` +SELECT JSON_OBJECT('actorID', actor.actor_id, + 'firstName', actor.first_name, + 'lastName', actor.last_name, + 'lastUpdate', actor.last_update, + 'LongestFilm', ( + SELECT JSON_OBJECT('filmID', film.film_id, + 'title', film.title, + 'description', film.description, + 'releaseYear', film.release_year, + 'languageID', film.language_id, + 'originalLanguageID', film.original_language_id, + 'rentalDuration', film.rental_duration, + 'rentalRate', film.rental_rate, + 'length', film.length, + 'replacementCost', film.replacement_cost, + 'rating', film.rating, + 'specialFeatures', film.special_features, + 'lastUpdate', film.last_update) AS "json" + FROM dvds.film_actor + INNER JOIN dvds.film ON (film.film_id = film_actor.film_id) + WHERE actor.actor_id = film_actor.actor_id + ORDER BY film.length DESC + LIMIT ? + )) AS "json" +FROM dvds.actor +WHERE actor.actor_id = ?; +`) + + var dest struct { + model.Actor + + LongestFilm model.Film + } + + err := stmt.QueryJSON(ctx, db, &dest) + require.Nil(t, err) + testutils.AssertJSON(t, dest, ` +{ + "ActorID": 2, + "FirstName": "NICK", + "LastName": "WAHLBERG", + "LastUpdate": "2006-02-15T04:34:33Z", + "LongestFilm": { + "FilmID": 958, + "Title": "WARDROBE PHANTOM", + "Description": "A Action-Packed Display of a Mad Cow And a Astronaut who must Kill a Car in Ancient India", + "ReleaseYear": 2006, + "LanguageID": 1, + "OriginalLanguageID": null, + "RentalDuration": 6, + "RentalRate": 2.99, + "Length": 178, + "ReplacementCost": 19.99, + "Rating": "G", + "SpecialFeatures": "Trailers,Commentaries", + "LastUpdate": "2006-02-15T05:03:42Z" + } +} +`) + +} + +func TestSelectJsonArr(t *testing.T) { + stmt := SELECT_JSON_ARR(Actor.AllColumns). + FROM(Actor). + ORDER_BY(Actor.ActorID) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT JSON_ARRAYAGG(JSON_OBJECT('actorID', actor.actor_id, + 'firstName', actor.first_name, + 'lastName', actor.last_name, + 'lastUpdate', actor.last_update)) AS "json" +FROM dvds.actor +ORDER BY actor.actor_id; +`) + + var dest []model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.Nil(t, err) + + testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/all_actors.json") + requireLogged(t, stmt) + requireQueryLogged(t, stmt, 1) +} + +func TestSelectJsonArr_NestedArr(t *testing.T) { + stmt := SELECT_JSON_ARR( + Actor.AllColumns, + + SELECT_JSON_ARR( + Film.AllColumns, + ).FROM( + FilmActor.INNER_JOIN( + Film, + Film.FilmID.EQ(FilmActor.FilmID).AND( + Actor.ActorID.EQ(FilmActor.ActorID)), + ), + ).WHERE( + Film.FilmID.MOD(Int(17)).EQ(Int(0)), + ).ORDER_BY( + Film.Length.DESC(), + ).AS("Films"), + ).FROM( + Actor, + ).WHERE( + Actor.ActorID.BETWEEN(Int(1), Int(3)), + ).ORDER_BY( + Actor.ActorID, + ) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT JSON_ARRAYAGG(JSON_OBJECT('actorID', actor.actor_id, + 'firstName', actor.first_name, + 'lastName', actor.last_name, + 'lastUpdate', actor.last_update, + 'Films', ( + SELECT JSON_ARRAYAGG(JSON_OBJECT('filmID', film.film_id, + 'title', film.title, + 'description', film.description, + 'releaseYear', film.release_year, + 'languageID', film.language_id, + 'originalLanguageID', film.original_language_id, + 'rentalDuration', film.rental_duration, + 'rentalRate', film.rental_rate, + 'length', film.length, + 'replacementCost', film.replacement_cost, + 'rating', film.rating, + 'specialFeatures', film.special_features, + 'lastUpdate', film.last_update)) AS "json" + FROM dvds.film_actor + INNER JOIN dvds.film ON ((film.film_id = film_actor.film_id) AND (actor.actor_id = film_actor.actor_id)) + WHERE (film.film_id % 17) = 0 + ORDER BY film.length DESC + ))) AS "json" +FROM dvds.actor +WHERE actor.actor_id BETWEEN 1 AND 3 +ORDER BY actor.actor_id; +`) + + var dest []struct { + model.Actor + + Films []model.Film + } + + err := stmt.QueryJSON(ctx, db, &dest) + fmt.Println(err) + require.Nil(t, err) + testutils.AssertJSON(t, dest, ` +[ + { + "ActorID": 1, + "FirstName": "PENELOPE", + "LastName": "GUINESS", + "LastUpdate": "2006-02-15T04:34:33Z", + "Films": null + }, + { + "ActorID": 2, + "FirstName": "NICK", + "LastName": "WAHLBERG", + "LastUpdate": "2006-02-15T04:34:33Z", + "Films": [ + { + "FilmID": 357, + "Title": "GILBERT PELICAN", + "Description": "A Fateful Tale of a Man And a Feminist who must Conquer a Crocodile in A Manhattan Penthouse", + "ReleaseYear": 2006, + "LanguageID": 1, + "OriginalLanguageID": null, + "RentalDuration": 7, + "RentalRate": 0.99, + "Length": 114, + "ReplacementCost": 13.99, + "Rating": "G", + "SpecialFeatures": "Trailers,Commentaries", + "LastUpdate": "2006-02-15T05:03:42Z" + }, + { + "FilmID": 561, + "Title": "MASK PEACH", + "Description": "A Boring Character Study of a Student And a Robot who must Meet a Woman in California", + "ReleaseYear": 2006, + "LanguageID": 1, + "OriginalLanguageID": null, + "RentalDuration": 6, + "RentalRate": 2.99, + "Length": 123, + "ReplacementCost": 26.99, + "Rating": "NC-17", + "SpecialFeatures": "Commentaries,Deleted Scenes", + "LastUpdate": "2006-02-15T05:03:42Z" + } + ] + }, + { + "ActorID": 3, + "FirstName": "ED", + "LastName": "CHASE", + "LastUpdate": "2006-02-15T04:34:33Z", + "Films": [ + { + "FilmID": 17, + "Title": "ALONE TRIP", + "Description": "A Fast-Paced Character Study of a Composer And a Dog who must Outgun a Boat in An Abandoned Fun House", + "ReleaseYear": 2006, + "LanguageID": 1, + "OriginalLanguageID": null, + "RentalDuration": 3, + "RentalRate": 0.99, + "Length": 82, + "ReplacementCost": 14.99, + "Rating": "R", + "SpecialFeatures": "Trailers,Behind the Scenes", + "LastUpdate": "2006-02-15T05:03:42Z" + }, + { + "FilmID": 289, + "Title": "EVE RESURRECTION", + "Description": "A Awe-Inspiring Yarn of a Pastry Chef And a Database Administrator who must Challenge a Teacher in A Baloon", + "ReleaseYear": 2006, + "LanguageID": 1, + "OriginalLanguageID": null, + "RentalDuration": 5, + "RentalRate": 4.99, + "Length": 66, + "ReplacementCost": 25.99, + "Rating": "G", + "SpecialFeatures": "Trailers,Commentaries,Deleted Scenes", + "LastUpdate": "2006-02-15T05:03:42Z" + } + ] + } +] +`) + +} + +func TestSelectJson_GroupBy(t *testing.T) { + skipForMariaDB(t) // scope issues with select without FROM + + subQuery := SELECT( + Customer.AllColumns, + + SUM(Payment.Amount).AS("sum"), + AVG(Payment.Amount).AS("avg"), + MAX(Payment.Amount).AS("max"), + MIN(Payment.Amount).AS("min"), + COUNT(Payment.Amount).AS("count"), + ).FROM( + Payment. + INNER_JOIN(Customer, Customer.CustomerID.EQ(Payment.CustomerID)), + ).GROUP_BY( + Customer.CustomerID, + ).HAVING( + SUMf(Payment.Amount).GT(Float(125)), + ).ORDER_BY( + Customer.CustomerID, SUM(Payment.Amount).ASC(), + ).AsTable("customers_info") + + stmt := SELECT_JSON_ARR( + subQuery.AllColumns().Except( // TODO: remove when ColumnList.From() is implemented + FloatColumn("sum"), + FloatColumn("avg"), + FloatColumn("max"), + FloatColumn("min"), + FloatColumn("count"), + ), + + SELECT_JSON_OBJ( + FloatColumn("sum").From(subQuery), + FloatColumn("avg").From(subQuery), + FloatColumn("max").From(subQuery), + FloatColumn("min").From(subQuery), + FloatColumn("count").From(subQuery), + ).AS("amount"), + ).FROM(subQuery) + + testutils.AssertDebugStatementSql(t, stmt, strings.ReplaceAll(` +SELECT JSON_ARRAYAGG(JSON_OBJECT('customerID', customers_info.""customer.customer_id"", + 'storeID', customers_info.""customer.store_id"", + 'firstName', customers_info.""customer.first_name"", + 'lastName', customers_info.""customer.last_name"", + 'email', customers_info.""customer.email"", + 'addressID', customers_info.""customer.address_id"", + 'active', customers_info.""customer.active"", + 'createDate', customers_info.""customer.create_date"", + 'lastUpdate', customers_info.""customer.last_update"", + 'amount', ( + SELECT JSON_OBJECT('sum', customers_info.sum, + 'avg', customers_info.avg, + 'max', customers_info.max, + 'min', customers_info.min, + 'count', customers_info.count) AS "json" + ))) AS "json" +FROM ( + SELECT customer.customer_id AS "customer.customer_id", + customer.store_id AS "customer.store_id", + customer.first_name AS "customer.first_name", + customer.last_name AS "customer.last_name", + customer.email AS "customer.email", + customer.address_id AS "customer.address_id", + customer.active AS "customer.active", + customer.create_date AS "customer.create_date", + customer.last_update AS "customer.last_update", + SUM(payment.amount) AS "sum", + AVG(payment.amount) AS "avg", + MAX(payment.amount) AS "max", + MIN(payment.amount) AS "min", + COUNT(payment.amount) AS "count" + FROM dvds.payment + INNER JOIN dvds.customer ON (customer.customer_id = payment.customer_id) + GROUP BY customer.customer_id + HAVING SUM(payment.amount) > 125 + ORDER BY customer.customer_id, SUM(payment.amount) ASC + ) AS customers_info; +`, `""`, "`")) + + var dest []struct { + model.Customer + + Amount struct { + Sum float64 + Avg float64 + Max float64 + Min float64 + Count int64 + } + } + + err := stmt.QueryJSON(ctx, db, &dest) + fmt.Println(err) + require.Nil(t, err) + + testutils.AssertJSONFile(t, dest, "./testdata/results/mysql/customer_payment_sum.json") + requireLogged(t, stmt) +} + +func TestSelectJsonObject_EmptyResult(t *testing.T) { + + t.Run("json obj", func(t *testing.T) { + stmt := SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.FirstName.EQ(String("Kowalski"))) + + var dest model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.ErrorIs(t, err, qrm.ErrNoRows) + }) + + t.Run("json arr", func(t *testing.T) { + stmt := SELECT_JSON_ARR(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.FirstName.EQ(String("Kowalski"))) + + var dest []model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + require.Empty(t, dest) + }) +} + +func TestSelectJson_ProjectionNotAliased(t *testing.T) { + + t.Run("expression not aliased", func(t *testing.T) { + testutils.AssertPanicErr(t, func() { + stmt := SELECT_JSON_ARR( + Int(2).ADD(Customer.CustomerID), + ).FROM(Customer) + + stmt.DebugSql() + + }, "jet: expression need to be aliased when used as SELECT JSON projection.") + }) +} diff --git a/tests/mysql/select_test.go b/tests/mysql/select_test.go index 6e4507d..e27c683 100644 --- a/tests/mysql/select_test.go +++ b/tests/mysql/select_test.go @@ -19,9 +19,9 @@ import ( ) func TestSelect_ScanToStruct(t *testing.T) { - query := Actor. - SELECT(Actor.AllColumns). + query := SELECT(Actor.AllColumns). DISTINCT(). + FROM(Actor). WHERE(Actor.ActorID.EQ(Int(2))) testutils.AssertStatementSql(t, query, ` @@ -50,9 +50,56 @@ var actor2 = model.Actor{ LastUpdate: *testutils.TimestampWithoutTimeZone("2006-02-15 04:34:33", 2), } +func TestSelect_NestedObject(t *testing.T) { + stmt := SELECT( + Actor.AllColumns, + Film.AllColumns, + ).FROM( + Actor. + LEFT_JOIN(FilmActor, FilmActor.ActorID.EQ(Actor.ActorID)). + LEFT_JOIN(Film, Film.FilmID.EQ(FilmActor.FilmID)), + ).WHERE( + Actor.ActorID.EQ(Int(2)), + ).ORDER_BY( + Film.LastUpdate.DESC(), + ).LIMIT(1) + + var dest struct { + model.Actor + + LatestFilm model.Film + } + + err := stmt.Query(db, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, ` +{ + "ActorID": 2, + "FirstName": "NICK", + "LastName": "WAHLBERG", + "LastUpdate": "2006-02-15T04:34:33Z", + "LatestFilm": { + "FilmID": 3, + "Title": "ADAPTATION HOLES", + "Description": "A Astounding Reflection of a Lumberjack And a Car who must Sink a Lumberjack in A Baloon Factory", + "ReleaseYear": 2006, + "LanguageID": 1, + "OriginalLanguageID": null, + "RentalDuration": 7, + "RentalRate": 2.99, + "Length": 50, + "ReplacementCost": 18.99, + "Rating": "NC-17", + "SpecialFeatures": "Trailers,Deleted Scenes", + "LastUpdate": "2006-02-15T05:03:42Z" + } +} +`) +} + func TestSelect_ScanToSlice(t *testing.T) { - query := Actor. - SELECT(Actor.AllColumns). + query := SELECT(Actor.AllColumns). + FROM(Actor). ORDER_BY(Actor.ActorID) testutils.AssertStatementSql(t, query, ` @@ -107,19 +154,20 @@ GROUP BY payment.customer_id HAVING SUM(payment.amount) > 125.6 ORDER BY payment.customer_id, SUM(payment.amount) ASC; ` - query := Payment. - INNER_JOIN(Customer, Customer.CustomerID.EQ(Payment.CustomerID)). - SELECT( - Customer.AllColumns, + query := SELECT( + Customer.AllColumns, - SUMf(Payment.Amount).AS("amount.sum"), - AVG(Payment.Amount).AS("amount.avg"), - MAX(Payment.PaymentDate).AS("amount.max_date"), - MAXf(Payment.Amount).AS("amount.max"), - MIN(Payment.PaymentDate).AS("amount.min_date"), - MINf(Payment.Amount).AS("amount.min"), - COUNT(Payment.Amount).AS("amount.count"), - ). + SUMf(Payment.Amount).AS("amount.sum"), + AVG(Payment.Amount).AS("amount.avg"), + MAX(Payment.PaymentDate).AS("amount.max_date"), + MAXf(Payment.Amount).AS("amount.max"), + MIN(Payment.PaymentDate).AS("amount.min_date"), + MINf(Payment.Amount).AS("amount.min"), + COUNT(Payment.Amount).AS("amount.count"), + ).FROM( + Payment. + INNER_JOIN(Customer, Customer.CustomerID.EQ(Payment.CustomerID)), + ). GROUP_BY(Payment.CustomerID). HAVING( SUMf(Payment.Amount).GT(Float(125.6)), @@ -1122,7 +1170,7 @@ WHERE payment.payment_id < ? WINDOW w1 AS (PARTITION BY payment.payment_date), w2 AS (w1), w3 AS (w2 ORDER BY payment.customer_id) ORDER BY payment.customer_id; ` - query := Payment.SELECT( + query := SELECT( AVG(Payment.Amount).OVER(), AVG(Payment.Amount).OVER(Window("w1")), AVG(Payment.Amount).OVER( @@ -1131,7 +1179,7 @@ ORDER BY payment.customer_id; RANGE(PRECEDING(UNBOUNDED), FOLLOWING(UNBOUNDED)), ), AVG(Payment.Amount).OVER(Window("w3").RANGE(PRECEDING(UNBOUNDED), FOLLOWING(UNBOUNDED))), - ). + ).FROM(Payment). WHERE(Payment.PaymentID.LT(Int(10))). WINDOW("w1").AS(PARTITION_BY(Payment.PaymentDate)). WINDOW("w2").AS(Window("w1")). diff --git a/tests/postgres/alltypes_test.go b/tests/postgres/alltypes_test.go index 6c3755a..e28322d 100644 --- a/tests/postgres/alltypes_test.go +++ b/tests/postgres/alltypes_test.go @@ -35,6 +35,58 @@ func TestAllTypesSelect(t *testing.T) { testutils.AssertDeepEqual(t, dest[1], allTypesRow1) } +func TestAllTypesSelectJson(t *testing.T) { + + stmt := SELECT_JSON_ARR( + AllTypesAllColumns.Except( + AllTypes.JSON, AllTypes.JSONPtr, + AllTypes.Jsonb, AllTypes.JsonbPtr, + AllTypes.TextArray, AllTypes.TextArrayPtr, + AllTypes.JsonbArray, AllTypes.IntegerArray, AllTypes.IntegerArrayPtr, + AllTypes.TextMultiDimArray, AllTypes.TextMultiDimArrayPtr, + ), + CAST(AllTypes.JSONPtr).AS_TEXT().AS("jsonPtr"), + CAST(AllTypes.JSON).AS_TEXT().AS("JSON"), + CAST(AllTypes.JsonbPtr).AS_TEXT().AS("jsonbPtr"), + CAST(AllTypes.Jsonb).AS_TEXT().AS("Jsonb"), + CAST(AllTypes.TextArrayPtr).AS_TEXT().AS("TextArrayPtr"), + CAST(AllTypes.TextArray).AS_TEXT().AS("TextArray"), + CAST(AllTypes.JsonbArray).AS_TEXT().AS("JsonbArray"), + CAST(AllTypes.IntegerArray).AS_TEXT().AS("IntegerArray"), + CAST(AllTypes.IntegerArrayPtr).AS_TEXT().AS("IntegerArrayPtr"), + CAST(AllTypes.TextMultiDimArray).AS_TEXT().AS("TextMultiDimArray"), + CAST(AllTypes.TextMultiDimArrayPtr).AS_TEXT().AS("TextMultiDimArrayPtr"), + ).FROM(AllTypes) + + //fmt.Println(stmt.DebugSql()) + + var dest []model.AllTypes + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + // fix inconsistencies between postgres and cockroachdb. + // cockroachdb returns char[N] columns with trailing whitespaces trimmed + if sourceIsCockroachDB() { + dest[0].Char = allTypesRow0.Char + dest[0].CharPtr = allTypesRow0.CharPtr + + dest[1].Char = allTypesRow1.Char + dest[1].CharPtr = allTypesRow1.CharPtr + } + + // set time local before comparison + dest[0].Timestampz = dest[0].Timestampz.Local() + + if dest[0].TimestampzPtr != nil { + dest[0].TimestampzPtr = ptr.Of(dest[0].TimestampzPtr.Local()) + } + dest[1].Timestampz = dest[1].Timestampz.Local() + + require.Equal(t, dest[0], allTypesRow0) + require.Equal(t, dest[1], allTypesRow1) +} + func TestAllTypesViewSelect(t *testing.T) { type AllTypesView model.AllTypes var dest []AllTypesView @@ -146,40 +198,42 @@ RETURNING all_types.bytea AS "all_types.bytea", all_types.bytea_ptr AS "all_types.bytea_ptr"; `, byteArrHex, byteArrBin) - var inserted model.AllTypes - err := insertStmt.Query(db, &inserted) - require.NoError(t, err) + testutils.ExecuteInTxAndRollback(t, db, func(tx qrm.DB) { + var inserted model.AllTypes + err := insertStmt.Query(tx, &inserted) + require.NoError(t, err) - require.Equal(t, string(*inserted.ByteaPtr), "Hello Gopher!") - // It is not possible to initiate bytea column using hex format '\xDEADBEEF' with pq driver. - // pq driver always encodes parameter string if destination column is of type bytea. - // Probably pq driver error. - // require.Equal(t, string(inserted.Bytea), "Hello Gopher!") + require.Equal(t, string(*inserted.ByteaPtr), "Hello Gopher!") + // It is not possible to initiate bytea column using hex format '\xDEADBEEF' with pq driver. + // pq driver always encodes parameter string if destination column is of type bytea. + // Probably pq driver error. + // require.Equal(t, string(inserted.Bytea), "Hello Gopher!") - stmt := SELECT( - AllTypes.Bytea, - AllTypes.ByteaPtr, - ).FROM( - AllTypes, - ).WHERE( - AllTypes.ByteaPtr.EQ(Bytea(byteArrBin)), - ) + stmt := SELECT( + AllTypes.Bytea, + AllTypes.ByteaPtr, + ).FROM( + AllTypes, + ).WHERE( + AllTypes.ByteaPtr.EQ(Bytea(byteArrBin)), + ) - testutils.AssertStatementSql(t, stmt, ` + testutils.AssertStatementSql(t, stmt, ` SELECT all_types.bytea AS "all_types.bytea", all_types.bytea_ptr AS "all_types.bytea_ptr" FROM test_sample.all_types WHERE all_types.bytea_ptr = $1::bytea; `, byteArrBin) - var dest model.AllTypes + var dest model.AllTypes - err = stmt.Query(db, &dest) - require.NoError(t, err) + err = stmt.Query(tx, &dest) + require.NoError(t, err) - require.Equal(t, string(*dest.ByteaPtr), "Hello Gopher!") - // Probably pq driver error. - // require.Equal(t, string(dest.Bytea), "Hello Gopher!") + require.Equal(t, string(*dest.ByteaPtr), "Hello Gopher!") + // Probably pq driver error. + // require.Equal(t, string(dest.Bytea), "Hello Gopher!") + }) } func TestAllTypesFromSubQuery(t *testing.T) { diff --git a/tests/postgres/chinook_db_test.go b/tests/postgres/chinook_db_test.go index 349a4ba..a001ab5 100644 --- a/tests/postgres/chinook_db_test.go +++ b/tests/postgres/chinook_db_test.go @@ -188,7 +188,130 @@ ORDER BY "Artist"."ArtistId", "Album"."AlbumId", "Track"."TrackId"; `) } +type AllArtistDetails []struct { //list of all artist + model.Artist + + Albums []struct { // list of albums per artist + model.Album + + Tracks []struct { // list of tracks per album + model.Track + + Genre model.Genre // track genre + MediaType model.MediaType // track media type + + Playlists []model.Playlist // list of playlist where track is used + + Invoices []struct { // list of invoices where track occurs + model.Invoice + + Customer struct { // customer data for invoice + model.Customer + + Employee *struct { // employee data for customer if exists + model.Employee + + Manager *model.Employee `alias:"Manager"` + } + } + } + } + } +} + +func BenchmarkJoinEverythingJSON(b *testing.B) { + for i := 0; i < b.N; i++ { + testJoinEverythingJSON(b) + } +} + +func TestJoinEverythingJSON(t *testing.T) { + testJoinEverythingJSON(t) +} + +func testJoinEverythingJSON(t require.TestingT) { + + manager := Employee.AS("Manager") + + stmt := SELECT_JSON_ARR( + Artist.AllColumns, + + SELECT_JSON_ARR( + Album.AllColumns, + + SELECT_JSON_ARR( + Track.AllColumns, + + SELECT_JSON_OBJ(Genre.AllColumns). + FROM(Genre). + WHERE(Genre.GenreId.EQ(Track.GenreId)).AS("Genre"), + + SELECT_JSON_OBJ(MediaType.AllColumns). + FROM(MediaType). + WHERE(MediaType.MediaTypeId.EQ(Track.MediaTypeId)).AS("MediaType"), + + SELECT_JSON_ARR(Playlist.AllColumns). + FROM(Playlist.INNER_JOIN( + PlaylistTrack, + Playlist.PlaylistId.EQ(PlaylistTrack.PlaylistId))). + WHERE(PlaylistTrack.TrackId.EQ(Track.TrackId)). + ORDER_BY(Playlist.PlaylistId).AS("Playlists"), + + SELECT_JSON_ARR( + Invoice.AllColumns, + + SELECT_JSON_OBJ( + Customer.AllColumns, + + SELECT_JSON_OBJ( + Employee.AllColumns, + + SELECT_JSON_OBJ(manager.AllColumns). + FROM(manager). + WHERE(manager.EmployeeId.EQ(Employee.ReportsTo)).AS("Manager"), + ).FROM(Employee). + WHERE(Employee.EmployeeId.EQ(Customer.SupportRepId)).AS("Employee"), + ).FROM(Customer). + WHERE(Customer.CustomerId.EQ(Invoice.CustomerId)).AS("Customer"), + ).FROM(Invoice.INNER_JOIN( + InvoiceLine, + InvoiceLine.InvoiceId.EQ(Invoice.InvoiceId)), + ).WHERE(InvoiceLine.TrackId.EQ(Track.TrackId)). + ORDER_BY(Invoice.InvoiceId).AS("Invoices"), + ).FROM(Track). + WHERE(Track.AlbumId.EQ(Album.AlbumId)). + ORDER_BY(Track.TrackId).AS("Tracks"), + ).FROM(Album). + WHERE(Album.ArtistId.EQ(Artist.ArtistId)). + ORDER_BY(Album.AlbumId).AS("Albums"), + ).FROM(Artist). + ORDER_BY(Artist.ArtistId) + + //fmt.Println(stmt.DebugSql()) + + var dest AllArtistDetails + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + require.Equal(t, len(dest), 275) + //testutils.SaveJSONFile(dest, "./testdata/results/postgres/joined_everything2.json") + testutils.AssertJSONFile(t, dest, "./testdata/results/postgres/joined_everything.json") + requireLogged(t, stmt) + requireQueryLogged(t, stmt, 1) +} + +func BenchmarkJoinEverything(b *testing.B) { + for i := 0; i < b.N; i++ { + testJoinEverything(b) + } +} + func TestJoinEverything(t *testing.T) { + testJoinEverything(t) +} + +func testJoinEverything(t require.TestingT) { manager := Employee.AS("Manager") @@ -223,37 +346,6 @@ func TestJoinEverything(t *testing.T) { Invoice.InvoiceId, Customer.CustomerId, ) - var dest []struct { //list of all artist - model.Artist - - Albums []struct { // list of albums per artist - model.Album - - Tracks []struct { // list of tracks per album - model.Track - - Genre model.Genre // track genre - MediaType model.MediaType // track media type - - Playlists []model.Playlist // list of playlist where track is used - - Invoices []struct { // list of invoices where track occurs - model.Invoice - - Customer struct { // customer data for invoice - model.Customer - - Employee *struct { // employee data for customer if exists - model.Employee - - Manager *model.Employee `alias:"Manager"` - } - } - } - } - } - } - testutils.AssertStatementSql(t, stmt, ` SELECT "Artist"."ArtistId" AS "Artist.ArtistId", "Artist"."Name" AS "Artist.Name", @@ -344,7 +436,7 @@ FROM chinook."Artist" LEFT JOIN chinook."Employee" AS "Manager" ON ("Manager"."EmployeeId" = "Employee"."ReportsTo") ORDER BY "Artist"."ArtistId", "Album"."AlbumId", "Track"."TrackId", "Genre"."GenreId", "MediaType"."MediaTypeId", "Playlist"."PlaylistId", "Invoice"."InvoiceId", "Customer"."CustomerId"; `) - + var dest AllArtistDetails err := stmt.QueryContext(context.Background(), db, &dest) require.NoError(t, err) diff --git a/tests/postgres/main_test.go b/tests/postgres/main_test.go index 46dfd94..a179bd4 100644 --- a/tests/postgres/main_test.go +++ b/tests/postgres/main_test.go @@ -119,14 +119,22 @@ func init() { }) } -func requireLogged(t *testing.T, statement postgres.Statement) { +func requireLogged(t require.TestingT, statement postgres.Statement) { + if _, ok := t.(*testing.B); ok { + return // skip assert for benchmarks + } + query, args := statement.Sql() require.Equal(t, loggedSQL, query) require.Equal(t, loggedSQLArgs, args) require.Equal(t, loggedDebugSQL, statement.DebugSql()) } -func requireQueryLogged(t *testing.T, statement postgres.Statement, rowsProcessed int64) { +func requireQueryLogged(t require.TestingT, statement postgres.Statement, rowsProcessed int64) { + if _, ok := t.(*testing.B); ok { + return // skip assert for benchmarks + } + query, args := statement.Sql() queryLogged, argsLogged := queryInfo.Statement.Sql() diff --git a/tests/postgres/northwind_test.go b/tests/postgres/northwind_test.go index 7aad0d4..63e1b1d 100644 --- a/tests/postgres/northwind_test.go +++ b/tests/postgres/northwind_test.go @@ -9,7 +9,50 @@ import ( "testing" ) -func TestNorthwindJoinEverything(t *testing.T) { +type Dest []struct { + model.Customers + + Demographics model.CustomerDemographics + + Orders []struct { + model.Orders + + Shipper model.Shippers + + Employee struct { + model.Employees + + Territories []struct { + model.Territories + + Region model.Region + } + } + + Details []struct { + model.OrderDetails + + Products struct { + model.Products + + Category model.Categories + Supplier model.Suppliers + } + } + } +} + +func BenchmarkTestNorthwindJoinEverything(b *testing.B) { + for i := 0; i < b.N; i++ { + testNorthwindJoinEverything(b) + } +} + +func TestTestNorthwindJoinEverything(t *testing.T) { + testNorthwindJoinEverything(t) +} + +func testNorthwindJoinEverything(t require.TestingT) { stmt := SELECT( @@ -21,6 +64,9 @@ func TestNorthwindJoinEverything(t *testing.T) { Products.AllColumns, Categories.AllColumns, Suppliers.AllColumns, + Employees.AllColumns, + Territories.AllColumns, + Region.AllColumns, ).FROM( Customers. LEFT_JOIN(CustomerCustomerDemo, Customers.CustomerID.EQ(CustomerCustomerDemo.CustomerID)). @@ -35,35 +81,110 @@ func TestNorthwindJoinEverything(t *testing.T) { LEFT_JOIN(EmployeeTerritories, EmployeeTerritories.EmployeeID.EQ(Employees.EmployeeID)). LEFT_JOIN(Territories, EmployeeTerritories.TerritoryID.EQ(Territories.TerritoryID)). LEFT_JOIN(Region, Territories.RegionID.EQ(Region.RegionID)), - ).ORDER_BY(Customers.CustomerID, Orders.OrderID, Products.ProductID) + ).ORDER_BY( + Customers.CustomerID, + Orders.OrderID, + Products.ProductID, + Territories.TerritoryID, + ) - var dest []struct { - model.Customers + //fmt.Println(stmt.DebugSql()) - Demographics model.CustomerDemographics - - Orders []struct { - model.Orders - - Shipper model.Shippers - - Details struct { - model.OrderDetails - - Products []struct { - model.Products - - Category model.Categories - Supplier model.Suppliers - } - } - } - } + var dest Dest err := stmt.Query(db, &dest) require.NoError(t, err) - //jsonSave("./testdata/northwind-all.json", dest) + //testutils.SaveJSONFile(dest, "./testdata/results/postgres/northwind-all.json") testutils.AssertJSONFile(t, dest, "./testdata/results/postgres/northwind-all.json") requireLogged(t, stmt) } + +func BenchmarkTestNorthwindJoinEverythingJson(b *testing.B) { + for i := 0; i < b.N; i++ { + testNorthwindJoinEverythingJson(b) + } +} + +func TestNorthwindJoinEverythingJson(t *testing.T) { + testNorthwindJoinEverythingJson(t) +} + +func testNorthwindJoinEverythingJson(t require.TestingT) { + + stmt := SELECT_JSON_ARR( + Customers.AllColumns, + + SELECT_JSON_OBJ(CustomerDemographics.AllColumns). + FROM(CustomerDemographics.INNER_JOIN(CustomerCustomerDemo, CustomerCustomerDemo.CustomerTypeID.EQ(CustomerDemographics.CustomerTypeID))). + WHERE(CustomerCustomerDemo.CustomerID.EQ(Customers.CustomerID)).AS("Demographics"), + + SELECT_JSON_ARR( + Orders.AllColumns, + + SELECT_JSON_OBJ(Shippers.AllColumns). + FROM(Shippers). + WHERE(Shippers.ShipperID.EQ(Orders.ShipVia)).AS("Shipper"), + + SELECT_JSON_OBJ( + Employees.AllColumns, + SELECT_JSON_ARR( + Territories.AllColumns, + + SELECT_JSON_OBJ(Region.AllColumns). + FROM(Region). + WHERE(Region.RegionID.EQ(Territories.RegionID)).AS("Region"), + ).FROM( + EmployeeTerritories.LEFT_JOIN( + Territories, + EmployeeTerritories.TerritoryID.EQ(Territories.TerritoryID)), + ).WHERE( + EmployeeTerritories.EmployeeID.EQ(Employees.EmployeeID), // TODO: move to join + ).AS("Territories"), + ).FROM(Employees). + WHERE(Orders.EmployeeID.EQ(Employees.EmployeeID)).AS("Employee"), + + SELECT_JSON_ARR( + OrderDetails.AllColumns, + + SELECT_JSON_OBJ( + Products.AllColumns, + + SELECT_JSON_OBJ( + Categories.AllColumns, + ).FROM(Categories). + WHERE(Categories.CategoryID.EQ(Products.CategoryID)).AS("Category"), + + SELECT_JSON_OBJ(Suppliers.AllColumns). + FROM(Suppliers). + WHERE(Suppliers.SupplierID.EQ(Products.SupplierID)).AS("Supplier"), + ).FROM(Products). + WHERE(Products.ProductID.EQ(OrderDetails.ProductID)).AS("Products"), + ).FROM( + OrderDetails, + ).WHERE( + OrderDetails.OrderID.EQ(Orders.OrderID), + ).AS("Details"), + ).FROM( + Orders, + ).WHERE( + Orders.CustomerID.EQ(Customers.CustomerID), + ).ORDER_BY( + Orders.OrderID, + ).AS("Orders"), + ).FROM( + Customers, + ).ORDER_BY( + Customers.CustomerID, + ) + + //fmt.Println(stmt.DebugSql()) + + var dest Dest + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + //testutils.SaveJSONFile(dest, "./testdata/results/postgres/northwind-all2.json") + testutils.AssertJSONFile(t, dest, "./testdata/results/postgres/northwind-all.json") +} diff --git a/tests/postgres/sample_test.go b/tests/postgres/sample_test.go index 0a252db..45eb7f3 100644 --- a/tests/postgres/sample_test.go +++ b/tests/postgres/sample_test.go @@ -220,20 +220,7 @@ func TestUUIDComplex(t *testing.T) { requireLogged(t, query) }) - t.Run("slice of structs left join", func(t *testing.T) { - leftQuery := Person.LEFT_JOIN(PersonPhone, PersonPhone.PersonID.EQ(Person.PersonID)). - SELECT(Person.AllColumns, PersonPhone.AllColumns). - ORDER_BY(Person.PersonID.ASC(), PersonPhone.PhoneID.ASC()) - var dest []struct { - model.Person - Phones []struct { - model.PersonPhone - } - } - err := leftQuery.Query(db, &dest) - - require.NoError(t, err) - testutils.AssertJSON(t, dest, ` + var expectedSliceOfStructsLeftJoin = ` [ { "PersonID": "b68dbff4-a87d-11e9-a7f2-98ded00c39c6", @@ -274,10 +261,50 @@ func TestUUIDComplex(t *testing.T) { ] } ] -`) +` + + t.Run("slice of structs left join", func(t *testing.T) { + leftQuery := Person.LEFT_JOIN(PersonPhone, PersonPhone.PersonID.EQ(Person.PersonID)). + SELECT(Person.AllColumns, PersonPhone.AllColumns). + ORDER_BY(Person.PersonID.ASC(), PersonPhone.PhoneID.ASC()) + var dest []struct { + model.Person + Phones []struct { + model.PersonPhone + } + } + err := leftQuery.Query(db, &dest) + + require.NoError(t, err) + testutils.AssertJSON(t, dest, expectedSliceOfStructsLeftJoin) requireLogged(t, leftQuery) }) + t.Run("select json", func(t *testing.T) { + jsonQuery := SELECT_JSON_ARR( + Person.AllColumns, + SELECT_JSON_ARR(PersonPhone.AllColumns). + FROM(PersonPhone). + WHERE(PersonPhone.PersonID.EQ(Person.PersonID)). + ORDER_BY(PersonPhone.PhoneID).AS("Phones"), + ).FROM( + Person, + ).ORDER_BY( + Person.PersonID.ASC(), + ) + + var dest []struct { + model.Person + Phones []struct { + model.PersonPhone + } + } + + err := jsonQuery.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, expectedSliceOfStructsLeftJoin) + }) + } func TestEnumType(t *testing.T) { query := Person. diff --git a/tests/postgres/scan_test.go b/tests/postgres/scan_test.go index 321fc38..a630145 100644 --- a/tests/postgres/scan_test.go +++ b/tests/postgres/scan_test.go @@ -209,7 +209,7 @@ func TestScanToStruct(t *testing.T) { err := query.Query(db, &dest) require.Error(t, err) - require.EqualError(t, err, "jet: can't scan int64('\\x01') to 'InventoryID uuid.UUID': Scan: unable to scan type int64 into UUID") + require.EqualError(t, err, "jet: can't assign int64('\\x01') to 'InventoryID uuid.UUID': Scan: unable to scan type int64 into UUID") }) t.Run("type mismatch base type", func(t *testing.T) { diff --git a/tests/postgres/select_json_test.go b/tests/postgres/select_json_test.go new file mode 100644 index 0000000..517df9a --- /dev/null +++ b/tests/postgres/select_json_test.go @@ -0,0 +1,908 @@ +package postgres + +import ( + "context" + "github.com/go-jet/jet/v2/internal/testutils" + "github.com/go-jet/jet/v2/internal/utils/ptr" + "github.com/go-jet/jet/v2/qrm" + "github.com/go-jet/jet/v2/tests/.gentestdata/jetdb/dvds/view" + "github.com/stretchr/testify/require" + "testing" + "time" + + . "github.com/go-jet/jet/v2/postgres" + "github.com/go-jet/jet/v2/tests/.gentestdata/jetdb/dvds/model" + . "github.com/go-jet/jet/v2/tests/.gentestdata/jetdb/dvds/table" +) + +var ctx = context.Background() + +func TestSelectJsonObject(t *testing.T) { + stmt := SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.ActorID.EQ(Int32(2))) + + testutils.AssertStatementSql(t, stmt, ` +SELECT row_to_json(records) AS "json" +FROM ( + SELECT actor.actor_id AS "actorID", + actor.first_name AS "firstName", + actor.last_name AS "lastName", + actor.last_update AS "lastUpdate" + FROM dvds.actor + WHERE actor.actor_id = $1::integer + ) AS records; +`, int32(2)) + + var dest model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + testutils.AssertDeepEqual(t, dest, actor2) + requireLogged(t, stmt) + + t.Run("scan to map", func(t *testing.T) { + var dest2 map[string]interface{} + + err = stmt.QueryJSON(ctx, db, &dest2) + require.NoError(t, err) + testutils.PrintJson(dest2) + testutils.AssertDeepEqual(t, dest2, map[string]interface{}{ + "actorID": float64(2), + "firstName": "Nick", + "lastName": "Wahlberg", + "lastUpdate": "2013-05-26T14:47:57.62", + }) + }) +} + +func TestSelectJsonArr(t *testing.T) { + stmt := SELECT_JSON_ARR( + Rental.StaffID, + Rental.CustomerID, + Rental.RentalID, + ).DISTINCT( + Rental.StaffID, + Rental.CustomerID, + ).FROM( + Rental, + ).WHERE( + Rental.CustomerID.LT(Int(2)), + ).ORDER_BY( + Rental.StaffID.ASC(), + Rental.CustomerID.ASC(), + Rental.RentalID.ASC(), + ) + + testutils.AssertStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT DISTINCT ON (rental.staff_id, rental.customer_id) rental.staff_id AS "staffID", + rental.customer_id AS "customerID", + rental.rental_id AS "rentalID" + FROM dvds.rental + WHERE rental.customer_id < $1 + ORDER BY rental.staff_id ASC, rental.customer_id ASC, rental.rental_id ASC + ) AS records; +`, int64(2)) + + var dest []model.Rental + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, ` +[ + { + "RentalID": 573, + "RentalDate": "0001-01-01T00:00:00Z", + "InventoryID": 0, + "CustomerID": 1, + "ReturnDate": null, + "StaffID": 1, + "LastUpdate": "0001-01-01T00:00:00Z" + }, + { + "RentalID": 76, + "RentalDate": "0001-01-01T00:00:00Z", + "InventoryID": 0, + "CustomerID": 1, + "ReturnDate": null, + "StaffID": 2, + "LastUpdate": "0001-01-01T00:00:00Z" + } +] +`) + t.Run("scan to array map", func(t *testing.T) { + var dest2 []map[string]interface{} + + err := stmt.QueryJSON(ctx, db, &dest2) + require.NoError(t, err) + testutils.AssertDeepEqual(t, dest2, []map[string]interface{}{ + { + "rentalID": 573., + "customerID": 1., + "staffID": 1., + }, + { + "rentalID": 76., + "customerID": 1., + "staffID": 2., + }, + }) + }) +} + +func TestSelectJsonArr_NestedArr(t *testing.T) { + + stmt := SELECT_JSON_ARR( + Customer.AllColumns, + + SELECT_JSON_ARR(Rental.AllColumns). + FROM(Rental). + WHERE(Rental.CustomerID.EQ(Customer.CustomerID)). + ORDER_BY(Rental.RentalID). + OFFSET_e(Int(1)).LIMIT(3).AS("Rentals"), + ).FROM( + Customer, + ).ORDER_BY( + Customer.CustomerID, + ).LIMIT(2).OFFSET(1) + + testutils.AssertStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT customer.customer_id AS "customerID", + customer.store_id AS "storeID", + customer.first_name AS "firstName", + customer.last_name AS "lastName", + customer.email AS "email", + customer.address_id AS "addressID", + customer.activebool AS "activebool", + customer.create_date AS "createDate", + customer.last_update AS "lastUpdate", + customer.active AS "active", + ( + SELECT json_agg(row_to_json(rentals_records)) AS "rentals_json" + FROM ( + SELECT rental.rental_id AS "rentalID", + rental.rental_date AS "rentalDate", + rental.inventory_id AS "inventoryID", + rental.customer_id AS "customerID", + rental.return_date AS "returnDate", + rental.staff_id AS "staffID", + rental.last_update AS "lastUpdate" + FROM dvds.rental + WHERE rental.customer_id = customer.customer_id + ORDER BY rental.rental_id + LIMIT $1 + OFFSET $2 + ) AS rentals_records + ) AS "Rentals" + FROM dvds.customer + ORDER BY customer.customer_id + LIMIT $3 + OFFSET $4 + ) AS records; +`) + + var dest []struct { + model.Customer + + Rentals []model.Rental + } + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + t.Run("partial select json", func(t *testing.T) { + + stmt := SELECT( + Customer.AllColumns, + + SELECT_JSON_ARR(Rental.AllColumns). + FROM(Rental). + WHERE(Rental.CustomerID.EQ(Customer.CustomerID)). + ORDER_BY(Rental.RentalID). + OFFSET_e(Int(1)).LIMIT(3).AS("Rentals"), + ).FROM( + Customer, + ).ORDER_BY( + Customer.CustomerID, + ).OFFSET(1).LIMIT(2) + + testutils.AssertStatementSql(t, stmt, ` +SELECT customer.customer_id AS "customer.customer_id", + customer.store_id AS "customer.store_id", + customer.first_name AS "customer.first_name", + customer.last_name AS "customer.last_name", + customer.email AS "customer.email", + customer.address_id AS "customer.address_id", + customer.activebool AS "customer.activebool", + customer.create_date AS "customer.create_date", + customer.last_update AS "customer.last_update", + customer.active AS "customer.active", + ( + SELECT json_agg(row_to_json(rentals_records)) AS "rentals_json" + FROM ( + SELECT rental.rental_id AS "rentalID", + rental.rental_date AS "rentalDate", + rental.inventory_id AS "inventoryID", + rental.customer_id AS "customerID", + rental.return_date AS "returnDate", + rental.staff_id AS "staffID", + rental.last_update AS "lastUpdate" + FROM dvds.rental + WHERE rental.customer_id = customer.customer_id + ORDER BY rental.rental_id + LIMIT $1 + OFFSET $2 + ) AS rentals_records + ) AS "Rentals" +FROM dvds.customer +ORDER BY customer.customer_id +LIMIT $3 +OFFSET $4; +`) + + var dest2 []struct { + model.Customer + + Rentals []model.Rental `json_column:"Rentals"` + } + + err := stmt.Query(db, &dest2) + require.NoError(t, err) + testutils.AssertJsonEqual(t, dest, dest2) + + var dest3 []struct { + model.Customer + + Rentals *[]model.Rental `json_column:"rentals"` + } + + err = stmt.Query(db, &dest3) + require.NoError(t, err) + testutils.AssertJsonEqual(t, dest, dest3) + + var dest4 []struct { + model.Customer + + Rentals []*model.Rental `json_column:"rentals"` + } + + err = stmt.Query(db, &dest4) + require.NoError(t, err) + testutils.AssertJsonEqual(t, dest, dest4) + }) +} + +func TestSelectJson_GroupByHaving(t *testing.T) { + stmt := SELECT_JSON_ARR( + Customer.AllColumns, + + SELECT_JSON_OBJ( + SUM(Payment.Amount).AS("sum"), + AVG(Payment.Amount).AS("avg"), + MAX(Payment.PaymentDate).AS("max_date"), + MAX(Payment.Amount).AS("max"), + MIN(Payment.PaymentDate).AS("min_date"), + MIN(Payment.Amount).AS("min"), + COUNT(Payment.Amount).AS("count"), + ).AS("amount"), + ).FROM( + Payment. + INNER_JOIN(Customer, Customer.CustomerID.EQ(Payment.CustomerID)), + ).GROUP_BY( + Customer.CustomerID, + ).HAVING( + SUMf(Payment.Amount).GT(Real(125)), + ).ORDER_BY( + Customer.CustomerID, SUM(Payment.Amount).ASC(), + ) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT customer.customer_id AS "customerID", + customer.store_id AS "storeID", + customer.first_name AS "firstName", + customer.last_name AS "lastName", + customer.email AS "email", + customer.address_id AS "addressID", + customer.activebool AS "activebool", + customer.create_date AS "createDate", + customer.last_update AS "lastUpdate", + customer.active AS "active", + ( + SELECT row_to_json(amount_records) AS "amount_json" + FROM ( + SELECT SUM(payment.amount) AS "sum", + AVG(payment.amount) AS "avg", + MAX(payment.payment_date) AS "max_date", + MAX(payment.amount) AS "max", + MIN(payment.payment_date) AS "min_date", + MIN(payment.amount) AS "min", + COUNT(payment.amount) AS "count" + ) AS amount_records + ) AS "amount" + FROM dvds.payment + INNER JOIN dvds.customer ON (customer.customer_id = payment.customer_id) + GROUP BY customer.customer_id + HAVING SUM(payment.amount) > 125::real + ORDER BY customer.customer_id, SUM(payment.amount) ASC + ) AS records; +`) + + var dest []struct { + model.Customer + + Amount struct { + Sum float64 + Avg float64 + Max float64 + Min float64 + Count int64 + } `alias:"amount"` + } + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + if sourceIsCockroachDB() { + return // small precision difference in result + } + + testutils.AssertJSONFile(t, dest, "./testdata/results/postgres/customer_payment_sum.json") +} + +func TestSelectQuickStartJSON(t *testing.T) { + + stmt := SELECT_JSON_ARR( + Actor.ActorID, Actor.FirstName, Actor.LastName, Actor.LastUpdate, + + SELECT_JSON_ARR( + Film.AllColumns.Except(Film.SpecialFeatures), + CAST(Film.SpecialFeatures).AS_TEXT().AS("SpecialFeatures"), + + SELECT_JSON_OBJ( + Language.AllColumns, + ).FROM( + Language, + ).WHERE( + Language.LanguageID.EQ(Film.LanguageID).AND( + Language.Name.EQ(Char(20)("English")), + ), + ).AS("Language"), + + SELECT_JSON_ARR( + Category.AllColumns, + ).FROM( + Category. + INNER_JOIN(FilmCategory, FilmCategory.CategoryID.EQ(Category.CategoryID)), + ).WHERE( + FilmCategory.FilmID.EQ(Film.FilmID).AND( + Category.Name.NOT_EQ(Text("Action")), + ), + ).AS("Categories"), + ).FROM( + Film. + INNER_JOIN(FilmActor, FilmActor.FilmID.EQ(Film.FilmID)), + ).WHERE( + FilmActor.ActorID.EQ(Actor.ActorID).AND(Film.Length.GT(Int32(180))), + ).ORDER_BY( + Film.FilmID.ASC(), + ).AS("Films"), + ).FROM( + Actor, + ).ORDER_BY( + Actor.ActorID.ASC(), + ) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT actor.actor_id AS "actorID", + actor.first_name AS "firstName", + actor.last_name AS "lastName", + actor.last_update AS "lastUpdate", + ( + SELECT json_agg(row_to_json(films_records)) AS "films_json" + FROM ( + SELECT film.film_id AS "filmID", + film.title AS "title", + film.description AS "description", + film.release_year AS "releaseYear", + film.language_id AS "languageID", + film.rental_duration AS "rentalDuration", + film.rental_rate AS "rentalRate", + film.length AS "length", + film.replacement_cost AS "replacementCost", + film.rating AS "rating", + film.last_update AS "lastUpdate", + film.fulltext AS "fulltext", + film.special_features::text AS "SpecialFeatures", + ( + SELECT row_to_json(language_records) AS "language_json" + FROM ( + SELECT language.language_id AS "languageID", + language.name AS "name", + language.last_update AS "lastUpdate" + FROM dvds.language + WHERE (language.language_id = film.language_id) AND (language.name = 'English'::char(20)) + ) AS language_records + ) AS "Language", + ( + SELECT json_agg(row_to_json(categories_records)) AS "categories_json" + FROM ( + SELECT category.category_id AS "categoryID", + category.name AS "name", + category.last_update AS "lastUpdate" + FROM dvds.category + INNER JOIN dvds.film_category ON (film_category.category_id = category.category_id) + WHERE (film_category.film_id = film.film_id) AND (category.name != 'Action'::text) + ) AS categories_records + ) AS "Categories" + FROM dvds.film + INNER JOIN dvds.film_actor ON (film_actor.film_id = film.film_id) + WHERE (film_actor.actor_id = actor.actor_id) AND (film.length > 180::integer) + ORDER BY film.film_id ASC + ) AS films_records + ) AS "Films" + FROM dvds.actor + ORDER BY actor.actor_id ASC + ) AS records; +`) + + var dest []struct { + model.Actor + + Films []struct { + model.Film + + Language model.Language + Categories []model.Category + } + } + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + require.Len(t, dest, 200) + + if sourceIsCockroachDB() { + return // char[n] columns whitespaces are trimmed when returned as json in cockroachdb + } + + //testutils.SaveJSONFile(dest, "./testdata/results/postgres/quick-start-json-dest2.json") + //testutils.AssertJSONFile(t, dest, "./testdata/results/postgres/quick-start-json-dest.json") +} + +func TestSelectJsonInReturning(t *testing.T) { + + stmt := Rental. + UPDATE(Rental.ReturnDate). + MODEL(model.Rental{ + ReturnDate: ptr.Of(time.Date(2010, 2, 4, 5, 6, 7, 8, time.UTC)), + }). + WHERE( + Rental.RentalID.EQ(Int(11496)), + ). + RETURNING( + Rental.AllColumns.Except(Rental.LastUpdate), + + SELECT_JSON_OBJ( + Customer.AllColumns, + ).FROM( + Customer, + ).WHERE( + Customer.CustomerID.EQ(Rental.CustomerID), + ).AS("Customer"), + ) + + testutils.AssertStatementSql(t, stmt, ` +UPDATE dvds.rental +SET return_date = $1 +WHERE rental.rental_id = $2 +RETURNING rental.rental_id AS "rental.rental_id", + rental.rental_date AS "rental.rental_date", + rental.inventory_id AS "rental.inventory_id", + rental.customer_id AS "rental.customer_id", + rental.return_date AS "rental.return_date", + rental.staff_id AS "rental.staff_id", + ( + SELECT row_to_json(customer_records) AS "customer_json" + FROM ( + SELECT customer.customer_id AS "customerID", + customer.store_id AS "storeID", + customer.first_name AS "firstName", + customer.last_name AS "lastName", + customer.email AS "email", + customer.address_id AS "addressID", + customer.activebool AS "activebool", + customer.create_date AS "createDate", + customer.last_update AS "lastUpdate", + customer.active AS "active" + FROM dvds.customer + WHERE customer.customer_id = rental.customer_id + ) AS customer_records + ) AS "Customer"; +`) + + testutils.ExecuteInTxAndRollback(t, db, func(tx qrm.DB) { + var dest struct { + model.Rental + + Customer model.Customer `json_column:"Customer"` + } + + err := stmt.Query(tx, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, ` +{ + "RentalID": 11496, + "RentalDate": "2006-02-14T15:16:03Z", + "InventoryID": 2047, + "CustomerID": 155, + "ReturnDate": "2010-02-04T05:06:07Z", + "StaffID": 1, + "LastUpdate": "0001-01-01T00:00:00Z", + "Customer": { + "CustomerID": 155, + "StoreID": 1, + "FirstName": "Gail", + "LastName": "Knight", + "Email": "gail.knight@sakilacustomer.org", + "AddressID": 159, + "Activebool": true, + "CreateDate": "2006-02-14T00:00:00Z", + "LastUpdate": "2013-05-26T14:49:45.738Z", + "Active": 1 + } +} +`) + }) +} + +func TestSelectJson_FetchFirst(t *testing.T) { + stmt := SELECT_JSON_ARR(Actor.AllColumns). + FROM(Actor). + ORDER_BY(Actor.ActorID). + OFFSET(2). + FETCH_FIRST(Int(3)).ROWS_ONLY() + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT actor.actor_id AS "actorID", + actor.first_name AS "firstName", + actor.last_name AS "lastName", + actor.last_update AS "lastUpdate" + FROM dvds.actor + ORDER BY actor.actor_id + OFFSET 2 + FETCH FIRST 3 ROWS ONLY + ) AS records; +`) + + var dest []model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, ` +[ + { + "ActorID": 3, + "FirstName": "Ed", + "LastName": "Chase", + "LastUpdate": "2013-05-26T14:47:57.62Z" + }, + { + "ActorID": 4, + "FirstName": "Jennifer", + "LastName": "Davis", + "LastUpdate": "2013-05-26T14:47:57.62Z" + }, + { + "ActorID": 5, + "FirstName": "Johnny", + "LastName": "Lollobrigida", + "LastUpdate": "2013-05-26T14:47:57.62Z" + } +] +`) +} + +func TestSelectJson_RowLock(t *testing.T) { + + stmt := SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.ActorID.EQ(Int(200))). + FOR(UPDATE().NOWAIT()) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT row_to_json(records) AS "json" +FROM ( + SELECT actor.actor_id AS "actorID", + actor.first_name AS "firstName", + actor.last_name AS "lastName", + actor.last_update AS "lastUpdate" + FROM dvds.actor + WHERE actor.actor_id = 200 + FOR UPDATE NOWAIT + ) AS records; +`) + + testutils.ExecuteInTxAndRollback(t, db, func(tx qrm.DB) { + var dest model.Actor + + err := stmt.QueryJSON(ctx, tx, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, ` +{ + "ActorID": 200, + "FirstName": "Thora", + "LastName": "Temple", + "LastUpdate": "2013-05-26T14:47:57.62Z" +} +`) + }) + +} + +func TestSelectJson_UNION(t *testing.T) { + + stmt := UNION_ALL( + SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.ActorID.EQ(Int(20))), + + SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.ActorID.EQ(Int(21))), + ) + + testutils.AssertDebugStatementSql(t, stmt, ` +( + SELECT row_to_json(records) AS "json" + FROM ( + SELECT actor.actor_id AS "actorID", + actor.first_name AS "firstName", + actor.last_name AS "lastName", + actor.last_update AS "lastUpdate" + FROM dvds.actor + WHERE actor.actor_id = 20 + ) AS records +) +UNION ALL +( + SELECT row_to_json(records) AS "json" + FROM ( + SELECT actor.actor_id AS "actorID", + actor.first_name AS "firstName", + actor.last_name AS "lastName", + actor.last_update AS "lastUpdate" + FROM dvds.actor + WHERE actor.actor_id = 21 + ) AS records +); +`) + + var dest []struct { + model.Actor `json_column:"json"` + } + + err := stmt.Query(db, &dest) + require.NoError(t, err) + testutils.AssertJSON(t, dest, ` +[ + { + "ActorID": 20, + "FirstName": "Lucille", + "LastName": "Tracy", + "LastUpdate": "2013-05-26T14:47:57.62Z" + }, + { + "ActorID": 21, + "FirstName": "Kirsten", + "LastName": "Paltrow", + "LastUpdate": "2013-05-26T14:47:57.62Z" + } +] +`) +} + +func TestSelectJson_Window(t *testing.T) { + stmt := SELECT_JSON_ARR( + AVG(Payment.Amount).OVER().AS("avgOver"), + AVG(Payment.Amount).OVER(Window("w1")).AS("avgOverW1"), + AVG(Payment.Amount).OVER( + Window("w2"). + ORDER_BY(Payment.CustomerID). + RANGE(PRECEDING(UNBOUNDED), FOLLOWING(UNBOUNDED)), + ).AS("avgOverW2"), + AVG(Payment.Amount).OVER(Window("w3").RANGE(PRECEDING(UNBOUNDED), FOLLOWING(UNBOUNDED))).AS("avgOverW3"), + ).FROM( + Payment, + ).WINDOW("w1").AS(PARTITION_BY(Payment.PaymentDate)). + WINDOW("w2").AS(Window("w1")). + WINDOW("w3").AS(Window("w2").ORDER_BY(Payment.CustomerID)). + ORDER_BY(Payment.CustomerID). + LIMIT(4) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT AVG(payment.amount) OVER () AS "avgOver", + AVG(payment.amount) OVER (w1) AS "avgOverW1", + AVG(payment.amount) OVER (w2 ORDER BY payment.customer_id RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS "avgOverW2", + AVG(payment.amount) OVER (w3 RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS "avgOverW3" + FROM dvds.payment + WINDOW w1 AS (PARTITION BY payment.payment_date), w2 AS (w1), w3 AS (w2 ORDER BY payment.customer_id) + ORDER BY payment.customer_id + LIMIT 4 + ) AS records; +`) + + var dest []struct { + AvgOver float64 + AvgOverW1 float64 + AvgOverW2 float64 + AvgOverW3 float64 + } + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) +} + +func TestSelectJson_QueryWithoutUnMarshaling(t *testing.T) { + stmt := SELECT_JSON_ARR( + view.CustomerList.AllColumns, + + SELECT_JSON_ARR(Rental.AllColumns). + FROM(Rental). + WHERE(view.CustomerList.ID.EQ(Rental.CustomerID)). + ORDER_BY(Rental.CustomerID). + AS("Rentals"), + ).FROM( + view.CustomerList, + ).WHERE( + view.CustomerList.ID.LT_EQ(Int(2)), + ).ORDER_BY( + view.CustomerList.ID, + ) + + //fmt.Println(stmt.DebugSql()) + + testutils.AssertDebugStatementSql(t, stmt, ` +SELECT json_agg(row_to_json(records)) AS "json" +FROM ( + SELECT customer_list.id AS "id", + customer_list.name AS "name", + customer_list.address AS "address", + customer_list."zip code" AS "zip code", + customer_list.phone AS "phone", + customer_list.city AS "city", + customer_list.country AS "country", + customer_list.notes AS "notes", + customer_list.sid AS "sid", + ( + SELECT json_agg(row_to_json(rentals_records)) AS "rentals_json" + FROM ( + SELECT rental.rental_id AS "rentalID", + rental.rental_date AS "rentalDate", + rental.inventory_id AS "inventoryID", + rental.customer_id AS "customerID", + rental.return_date AS "returnDate", + rental.staff_id AS "staffID", + rental.last_update AS "lastUpdate" + FROM dvds.rental + WHERE customer_list.id = rental.customer_id + ORDER BY rental.customer_id + ) AS rentals_records + ) AS "Rentals" + FROM dvds.customer_list + WHERE customer_list.id <= 2 + ORDER BY customer_list.id + ) AS records; +`) + + var dest struct { + Json []byte + } + + err := stmt.Query(db, &dest) + require.NoError(t, err) + + if sourceIsCockroachDB() { + require.Equal(t, string(dest.Json), `[{"Rentals": [{"customerID": 1, "inventoryID": 3021, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-05-25T11:30:37", "rentalID": 76, "returnDate": "2005-06-03T12:00:37", "staffID": 2}, {"customerID": 1, "inventoryID": 4020, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-05-28T10:35:23", "rentalID": 573, "returnDate": "2005-06-03T06:32:23", "staffID": 1}, {"customerID": 1, "inventoryID": 2785, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-15T00:54:12", "rentalID": 1185, "returnDate": "2005-06-23T02:42:12", "staffID": 2}, {"customerID": 1, "inventoryID": 1021, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-15T18:02:53", "rentalID": 1422, "returnDate": "2005-06-19T15:54:53", "staffID": 2}, {"customerID": 1, "inventoryID": 1407, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-15T21:08:46", "rentalID": 1476, "returnDate": "2005-06-25T02:26:46", "staffID": 1}, {"customerID": 1, "inventoryID": 726, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-16T15:18:57", "rentalID": 1725, "returnDate": "2005-06-17T21:05:57", "staffID": 1}, {"customerID": 1, "inventoryID": 197, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-18T08:41:48", "rentalID": 2308, "returnDate": "2005-06-22T03:36:48", "staffID": 2}, {"customerID": 1, "inventoryID": 3497, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-18T13:33:59", "rentalID": 2363, "returnDate": "2005-06-19T17:40:59", "staffID": 1}, {"customerID": 1, "inventoryID": 4566, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-21T06:24:45", "rentalID": 3284, "returnDate": "2005-06-28T03:28:45", "staffID": 1}, {"customerID": 1, "inventoryID": 1443, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-08T03:17:05", "rentalID": 4526, "returnDate": "2005-07-14T01:19:05", "staffID": 2}, {"customerID": 1, "inventoryID": 3486, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-08T07:33:56", "rentalID": 4611, "returnDate": "2005-07-12T13:25:56", "staffID": 2}, {"customerID": 1, "inventoryID": 3726, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-09T13:24:07", "rentalID": 5244, "returnDate": "2005-07-14T14:01:07", "staffID": 2}, {"customerID": 1, "inventoryID": 797, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-09T16:38:01", "rentalID": 5326, "returnDate": "2005-07-13T18:02:01", "staffID": 1}, {"customerID": 1, "inventoryID": 1330, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-11T10:13:46", "rentalID": 6163, "returnDate": "2005-07-19T13:15:46", "staffID": 2}, {"customerID": 1, "inventoryID": 2465, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-27T11:31:22", "rentalID": 7273, "returnDate": "2005-07-31T06:50:22", "staffID": 1}, {"customerID": 1, "inventoryID": 1092, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-28T09:04:45", "rentalID": 7841, "returnDate": "2005-07-30T12:37:45", "staffID": 2}, {"customerID": 1, "inventoryID": 4268, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-28T16:18:23", "rentalID": 8033, "returnDate": "2005-07-30T17:56:23", "staffID": 1}, {"customerID": 1, "inventoryID": 1558, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-28T17:33:39", "rentalID": 8074, "returnDate": "2005-07-29T20:17:39", "staffID": 1}, {"customerID": 1, "inventoryID": 4497, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-28T19:20:07", "rentalID": 8116, "returnDate": "2005-07-29T22:54:07", "staffID": 1}, {"customerID": 1, "inventoryID": 108, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-29T03:58:49", "rentalID": 8326, "returnDate": "2005-08-01T05:16:49", "staffID": 2}, {"customerID": 1, "inventoryID": 2219, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-31T02:42:18", "rentalID": 9571, "returnDate": "2005-08-02T23:26:18", "staffID": 2}, {"customerID": 1, "inventoryID": 14, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-01T08:51:04", "rentalID": 10437, "returnDate": "2005-08-10T12:12:04", "staffID": 1}, {"customerID": 1, "inventoryID": 3232, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-02T15:36:52", "rentalID": 11299, "returnDate": "2005-08-10T16:40:52", "staffID": 2}, {"customerID": 1, "inventoryID": 1440, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-02T18:01:38", "rentalID": 11367, "returnDate": "2005-08-04T13:19:38", "staffID": 1}, {"customerID": 1, "inventoryID": 2639, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-17T12:37:54", "rentalID": 11824, "returnDate": "2005-08-19T10:11:54", "staffID": 2}, {"customerID": 1, "inventoryID": 921, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-18T03:57:29", "rentalID": 12250, "returnDate": "2005-08-22T23:05:29", "staffID": 1}, {"customerID": 1, "inventoryID": 3019, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-19T09:55:16", "rentalID": 13068, "returnDate": "2005-08-20T14:44:16", "staffID": 2}, {"customerID": 1, "inventoryID": 2269, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-19T13:56:54", "rentalID": 13176, "returnDate": "2005-08-23T08:50:54", "staffID": 2}, {"customerID": 1, "inventoryID": 4249, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-21T23:33:57", "rentalID": 14762, "returnDate": "2005-08-23T01:30:57", "staffID": 1}, {"customerID": 1, "inventoryID": 1449, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-22T01:27:57", "rentalID": 14825, "returnDate": "2005-08-27T07:01:57", "staffID": 2}, {"customerID": 1, "inventoryID": 1446, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-22T19:41:37", "rentalID": 15298, "returnDate": "2005-08-28T22:49:37", "staffID": 1}, {"customerID": 1, "inventoryID": 312, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-22T20:03:46", "rentalID": 15315, "returnDate": "2005-08-30T01:51:46", "staffID": 2}], "address": "1913 Hanoi Way", "city": "Sasebo", "country": "Japan", "id": 1, "name": "Mary Smith", "notes": "active", "phone": "28303384290", "sid": 1, "zip code": "35200"}, {"Rentals": [{"customerID": 2, "inventoryID": 1090, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-05-27T00:09:24", "rentalID": 320, "returnDate": "2005-05-28T04:30:24", "staffID": 2}, {"customerID": 2, "inventoryID": 352, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-06-17T20:54:58", "rentalID": 2128, "returnDate": "2005-06-24T00:41:58", "staffID": 2}, {"customerID": 2, "inventoryID": 4116, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-10T06:31:24", "rentalID": 5636, "returnDate": "2005-07-13T02:36:24", "staffID": 1}, {"customerID": 2, "inventoryID": 2760, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-10T12:38:56", "rentalID": 5755, "returnDate": "2005-07-19T17:02:56", "staffID": 1}, {"customerID": 2, "inventoryID": 741, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-27T14:30:42", "rentalID": 7346, "returnDate": "2005-08-02T16:48:42", "staffID": 1}, {"customerID": 2, "inventoryID": 488, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-27T15:23:02", "rentalID": 7376, "returnDate": "2005-08-04T10:35:02", "staffID": 2}, {"customerID": 2, "inventoryID": 2053, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-27T18:40:20", "rentalID": 7459, "returnDate": "2005-08-02T21:07:20", "staffID": 2}, {"customerID": 2, "inventoryID": 1937, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-29T00:12:59", "rentalID": 8230, "returnDate": "2005-08-06T19:52:59", "staffID": 2}, {"customerID": 2, "inventoryID": 626, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-29T12:56:59", "rentalID": 8598, "returnDate": "2005-08-01T08:39:59", "staffID": 2}, {"customerID": 2, "inventoryID": 4038, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-29T17:14:29", "rentalID": 8705, "returnDate": "2005-08-02T16:01:29", "staffID": 1}, {"customerID": 2, "inventoryID": 2377, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-30T06:06:10", "rentalID": 9031, "returnDate": "2005-08-04T10:45:10", "staffID": 2}, {"customerID": 2, "inventoryID": 4030, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-30T13:47:43", "rentalID": 9236, "returnDate": "2005-08-08T18:52:43", "staffID": 1}, {"customerID": 2, "inventoryID": 1382, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-30T14:14:11", "rentalID": 9248, "returnDate": "2005-08-05T11:19:11", "staffID": 1}, {"customerID": 2, "inventoryID": 4088, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-30T16:21:13", "rentalID": 9296, "returnDate": "2005-08-08T11:57:13", "staffID": 1}, {"customerID": 2, "inventoryID": 3084, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-30T22:39:53", "rentalID": 9465, "returnDate": "2005-08-06T16:43:53", "staffID": 2}, {"customerID": 2, "inventoryID": 3142, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-07-31T21:58:56", "rentalID": 10136, "returnDate": "2005-08-03T19:44:56", "staffID": 1}, {"customerID": 2, "inventoryID": 138, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-01T09:45:26", "rentalID": 10466, "returnDate": "2005-08-06T06:28:26", "staffID": 1}, {"customerID": 2, "inventoryID": 3418, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-02T02:10:56", "rentalID": 10918, "returnDate": "2005-08-02T21:23:56", "staffID": 1}, {"customerID": 2, "inventoryID": 654, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-02T07:41:41", "rentalID": 11087, "returnDate": "2005-08-10T10:37:41", "staffID": 2}, {"customerID": 2, "inventoryID": 1149, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-02T10:43:48", "rentalID": 11177, "returnDate": "2005-08-10T10:55:48", "staffID": 2}, {"customerID": 2, "inventoryID": 2060, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-02T13:44:53", "rentalID": 11256, "returnDate": "2005-08-04T16:39:53", "staffID": 1}, {"customerID": 2, "inventoryID": 805, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-17T03:52:18", "rentalID": 11614, "returnDate": "2005-08-20T07:04:18", "staffID": 1}, {"customerID": 2, "inventoryID": 1521, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-19T06:26:04", "rentalID": 12963, "returnDate": "2005-08-23T11:37:04", "staffID": 2}, {"customerID": 2, "inventoryID": 3164, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-21T13:24:32", "rentalID": 14475, "returnDate": "2005-08-27T08:59:32", "staffID": 2}, {"customerID": 2, "inventoryID": 4570, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-21T22:41:56", "rentalID": 14743, "returnDate": "2005-08-29T00:18:56", "staffID": 1}, {"customerID": 2, "inventoryID": 2179, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-22T13:53:04", "rentalID": 15145, "returnDate": "2005-08-31T15:51:04", "staffID": 1}, {"customerID": 2, "inventoryID": 2898, "lastUpdate": "2006-02-16T02:30:53", "rentalDate": "2005-08-23T17:39:35", "rentalID": 15907, "returnDate": "2005-08-25T23:23:35", "staffID": 1}], "address": "1121 Loja Avenue", "city": "San Bernardino", "country": "United States", "id": 2, "name": "Patricia Johnson", "notes": "active", "phone": "838635286649", "sid": 1, "zip code": "17886"}]`) + } else { + require.Equal(t, string(dest.Json), `[{"id":1,"name":"Mary Smith","address":"1913 Hanoi Way","zip code":"35200","phone":"28303384290","city":"Sasebo","country":"Japan","notes":"active","sid":1,"Rentals":[{"rentalID":76,"rentalDate":"2005-05-25T11:30:37","inventoryID":3021,"customerID":1,"returnDate":"2005-06-03T12:00:37","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":573,"rentalDate":"2005-05-28T10:35:23","inventoryID":4020,"customerID":1,"returnDate":"2005-06-03T06:32:23","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":1185,"rentalDate":"2005-06-15T00:54:12","inventoryID":2785,"customerID":1,"returnDate":"2005-06-23T02:42:12","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":1422,"rentalDate":"2005-06-15T18:02:53","inventoryID":1021,"customerID":1,"returnDate":"2005-06-19T15:54:53","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":1476,"rentalDate":"2005-06-15T21:08:46","inventoryID":1407,"customerID":1,"returnDate":"2005-06-25T02:26:46","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":1725,"rentalDate":"2005-06-16T15:18:57","inventoryID":726,"customerID":1,"returnDate":"2005-06-17T21:05:57","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":2308,"rentalDate":"2005-06-18T08:41:48","inventoryID":197,"customerID":1,"returnDate":"2005-06-22T03:36:48","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":2363,"rentalDate":"2005-06-18T13:33:59","inventoryID":3497,"customerID":1,"returnDate":"2005-06-19T17:40:59","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":3284,"rentalDate":"2005-06-21T06:24:45","inventoryID":4566,"customerID":1,"returnDate":"2005-06-28T03:28:45","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":4526,"rentalDate":"2005-07-08T03:17:05","inventoryID":1443,"customerID":1,"returnDate":"2005-07-14T01:19:05","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":4611,"rentalDate":"2005-07-08T07:33:56","inventoryID":3486,"customerID":1,"returnDate":"2005-07-12T13:25:56","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":5244,"rentalDate":"2005-07-09T13:24:07","inventoryID":3726,"customerID":1,"returnDate":"2005-07-14T14:01:07","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":5326,"rentalDate":"2005-07-09T16:38:01","inventoryID":797,"customerID":1,"returnDate":"2005-07-13T18:02:01","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":6163,"rentalDate":"2005-07-11T10:13:46","inventoryID":1330,"customerID":1,"returnDate":"2005-07-19T13:15:46","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":7273,"rentalDate":"2005-07-27T11:31:22","inventoryID":2465,"customerID":1,"returnDate":"2005-07-31T06:50:22","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":7841,"rentalDate":"2005-07-28T09:04:45","inventoryID":1092,"customerID":1,"returnDate":"2005-07-30T12:37:45","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8033,"rentalDate":"2005-07-28T16:18:23","inventoryID":4268,"customerID":1,"returnDate":"2005-07-30T17:56:23","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8074,"rentalDate":"2005-07-28T17:33:39","inventoryID":1558,"customerID":1,"returnDate":"2005-07-29T20:17:39","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8116,"rentalDate":"2005-07-28T19:20:07","inventoryID":4497,"customerID":1,"returnDate":"2005-07-29T22:54:07","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8326,"rentalDate":"2005-07-29T03:58:49","inventoryID":108,"customerID":1,"returnDate":"2005-08-01T05:16:49","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":9571,"rentalDate":"2005-07-31T02:42:18","inventoryID":2219,"customerID":1,"returnDate":"2005-08-02T23:26:18","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":10437,"rentalDate":"2005-08-01T08:51:04","inventoryID":14,"customerID":1,"returnDate":"2005-08-10T12:12:04","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11299,"rentalDate":"2005-08-02T15:36:52","inventoryID":3232,"customerID":1,"returnDate":"2005-08-10T16:40:52","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11367,"rentalDate":"2005-08-02T18:01:38","inventoryID":1440,"customerID":1,"returnDate":"2005-08-04T13:19:38","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11824,"rentalDate":"2005-08-17T12:37:54","inventoryID":2639,"customerID":1,"returnDate":"2005-08-19T10:11:54","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":12250,"rentalDate":"2005-08-18T03:57:29","inventoryID":921,"customerID":1,"returnDate":"2005-08-22T23:05:29","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":13068,"rentalDate":"2005-08-19T09:55:16","inventoryID":3019,"customerID":1,"returnDate":"2005-08-20T14:44:16","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":13176,"rentalDate":"2005-08-19T13:56:54","inventoryID":2269,"customerID":1,"returnDate":"2005-08-23T08:50:54","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":14762,"rentalDate":"2005-08-21T23:33:57","inventoryID":4249,"customerID":1,"returnDate":"2005-08-23T01:30:57","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":14825,"rentalDate":"2005-08-22T01:27:57","inventoryID":1449,"customerID":1,"returnDate":"2005-08-27T07:01:57","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":15298,"rentalDate":"2005-08-22T19:41:37","inventoryID":1446,"customerID":1,"returnDate":"2005-08-28T22:49:37","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":15315,"rentalDate":"2005-08-22T20:03:46","inventoryID":312,"customerID":1,"returnDate":"2005-08-30T01:51:46","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}]}, {"id":2,"name":"Patricia Johnson","address":"1121 Loja Avenue","zip code":"17886","phone":"838635286649","city":"San Bernardino","country":"United States","notes":"active","sid":1,"Rentals":[{"rentalID":320,"rentalDate":"2005-05-27T00:09:24","inventoryID":1090,"customerID":2,"returnDate":"2005-05-28T04:30:24","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":2128,"rentalDate":"2005-06-17T20:54:58","inventoryID":352,"customerID":2,"returnDate":"2005-06-24T00:41:58","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":5636,"rentalDate":"2005-07-10T06:31:24","inventoryID":4116,"customerID":2,"returnDate":"2005-07-13T02:36:24","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":5755,"rentalDate":"2005-07-10T12:38:56","inventoryID":2760,"customerID":2,"returnDate":"2005-07-19T17:02:56","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":7346,"rentalDate":"2005-07-27T14:30:42","inventoryID":741,"customerID":2,"returnDate":"2005-08-02T16:48:42","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":7376,"rentalDate":"2005-07-27T15:23:02","inventoryID":488,"customerID":2,"returnDate":"2005-08-04T10:35:02","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":7459,"rentalDate":"2005-07-27T18:40:20","inventoryID":2053,"customerID":2,"returnDate":"2005-08-02T21:07:20","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8230,"rentalDate":"2005-07-29T00:12:59","inventoryID":1937,"customerID":2,"returnDate":"2005-08-06T19:52:59","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8598,"rentalDate":"2005-07-29T12:56:59","inventoryID":626,"customerID":2,"returnDate":"2005-08-01T08:39:59","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":8705,"rentalDate":"2005-07-29T17:14:29","inventoryID":4038,"customerID":2,"returnDate":"2005-08-02T16:01:29","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":9031,"rentalDate":"2005-07-30T06:06:10","inventoryID":2377,"customerID":2,"returnDate":"2005-08-04T10:45:10","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":9236,"rentalDate":"2005-07-30T13:47:43","inventoryID":4030,"customerID":2,"returnDate":"2005-08-08T18:52:43","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":9248,"rentalDate":"2005-07-30T14:14:11","inventoryID":1382,"customerID":2,"returnDate":"2005-08-05T11:19:11","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":9296,"rentalDate":"2005-07-30T16:21:13","inventoryID":4088,"customerID":2,"returnDate":"2005-08-08T11:57:13","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":9465,"rentalDate":"2005-07-30T22:39:53","inventoryID":3084,"customerID":2,"returnDate":"2005-08-06T16:43:53","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":10136,"rentalDate":"2005-07-31T21:58:56","inventoryID":3142,"customerID":2,"returnDate":"2005-08-03T19:44:56","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":10466,"rentalDate":"2005-08-01T09:45:26","inventoryID":138,"customerID":2,"returnDate":"2005-08-06T06:28:26","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":10918,"rentalDate":"2005-08-02T02:10:56","inventoryID":3418,"customerID":2,"returnDate":"2005-08-02T21:23:56","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11087,"rentalDate":"2005-08-02T07:41:41","inventoryID":654,"customerID":2,"returnDate":"2005-08-10T10:37:41","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11177,"rentalDate":"2005-08-02T10:43:48","inventoryID":1149,"customerID":2,"returnDate":"2005-08-10T10:55:48","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11256,"rentalDate":"2005-08-02T13:44:53","inventoryID":2060,"customerID":2,"returnDate":"2005-08-04T16:39:53","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":11614,"rentalDate":"2005-08-17T03:52:18","inventoryID":805,"customerID":2,"returnDate":"2005-08-20T07:04:18","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":12963,"rentalDate":"2005-08-19T06:26:04","inventoryID":1521,"customerID":2,"returnDate":"2005-08-23T11:37:04","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":14475,"rentalDate":"2005-08-21T13:24:32","inventoryID":3164,"customerID":2,"returnDate":"2005-08-27T08:59:32","staffID":2,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":14743,"rentalDate":"2005-08-21T22:41:56","inventoryID":4570,"customerID":2,"returnDate":"2005-08-29T00:18:56","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":15145,"rentalDate":"2005-08-22T13:53:04","inventoryID":2179,"customerID":2,"returnDate":"2005-08-31T15:51:04","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}, {"rentalID":15907,"rentalDate":"2005-08-23T17:39:35","inventoryID":2898,"customerID":2,"returnDate":"2005-08-25T23:23:35","staffID":1,"lastUpdate":"2006-02-16T02:30:53"}]}]`) + } +} + +func TestSelectJsonObject_EmptyResult(t *testing.T) { + t.Run("json obj", func(t *testing.T) { + stmt := SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.FirstName.EQ(Text("Kowalski"))) + + var dest model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.ErrorIs(t, err, qrm.ErrNoRows) + }) + + t.Run("json arr", func(t *testing.T) { + stmt := SELECT_JSON_ARR(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.FirstName.EQ(Text("Kowalski"))) + + var dest []model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + require.Empty(t, dest) + }) +} + +func TestSelectJson_InvalidDestination(t *testing.T) { + t.Run("json obj", func(t *testing.T) { + stmt := SELECT_JSON_OBJ(Actor.AllColumns). + FROM(Actor) + + testutils.AssertQueryJsonPanicErr(t, stmt, db, &[]model.Actor{}, "jet: destination has to be a pointer to struct or pointer to map[string]any") + testutils.AssertQueryJsonPanicErr(t, stmt, db, model.Actor{}, "jet: destination has to be a pointer to struct or pointer to map[string]any") + testutils.AssertQueryJsonPanicErr(t, stmt, nil, &model.Actor{}, "jet: db is nil") + testutils.AssertQueryJsonPanicErr(t, stmt, db, nil, "jet: destination is nil") + }) + + t.Run("json arr", func(t *testing.T) { + stmt := SELECT_JSON_ARR(Actor.AllColumns). + FROM(Actor) + + testutils.AssertQueryJsonPanicErr(t, stmt, db, &model.Actor{}, "jet: destination has to be a pointer to slice of struct or pointer to []map[string]any") + testutils.AssertQueryJsonPanicErr(t, stmt, db, []model.Actor{}, "jet: destination has to be a pointer to slice of struct or pointer to []map[string]any") + testutils.AssertQueryJsonPanicErr(t, stmt, nil, &[]model.Actor{}, "jet: db is nil") + testutils.AssertQueryJsonPanicErr(t, stmt, db, nil, "jet: destination is nil") + }) +} + +func TestSelectJson_ProjectionNotAliased(t *testing.T) { + t.Run("statement not aliased", func(t *testing.T) { + testutils.AssertPanicErr(t, func() { + stmt := SELECT_JSON_ARR( + Customer.AllColumns, + + SELECT_JSON_ARR(Rental.AllColumns). + FROM(Rental). + WHERE(Rental.CustomerID.EQ(Customer.CustomerID)), + ).FROM(Customer) + + stmt.DebugSql() + + }, "jet: SELECT JSON statements need to be aliased when used as a projection.") + }) + + t.Run("expression not aliased", func(t *testing.T) { + testutils.AssertPanicErr(t, func() { + stmt := SELECT_JSON_ARR( + Int(2).ADD(Customer.CustomerID), + ).FROM(Customer) + + stmt.DebugSql() + + }, "jet: expression need to be aliased when used as SELECT JSON projection.") + }) +} + +func TestSelectJson_MoreThenOneRowErr(t *testing.T) { + actors := SELECT_JSON_ARR(Actor.AllColumns). + FROM(Actor). + WHERE(Actor.ActorID.BETWEEN(Int(20), Int(30))) + + stmt := UNION_ALL(actors, actors) + + var dest []model.Actor + + err := stmt.QueryJSON(ctx, db, &dest) + require.ErrorContains(t, err, "jet: query returned more then one row") +} diff --git a/tests/postgres/select_test.go b/tests/postgres/select_test.go index d39987d..ca46fc7 100644 --- a/tests/postgres/select_test.go +++ b/tests/postgres/select_test.go @@ -21,36 +21,36 @@ import ( ) func TestSelect_ScanToStruct(t *testing.T) { - expectedSQL := ` + + t.Run("standard", func(t *testing.T) { + stmt := SELECT(Actor.AllColumns). + DISTINCT(). + FROM(Actor). + WHERE(Actor.ActorID.EQ(Int(2))) + + testutils.AssertDebugStatementSql(t, stmt, ` SELECT DISTINCT actor.actor_id AS "actor.actor_id", actor.first_name AS "actor.first_name", actor.last_name AS "actor.last_name", actor.last_update AS "actor.last_update" FROM dvds.actor WHERE actor.actor_id = 2; -` +`, int64(2)) - query := SELECT(Actor.AllColumns). - DISTINCT(). - FROM(Actor). - WHERE(Actor.ActorID.EQ(Int(2))) + var dest model.Actor + err := stmt.Query(db, &dest) - testutils.AssertDebugStatementSql(t, query, expectedSQL, int64(2)) + require.NoError(t, err) + testutils.AssertDeepEqual(t, dest, actor2) + requireLogged(t, stmt) + }) +} - actor := model.Actor{} - err := query.Query(db, &actor) - - require.NoError(t, err) - - expectedActor := model.Actor{ - ActorID: 2, - FirstName: "Nick", - LastName: "Wahlberg", - LastUpdate: *testutils.TimestampWithoutTimeZone("2013-05-26 14:47:57.62", 2), - } - - testutils.AssertDeepEqual(t, actor, expectedActor) - requireLogged(t, query) +var actor2 = model.Actor{ + ActorID: 2, + FirstName: "Nick", + LastName: "Wahlberg", + LastUpdate: *testutils.TimestampWithoutTimeZone("2013-05-26 14:47:57.62", 2), } func TestSelectDistinctOn(t *testing.T) { @@ -85,7 +85,6 @@ ORDER BY rental.staff_id ASC, rental.customer_id ASC, rental.rental_id ASC; err := stmt.Query(db, &dest) require.NoError(t, err) - testutils.AssertJSON(t, dest, ` [ { @@ -187,6 +186,21 @@ ORDER BY customer.customer_id ASC; testutils.AssertDeepEqual(t, lastCustomer, customers[598]) requireLogged(t, query) + + t.Run("select json", func(t *testing.T) { + stmt := SELECT_JSON_ARR( + Customer.AllColumns, + ).FROM( + Customer, + ).ORDER_BY(Customer.CustomerID.ASC()) + + var dest []model.Customer + + err := stmt.QueryJSON(ctx, db, &dest) + require.NoError(t, err) + + testutils.AssertDeepEqual(t, customers, dest) + }) } func TestSelectAndUnionInProjection(t *testing.T) { @@ -217,15 +231,14 @@ FROM dvds.payment LIMIT 12; ` - query := Payment. - SELECT( - Payment.PaymentID, - Customer.SELECT(Customer.CustomerID).LIMIT(1), - UNION( - Payment.SELECT(Payment.PaymentID).LIMIT(1).OFFSET(10), - Payment.SELECT(Payment.PaymentID).LIMIT(1).OFFSET(2), - ).LIMIT(1), - ). + query := SELECT( + Payment.PaymentID, + Customer.SELECT(Customer.CustomerID).LIMIT(1), + UNION( + Payment.SELECT(Payment.PaymentID).LIMIT(1).OFFSET(10), + Payment.SELECT(Payment.PaymentID).LIMIT(1).OFFSET(2), + ).LIMIT(1), + ).FROM(Payment). LIMIT(12) //fmt.Println(query.DebugSql()) @@ -2771,7 +2784,8 @@ ORDER BY actor.actor_id ASC, film.film_id ASC; err := stmt.Query(db, &dest) require.NoError(t, err) - //jsonSave("./testdata/quick-start-dest.json", dest) + //testutils.SaveJSONFile(dest, "./testdata/results/postgres/quick-start-dest.json") + testutils.AssertJSONFile(t, dest, "./testdata/results/postgres/quick-start-dest.json") var dest2 []struct { @@ -2784,7 +2798,7 @@ ORDER BY actor.actor_id ASC, film.film_id ASC; err = stmt.Query(db, &dest2) require.NoError(t, err) - //jsonSave("./testdata/quick-start-dest2.json", dest2) + //testutils.SaveJSONFile(dest2, "./testdata/results/postgres/quick-start-dest2.json") testutils.AssertJSONFile(t, dest2, "./testdata/results/postgres/quick-start-dest2.json") } @@ -2966,7 +2980,7 @@ WHERE payment.payment_id < $1 WINDOW w1 AS (PARTITION BY payment.payment_date), w2 AS (w1), w3 AS (w2 ORDER BY payment.customer_id) ORDER BY payment.customer_id; ` - query := Payment.SELECT( + query := SELECT( AVG(Payment.Amount).OVER(), AVG(Payment.Amount).OVER(Window("w1")), AVG(Payment.Amount).OVER( @@ -2976,6 +2990,7 @@ ORDER BY payment.customer_id; ), AVG(Payment.Amount).OVER(Window("w3").RANGE(PRECEDING(UNBOUNDED), FOLLOWING(UNBOUNDED))), ). + FROM(Payment). WHERE(Payment.PaymentID.LT(Int(10))). WINDOW("w1").AS(PARTITION_BY(Payment.PaymentDate)). WINDOW("w2").AS(Window("w1")). diff --git a/tests/testdata b/tests/testdata index 1c501ac..0997c82 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 1c501acb72bea389788404988ef0130b733f9cee +Subproject commit 0997c825e6569fc49b69ffbef959eadab9013e00 From c93c9f2888259efb615c41d59e68aaa135bc4607 Mon Sep 17 00:00:00 2001 From: go-jet Date: Fri, 21 Feb 2025 19:58:26 +0100 Subject: [PATCH 02/18] Add encoding/json from standard library --- internal/3rdparty/json/bench_test.go | 584 ++++ internal/3rdparty/json/decode.go | 1302 ++++++++ internal/3rdparty/json/decode_test.go | 2621 +++++++++++++++++ internal/3rdparty/json/encode.go | 1286 ++++++++ internal/3rdparty/json/encode_test.go | 1221 ++++++++ .../3rdparty/json/example_marshaling_test.go | 73 + internal/3rdparty/json/example_test.go | 310 ++ .../json/example_text_marshaling_test.go | 67 + internal/3rdparty/json/fold.go | 48 + internal/3rdparty/json/fold_test.go | 50 + internal/3rdparty/json/fuzz_test.go | 83 + internal/3rdparty/json/indent.go | 182 ++ internal/3rdparty/json/number_test.go | 118 + internal/3rdparty/json/scanner.go | 610 ++++ internal/3rdparty/json/scanner_test.go | 304 ++ internal/3rdparty/json/stream.go | 512 ++++ internal/3rdparty/json/stream_test.go | 522 ++++ internal/3rdparty/json/tables.go | 218 ++ internal/3rdparty/json/tagkey_test.go | 121 + internal/3rdparty/json/tags.go | 38 + internal/3rdparty/json/tags_test.go | 28 + internal/3rdparty/json/testdata/code.json.gz | Bin 0 -> 120432 bytes 22 files changed, 10298 insertions(+) create mode 100644 internal/3rdparty/json/bench_test.go create mode 100644 internal/3rdparty/json/decode.go create mode 100644 internal/3rdparty/json/decode_test.go create mode 100644 internal/3rdparty/json/encode.go create mode 100644 internal/3rdparty/json/encode_test.go create mode 100644 internal/3rdparty/json/example_marshaling_test.go create mode 100644 internal/3rdparty/json/example_test.go create mode 100644 internal/3rdparty/json/example_text_marshaling_test.go create mode 100644 internal/3rdparty/json/fold.go create mode 100644 internal/3rdparty/json/fold_test.go create mode 100644 internal/3rdparty/json/fuzz_test.go create mode 100644 internal/3rdparty/json/indent.go create mode 100644 internal/3rdparty/json/number_test.go create mode 100644 internal/3rdparty/json/scanner.go create mode 100644 internal/3rdparty/json/scanner_test.go create mode 100644 internal/3rdparty/json/stream.go create mode 100644 internal/3rdparty/json/stream_test.go create mode 100644 internal/3rdparty/json/tables.go create mode 100644 internal/3rdparty/json/tagkey_test.go create mode 100644 internal/3rdparty/json/tags.go create mode 100644 internal/3rdparty/json/tags_test.go create mode 100644 internal/3rdparty/json/testdata/code.json.gz diff --git a/internal/3rdparty/json/bench_test.go b/internal/3rdparty/json/bench_test.go new file mode 100644 index 0000000..032114c --- /dev/null +++ b/internal/3rdparty/json/bench_test.go @@ -0,0 +1,584 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Large data benchmark. +// The JSON data is a summary of agl's changes in the +// go, webkit, and chromium open source projects. +// We benchmark converting between the JSON form +// and in-memory data structures. + +package json + +import ( + "bytes" + "compress/gzip" + "fmt" + "internal/testenv" + "io" + "os" + "reflect" + "regexp" + "runtime" + "strings" + "sync" + "testing" +) + +type codeResponse struct { + Tree *codeNode `json:"tree"` + Username string `json:"username"` +} + +type codeNode struct { + Name string `json:"name"` + Kids []*codeNode `json:"kids"` + CLWeight float64 `json:"cl_weight"` + Touches int `json:"touches"` + MinT int64 `json:"min_t"` + MaxT int64 `json:"max_t"` + MeanT int64 `json:"mean_t"` +} + +var codeJSON []byte +var codeStruct codeResponse + +func codeInit() { + f, err := os.Open("testdata/code.json.gz") + if err != nil { + panic(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + panic(err) + } + data, err := io.ReadAll(gz) + if err != nil { + panic(err) + } + + codeJSON = data + + if err := Unmarshal(codeJSON, &codeStruct); err != nil { + panic("unmarshal code.json: " + err.Error()) + } + + if data, err = Marshal(&codeStruct); err != nil { + panic("marshal code.json: " + err.Error()) + } + + if !bytes.Equal(data, codeJSON) { + println("different lengths", len(data), len(codeJSON)) + for i := 0; i < len(data) && i < len(codeJSON); i++ { + if data[i] != codeJSON[i] { + println("re-marshal: changed at byte", i) + println("orig: ", string(codeJSON[i-10:i+10])) + println("new: ", string(data[i-10:i+10])) + break + } + } + panic("re-marshal code.json: different result") + } +} + +func BenchmarkCodeEncoder(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(io.Discard) + for pb.Next() { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatalf("Encode error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeEncoderError(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + + // Trigger an error in Marshal with cyclic data. + type Dummy struct { + Name string + Next *Dummy + } + dummy := Dummy{Name: "Dummy"} + dummy.Next = &dummy + + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(io.Discard) + for pb.Next() { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatalf("Encode error: %v", err) + } + if _, err := Marshal(dummy); err == nil { + b.Fatal("Marshal error: got nil, want non-nil") + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeMarshal(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatalf("Marshal error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeMarshalError(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + + // Trigger an error in Marshal with cyclic data. + type Dummy struct { + Name string + Next *Dummy + } + dummy := Dummy{Name: "Dummy"} + dummy.Next = &dummy + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatalf("Marshal error: %v", err) + } + if _, err := Marshal(dummy); err == nil { + b.Fatal("Marshal error: got nil, want non-nil") + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func benchMarshalBytes(n int) func(*testing.B) { + sample := []byte("hello world") + // Use a struct pointer, to avoid an allocation when passing it as an + // interface parameter to Marshal. + v := &struct { + Bytes []byte + }{ + bytes.Repeat(sample, (n/len(sample))+1)[:n], + } + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := Marshal(v); err != nil { + b.Fatalf("Marshal error: %v", err) + } + } + } +} + +func benchMarshalBytesError(n int) func(*testing.B) { + sample := []byte("hello world") + // Use a struct pointer, to avoid an allocation when passing it as an + // interface parameter to Marshal. + v := &struct { + Bytes []byte + }{ + bytes.Repeat(sample, (n/len(sample))+1)[:n], + } + + // Trigger an error in Marshal with cyclic data. + type Dummy struct { + Name string + Next *Dummy + } + dummy := Dummy{Name: "Dummy"} + dummy.Next = &dummy + + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := Marshal(v); err != nil { + b.Fatalf("Marshal error: %v", err) + } + if _, err := Marshal(dummy); err == nil { + b.Fatal("Marshal error: got nil, want non-nil") + } + } + } +} + +func BenchmarkMarshalBytes(b *testing.B) { + b.ReportAllocs() + // 32 fits within encodeState.scratch. + b.Run("32", benchMarshalBytes(32)) + // 256 doesn't fit in encodeState.scratch, but is small enough to + // allocate and avoid the slower base64.NewEncoder. + b.Run("256", benchMarshalBytes(256)) + // 4096 is large enough that we want to avoid allocating for it. + b.Run("4096", benchMarshalBytes(4096)) +} + +func BenchmarkMarshalBytesError(b *testing.B) { + b.ReportAllocs() + // 32 fits within encodeState.scratch. + b.Run("32", benchMarshalBytesError(32)) + // 256 doesn't fit in encodeState.scratch, but is small enough to + // allocate and avoid the slower base64.NewEncoder. + b.Run("256", benchMarshalBytesError(256)) + // 4096 is large enough that we want to avoid allocating for it. + b.Run("4096", benchMarshalBytesError(4096)) +} + +func BenchmarkMarshalMap(b *testing.B) { + b.ReportAllocs() + m := map[string]int{ + "key3": 3, + "key2": 2, + "key1": 1, + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(m); err != nil { + b.Fatal("Marshal:", err) + } + } + }) +} + +func BenchmarkCodeDecoder(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + dec := NewDecoder(&buf) + var r codeResponse + for pb.Next() { + buf.Write(codeJSON) + // hide EOF + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteByte('\n') + if err := dec.Decode(&r); err != nil { + b.Fatalf("Decode error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkUnicodeDecoder(b *testing.B) { + b.ReportAllocs() + j := []byte(`"\uD83D\uDE01"`) + b.SetBytes(int64(len(j))) + r := bytes.NewReader(j) + dec := NewDecoder(r) + var out string + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := dec.Decode(&out); err != nil { + b.Fatalf("Decode error: %v", err) + } + r.Seek(0, 0) + } +} + +func BenchmarkDecoderStream(b *testing.B) { + b.ReportAllocs() + b.StopTimer() + var buf bytes.Buffer + dec := NewDecoder(&buf) + buf.WriteString(`"` + strings.Repeat("x", 1000000) + `"` + "\n\n\n") + var x any + if err := dec.Decode(&x); err != nil { + b.Fatalf("Decode error: %v", err) + } + ones := strings.Repeat(" 1\n", 300000) + "\n\n\n" + b.StartTimer() + for i := 0; i < b.N; i++ { + if i%300000 == 0 { + buf.WriteString(ones) + } + x = nil + switch err := dec.Decode(&x); { + case err != nil: + b.Fatalf("Decode error: %v", err) + case x != 1.0: + b.Fatalf("Decode: got %v want 1.0", i) + } + } +} + +func BenchmarkCodeUnmarshal(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var r codeResponse + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeUnmarshalReuse(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + var r codeResponse + for pb.Next() { + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkUnmarshalString(b *testing.B) { + b.ReportAllocs() + data := []byte(`"hello, world"`) + b.RunParallel(func(pb *testing.PB) { + var s string + for pb.Next() { + if err := Unmarshal(data, &s); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmarshalFloat64(b *testing.B) { + b.ReportAllocs() + data := []byte(`3.14`) + b.RunParallel(func(pb *testing.PB) { + var f float64 + for pb.Next() { + if err := Unmarshal(data, &f); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmarshalInt64(b *testing.B) { + b.ReportAllocs() + data := []byte(`3`) + b.RunParallel(func(pb *testing.PB) { + var x int64 + for pb.Next() { + if err := Unmarshal(data, &x); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmarshalMap(b *testing.B) { + b.ReportAllocs() + data := []byte(`{"key1":"value1","key2":"value2","key3":"value3"}`) + b.RunParallel(func(pb *testing.PB) { + x := make(map[string]string, 3) + for pb.Next() { + if err := Unmarshal(data, &x); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkIssue10335(b *testing.B) { + b.ReportAllocs() + j := []byte(`{"a":{ }}`) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkIssue34127(b *testing.B) { + b.ReportAllocs() + j := struct { + Bar string `json:"bar,string"` + }{ + Bar: `foobar`, + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&j); err != nil { + b.Fatalf("Marshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmapped(b *testing.B) { + b.ReportAllocs() + j := []byte(`{"s": "hello", "y": 2, "o": {"x": 0}, "a": [1, 99, {"x": 1}]}`) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkTypeFieldsCache(b *testing.B) { + b.ReportAllocs() + var maxTypes int = 1e6 + if testenv.Builder() != "" { + maxTypes = 1e3 // restrict cache sizes on builders + } + + // Dynamically generate many new types. + types := make([]reflect.Type, maxTypes) + fs := []reflect.StructField{{ + Type: reflect.TypeFor[string](), + Index: []int{0}, + }} + for i := range types { + fs[0].Name = fmt.Sprintf("TypeFieldsCache%d", i) + types[i] = reflect.StructOf(fs) + } + + // clearClear clears the cache. Other JSON operations, must not be running. + clearCache := func() { + fieldCache = sync.Map{} + } + + // MissTypes tests the performance of repeated cache misses. + // This measures the time to rebuild a cache of size nt. + for nt := 1; nt <= maxTypes; nt *= 10 { + ts := types[:nt] + b.Run(fmt.Sprintf("MissTypes%d", nt), func(b *testing.B) { + nc := runtime.GOMAXPROCS(0) + for i := 0; i < b.N; i++ { + clearCache() + var wg sync.WaitGroup + for j := 0; j < nc; j++ { + wg.Add(1) + go func(j int) { + for _, t := range ts[(j*len(ts))/nc : ((j+1)*len(ts))/nc] { + cachedTypeFields(t) + } + wg.Done() + }(j) + } + wg.Wait() + } + }) + } + + // HitTypes tests the performance of repeated cache hits. + // This measures the average time of each cache lookup. + for nt := 1; nt <= maxTypes; nt *= 10 { + // Pre-warm a cache of size nt. + clearCache() + for _, t := range types[:nt] { + cachedTypeFields(t) + } + b.Run(fmt.Sprintf("HitTypes%d", nt), func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + cachedTypeFields(types[0]) + } + }) + }) + } +} + +func BenchmarkEncodeMarshaler(b *testing.B) { + b.ReportAllocs() + + m := struct { + A int + B RawMessage + }{} + + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(io.Discard) + + for pb.Next() { + if err := enc.Encode(&m); err != nil { + b.Fatalf("Encode error: %v", err) + } + } + }) +} + +func BenchmarkEncoderEncode(b *testing.B) { + b.ReportAllocs() + type T struct { + X, Y string + } + v := &T{"foo", "bar"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := NewEncoder(io.Discard).Encode(v); err != nil { + b.Fatalf("Encode error: %v", err) + } + } + }) +} + +func BenchmarkNumberIsValid(b *testing.B) { + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + isValidNumber(s) + } +} + +func BenchmarkNumberIsValidRegexp(b *testing.B) { + var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + jsonNumberRegexp.MatchString(s) + } +} + +func BenchmarkUnmarshalNumber(b *testing.B) { + b.ReportAllocs() + data := []byte(`"-61657.61667E+61673"`) + var number Number + for i := 0; i < b.N; i++ { + if err := Unmarshal(data, &number); err != nil { + b.Fatal("Unmarshal:", err) + } + } +} diff --git a/internal/3rdparty/json/decode.go b/internal/3rdparty/json/decode.go new file mode 100644 index 0000000..f820570 --- /dev/null +++ b/internal/3rdparty/json/decode.go @@ -0,0 +1,1302 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "encoding" + "encoding/base64" + "fmt" + "reflect" + "strconv" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" + _ "unsafe" // for linkname +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an [InvalidUnmarshalError]. +// +// Unmarshal uses the inverse of the encodings that +// [Marshal] uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a value implementing [Unmarshaler], +// Unmarshal calls that value's [Unmarshaler.UnmarshalJSON] method, including +// when the input is a JSON null. +// Otherwise, if the value implements [encoding.TextUnmarshaler] +// and the input is a JSON quoted string, Unmarshal calls +// [encoding.TextUnmarshaler.UnmarshalText] with the unquoted form of the string. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by [Marshal] (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. By +// default, object keys which don't have a corresponding struct field are +// ignored (see [Decoder.DisallowUnknownFields] for an alternative). +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// - bool, for JSON booleans +// - float64, for JSON numbers +// - string, for JSON strings +// - []interface{}, for JSON arrays +// - map[string]interface{}, for JSON objects +// - nil for JSON null +// +// To unmarshal a JSON array into a slice, Unmarshal resets the slice length +// to zero and then appends each element to the slice. +// As a special case, to unmarshal an empty JSON array into a slice, +// Unmarshal replaces the slice with a new empty slice. +// +// To unmarshal a JSON array into a Go array, Unmarshal decodes +// JSON array elements into corresponding Go array elements. +// If the Go array is smaller than the JSON array, +// the additional JSON array elements are discarded. +// If the JSON array is smaller than the Go array, +// the additional Go array elements are set to zero values. +// +// To unmarshal a JSON object into a map, Unmarshal first establishes a map to +// use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must +// either be any string type, an integer, or implement [encoding.TextUnmarshaler]. +// +// If the JSON-encoded data contain a syntax error, Unmarshal returns a [SyntaxError]. +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshaling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an [UnmarshalTypeError] describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// “not present,” unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +func Unmarshal(data []byte, v any) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by types +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +// +// By convention, to approximate the behavior of [Unmarshal] itself, +// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes + Struct string // name of the struct type containing the field + Field string // the full path from root node to the field +} + +func (e *UnmarshalTypeError) Error() string { + if e.Struct != "" || e.Field != "" { + return "json: cannot unmarshal " + e.Value + " into Go struct field " + e.Struct + "." + e.Field + " of type " + e.Type.String() + } + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// +// Deprecated: No longer used; kept for compatibility. +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to [Unmarshal]. +// (The argument to [Unmarshal] must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Pointer { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v any) error { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + d.scanWhile(scanSkipSpace) + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + err := d.value(rv) + if err != nil { + return d.addErrorContext(err) + } + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +// An errorContext provides context for type errors during decoding. +type errorContext struct { + Struct reflect.Type + FieldStack []string +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // next read offset in data + opcode int // last read result + scan scanner + errorContext *errorContext + savedError error + useNumber bool + disallowUnknownFields bool +} + +// readIndex returns the position of the last byte read. +func (d *decodeState) readIndex() int { + return d.off - 1 +} + +// phasePanicMsg is used as a panic message when we end up with something that +// shouldn't happen. It can indicate a bug in the JSON decoder, or that +// something is editing the data slice while the decoder executes. +const phasePanicMsg = "JSON decoder out of sync - data changing underfoot?" + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + if d.errorContext != nil { + d.errorContext.Struct = nil + // Reuse the allocated space for the FieldStack slice. + d.errorContext.FieldStack = d.errorContext.FieldStack[:0] + } + return d +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + d.savedError = d.addErrorContext(err) + } +} + +// addErrorContext returns a new error enhanced with information from d.errorContext +func (d *decodeState) addErrorContext(err error) error { + if d.errorContext != nil && (d.errorContext.Struct != nil || len(d.errorContext.FieldStack) > 0) { + switch err := err.(type) { + case *UnmarshalTypeError: + err.Struct = d.errorContext.Struct.Name() + err.Field = strings.Join(d.errorContext.FieldStack, ".") + } + } + return err +} + +// skip scans to the end of what was started. +func (d *decodeState) skip() { + s, data, i := &d.scan, d.data, d.off + depth := len(s.parseState) + for { + op := s.step(s, data[i]) + i++ + if len(s.parseState) < depth { + d.off = i + d.opcode = op + return + } + } +} + +// scanNext processes the byte at d.data[d.off]. +func (d *decodeState) scanNext() { + if d.off < len(d.data) { + d.opcode = d.scan.step(&d.scan, d.data[d.off]) + d.off++ + } else { + d.opcode = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +func (d *decodeState) scanWhile(op int) { + s, data, i := &d.scan, d.data, d.off + for i < len(data) { + newOp := s.step(s, data[i]) + i++ + if newOp != op { + d.opcode = newOp + d.off = i + return + } + } + + d.off = len(data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() +} + +// rescanLiteral is similar to scanWhile(scanContinue), but it specialises the +// common case where we're decoding a literal. The decoder scans the input +// twice, once for syntax errors and to check the length of the value, and the +// second to perform the decoding. +// +// Only in the second step do we use decodeState to tokenize literals, so we +// know there aren't any syntax errors. We can take advantage of that knowledge, +// and scan a literal's bytes much more quickly. +func (d *decodeState) rescanLiteral() { + data, i := d.data, d.off +Switch: + switch data[i-1] { + case '"': // string + for ; i < len(data); i++ { + switch data[i] { + case '\\': + i++ // escaped char + case '"': + i++ // tokenize the closing quote too + break Switch + } + } + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': // number + for ; i < len(data); i++ { + switch data[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '.', 'e', 'E', '+', '-': + default: + break Switch + } + } + case 't': // true + i += len("rue") + case 'f': // false + i += len("alse") + case 'n': // null + i += len("ull") + } + if i < len(data) { + d.opcode = stateEndValue(&d.scan, data[i]) + } else { + d.opcode = scanEnd + } + d.off = i + 1 +} + +// value consumes a JSON value from d.data[d.off-1:], decoding into v, and +// reads the following byte ahead. If v is invalid, the value is discarded. +// The first byte of the value has been read already. +func (d *decodeState) value(v reflect.Value) error { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray: + if v.IsValid() { + if err := d.array(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginObject: + if v.IsValid() { + if err := d.object(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginLiteral: + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + if v.IsValid() { + if err := d.literalStore(d.data[start:d.readIndex()], v, false); err != nil { + return err + } + } + } + return nil +} + +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() any { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray, scanBeginObject: + d.skip() + d.scanNext() + + case scanBeginLiteral: + v := d.literalInterface() + switch v.(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// If it encounters an Unmarshaler, indirect stops and returns that. +// If decodingNull is true, indirect stops at the first settable pointer so it +// can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // Issue #24153 indicates that it is generally not a guaranteed property + // that you may round-trip a reflect.Value by calling Value.Addr().Elem() + // and expect the value to still be settable for values derived from + // unexported embedded struct fields. + // + // The logic below effectively does this when it first addresses the value + // (to satisfy possible pointer methods) and continues to dereference + // subsequent pointers as necessary. + // + // After the first round-trip, we set v back to the original value to + // preserve the original RW flags contained in reflect.Value. + v0 := v + haveAddr := false + + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Pointer && v.Type().Name() != "" && v.CanAddr() { + haveAddr = true + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Pointer && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Pointer) { + haveAddr = false + v = e + continue + } + } + + if v.Kind() != reflect.Pointer { + break + } + + if decodingNull && v.CanSet() { + break + } + + // Prevent infinite loop if v is an interface pointing to its own address: + // var v interface{} + // v = &v + if v.Elem().Kind() == reflect.Interface && v.Elem().Elem() == v { + v = v.Elem() + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 && v.CanInterface() { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if !decodingNull { + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + } + + if haveAddr { + v = v0 // restore original value after round-trip Value.Addr().Elem() + haveAddr = false + } else { + v = v.Elem() + } + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into v. +// The first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + ai := d.arrayInterface() + v.Set(reflect.ValueOf(ai)) + return nil + } + // Otherwise it's invalid. + fallthrough + default: + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + case reflect.Array, reflect.Slice: + break + } + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + // Expand slice length, growing the slice if necessary. + if v.Kind() == reflect.Slice { + if i >= v.Cap() { + v.Grow(1) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + if err := d.value(v.Index(i)); err != nil { + return err + } + } else { + // Ran out of fixed array: skip. + if err := d.value(reflect.Value{}); err != nil { + return err + } + } + i++ + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + } + + if i < v.Len() { + if v.Kind() == reflect.Array { + for ; i < v.Len(); i++ { + v.Index(i).SetZero() // zero remainder of array + } + } else { + v.SetLen(i) // truncate the slice + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } + return nil +} + +var nullLiteral = []byte("null") +var textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]() + +// object consumes an object from d.data[d.off-1:], decoding into v. +// The first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + t := v.Type() + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + oi := d.objectInterface() + v.Set(reflect.ValueOf(oi)) + return nil + } + + var fields structFields + + // Check type of target: + // struct or + // map[T1]T2 where T1 is string, an integer type, + // or an encoding.TextUnmarshaler + switch v.Kind() { + case reflect.Map: + // Map key must either have string kind, have an integer kind, + // or be an encoding.TextUnmarshaler. + switch t.Key().Kind() { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + if !reflect.PointerTo(t.Key()).Implements(textUnmarshalerType) { + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + fields = cachedTypeFields(t) + // ok + default: + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + + var mapElem reflect.Value + var origErrorContext errorContext + if d.errorContext != nil { + origErrorContext = *d.errorContext + } + + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquoteBytes(item) + if !ok { + panic(phasePanicMsg) + } + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := t.Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.SetZero() + } + subv = mapElem + } else { + f := fields.byExactName[string(key)] + if f == nil { + f = fields.byFoldedName[string(foldName(key))] + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Pointer { + if subv.IsNil() { + // If a struct embeds a pointer to an unexported type, + // it is not possible to set a newly allocated value + // since the field is unexported. + // + // See https://golang.org/issue/21357 + if !subv.CanSet() { + d.saveError(fmt.Errorf("json: cannot set embedded pointer to unexported struct: %v", subv.Type().Elem())) + // Invalidate subv to ensure d.value(subv) skips over + // the JSON value without assigning it to subv. + subv = reflect.Value{} + destring = false + break + } + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + if d.errorContext == nil { + d.errorContext = new(errorContext) + } + d.errorContext.FieldStack = append(d.errorContext.FieldStack, f.name) + d.errorContext.Struct = t + } else if d.disallowUnknownFields { + d.saveError(fmt.Errorf("json: unknown field %q", key)) + } + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + if destring { + switch qv := d.valueQuoted().(type) { + case nil: + if err := d.literalStore(nullLiteral, subv, false); err != nil { + return err + } + case string: + if err := d.literalStore([]byte(qv), subv, true); err != nil { + return err + } + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) + } + } else { + if err := d.value(subv); err != nil { + return err + } + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kt := t.Key() + var kv reflect.Value + if reflect.PointerTo(kt).Implements(textUnmarshalerType) { + kv = reflect.New(kt) + if err := d.literalStore(item, kv, true); err != nil { + return err + } + kv = kv.Elem() + } else { + switch kt.Kind() { + case reflect.String: + kv = reflect.New(kt).Elem() + kv.SetString(string(key)) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + s := string(key) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || kt.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.New(kt).Elem() + kv.SetInt(n) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + s := string(key) + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || kt.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.New(kt).Elem() + kv.SetUint(n) + default: + panic("json: Unexpected key type") // should never occur + } + } + if kv.IsValid() { + v.SetMapIndex(kv, subv) + } + } + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.errorContext != nil { + // Reset errorContext to its original state. + // Keep the same underlying array for FieldStack, to reuse the + // space and avoid unnecessary allocs. + d.errorContext.FieldStack = d.errorContext.FieldStack[:len(origErrorContext.FieldStack)] + d.errorContext.Struct = origErrorContext.Struct + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + return nil +} + +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (any, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{Value: "number " + s, Type: reflect.TypeFor[float64](), Offset: int64(d.off)} + } + return f, nil +} + +var numberType = reflect.TypeFor[Number]() + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) error { + // Check for unmarshaler. + if len(item) == 0 { + // Empty string given. + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + isNull := item[0] == 'n' // null + u, ut, pv := indirect(v, isNull) + if u != nil { + return u.UnmarshalJSON(item) + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + val := "number" + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" + } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) + return nil + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + return ut.UnmarshalText(s) + } + + v = pv + + switch c := item[0]; c { + case 'n': // null + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "null" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + case reflect.Interface, reflect.Pointer, reflect.Map, reflect.Slice: + v.SetZero() + // otherwise, ignore null for primitives/string + } + case 't', 'f': // true, false + value := item[0] == 't' + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "true" && string(item) != "false" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Slice: + if v.Type().Elem().Kind() != reflect.Uint8 { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.SetBytes(b[:n]) + case reflect.String: + t := string(s) + if v.Type() == numberType && !isValidNumber(t) { + return fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item) + } + v.SetString(t) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + switch v.Kind() { + default: + if v.Kind() == reflect.String && v.Type() == numberType { + // s must be a valid number, because it's + // already been tokenized. + v.SetString(string(item)) + break + } + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Interface: + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(string(item), 10, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + string(item), Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(string(item), 10, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + string(item), Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(string(item), v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + string(item), Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetFloat(n) + } + } + return nil +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() (val any) { + switch d.opcode { + default: + panic(phasePanicMsg) + case scanBeginArray: + val = d.arrayInterface() + d.scanNext() + case scanBeginObject: + val = d.objectInterface() + d.scanNext() + case scanBeginLiteral: + val = d.literalInterface() + } + return +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []any { + var v = make([]any, 0) + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + v = append(v, d.valueInterface()) + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + } + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]any { + m := make(map[string]any) + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read string key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + return m +} + +// literalInterface consumes and returns a literal from d.data[d.off-1:] and +// it reads the following byte ahead. The first byte of the literal has been +// read already (that's how the caller knows it's a literal). +func (d *decodeState) literalInterface() any { + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + item := d.data[start:d.readIndex()] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + panic(phasePanicMsg) + } + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + } +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + var r rune + for _, c := range s[2:6] { + switch { + case '0' <= c && c <= '9': + c = c - '0' + case 'a' <= c && c <= 'f': + c = c - 'a' + 10 + case 'A' <= c && c <= 'F': + c = c - 'A' + 10 + default: + return -1 + } + r = r*16 + rune(c) + } + return r +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +// unquoteBytes should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname unquoteBytes +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/internal/3rdparty/json/decode_test.go b/internal/3rdparty/json/decode_test.go new file mode 100644 index 0000000..f5b4467 --- /dev/null +++ b/internal/3rdparty/json/decode_test.go @@ -0,0 +1,2621 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "encoding" + "errors" + "fmt" + "image" + "math" + "math/big" + "net" + "reflect" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +type T struct { + X string + Y int + Z int `json:"-"` +} + +type U struct { + Alphabet string `json:"alpha"` +} + +type V struct { + F1 any + F2 int32 + F3 Number + F4 *VOuter +} + +type VOuter struct { + V V +} + +type W struct { + S SS +} + +type P struct { + PP PP +} + +type PP struct { + T T + Ts []T +} + +type SS string + +func (*SS) UnmarshalJSON(data []byte) error { + return &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[SS]()} +} + +// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and +// without UseNumber +var ifaceNumAsFloat64 = map[string]any{ + "k1": float64(1), + "k2": "s", + "k3": []any{float64(1), float64(2.0), float64(3e-3)}, + "k4": map[string]any{"kk1": "s", "kk2": float64(2)}, +} + +var ifaceNumAsNumber = map[string]any{ + "k1": Number("1"), + "k2": "s", + "k3": []any{Number("1"), Number("2.0"), Number("3e-3")}, + "k4": map[string]any{"kk1": "s", "kk2": Number("2")}, +} + +type tx struct { + x int +} + +type u8 uint8 + +// A type that can unmarshal itself. + +type unmarshaler struct { + T bool +} + +func (u *unmarshaler) UnmarshalJSON(b []byte) error { + *u = unmarshaler{true} // All we need to see that UnmarshalJSON is called. + return nil +} + +type ustruct struct { + M unmarshaler +} + +type unmarshalerText struct { + A, B string +} + +// needed for re-marshaling tests +func (u unmarshalerText) MarshalText() ([]byte, error) { + return []byte(u.A + ":" + u.B), nil +} + +func (u *unmarshalerText) UnmarshalText(b []byte) error { + pos := bytes.IndexByte(b, ':') + if pos == -1 { + return errors.New("missing separator") + } + u.A, u.B = string(b[:pos]), string(b[pos+1:]) + return nil +} + +var _ encoding.TextUnmarshaler = (*unmarshalerText)(nil) + +type ustructText struct { + M unmarshalerText +} + +// u8marshal is an integer type that can marshal/unmarshal itself. +type u8marshal uint8 + +func (u8 u8marshal) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("u%d", u8)), nil +} + +var errMissingU8Prefix = errors.New("missing 'u' prefix") + +func (u8 *u8marshal) UnmarshalText(b []byte) error { + if !bytes.HasPrefix(b, []byte{'u'}) { + return errMissingU8Prefix + } + n, err := strconv.Atoi(string(b[1:])) + if err != nil { + return err + } + *u8 = u8marshal(n) + return nil +} + +var _ encoding.TextUnmarshaler = (*u8marshal)(nil) + +var ( + umtrue = unmarshaler{true} + umslice = []unmarshaler{{true}} + umstruct = ustruct{unmarshaler{true}} + + umtrueXY = unmarshalerText{"x", "y"} + umsliceXY = []unmarshalerText{{"x", "y"}} + umstructXY = ustructText{unmarshalerText{"x", "y"}} + + ummapXY = map[unmarshalerText]bool{{"x", "y"}: true} +) + +// Test data structures for anonymous fields. + +type Point struct { + Z int +} + +type Top struct { + Level0 int + Embed0 + *Embed0a + *Embed0b `json:"e,omitempty"` // treated as named + Embed0c `json:"-"` // ignored + Loop + Embed0p // has Point with X, Y, used + Embed0q // has Point with Z, used + embed // contains exported field +} + +type Embed0 struct { + Level1a int // overridden by Embed0a's Level1a with json tag + Level1b int // used because Embed0a's Level1b is renamed + Level1c int // used because Embed0a's Level1c is ignored + Level1d int // annihilated by Embed0a's Level1d + Level1e int `json:"x"` // annihilated by Embed0a.Level1e +} + +type Embed0a struct { + Level1a int `json:"Level1a,omitempty"` + Level1b int `json:"LEVEL1B,omitempty"` + Level1c int `json:"-"` + Level1d int // annihilated by Embed0's Level1d + Level1f int `json:"x"` // annihilated by Embed0's Level1e +} + +type Embed0b Embed0 + +type Embed0c Embed0 + +type Embed0p struct { + image.Point +} + +type Embed0q struct { + Point +} + +type embed struct { + Q int +} + +type Loop struct { + Loop1 int `json:",omitempty"` + Loop2 int `json:",omitempty"` + *Loop +} + +// From reflect test: +// The X in S6 and S7 annihilate, but they also block the X in S8.S9. +type S5 struct { + S6 + S7 + S8 +} + +type S6 struct { + X int +} + +type S7 S6 + +type S8 struct { + S9 +} + +type S9 struct { + X int + Y int +} + +// From reflect test: +// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. +type S10 struct { + S11 + S12 + S13 +} + +type S11 struct { + S6 +} + +type S12 struct { + S6 +} + +type S13 struct { + S8 +} + +type Ambig struct { + // Given "hello", the first match should win. + First int `json:"HELLO"` + Second int `json:"Hello"` +} + +type XYZ struct { + X any + Y any + Z any +} + +type unexportedWithMethods struct{} + +func (unexportedWithMethods) F() {} + +type byteWithMarshalJSON byte + +func (b byteWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, byte(b))), nil +} + +func (b *byteWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalJSON(i) + return nil +} + +type byteWithPtrMarshalJSON byte + +func (b *byteWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return byteWithMarshalJSON(*b).MarshalJSON() +} + +func (b *byteWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*byteWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type byteWithMarshalText byte + +func (b byteWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, byte(b))), nil +} + +func (b *byteWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalText(i) + return nil +} + +type byteWithPtrMarshalText byte + +func (b *byteWithPtrMarshalText) MarshalText() ([]byte, error) { + return byteWithMarshalText(*b).MarshalText() +} + +func (b *byteWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*byteWithMarshalText)(b).UnmarshalText(data) +} + +type intWithMarshalJSON int + +func (b intWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, int(b))), nil +} + +func (b *intWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalJSON(i) + return nil +} + +type intWithPtrMarshalJSON int + +func (b *intWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return intWithMarshalJSON(*b).MarshalJSON() +} + +func (b *intWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*intWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type intWithMarshalText int + +func (b intWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, int(b))), nil +} + +func (b *intWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalText(i) + return nil +} + +type intWithPtrMarshalText int + +func (b *intWithPtrMarshalText) MarshalText() ([]byte, error) { + return intWithMarshalText(*b).MarshalText() +} + +func (b *intWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*intWithMarshalText)(b).UnmarshalText(data) +} + +type mapStringToStringData struct { + Data map[string]string `json:"data"` +} + +type B struct { + B bool `json:",string"` +} + +type DoublePtr struct { + I **int + J **int +} + +var unmarshalTests = []struct { + CaseName + in string + ptr any // new(type) + out any + err error + useNumber bool + golden bool + disallowUnknownFields bool +}{ + // basic types + {CaseName: Name(""), in: `true`, ptr: new(bool), out: true}, + {CaseName: Name(""), in: `1`, ptr: new(int), out: 1}, + {CaseName: Name(""), in: `1.2`, ptr: new(float64), out: 1.2}, + {CaseName: Name(""), in: `-5`, ptr: new(int16), out: int16(-5)}, + {CaseName: Name(""), in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, + {CaseName: Name(""), in: `2`, ptr: new(Number), out: Number("2")}, + {CaseName: Name(""), in: `2`, ptr: new(any), out: float64(2.0)}, + {CaseName: Name(""), in: `2`, ptr: new(any), out: Number("2"), useNumber: true}, + {CaseName: Name(""), in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + {CaseName: Name(""), in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + {CaseName: Name(""), in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + {CaseName: Name(""), in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + {CaseName: Name(""), in: "null", ptr: new(any), out: nil}, + {CaseName: Name(""), in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeFor[string](), 7, "T", "X"}}, + {CaseName: Name(""), in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[string](), 8, "T", "X"}}, + {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, + {CaseName: Name(""), in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[SS](), 0, "W", "S"}}, + {CaseName: Name(""), in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, + {CaseName: Name(""), in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, + {CaseName: Name(""), in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsFloat64}, + {CaseName: Name(""), in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsNumber, useNumber: true}, + + // raw values with whitespace + {CaseName: Name(""), in: "\n true ", ptr: new(bool), out: true}, + {CaseName: Name(""), in: "\t 1 ", ptr: new(int), out: 1}, + {CaseName: Name(""), in: "\r 1.2 ", ptr: new(float64), out: 1.2}, + {CaseName: Name(""), in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, + {CaseName: Name(""), in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, + + // Z has a "-" tag. + {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, + + {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + {CaseName: Name(""), in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, + {CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + + // syntax errors + {CaseName: Name(""), in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, + {CaseName: Name(""), in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, + {CaseName: Name(""), in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, + {CaseName: Name(""), in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}}, + {CaseName: Name(""), in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}}, + + // raw value errors + {CaseName: Name(""), in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {CaseName: Name(""), in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 5}}, + {CaseName: Name(""), in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {CaseName: Name(""), in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 8}}, + {CaseName: Name(""), in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {CaseName: Name(""), in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 6}}, + {CaseName: Name(""), in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {CaseName: Name(""), in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 11}}, + + // array tests + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, + + // empty array to interface test + {CaseName: Name(""), in: `[]`, ptr: new([]any), out: []any{}}, + {CaseName: Name(""), in: `null`, ptr: new([]any), out: []any(nil)}, + {CaseName: Name(""), in: `{"T":[]}`, ptr: new(map[string]any), out: map[string]any{"T": []any{}}}, + {CaseName: Name(""), in: `{"T":null}`, ptr: new(map[string]any), out: map[string]any{"T": any(nil)}}, + + // composite tests + {CaseName: Name(""), in: allValueIndent, ptr: new(All), out: allValue}, + {CaseName: Name(""), in: allValueCompact, ptr: new(All), out: allValue}, + {CaseName: Name(""), in: allValueIndent, ptr: new(*All), out: &allValue}, + {CaseName: Name(""), in: allValueCompact, ptr: new(*All), out: &allValue}, + {CaseName: Name(""), in: pallValueIndent, ptr: new(All), out: pallValue}, + {CaseName: Name(""), in: pallValueCompact, ptr: new(All), out: pallValue}, + {CaseName: Name(""), in: pallValueIndent, ptr: new(*All), out: &pallValue}, + {CaseName: Name(""), in: pallValueCompact, ptr: new(*All), out: &pallValue}, + + // unmarshal interface test + {CaseName: Name(""), in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + {CaseName: Name(""), in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, + {CaseName: Name(""), in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, + {CaseName: Name(""), in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, + {CaseName: Name(""), in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, + + // UnmarshalText interface test + {CaseName: Name(""), in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, + {CaseName: Name(""), in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, + {CaseName: Name(""), in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, + {CaseName: Name(""), in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, + {CaseName: Name(""), in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, + + // integer-keyed map test + { + CaseName: Name(""), + in: `{"-1":"a","0":"b","1":"c"}`, + ptr: new(map[int]string), + out: map[int]string{-1: "a", 0: "b", 1: "c"}, + }, + { + CaseName: Name(""), + in: `{"0":"a","10":"c","9":"b"}`, + ptr: new(map[u8]string), + out: map[u8]string{0: "a", 9: "b", 10: "c"}, + }, + { + CaseName: Name(""), + in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, + ptr: new(map[int64]string), + out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, + }, + { + CaseName: Name(""), + in: `{"18446744073709551615":"max"}`, + ptr: new(map[uint64]string), + out: map[uint64]string{math.MaxUint64: "max"}, + }, + { + CaseName: Name(""), + in: `{"0":false,"10":true}`, + ptr: new(map[uintptr]bool), + out: map[uintptr]bool{0: false, 10: true}, + }, + + // Check that MarshalText and UnmarshalText take precedence + // over default integer handling in map keys. + { + CaseName: Name(""), + in: `{"u2":4}`, + ptr: new(map[u8marshal]int), + out: map[u8marshal]int{2: 4}, + }, + { + CaseName: Name(""), + in: `{"2":4}`, + ptr: new(map[u8marshal]int), + err: errMissingU8Prefix, + }, + + // integer-keyed map errors + { + CaseName: Name(""), + in: `{"abc":"abc"}`, + ptr: new(map[int]string), + err: &UnmarshalTypeError{Value: "number abc", Type: reflect.TypeFor[int](), Offset: 2}, + }, + { + CaseName: Name(""), + in: `{"256":"abc"}`, + ptr: new(map[uint8]string), + err: &UnmarshalTypeError{Value: "number 256", Type: reflect.TypeFor[uint8](), Offset: 2}, + }, + { + CaseName: Name(""), + in: `{"128":"abc"}`, + ptr: new(map[int8]string), + err: &UnmarshalTypeError{Value: "number 128", Type: reflect.TypeFor[int8](), Offset: 2}, + }, + { + CaseName: Name(""), + in: `{"-1":"abc"}`, + ptr: new(map[uint8]string), + err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeFor[uint8](), Offset: 2}, + }, + { + CaseName: Name(""), + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[int]int), + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeFor[int](), Offset: 7}, + }, + { + CaseName: Name(""), + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[uint]int), + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeFor[uint](), Offset: 7}, + }, + + // Map keys can be encoding.TextUnmarshalers. + {CaseName: Name(""), in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + // If multiple values for the same key exists, only the most recent value is used. + {CaseName: Name(""), in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + + { + CaseName: Name(""), + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + }, + }, + { + CaseName: Name(""), + in: `{"hello": 1}`, + ptr: new(Ambig), + out: Ambig{First: 1}, + }, + + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9: S9{Y: 2}}}, + }, + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + }, + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + CaseName: Name(""), + in: `{"I": 0, "I": null, "J": null}`, + ptr: new(DoublePtr), + out: DoublePtr{I: nil, J: nil}, + }, + + // invalid UTF-8 is coerced to valid UTF-8. + { + CaseName: Name(""), + in: "\"hello\xffworld\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\xc2\xc2world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\xc2\xffworld\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\\ud800world\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", + ptr: new(string), + out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", + }, + + // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. + { + CaseName: Name(""), + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[time.Time]string), + out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, + }, + + // issue 8305 + { + CaseName: Name(""), + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[Point]string), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeFor[map[Point]string](), Offset: 1}, + }, + { + CaseName: Name(""), + in: `{"asdf": "hello world"}`, + ptr: new(map[unmarshaler]string), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeFor[map[unmarshaler]string](), Offset: 1}, + }, + + // related to issue 13783. + // Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type, + // similar to marshaling a slice of typed int. + // These tests check that, assuming the byte type also has valid decoding methods, + // either the old base64 string encoding or the new per-element encoding can be + // successfully unmarshaled. The custom unmarshalers were accessible in earlier + // versions of Go, even though the custom marshaler was not. + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + // ints work with the marshaler but not the base64 []byte case + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalJSON), + out: []intWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalText), + out: []intWithMarshalText{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalJSON), + out: []intWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalText), + out: []intWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + {CaseName: Name(""), in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + {CaseName: Name(""), in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + {CaseName: Name(""), in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + {CaseName: Name(""), in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + {CaseName: Name(""), in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + {CaseName: Name(""), in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + {CaseName: Name(""), in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + {CaseName: Name(""), in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + {CaseName: Name(""), in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + {CaseName: Name(""), in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + {CaseName: Name(""), in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + { + CaseName: Name(""), + in: `{"V": {"F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "V.F2", + Type: reflect.TypeFor[int32](), + Offset: 20, + }, + }, + { + CaseName: Name(""), + in: `{"V": {"F4": {}, "F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "V.F2", + Type: reflect.TypeFor[int32](), + Offset: 30, + }, + }, + + // issue 15146. + // invalid inputs in wrongStringTests below. + {CaseName: Name(""), in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + {CaseName: Name(""), in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + {CaseName: Name(""), in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, + {CaseName: Name(""), in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, + {CaseName: Name(""), in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, + {CaseName: Name(""), in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + {CaseName: Name(""), in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, + {CaseName: Name(""), in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, + + // additional tests for disallowUnknownFields + { + CaseName: Name(""), + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18, + "extra": true + }`, + ptr: new(Top), + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + { + CaseName: Name(""), + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12, + "extra": null + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + // issue 26444 + // UnmarshalTypeError without field & struct values + { + CaseName: Name(""), + in: `{"data":{"test1": "bob", "test2": 123}}`, + ptr: new(mapStringToStringData), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[string](), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, + }, + { + CaseName: Name(""), + in: `{"data":{"test1": 123, "test2": "bob"}}`, + ptr: new(mapStringToStringData), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[string](), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, + }, + + // trying to decode JSON arrays or objects via TextUnmarshaler + { + CaseName: Name(""), + in: `[1, 2, 3]`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "array", Type: reflect.TypeFor[*MustNotUnmarshalText](), Offset: 1}, + }, + { + CaseName: Name(""), + in: `{"foo": "bar"}`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeFor[*MustNotUnmarshalText](), Offset: 1}, + }, + // #22369 + { + CaseName: Name(""), + in: `{"PP": {"T": {"Y": "bad-type"}}}`, + ptr: new(P), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "T", + Field: "PP.T.Y", + Type: reflect.TypeFor[int](), + Offset: 29, + }, + }, + { + CaseName: Name(""), + in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, + ptr: new(PP), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "T", + Field: "Ts.Y", + Type: reflect.TypeFor[int](), + Offset: 29, + }, + }, + // #14702 + { + CaseName: Name(""), + in: `invalid`, + ptr: new(Number), + err: &SyntaxError{ + msg: "invalid character 'i' looking for beginning of value", + Offset: 1, + }, + }, + { + CaseName: Name(""), + in: `"invalid"`, + ptr: new(Number), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + { + CaseName: Name(""), + in: `{"A":"invalid"}`, + ptr: new(struct{ A Number }), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + { + CaseName: Name(""), + in: `{"A":"invalid"}`, + ptr: new(struct { + A Number `json:",string"` + }), + err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number", `invalid`), + }, + { + CaseName: Name(""), + in: `{"A":"invalid"}`, + ptr: new(map[string]Number), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, +} + +func TestMarshal(t *testing.T) { + b, err := Marshal(allValue) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if string(b) != allValueCompact { + t.Errorf("Marshal:") + diff(t, b, []byte(allValueCompact)) + return + } + + b, err = Marshal(pallValue) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if string(b) != pallValueCompact { + t.Errorf("Marshal:") + diff(t, b, []byte(pallValueCompact)) + return + } +} + +func TestMarshalInvalidUTF8(t *testing.T) { + tests := []struct { + CaseName + in string + want string + }{ + {Name(""), "hello\xffworld", `"hello\ufffdworld"`}, + {Name(""), "", `""`}, + {Name(""), "\xff", `"\ufffd"`}, + {Name(""), "\xff\xff", `"\ufffd\ufffd"`}, + {Name(""), "a\xffb", `"a\ufffdb"`}, + {Name(""), "\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"日本\ufffd\ufffd\ufffd"`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + got, err := Marshal(tt.in) + if string(got) != tt.want || err != nil { + t.Errorf("%s: Marshal(%q):\n\tgot: (%q, %v)\n\twant: (%q, nil)", tt.Where, tt.in, got, err, tt.want) + } + }) + } +} + +func TestMarshalNumberZeroVal(t *testing.T) { + var n Number + out, err := Marshal(n) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + got := string(out) + if got != "0" { + t.Fatalf("Marshal: got %s, want 0", got) + } +} + +func TestMarshalEmbeds(t *testing.T) { + top := &Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + } + got, err := Marshal(top) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + want := "{\"Level0\":1,\"Level1b\":2,\"Level1c\":3,\"Level1a\":5,\"LEVEL1B\":6,\"e\":{\"Level1a\":8,\"Level1b\":9,\"Level1c\":10,\"Level1d\":11,\"x\":12},\"Loop1\":13,\"Loop2\":14,\"X\":15,\"Y\":16,\"Z\":17,\"Q\":18}" + if string(got) != want { + t.Errorf("Marshal:\n\tgot: %s\n\twant: %s", got, want) + } +} + +func equalError(a, b error) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Error() == b.Error() +} + +func TestUnmarshal(t *testing.T) { + for _, tt := range unmarshalTests { + t.Run(tt.Name, func(t *testing.T) { + in := []byte(tt.in) + var scan scanner + if err := checkValid(in, &scan); err != nil { + if !equalError(err, tt.err) { + t.Fatalf("%s: checkValid error: %#v", tt.Where, err) + } + } + if tt.ptr == nil { + return + } + + typ := reflect.TypeOf(tt.ptr) + if typ.Kind() != reflect.Pointer { + t.Fatalf("%s: unmarshalTest.ptr %T is not a pointer type", tt.Where, tt.ptr) + } + typ = typ.Elem() + + // v = new(right-type) + v := reflect.New(typ) + + if !reflect.DeepEqual(tt.ptr, v.Interface()) { + // There's no reason for ptr to point to non-zero data, + // as we decode into new(right-type), so the data is + // discarded. + // This can easily mean tests that silently don't test + // what they should. To test decoding into existing + // data, see TestPrefilled. + t.Fatalf("%s: unmarshalTest.ptr %#v is not a pointer to a zero value", tt.Where, tt.ptr) + } + + dec := NewDecoder(bytes.NewReader(in)) + if tt.useNumber { + dec.UseNumber() + } + if tt.disallowUnknownFields { + dec.DisallowUnknownFields() + } + if err := dec.Decode(v.Interface()); !equalError(err, tt.err) { + t.Fatalf("%s: Decode error:\n\tgot: %v\n\twant: %v", tt.Where, err, tt.err) + } else if err != nil { + return + } + if got := v.Elem().Interface(); !reflect.DeepEqual(got, tt.out) { + gotJSON, _ := Marshal(got) + wantJSON, _ := Marshal(tt.out) + t.Fatalf("%s: Decode:\n\tgot: %#+v\n\twant: %#+v\n\n\tgotJSON: %s\n\twantJSON: %s", tt.Where, got, tt.out, gotJSON, wantJSON) + } + + // Check round trip also decodes correctly. + if tt.err == nil { + enc, err := Marshal(v.Interface()) + if err != nil { + t.Fatalf("%s: Marshal error after roundtrip: %v", tt.Where, err) + } + if tt.golden && !bytes.Equal(enc, in) { + t.Errorf("%s: Marshal:\n\tgot: %s\n\twant: %s", tt.Where, enc, in) + } + vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec = NewDecoder(bytes.NewReader(enc)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(vv.Interface()); err != nil { + t.Fatalf("%s: Decode(%#q) error after roundtrip: %v", tt.Where, enc, err) + } + if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { + t.Fatalf("%s: Decode:\n\tgot: %#+v\n\twant: %#+v\n\n\tgotJSON: %s\n\twantJSON: %s", + tt.Where, v.Elem().Interface(), vv.Elem().Interface(), + stripWhitespace(string(enc)), stripWhitespace(string(in))) + } + } + }) + } +} + +func TestUnmarshalMarshal(t *testing.T) { + initBig() + var v any + if err := Unmarshal(jsonBig, &v); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + b, err := Marshal(v) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if !bytes.Equal(jsonBig, b) { + t.Errorf("Marshal:") + diff(t, b, jsonBig) + return + } +} + +// Independent of Decode, basic coverage of the accessors in Number +func TestNumberAccessors(t *testing.T) { + tests := []struct { + CaseName + in string + i int64 + intErr string + f float64 + floatErr string + }{ + {CaseName: Name(""), in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, + {CaseName: Name(""), in: "-12", i: -12, f: -12.0}, + {CaseName: Name(""), in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + n := Number(tt.in) + if got := n.String(); got != tt.in { + t.Errorf("%s: Number(%q).String() = %s, want %s", tt.Where, tt.in, got, tt.in) + } + if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { + t.Errorf("%s: Number(%q).Int64() = %d, want %d", tt.Where, tt.in, i, tt.i) + } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { + t.Errorf("%s: Number(%q).Int64() error:\n\tgot: %v\n\twant: %v", tt.Where, tt.in, err, tt.intErr) + } + if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { + t.Errorf("%s: Number(%q).Float64() = %g, want %g", tt.Where, tt.in, f, tt.f) + } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { + t.Errorf("%s: Number(%q).Float64() error:\n\tgot %v\n\twant: %v", tt.Where, tt.in, err, tt.floatErr) + } + }) + } +} + +func TestLargeByteSlice(t *testing.T) { + s0 := make([]byte, 2000) + for i := range s0 { + s0[i] = byte(i) + } + b, err := Marshal(s0) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var s1 []byte + if err := Unmarshal(b, &s1); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !bytes.Equal(s0, s1) { + t.Errorf("Marshal:") + diff(t, s0, s1) + } +} + +type Xint struct { + X int +} + +func TestUnmarshalInterface(t *testing.T) { + var xint Xint + var i any = &xint + if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if xint.X != 1 { + t.Fatalf("xint.X = %d, want 1", xint.X) + } +} + +func TestUnmarshalPtrPtr(t *testing.T) { + var xint Xint + pxint := &xint + if err := Unmarshal([]byte(`{"X":1}`), &pxint); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if xint.X != 1 { + t.Fatalf("xint.X = %d, want 1", xint.X) + } +} + +func TestEscape(t *testing.T) { + const input = `"foobar"` + " [\u2028 \u2029]" + const want = `"\"foobar\"\u003chtml\u003e [\u2028 \u2029]"` + got, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if string(got) != want { + t.Errorf("Marshal(%#q):\n\tgot: %s\n\twant: %s", input, got, want) + } +} + +// If people misuse the ,string modifier, the error message should be +// helpful, telling the user that they're doing it wrong. +func TestErrorMessageFromMisusedString(t *testing.T) { + // WrongString is a struct that's misusing the ,string modifier. + type WrongString struct { + Message string `json:"result,string"` + } + tests := []struct { + CaseName + in, err string + }{ + {Name(""), `{"result":"x"}`, `json: invalid use of ,string struct tag, trying to unmarshal "x" into string`}, + {Name(""), `{"result":"foo"}`, `json: invalid use of ,string struct tag, trying to unmarshal "foo" into string`}, + {Name(""), `{"result":"123"}`, `json: invalid use of ,string struct tag, trying to unmarshal "123" into string`}, + {Name(""), `{"result":123}`, `json: invalid use of ,string struct tag, trying to unmarshal unquoted value into string`}, + {Name(""), `{"result":"\""}`, `json: invalid use of ,string struct tag, trying to unmarshal "\"" into string`}, + {Name(""), `{"result":"\"foo"}`, `json: invalid use of ,string struct tag, trying to unmarshal "\"foo" into string`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + r := strings.NewReader(tt.in) + var s WrongString + err := NewDecoder(r).Decode(&s) + got := fmt.Sprintf("%v", err) + if got != tt.err { + t.Errorf("%s: Decode error:\n\tgot: %s\n\twant: %s", tt.Where, got, tt.err) + } + }) + } +} + +type All struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Uintptr uintptr + Float32 float32 + Float64 float64 + + Foo string `json:"bar"` + Foo2 string `json:"bar2,dummyopt"` + + IntStr int64 `json:",string"` + UintptrStr uintptr `json:",string"` + + PBool *bool + PInt *int + PInt8 *int8 + PInt16 *int16 + PInt32 *int32 + PInt64 *int64 + PUint *uint + PUint8 *uint8 + PUint16 *uint16 + PUint32 *uint32 + PUint64 *uint64 + PUintptr *uintptr + PFloat32 *float32 + PFloat64 *float64 + + String string + PString *string + + Map map[string]Small + MapP map[string]*Small + PMap *map[string]Small + PMapP *map[string]*Small + + EmptyMap map[string]Small + NilMap map[string]Small + + Slice []Small + SliceP []*Small + PSlice *[]Small + PSliceP *[]*Small + + EmptySlice []Small + NilSlice []Small + + StringSlice []string + ByteSlice []byte + + Small Small + PSmall *Small + PPSmall **Small + + Interface any + PInterface *any + + unexported int +} + +type Small struct { + Tag string +} + +var allValue = All{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Uintptr: 12, + Float32: 14.1, + Float64: 15.1, + Foo: "foo", + Foo2: "foo2", + IntStr: 42, + UintptrStr: 44, + String: "16", + Map: map[string]Small{ + "17": {Tag: "tag17"}, + "18": {Tag: "tag18"}, + }, + MapP: map[string]*Small{ + "19": {Tag: "tag19"}, + "20": nil, + }, + EmptyMap: map[string]Small{}, + Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}}, + SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}}, + EmptySlice: []Small{}, + StringSlice: []string{"str24", "str25", "str26"}, + ByteSlice: []byte{27, 28, 29}, + Small: Small{Tag: "tag30"}, + PSmall: &Small{Tag: "tag31"}, + Interface: 5.2, +} + +var pallValue = All{ + PBool: &allValue.Bool, + PInt: &allValue.Int, + PInt8: &allValue.Int8, + PInt16: &allValue.Int16, + PInt32: &allValue.Int32, + PInt64: &allValue.Int64, + PUint: &allValue.Uint, + PUint8: &allValue.Uint8, + PUint16: &allValue.Uint16, + PUint32: &allValue.Uint32, + PUint64: &allValue.Uint64, + PUintptr: &allValue.Uintptr, + PFloat32: &allValue.Float32, + PFloat64: &allValue.Float64, + PString: &allValue.String, + PMap: &allValue.Map, + PMapP: &allValue.MapP, + PSlice: &allValue.Slice, + PSliceP: &allValue.SliceP, + PPSmall: &allValue.PSmall, + PInterface: &allValue.Interface, +} + +var allValueIndent = `{ + "Bool": true, + "Int": 2, + "Int8": 3, + "Int16": 4, + "Int32": 5, + "Int64": 6, + "Uint": 7, + "Uint8": 8, + "Uint16": 9, + "Uint32": 10, + "Uint64": 11, + "Uintptr": 12, + "Float32": 14.1, + "Float64": 15.1, + "bar": "foo", + "bar2": "foo2", + "IntStr": "42", + "UintptrStr": "44", + "PBool": null, + "PInt": null, + "PInt8": null, + "PInt16": null, + "PInt32": null, + "PInt64": null, + "PUint": null, + "PUint8": null, + "PUint16": null, + "PUint32": null, + "PUint64": null, + "PUintptr": null, + "PFloat32": null, + "PFloat64": null, + "String": "16", + "PString": null, + "Map": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "MapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "PMap": null, + "PMapP": null, + "EmptyMap": {}, + "NilMap": null, + "Slice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "SliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "PSlice": null, + "PSliceP": null, + "EmptySlice": [], + "NilSlice": null, + "StringSlice": [ + "str24", + "str25", + "str26" + ], + "ByteSlice": "Gxwd", + "Small": { + "Tag": "tag30" + }, + "PSmall": { + "Tag": "tag31" + }, + "PPSmall": null, + "Interface": 5.2, + "PInterface": null +}` + +var allValueCompact = stripWhitespace(allValueIndent) + +var pallValueIndent = `{ + "Bool": false, + "Int": 0, + "Int8": 0, + "Int16": 0, + "Int32": 0, + "Int64": 0, + "Uint": 0, + "Uint8": 0, + "Uint16": 0, + "Uint32": 0, + "Uint64": 0, + "Uintptr": 0, + "Float32": 0, + "Float64": 0, + "bar": "", + "bar2": "", + "IntStr": "0", + "UintptrStr": "0", + "PBool": true, + "PInt": 2, + "PInt8": 3, + "PInt16": 4, + "PInt32": 5, + "PInt64": 6, + "PUint": 7, + "PUint8": 8, + "PUint16": 9, + "PUint32": 10, + "PUint64": 11, + "PUintptr": 12, + "PFloat32": 14.1, + "PFloat64": 15.1, + "String": "", + "PString": "16", + "Map": null, + "MapP": null, + "PMap": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "PMapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "EmptyMap": null, + "NilMap": null, + "Slice": null, + "SliceP": null, + "PSlice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "PSliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "EmptySlice": null, + "NilSlice": null, + "StringSlice": null, + "ByteSlice": null, + "Small": { + "Tag": "" + }, + "PSmall": null, + "PPSmall": { + "Tag": "tag31" + }, + "Interface": null, + "PInterface": 5.2 +}` + +var pallValueCompact = stripWhitespace(pallValueIndent) + +func TestRefUnmarshal(t *testing.T) { + type S struct { + // Ref is defined in encode_test.go. + R0 Ref + R1 *Ref + R2 RefText + R3 *RefText + } + want := S{ + R0: 12, + R1: new(Ref), + R2: 13, + R3: new(RefText), + } + *want.R1 = 12 + *want.R3 = 13 + + var got S + if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref","R2":"ref","R3":"ref"}`), &got); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Unmarsha:\n\tgot: %+v\n\twant: %+v", got, want) + } +} + +// Test that the empty string doesn't panic decoding when ,string is specified +// Issue 3450 +func TestEmptyString(t *testing.T) { + type T2 struct { + Number1 int `json:",string"` + Number2 int `json:",string"` + } + data := `{"Number1":"1", "Number2":""}` + dec := NewDecoder(strings.NewReader(data)) + var got T2 + switch err := dec.Decode(&got); { + case err == nil: + t.Fatalf("Decode error: got nil, want non-nil") + case got.Number1 != 1: + t.Fatalf("Decode: got.Number1 = %d, want 1", got.Number1) + } +} + +// Test that a null for ,string is not replaced with the previous quoted string (issue 7046). +// It should also not be an error (issue 2540, issue 8587). +func TestNullString(t *testing.T) { + type T struct { + A int `json:",string"` + B int `json:",string"` + C *int `json:",string"` + } + data := []byte(`{"A": "1", "B": null, "C": null}`) + var s T + s.B = 1 + s.C = new(int) + *s.C = 2 + switch err := Unmarshal(data, &s); { + case err != nil: + t.Fatalf("Unmarshal error: %v", err) + case s.B != 1: + t.Fatalf("Unmarshal: s.B = %d, want 1", s.B) + case s.C != nil: + t.Fatalf("Unmarshal: s.C = %d, want non-nil", s.C) + } +} + +func intp(x int) *int { + p := new(int) + *p = x + return p +} + +func intpp(x *int) **int { + pp := new(*int) + *pp = x + return pp +} + +func TestInterfaceSet(t *testing.T) { + tests := []struct { + CaseName + pre any + json string + post any + }{ + {Name(""), "foo", `"bar"`, "bar"}, + {Name(""), "foo", `2`, 2.0}, + {Name(""), "foo", `true`, true}, + {Name(""), "foo", `null`, nil}, + + {Name(""), nil, `null`, nil}, + {Name(""), new(int), `null`, nil}, + {Name(""), (*int)(nil), `null`, nil}, + {Name(""), new(*int), `null`, new(*int)}, + {Name(""), (**int)(nil), `null`, nil}, + {Name(""), intp(1), `null`, nil}, + {Name(""), intpp(nil), `null`, intpp(nil)}, + {Name(""), intpp(intp(1)), `null`, intpp(nil)}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + b := struct{ X any }{tt.pre} + blob := `{"X":` + tt.json + `}` + if err := Unmarshal([]byte(blob), &b); err != nil { + t.Fatalf("%s: Unmarshal(%#q) error: %v", tt.Where, blob, err) + } + if !reflect.DeepEqual(b.X, tt.post) { + t.Errorf("%s: Unmarshal(%#q):\n\tpre.X: %#v\n\tgot.X: %#v\n\twant.X: %#v", tt.Where, blob, tt.pre, b.X, tt.post) + } + }) + } +} + +type NullTest struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + String string + PBool *bool + Map map[string]string + Slice []string + Interface any + + PRaw *RawMessage + PTime *time.Time + PBigInt *big.Int + PText *MustNotUnmarshalText + PBuffer *bytes.Buffer // has methods, just not relevant ones + PStruct *struct{} + + Raw RawMessage + Time time.Time + BigInt big.Int + Text MustNotUnmarshalText + Buffer bytes.Buffer + Struct struct{} +} + +// JSON null values should be ignored for primitives and string values instead of resulting in an error. +// Issue 2540 +func TestUnmarshalNulls(t *testing.T) { + // Unmarshal docs: + // The JSON null value unmarshals into an interface, map, pointer, or slice + // by setting that Go value to nil. Because null is often used in JSON to mean + // ``not present,'' unmarshaling a JSON null into any other Go type has no effect + // on the value and produces no error. + + jsonData := []byte(`{ + "Bool" : null, + "Int" : null, + "Int8" : null, + "Int16" : null, + "Int32" : null, + "Int64" : null, + "Uint" : null, + "Uint8" : null, + "Uint16" : null, + "Uint32" : null, + "Uint64" : null, + "Float32" : null, + "Float64" : null, + "String" : null, + "PBool": null, + "Map": null, + "Slice": null, + "Interface": null, + "PRaw": null, + "PTime": null, + "PBigInt": null, + "PText": null, + "PBuffer": null, + "PStruct": null, + "Raw": null, + "Time": null, + "BigInt": null, + "Text": null, + "Buffer": null, + "Struct": null + }`) + nulls := NullTest{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Float32: 12.1, + Float64: 13.1, + String: "14", + PBool: new(bool), + Map: map[string]string{}, + Slice: []string{}, + Interface: new(MustNotUnmarshalJSON), + PRaw: new(RawMessage), + PTime: new(time.Time), + PBigInt: new(big.Int), + PText: new(MustNotUnmarshalText), + PStruct: new(struct{}), + PBuffer: new(bytes.Buffer), + Raw: RawMessage("123"), + Time: time.Unix(123456789, 0), + BigInt: *big.NewInt(123), + } + + before := nulls.Time.String() + + err := Unmarshal(jsonData, &nulls) + if err != nil { + t.Errorf("Unmarshal of null values failed: %v", err) + } + if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || + nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || + nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { + t.Errorf("Unmarshal of null values affected primitives") + } + + if nulls.PBool != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBool") + } + if nulls.Map != nil { + t.Errorf("Unmarshal of null did not clear nulls.Map") + } + if nulls.Slice != nil { + t.Errorf("Unmarshal of null did not clear nulls.Slice") + } + if nulls.Interface != nil { + t.Errorf("Unmarshal of null did not clear nulls.Interface") + } + if nulls.PRaw != nil { + t.Errorf("Unmarshal of null did not clear nulls.PRaw") + } + if nulls.PTime != nil { + t.Errorf("Unmarshal of null did not clear nulls.PTime") + } + if nulls.PBigInt != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBigInt") + } + if nulls.PText != nil { + t.Errorf("Unmarshal of null did not clear nulls.PText") + } + if nulls.PBuffer != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBuffer") + } + if nulls.PStruct != nil { + t.Errorf("Unmarshal of null did not clear nulls.PStruct") + } + + if string(nulls.Raw) != "null" { + t.Errorf("Unmarshal of RawMessage null did not record null: %v", string(nulls.Raw)) + } + if nulls.Time.String() != before { + t.Errorf("Unmarshal of time.Time null set time to %v", nulls.Time.String()) + } + if nulls.BigInt.String() != "123" { + t.Errorf("Unmarshal of big.Int null set int to %v", nulls.BigInt.String()) + } +} + +type MustNotUnmarshalJSON struct{} + +func (x MustNotUnmarshalJSON) UnmarshalJSON(data []byte) error { + return errors.New("MustNotUnmarshalJSON was used") +} + +type MustNotUnmarshalText struct{} + +func (x MustNotUnmarshalText) UnmarshalText(text []byte) error { + return errors.New("MustNotUnmarshalText was used") +} + +func TestStringKind(t *testing.T) { + type stringKind string + want := map[stringKind]int{"foo": 42} + data, err := Marshal(want) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got map[stringKind]int + err = Unmarshal(data, &got) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v", got, want) + } +} + +// Custom types with []byte as underlying type could not be marshaled +// and then unmarshaled. +// Issue 8962. +func TestByteKind(t *testing.T) { + type byteKind []byte + want := byteKind("hello") + data, err := Marshal(want) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got byteKind + err = Unmarshal(data, &got) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !slices.Equal(got, want) { + t.Fatalf("Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v", got, want) + } +} + +// The fix for issue 8962 introduced a regression. +// Issue 12921. +func TestSliceOfCustomByte(t *testing.T) { + type Uint8 uint8 + want := []Uint8("hello") + data, err := Marshal(want) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got []Uint8 + err = Unmarshal(data, &got) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !slices.Equal(got, want) { + t.Fatalf("Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v", got, want) + } +} + +func TestUnmarshalTypeError(t *testing.T) { + tests := []struct { + CaseName + dest any + in string + }{ + {Name(""), new(string), `{"user": "name"}`}, // issue 4628. + {Name(""), new(error), `{}`}, // issue 4222 + {Name(""), new(error), `[]`}, + {Name(""), new(error), `""`}, + {Name(""), new(error), `123`}, + {Name(""), new(error), `true`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.in), tt.dest) + if _, ok := err.(*UnmarshalTypeError); !ok { + t.Errorf("%s: Unmarshal(%#q, %T):\n\tgot: %T\n\twant: %T", + tt.Where, tt.in, tt.dest, err, new(UnmarshalTypeError)) + } + }) + } +} + +func TestUnmarshalSyntax(t *testing.T) { + var x any + tests := []struct { + CaseName + in string + }{ + {Name(""), "tru"}, + {Name(""), "fals"}, + {Name(""), "nul"}, + {Name(""), "123e"}, + {Name(""), `"hello`}, + {Name(""), `[1,2,3`}, + {Name(""), `{"key":1`}, + {Name(""), `{"key":1,`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.in), &x) + if _, ok := err.(*SyntaxError); !ok { + t.Errorf("%s: Unmarshal(%#q, any):\n\tgot: %T\n\twant: %T", + tt.Where, tt.in, err, new(SyntaxError)) + } + }) + } +} + +// Test handling of unexported fields that should be ignored. +// Issue 4660 +type unexportedFields struct { + Name string + m map[string]any `json:"-"` + m2 map[string]any `json:"abcd"` + + s []int `json:"-"` +} + +func TestUnmarshalUnexported(t *testing.T) { + input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}, "s": [2, 3]}` + want := &unexportedFields{Name: "Bob"} + + out := &unexportedFields{} + err := Unmarshal([]byte(input), out) + if err != nil { + t.Errorf("Unmarshal error: %v", err) + } + if !reflect.DeepEqual(out, want) { + t.Errorf("Unmarshal:\n\tgot: %+v\n\twant: %+v", out, want) + } +} + +// Time3339 is a time.Time which encodes to and from JSON +// as an RFC 3339 time in UTC. +type Time3339 time.Time + +func (t *Time3339) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) + } + tm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1])) + if err != nil { + return err + } + *t = Time3339(tm) + return nil +} + +func TestUnmarshalJSONLiteralError(t *testing.T) { + var t3 Time3339 + switch err := Unmarshal([]byte(`"0000-00-00T00:00:00Z"`), &t3); { + case err == nil: + t.Fatalf("Unmarshal error: got nil, want non-nil") + case !strings.Contains(err.Error(), "range"): + t.Errorf("Unmarshal error:\n\tgot: %v\n\twant: out of range", err) + } +} + +// Test that extra object elements in an array do not result in a +// "data changing underfoot" error. +// Issue 3717 +func TestSkipArrayObjects(t *testing.T) { + json := `[{}]` + var dest [0]any + + err := Unmarshal([]byte(json), &dest) + if err != nil { + t.Errorf("Unmarshal error: %v", err) + } +} + +// Test semantics of pre-filled data, such as struct fields, map elements, +// slices, and arrays. +// Issues 4900 and 8837, among others. +func TestPrefilled(t *testing.T) { + // Values here change, cannot reuse table across runs. + tests := []struct { + CaseName + in string + ptr any + out any + }{{ + CaseName: Name(""), + in: `{"X": 1, "Y": 2}`, + ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5}, + out: &XYZ{X: float64(1), Y: float64(2), Z: 1.5}, + }, { + CaseName: Name(""), + in: `{"X": 1, "Y": 2}`, + ptr: &map[string]any{"X": float32(3), "Y": int16(4), "Z": 1.5}, + out: &map[string]any{"X": float64(1), "Y": float64(2), "Z": 1.5}, + }, { + CaseName: Name(""), + in: `[2]`, + ptr: &[]int{1}, + out: &[]int{2}, + }, { + CaseName: Name(""), + in: `[2, 3]`, + ptr: &[]int{1}, + out: &[]int{2, 3}, + }, { + CaseName: Name(""), + in: `[2, 3]`, + ptr: &[...]int{1}, + out: &[...]int{2}, + }, { + CaseName: Name(""), + in: `[3]`, + ptr: &[...]int{1, 2}, + out: &[...]int{3, 0}, + }} + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + ptrstr := fmt.Sprintf("%v", tt.ptr) + err := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here + if err != nil { + t.Errorf("%s: Unmarshal error: %v", tt.Where, err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("%s: Unmarshal(%#q, %T):\n\tgot: %v\n\twant: %v", tt.Where, tt.in, ptrstr, tt.ptr, tt.out) + } + }) + } +} + +func TestInvalidUnmarshal(t *testing.T) { + buf := []byte(`{"a":"1"}`) + tests := []struct { + CaseName + v any + want string + }{ + {Name(""), nil, "json: Unmarshal(nil)"}, + {Name(""), struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {Name(""), (*int)(nil), "json: Unmarshal(nil *int)"}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Fatalf("%s: Unmarshal error: got nil, want non-nil", tt.Where) + } + if got := err.Error(); got != tt.want { + t.Errorf("%s: Unmarshal error:\n\tgot: %s\n\twant: %s", tt.Where, got, tt.want) + } + }) + } +} + +func TestInvalidUnmarshalText(t *testing.T) { + buf := []byte(`123`) + tests := []struct { + CaseName + v any + want string + }{ + {Name(""), nil, "json: Unmarshal(nil)"}, + {Name(""), struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {Name(""), (*int)(nil), "json: Unmarshal(nil *int)"}, + {Name(""), new(net.IP), "json: cannot unmarshal number into Go value of type *net.IP"}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Fatalf("%s: Unmarshal error: got nil, want non-nil", tt.Where) + } + if got := err.Error(); got != tt.want { + t.Errorf("%s: Unmarshal error:\n\tgot: %s\n\twant: %s", tt.Where, got, tt.want) + } + }) + } +} + +// Test that string option is ignored for invalid types. +// Issue 9812. +func TestInvalidStringOption(t *testing.T) { + num := 0 + item := struct { + T time.Time `json:",string"` + M map[string]string `json:",string"` + S []string `json:",string"` + A [1]string `json:",string"` + I any `json:",string"` + P *int `json:",string"` + }{M: make(map[string]string), S: make([]string, 0), I: num, P: &num} + + data, err := Marshal(item) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + + err = Unmarshal(data, &item) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } +} + +// Test unmarshal behavior with regards to embedded unexported structs. +// +// (Issue 21357) If the embedded struct is a pointer and is unallocated, +// this returns an error because unmarshal cannot set the field. +// +// (Issue 24152) If the embedded struct is given an explicit name, +// ensure that the normal unmarshal logic does not panic in reflect. +// +// (Issue 28145) If the embedded struct is given an explicit name and has +// exported methods, don't cause a panic trying to get its value. +func TestUnmarshalEmbeddedUnexported(t *testing.T) { + type ( + embed1 struct{ Q int } + embed2 struct{ Q int } + embed3 struct { + Q int64 `json:",string"` + } + S1 struct { + *embed1 + R int + } + S2 struct { + *embed1 + Q int + } + S3 struct { + embed1 + R int + } + S4 struct { + *embed1 + embed2 + } + S5 struct { + *embed3 + R int + } + S6 struct { + embed1 `json:"embed1"` + } + S7 struct { + embed1 `json:"embed1"` + embed2 + } + S8 struct { + embed1 `json:"embed1"` + embed2 `json:"embed2"` + Q int + } + S9 struct { + unexportedWithMethods `json:"embed"` + } + ) + + tests := []struct { + CaseName + in string + ptr any + out any + err error + }{{ + // Error since we cannot set S1.embed1, but still able to set S1.R. + CaseName: Name(""), + in: `{"R":2,"Q":1}`, + ptr: new(S1), + out: &S1{R: 2}, + err: fmt.Errorf("json: cannot set embedded pointer to unexported struct: json.embed1"), + }, { + // The top level Q field takes precedence. + CaseName: Name(""), + in: `{"Q":1}`, + ptr: new(S2), + out: &S2{Q: 1}, + }, { + // No issue with non-pointer variant. + CaseName: Name(""), + in: `{"R":2,"Q":1}`, + ptr: new(S3), + out: &S3{embed1: embed1{Q: 1}, R: 2}, + }, { + // No error since both embedded structs have field R, which annihilate each other. + // Thus, no attempt is made at setting S4.embed1. + CaseName: Name(""), + in: `{"R":2}`, + ptr: new(S4), + out: new(S4), + }, { + // Error since we cannot set S5.embed1, but still able to set S5.R. + CaseName: Name(""), + in: `{"R":2,"Q":1}`, + ptr: new(S5), + out: &S5{R: 2}, + err: fmt.Errorf("json: cannot set embedded pointer to unexported struct: json.embed3"), + }, { + // Issue 24152, ensure decodeState.indirect does not panic. + CaseName: Name(""), + in: `{"embed1": {"Q": 1}}`, + ptr: new(S6), + out: &S6{embed1{1}}, + }, { + // Issue 24153, check that we can still set forwarded fields even in + // the presence of a name conflict. + // + // This relies on obscure behavior of reflect where it is possible + // to set a forwarded exported field on an unexported embedded struct + // even though there is a name conflict, even when it would have been + // impossible to do so according to Go visibility rules. + // Go forbids this because it is ambiguous whether S7.Q refers to + // S7.embed1.Q or S7.embed2.Q. Since embed1 and embed2 are unexported, + // it should be impossible for an external package to set either Q. + // + // It is probably okay for a future reflect change to break this. + CaseName: Name(""), + in: `{"embed1": {"Q": 1}, "Q": 2}`, + ptr: new(S7), + out: &S7{embed1{1}, embed2{2}}, + }, { + // Issue 24153, similar to the S7 case. + CaseName: Name(""), + in: `{"embed1": {"Q": 1}, "embed2": {"Q": 2}, "Q": 3}`, + ptr: new(S8), + out: &S8{embed1{1}, embed2{2}, 3}, + }, { + // Issue 228145, similar to the cases above. + CaseName: Name(""), + in: `{"embed": {}}`, + ptr: new(S9), + out: &S9{}, + }} + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.in), tt.ptr) + if !equalError(err, tt.err) { + t.Errorf("%s: Unmarshal error:\n\tgot: %v\n\twant: %v", tt.Where, err, tt.err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("%s: Unmarshal:\n\tgot: %#+v\n\twant: %#+v", tt.Where, tt.ptr, tt.out) + } + }) + } +} + +func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) { + tests := []struct { + CaseName + in string + err error + }{{ + CaseName: Name(""), + in: `1 false null :`, + err: &SyntaxError{"invalid character ':' looking for beginning of value", 14}, + }, { + CaseName: Name(""), + in: `1 [] [,]`, + err: &SyntaxError{"invalid character ',' looking for beginning of value", 7}, + }, { + CaseName: Name(""), + in: `1 [] [true:]`, + err: &SyntaxError{"invalid character ':' after array element", 11}, + }, { + CaseName: Name(""), + in: `1 {} {"x"=}`, + err: &SyntaxError{"invalid character '=' after object key", 14}, + }, { + CaseName: Name(""), + in: `falsetruenul#`, + err: &SyntaxError{"invalid character '#' in literal null (expecting 'l')", 13}, + }} + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + dec := NewDecoder(strings.NewReader(tt.in)) + var err error + for err == nil { + var v any + err = dec.Decode(&v) + } + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("%s: Decode error:\n\tgot: %v\n\twant: %v", tt.Where, err, tt.err) + } + }) + } +} + +type unmarshalPanic struct{} + +func (unmarshalPanic) UnmarshalJSON([]byte) error { panic(0xdead) } + +func TestUnmarshalPanic(t *testing.T) { + defer func() { + if got := recover(); !reflect.DeepEqual(got, 0xdead) { + t.Errorf("panic() = (%T)(%v), want 0xdead", got, got) + } + }() + Unmarshal([]byte("{}"), &unmarshalPanic{}) + t.Fatalf("Unmarshal should have panicked") +} + +// The decoder used to hang if decoding into an interface pointing to its own address. +// See golang.org/issues/31740. +func TestUnmarshalRecursivePointer(t *testing.T) { + var v any + v = &v + data := []byte(`{"a": "b"}`) + + if err := Unmarshal(data, v); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } +} + +type textUnmarshalerString string + +func (m *textUnmarshalerString) UnmarshalText(text []byte) error { + *m = textUnmarshalerString(strings.ToLower(string(text))) + return nil +} + +// Test unmarshal to a map, where the map key is a user defined type. +// See golang.org/issues/34437. +func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"FOO": "1"}`), &p); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + + if _, ok := p["foo"]; !ok { + t.Errorf(`key "foo" missing in map: %v`, p) + } +} + +func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { + // See golang.org/issues/38105. + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if _, ok := p["开源"]; !ok { + t.Errorf(`key "开源" missing in map: %v`, p) + } + + // See golang.org/issues/38126. + type T struct { + F1 string `json:"F1,string"` + } + wantT := T{"aaa\tbbb"} + + b, err := Marshal(wantT) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var gotT T + if err := Unmarshal(b, &gotT); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if gotT != wantT { + t.Errorf("Marshal/Unmarshal roundtrip:\n\tgot: %q\n\twant: %q", gotT, wantT) + } + + // See golang.org/issues/39555. + input := map[textUnmarshalerString]string{"FOO": "", `"`: ""} + + encoded, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got map[textUnmarshalerString]string + if err := Unmarshal(encoded, &got); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + want := map[textUnmarshalerString]string{"foo": "", `"`: ""} + if !reflect.DeepEqual(got, want) { + t.Errorf("Marshal/Unmarshal roundtrip:\n\tgot: %q\n\twant: %q", gotT, wantT) + } +} + +func TestUnmarshalMaxDepth(t *testing.T) { + tests := []struct { + CaseName + data string + errMaxDepth bool + }{{ + CaseName: Name("ArrayUnderMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`[`, 10000-1) + strings.Repeat(`]`, 10000-1) + `}`, + errMaxDepth: false, + }, { + CaseName: Name("ArrayOverMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`[`, 10000) + strings.Repeat(`]`, 10000) + `}`, + errMaxDepth: true, + }, { + CaseName: Name("ArrayOverStackDepth"), + data: `{"a":` + strings.Repeat(`[`, 3000000) + strings.Repeat(`]`, 3000000) + `}`, + errMaxDepth: true, + }, { + CaseName: Name("ObjectUnderMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`{"a":`, 10000-1) + `0` + strings.Repeat(`}`, 10000-1) + `}`, + errMaxDepth: false, + }, { + CaseName: Name("ObjectOverMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`{"a":`, 10000) + `0` + strings.Repeat(`}`, 10000) + `}`, + errMaxDepth: true, + }, { + CaseName: Name("ObjectOverStackDepth"), + data: `{"a":` + strings.Repeat(`{"a":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`, + errMaxDepth: true, + }} + + targets := []struct { + CaseName + newValue func() any + }{{ + CaseName: Name("unstructured"), + newValue: func() any { + var v any + return &v + }, + }, { + CaseName: Name("typed named field"), + newValue: func() any { + v := struct { + A any `json:"a"` + }{} + return &v + }, + }, { + CaseName: Name("typed missing field"), + newValue: func() any { + v := struct { + B any `json:"b"` + }{} + return &v + }, + }, { + CaseName: Name("custom unmarshaler"), + newValue: func() any { + v := unmarshaler{} + return &v + }, + }} + + for _, tt := range tests { + for _, target := range targets { + t.Run(target.Name+"-"+tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.data), target.newValue()) + if !tt.errMaxDepth { + if err != nil { + t.Errorf("%s: %s: Unmarshal error: %v", tt.Where, target.Where, err) + } + } else { + if err == nil || !strings.Contains(err.Error(), "exceeded max depth") { + t.Errorf("%s: %s: Unmarshal error:\n\tgot: %v\n\twant: exceeded max depth", tt.Where, target.Where, err) + } + } + }) + } + } +} diff --git a/internal/3rdparty/json/encode.go b/internal/3rdparty/json/encode.go new file mode 100644 index 0000000..7bee1a6 --- /dev/null +++ b/internal/3rdparty/json/encode.go @@ -0,0 +1,1286 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON as defined in +// RFC 7159. The mapping between JSON and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// https://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "cmp" + "encoding" + "encoding/base64" + "fmt" + "math" + "reflect" + "slices" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" + _ "unsafe" // for linkname +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements [Marshaler] +// and is not a nil pointer, Marshal calls [Marshaler.MarshalJSON] +// to produce JSON. If no [Marshaler.MarshalJSON] method is present but the +// value implements [encoding.TextMarshaler] instead, Marshal calls +// [encoding.TextMarshaler.MarshalText] and encodes the result as a JSON string. +// The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// [Unmarshaler.UnmarshalJSON]. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and [Number] values encode as JSON numbers. +// NaN and +/-Inf values will return an [UnsupportedValueError]. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// So that the JSON will be safe to embed inside HTML