From 1db6b120742ea43d7c4cc0787098e21547455219 Mon Sep 17 00:00:00 2001 From: go-jet Date: Tue, 11 Mar 2025 13:33:39 +0100 Subject: [PATCH] Add support for configuring the JSON unmarshal function used when querying SELECT_JSON statements. --- go.mod | 6 ++++++ go.sum | 16 ++++++++++++++++ qrm/qrm.go | 13 ++++++++++--- tests/postgres/chinook_db_test.go | 15 +++++++++++++++ tests/postgres/main_test.go | 11 +++++++++++ tests/postgres/northwind_test.go | 16 ++++++++++++++++ 6 files changed, 74 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 23d0e07..9ef9dd7 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( // used in tests require ( + github.com/bytedance/sonic v1.13.1 github.com/google/go-cmp v0.7.0 github.com/pkg/profile v1.7.0 github.com/shopspring/decimal v1.4.0 @@ -25,6 +26,8 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/friendsofgo/errors v0.9.2 // indirect @@ -35,10 +38,13 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/volatiletech/inflect v0.0.1 // indirect github.com/volatiletech/randomize v0.0.1 // indirect github.com/volatiletech/strmangle v0.0.1 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 5f5b0f8..104224e 100644 --- a/go.sum +++ b/go.sum @@ -3,9 +3,17 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g= +github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -85,6 +93,9 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -138,6 +149,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU= github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA= github.com/volatiletech/null/v8 v8.1.2 h1:kiTiX1PpwvuugKwfvUNX/SU/5A2KGZMXfGD0DUHdKEI= @@ -159,6 +172,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -254,3 +269,4 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/qrm/qrm.go b/qrm/qrm.go index 192ad12..6fc081f 100644 --- a/qrm/qrm.go +++ b/qrm/qrm.go @@ -17,12 +17,19 @@ type Config struct { // to a field in the destination struct. // Does not apply to statements build with SELECT_JSON_OBJ or SELECT_JSON_ARR StrictScan bool + + // JsonUnmarshalFunc is called by the Query method to unmarshal JSON query results created by + // SELECT_JSON_OBJ and SELECT_JSON_ARR statements. + // It can be replaced with any implementation that matches the standard "encoding/json" `Unmarshal` function signature. + // By default, it uses the `Unmarshal` function from Go's standard `encoding/json` package. + JsonUnmarshalFunc func(data []byte, v any) error } // GlobalConfig is the package-wide configuration for SQL scanning. -// This variable should be modified only once, for instance, during application initialization. +// This variable is not thread safe, and it should be modified only once, for instance, during application initialization. var GlobalConfig = Config{ - StrictScan: false, + StrictScan: false, + JsonUnmarshalFunc: json.Unmarshal, } // ErrNoRows is returned by Query when query result set is empty @@ -116,7 +123,7 @@ func queryJson(ctx context.Context, db Queryable, query string, args []interface return 1, nil } - err = json.Unmarshal(jsonData, &destPtr) + err = GlobalConfig.JsonUnmarshalFunc(jsonData, &destPtr) if err != nil { return 1, fmt.Errorf("jet: invalid json, %w", err) diff --git a/tests/postgres/chinook_db_test.go b/tests/postgres/chinook_db_test.go index c28623d..3dad8c6 100644 --- a/tests/postgres/chinook_db_test.go +++ b/tests/postgres/chinook_db_test.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "github.com/bytedance/sonic" "github.com/go-jet/jet/v2/internal/testutils" "github.com/go-jet/jet/v2/internal/utils/ptr" . "github.com/go-jet/jet/v2/postgres" @@ -225,10 +226,24 @@ func BenchmarkJoinEverythingJSON(b *testing.B) { } } +func BenchmarkJoinEverythingJsonSonic(b *testing.B) { + useJsonUnmarshalFunc(sonic.Unmarshal, func() { + for i := 0; i < b.N; i++ { + testJoinEverythingJSON(b) + } + }) +} + func TestJoinEverythingJSON(t *testing.T) { testJoinEverythingJSON(t) } +func TestJoinEverythingJSONSonic(t *testing.T) { + useJsonUnmarshalFunc(sonic.Unmarshal, func() { + testJoinEverythingJSON(t) + }) +} + func testJoinEverythingJSON(t require.TestingT) { manager := Employee.AS("Manager") diff --git a/tests/postgres/main_test.go b/tests/postgres/main_test.go index c4b06ba..c3ebecb 100644 --- a/tests/postgres/main_test.go +++ b/tests/postgres/main_test.go @@ -3,6 +3,7 @@ package postgres import ( "context" "database/sql" + "encoding/json" "fmt" "github.com/go-jet/jet/v2/qrm" "github.com/go-jet/jet/v2/stmtcache" @@ -108,6 +109,16 @@ func allowUnusedColumns(f func()) { f() } +func useJsonUnmarshalFunc(unmarshalJson func(data []byte, v any) error, f func()) { + defer func() { + qrm.GlobalConfig.JsonUnmarshalFunc = json.Unmarshal + }() + + qrm.GlobalConfig.JsonUnmarshalFunc = unmarshalJson + + f() +} + var loggedSQL string var loggedSQLArgs []interface{} var loggedDebugSQL string diff --git a/tests/postgres/northwind_test.go b/tests/postgres/northwind_test.go index 5615f45..2b55963 100644 --- a/tests/postgres/northwind_test.go +++ b/tests/postgres/northwind_test.go @@ -1,6 +1,7 @@ package postgres import ( + "github.com/bytedance/sonic" "github.com/go-jet/jet/v2/internal/testutils" . "github.com/go-jet/jet/v2/postgres" "github.com/go-jet/jet/v2/tests/.gentestdata/jetdb/northwind/model" @@ -110,6 +111,21 @@ func TestNorthwindJoinEverythingJson(t *testing.T) { testNorthwindJoinEverythingJson(t) } +func BenchmarkTestNorthwindJoinEverythingSonicJson(b *testing.B) { + useJsonUnmarshalFunc(sonic.Unmarshal, func() { + for i := 0; i < b.N; i++ { + testNorthwindJoinEverythingJson(b) + } + }) +} + +// uncomment when bug is fixed: https://github.com/bytedance/sonic/issues/774 +//func TestNorthwindJoinEverythingJsonSonic(t *testing.T) { +// useJsonUnmarshalFunc(sonic.Unmarshal, func() { +// testNorthwindJoinEverythingJson(t) +// }) +//} + func testNorthwindJoinEverythingJson(t require.TestingT) { stmt := SELECT_JSON_ARR(