From b7318ae973f6d51fa543882d062b343d05b42da8 Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Fri, 21 Nov 2025 16:28:03 +0000 Subject: [PATCH] Only get the latest 10 trap collections Otherwise the page gets really swamped --- .gitmodules | 3 + bob | 1 + database.go | 10 -- html.go | 60 +++++++++- model_conversion.go | 9 +- sql/trapcount_by_location_id.bob.go | 4 +- sql/trapcount_by_location_id.bob.sql | 2 +- sql/trapcount_by_location_id.sql | 4 +- sql/trapdata_by_location_id_recent.bob.go | 108 ++++++++++++++++++ sql/trapdata_by_location_id_recent.bob.sql | 15 +++ ...trapdata_by_location_id_recent.bob_test.go | 103 +++++++++++++++++ sql/trapdata_by_location_id_recent.sql | 12 ++ 12 files changed, 304 insertions(+), 27 deletions(-) create mode 160000 bob create mode 100644 sql/trapdata_by_location_id_recent.bob.go create mode 100644 sql/trapdata_by_location_id_recent.bob.sql create mode 100644 sql/trapdata_by_location_id_recent.bob_test.go create mode 100644 sql/trapdata_by_location_id_recent.sql diff --git a/.gitmodules b/.gitmodules index dc0c25cc..a95655f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "go-geojson2h3"] path = go-geojson2h3 url = git@github.com:Gleipnir-Technology/go-geojson2h3.git +[submodule "bob"] + path = bob + url = git@github.com:Gleipnir-Technology/bob.git diff --git a/bob b/bob new file mode 160000 index 00000000..96da65fd --- /dev/null +++ b/bob @@ -0,0 +1 @@ +Subproject commit 96da65fd88a50ae532079e8ea69746183f4af3a1 diff --git a/database.go b/database.go index 649ffb69..021524e9 100644 --- a/database.go +++ b/database.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "database/sql" "embed" @@ -197,12 +196,3 @@ func updateSummaryTables(ctx context.Context, org *models.Organization) { } } } - -func insertQueryToString(query bob.BaseQuery[*dialect.InsertQuery]) string { - buf := new(bytes.Buffer) - _, err := query.WriteQuery(context.TODO(), buf, 0) - if err != nil { - return fmt.Sprintf("Failed to write query: %v", err) - } - return buf.String() -} diff --git a/html.go b/html.go index aef5138b..428db052 100644 --- a/html.go +++ b/html.go @@ -512,6 +512,9 @@ func htmlSource(w http.ResponseWriter, r *http.Request, user *models.User, id st } cadence, deltas := calculateCadenceVariance(treatment_times) for i, treatment := range treatments { + if i >= len(deltas) { + break + } treatment.CadenceDelta = deltas[i] treatments[i] = treatment } @@ -782,12 +785,57 @@ func trapsBySource(ctx context.Context, org *models.Organization, sourceID strin location_ids = append(location_ids, location.TrapLocationGlobalid) args = append(args, psql.Arg(location.TrapLocationGlobalid)) } - trap_data, err := org.FSTrapdata( - sm.Where( - models.FSTrapdata.Columns.LocID.In(args...), - ), - sm.OrderBy("enddatetime"), - ).All(ctx, PGInstance.BobDB) + /* + trap_data, err := org.FSTrapdata( + sm.Where( + models.FSTrapdata.Columns.LocID.In(args...), + ), + sm.OrderBy("enddatetime"), + ).All(ctx, PGInstance.BobDB) + */ + + /* + query := org.FSTrapdata( + sm.From( + psql.Select( + sm.From(psql.F("ROW_NUMBER")( + fm.Over( + wm.PartitionBy(models.FSTrapdata.Columns.LocID), + wm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(), + ), + )).As("row_num"), + sm.Where(models.FSTrapdata.Columns.LocID.In(args...))), + ), + sm.Where(psql.Quote("row_num").LTE(psql.Arg(10))), + sm.OrderBy(models.FSTrapdata.Columns.LocID), + sm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(), + ) + */ + /* + query := psql.Select( + sm.From( + psql.Select( + sm.Columns( + models.FSTrapdata.Columns.Globalid, + psql.F("ROW_NUMBER")( + fm.Over( + wm.PartitionBy(models.FSTrapdata.Columns.LocID), + wm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(), + ), + ).As("row_num"), + sm.From(models.FSTrapdata.Name()), + ), + sm.Where(models.FSTrapdata.Columns.LocID.In(args...))), + ), + sm.Where(psql.Quote("row_num").LTE(psql.Arg(10))), + sm.OrderBy(models.FSTrapdata.Columns.LocID), + sm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(), + ) + log.Info().Str("trapdata", queryToString(query)).Msg("Getting trap data") + trap_data, err := query.Exec(ctx, PGInstance.BobDB) + */ + + trap_data, err := sql.TrapDataByLocationIDRecent(org.ID, location_ids).All(ctx, PGInstance.BobDB) if err != nil { return nil, fmt.Errorf("Failed to query trap data: %w", err) } diff --git a/model_conversion.go b/model_conversion.go index 387a73e3..0d6a4251 100644 --- a/model_conversion.go +++ b/model_conversion.go @@ -162,7 +162,7 @@ type Treatment struct { Product string } -func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data models.FSTrapdatumSlice, counts []sql.TrapCountByLocationIDRow) ([]TrapNearby, error) { +func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data []sql.TrapDataByLocationIDRecentRow, counts []sql.TrapCountByLocationIDRow) ([]TrapNearby, error) { results := make([]TrapNearby, 0) count_by_trap_data_id := make(map[string]*sql.TrapCountByLocationIDRow) for _, c := range counts { @@ -174,12 +174,9 @@ func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data models if !ok { return results, errors.New(fmt.Sprintf("Failed to find trap count for %s", td.Globalid)) } - if td.LocID.IsNull() { - return results, errors.New("Got a trap data with no location ID") - } - loc_id := td.LocID.MustGet() + loc_id := td.LocID count := &TrapCount{ - Ended: fsToTime(td.Enddatetime), + Ended: time.UnixMilli(td.Enddatetime), Females: int(c.TotalFemales.IntPart()), ID: td.Globalid, Males: int(c.TotalMales), diff --git a/sql/trapcount_by_location_id.bob.go b/sql/trapcount_by_location_id.bob.go index c1ec643d..1469e506 100644 --- a/sql/trapcount_by_location_id.bob.go +++ b/sql/trapcount_by_location_id.bob.go @@ -22,7 +22,7 @@ import ( //go:embed trapcount_by_location_id.bob.sql var formattedQueries_trapcount_by_location_id string -var trapCountByLocationIDSQL = formattedQueries_trapcount_by_location_id[159:580] +var trapCountByLocationIDSQL = formattedQueries_trapcount_by_location_id[159:591] type TrapCountByLocationIDQuery = orm.ModQuery[*dialect.SelectQuery, trapCountByLocationID, TrapCountByLocationIDRow, []TrapCountByLocationIDRow, trapCountByLocationIDTransformer] @@ -59,7 +59,7 @@ func TrapCountByLocationID(OrganizationID int32, LocID []string) *TrapCountByLoc q.AppendSelect(expressionTypArgs.subExpr(12, 223)) q.SetTable(expressionTypArgs.subExpr(234, 318)) q.AppendWhere(expressionTypArgs.subExpr(330, 379)) - q.AppendGroup(expressionTypArgs.subExpr(394, 421)) + q.AppendGroup(expressionTypArgs.subExpr(394, 432)) }), } } diff --git a/sql/trapcount_by_location_id.bob.sql b/sql/trapcount_by_location_id.bob.sql index ebb88083..84025c8e 100644 --- a/sql/trapcount_by_location_id.bob.sql +++ b/sql/trapcount_by_location_id.bob.sql @@ -16,4 +16,4 @@ WHERE td.organization_id = $1 AND td.loc_id IN ($2) GROUP BY - td.globalid, td.enddatetime; + td.globalid, td.loc_id, td.enddatetime; diff --git a/sql/trapcount_by_location_id.sql b/sql/trapcount_by_location_id.sql index daa69e72..0b0bbc58 100644 --- a/sql/trapcount_by_location_id.sql +++ b/sql/trapcount_by_location_id.sql @@ -1,6 +1,6 @@ -- TrapCountByLocationID SELECT - td.loc_id AS trapdata_globalid, + td.globalid AS trapdata_globalid, td.enddatetime AS trapdata_enddate, COALESCE(SUM(sa.females), 0) AS total_females, COALESCE(SUM(sa.males), 0) AS total_males, @@ -13,5 +13,5 @@ WHERE td.organization_id = $1 AND td.loc_id IN ($2) GROUP BY - td.globalid, td.enddatetime; + td.globalid, td.loc_id, td.enddatetime; diff --git a/sql/trapdata_by_location_id_recent.bob.go b/sql/trapdata_by_location_id_recent.bob.go new file mode 100644 index 00000000..3d19bef1 --- /dev/null +++ b/sql/trapdata_by_location_id_recent.bob.go @@ -0,0 +1,108 @@ +// Code generated by BobGen psql v0.41.1. DO NOT EDIT. +// This file is meant to be re-generated in place and/or deleted at any time. + +package sql + +import ( + "context" + _ "embed" + "io" + "iter" + + "github.com/stephenafamo/bob" + "github.com/stephenafamo/bob/dialect/psql" + "github.com/stephenafamo/bob/dialect/psql/dialect" + "github.com/stephenafamo/bob/expr" + "github.com/stephenafamo/bob/orm" + "github.com/stephenafamo/scan" +) + +//go:embed trapdata_by_location_id_recent.bob.sql +var formattedQueries_trapdata_by_location_id_recent string + +var trapDataByLocationIDRecentSQL = formattedQueries_trapdata_by_location_id_recent[164:454] + +type TrapDataByLocationIDRecentQuery = orm.ModQuery[*dialect.SelectQuery, trapDataByLocationIDRecent, TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow, trapDataByLocationIDRecentTransformer] + +func TrapDataByLocationIDRecent(OrganizationID int32, LocID []string) *TrapDataByLocationIDRecentQuery { + var expressionTypArgs trapDataByLocationIDRecent + + expressionTypArgs.OrganizationID = psql.Arg(OrganizationID) + expressionTypArgs.LocID = expr.ToArgs(LocID...) + + return &TrapDataByLocationIDRecentQuery{ + Query: orm.Query[trapDataByLocationIDRecent, TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow, trapDataByLocationIDRecentTransformer]{ + ExecQuery: orm.ExecQuery[trapDataByLocationIDRecent]{ + BaseQuery: bob.BaseQuery[trapDataByLocationIDRecent]{ + Expression: expressionTypArgs, + Dialect: dialect.Dialect, + QueryType: bob.QueryTypeSelect, + }, + }, + Scanner: func(context.Context, []string) (func(*scan.Row) (any, error), func(any) (TrapDataByLocationIDRecentRow, error)) { + return func(row *scan.Row) (any, error) { + var t TrapDataByLocationIDRecentRow + row.ScheduleScanByIndex(0, &t.Enddatetime) + row.ScheduleScanByIndex(1, &t.Globalid) + row.ScheduleScanByIndex(2, &t.LocID) + return &t, nil + }, func(v any) (TrapDataByLocationIDRecentRow, error) { + return *(v.(*TrapDataByLocationIDRecentRow)), nil + } + }, + }, + Mod: bob.ModFunc[*dialect.SelectQuery](func(q *dialect.SelectQuery) { + q.AppendSelect(expressionTypArgs.subExpr(7, 36)) + q.SetTable(expressionTypArgs.subExpr(42, 244)) + q.AppendWhere(expressionTypArgs.subExpr(251, 264)) + q.CombinedOrder.AppendOrder(expressionTypArgs.subExpr(274, 290)) + }), + } +} + +type TrapDataByLocationIDRecentRow = struct { + Enddatetime int64 `db:"enddatetime"` + Globalid string `db:"globalid"` + LocID string `db:"loc_id"` +} + +type trapDataByLocationIDRecentTransformer = bob.SliceTransformer[TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow] + +type trapDataByLocationIDRecent struct { + OrganizationID bob.Expression + LocID bob.Expression +} + +func (o trapDataByLocationIDRecent) args() iter.Seq[orm.ArgWithPosition] { + return func(yield func(arg orm.ArgWithPosition) bool) { + if !yield(orm.ArgWithPosition{ + Name: "organizationID", + Start: 207, + Stop: 209, + Expression: o.OrganizationID, + }) { + return + } + + if !yield(orm.ArgWithPosition{ + Name: "locID", + Start: 227, + Stop: 229, + Expression: o.LocID, + }) { + return + } + } +} + +func (o trapDataByLocationIDRecent) raw(from, to int) string { + return trapDataByLocationIDRecentSQL[from:to] +} + +func (o trapDataByLocationIDRecent) subExpr(from, to int) bob.Expression { + return orm.ArgsToExpression(trapDataByLocationIDRecentSQL, from, to, o.args()) +} + +func (o trapDataByLocationIDRecent) WriteSQL(ctx context.Context, w io.Writer, d bob.Dialect, start int) ([]any, error) { + return o.subExpr(0, len(trapDataByLocationIDRecentSQL)).WriteSQL(ctx, w, d, start) +} diff --git a/sql/trapdata_by_location_id_recent.bob.sql b/sql/trapdata_by_location_id_recent.bob.sql new file mode 100644 index 00000000..2c3e2eb1 --- /dev/null +++ b/sql/trapdata_by_location_id_recent.bob.sql @@ -0,0 +1,15 @@ +-- Code generated by BobGen psql v0.41.1. DO NOT EDIT. +-- This file is meant to be re-generated in place and/or deleted at any time. + +-- TrapDataByLocationIDRecent +SELECT enddatetime, globalid, loc_id +FROM ( + SELECT enddatetime, globalid, loc_id, ROW_NUMBER() + OVER (PARTITION BY loc_id ORDER BY enddatetime DESC) as row_num + FROM fs_trapdata + WHERE + organization_id = $1 AND + loc_id IN ($2) +) ranked_data +WHERE row_num <= 10 +ORDER BY enddatetime DESC; diff --git a/sql/trapdata_by_location_id_recent.bob_test.go b/sql/trapdata_by_location_id_recent.bob_test.go new file mode 100644 index 00000000..d4735df8 --- /dev/null +++ b/sql/trapdata_by_location_id_recent.bob_test.go @@ -0,0 +1,103 @@ +// Code generated by BobGen psql v0.41.1. DO NOT EDIT. +// This file is meant to be re-generated in place and/or deleted at any time. + +package sql + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stephenafamo/bob" + "github.com/stephenafamo/bob/dialect/psql" + testutils "github.com/stephenafamo/bob/test/utils" +) + +func TestTrapDataByLocationIDRecent(t *testing.T) { + t.Run("Base", func(t *testing.T) { + var sb strings.Builder + + query := TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)}) + + if _, err := query.WriteQuery(t.Context(), &sb, 1); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(trapDataByLocationIDRecentSQL, sb.String()); diff != "" { + t.Fatalf("unexpected result (-got +want):\n%s", diff) + } + }) + + t.Run("Mod", func(t *testing.T) { + var sb strings.Builder + + query := TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)}) + + if _, err := psql.Select(query).WriteQuery(t.Context(), &sb, 1); err != nil { + t.Fatal(err) + } + + queryDiff, err := testutils.QueryDiff(trapDataByLocationIDRecentSQL, sb.String(), formatQuery) + if err != nil { + t.Fatal(err) + } + if queryDiff != "" { + fmt.Println(sb.String()) + t.Fatalf("unexpected result (-got +want):\n%s", queryDiff) + } + }) + + t.Run("Scanning", func(t *testing.T) { + if testDB == nil { + t.Skip("skipping test, no DSN provided") + } + + ctxTx, cancel := context.WithCancel(t.Context()) + defer cancel() + + tx, err := testDB.Begin(ctxTx) + if err != nil { + t.Fatalf("Error starting transaction: %v", err) + } + + defer func() { + if err := tx.Rollback(ctxTx); err != nil { + t.Fatalf("Error rolling back transaction: %v", err) + } + }() + + query, args, err := bob.Build(ctxTx, psql.Select(TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)}))) + if err != nil { + t.Fatal(err) + } + + rows, err := tx.QueryContext(ctxTx, query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + columns, err := rows.Columns() + if err != nil { + t.Fatal(err) + } + + if len(columns) != 3 { + t.Fatalf("expected %d columns, got %d", 3, len(columns)) + } + + if columns[0] != "enddatetime" { + t.Fatalf("expected column %d to be %s, got %s", 0, "enddatetime", columns[0]) + } + + if columns[1] != "globalid" { + t.Fatalf("expected column %d to be %s, got %s", 1, "globalid", columns[1]) + } + + if columns[2] != "loc_id" { + t.Fatalf("expected column %d to be %s, got %s", 2, "loc_id", columns[2]) + } + }) +} diff --git a/sql/trapdata_by_location_id_recent.sql b/sql/trapdata_by_location_id_recent.sql new file mode 100644 index 00000000..e3d9afb6 --- /dev/null +++ b/sql/trapdata_by_location_id_recent.sql @@ -0,0 +1,12 @@ +-- TrapDataByLocationIDRecent +SELECT enddatetime, globalid, loc_id +FROM ( + SELECT enddatetime, globalid, loc_id, ROW_NUMBER() + OVER (PARTITION BY loc_id ORDER BY enddatetime DESC) as row_num + FROM fs_trapdata + WHERE + organization_id = $1 AND + loc_id IN ($2) +) ranked_data +WHERE row_num <= 10 +ORDER BY enddatetime DESC;