Only get the latest 10 trap collections

Otherwise the page gets really swamped

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "go-geojson2h3"]
 	path = go-geojson2h3
 	url = git@github.com:Gleipnir-Technology/go-geojson2h3.git
+[submodule "bob"]
+	path = bob
+	url = git@github.com:Gleipnir-Technology/bob.git

bob

@@ -0,0 +1 @@
+Subproject commit 96da65fd88a50ae532079e8ea69746183f4af3a1

database.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"database/sql"
 	"embed"
@@ -197,12 +196,3 @@ func updateSummaryTables(ctx context.Context, org *models.Organization) {
 		}
 	}
 }
-
-func insertQueryToString(query bob.BaseQuery[*dialect.InsertQuery]) string {
-	buf := new(bytes.Buffer)
-	_, err := query.WriteQuery(context.TODO(), buf, 0)
-	if err != nil {
-		return fmt.Sprintf("Failed to write query: %v", err)
-	}
-	return buf.String()
-}

html.go

@@ -512,6 +512,9 @@ func htmlSource(w http.ResponseWriter, r *http.Request, user *models.User, id st
 	}
 	cadence, deltas := calculateCadenceVariance(treatment_times)
 	for i, treatment := range treatments {
+		if i >= len(deltas) {
+			break
+		}
 		treatment.CadenceDelta = deltas[i]
 		treatments[i] = treatment
 	}
@@ -782,12 +785,57 @@ func trapsBySource(ctx context.Context, org *models.Organization, sourceID strin
 		location_ids = append(location_ids, location.TrapLocationGlobalid)
 		args = append(args, psql.Arg(location.TrapLocationGlobalid))
 	}
-	trap_data, err := org.FSTrapdata(
+	/*
-		sm.Where(
+		trap_data, err := org.FSTrapdata(
-			models.FSTrapdata.Columns.LocID.In(args...),
+			sm.Where(
-		),
+				models.FSTrapdata.Columns.LocID.In(args...),
-		sm.OrderBy("enddatetime"),
+			),
-	).All(ctx, PGInstance.BobDB)
+			sm.OrderBy("enddatetime"),
+		).All(ctx, PGInstance.BobDB)
+	*/
+
+	/*
+		query := org.FSTrapdata(
+			sm.From(
+				psql.Select(
+					sm.From(psql.F("ROW_NUMBER")(
+						fm.Over(
+							wm.PartitionBy(models.FSTrapdata.Columns.LocID),
+							wm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
+						),
+					)).As("row_num"),
+				sm.Where(models.FSTrapdata.Columns.LocID.In(args...))),
+			),
+			sm.Where(psql.Quote("row_num").LTE(psql.Arg(10))),
+			sm.OrderBy(models.FSTrapdata.Columns.LocID),
+			sm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
+		)
+	*/
+	/*
+		query := psql.Select(
+			sm.From(
+				psql.Select(
+					sm.Columns(
+						models.FSTrapdata.Columns.Globalid,
+						psql.F("ROW_NUMBER")(
+						fm.Over(
+							wm.PartitionBy(models.FSTrapdata.Columns.LocID),
+							wm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
+						),
+					).As("row_num"),
+					sm.From(models.FSTrapdata.Name()),
+				),
+				sm.Where(models.FSTrapdata.Columns.LocID.In(args...))),
+			),
+			sm.Where(psql.Quote("row_num").LTE(psql.Arg(10))),
+			sm.OrderBy(models.FSTrapdata.Columns.LocID),
+			sm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
+		)
+		log.Info().Str("trapdata", queryToString(query)).Msg("Getting trap data")
+		trap_data, err := query.Exec(ctx, PGInstance.BobDB)
+	*/
+
+	trap_data, err := sql.TrapDataByLocationIDRecent(org.ID, location_ids).All(ctx, PGInstance.BobDB)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to query trap data: %w", err)
 	}

model_conversion.go

@@ -162,7 +162,7 @@ type Treatment struct {
 	Product      string
 }
 
-func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data models.FSTrapdatumSlice, counts []sql.TrapCountByLocationIDRow) ([]TrapNearby, error) {
+func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data []sql.TrapDataByLocationIDRecentRow, counts []sql.TrapCountByLocationIDRow) ([]TrapNearby, error) {
 	results := make([]TrapNearby, 0)
 	count_by_trap_data_id := make(map[string]*sql.TrapCountByLocationIDRow)
 	for _, c := range counts {
@@ -174,12 +174,9 @@ func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data models
 		if !ok {
 			return results, errors.New(fmt.Sprintf("Failed to find trap count for %s", td.Globalid))
 		}
-		if td.LocID.IsNull() {
+		loc_id := td.LocID
-			return results, errors.New("Got a trap data with no location ID")
-		}
-		loc_id := td.LocID.MustGet()
 		count := &TrapCount{
-			Ended:   fsToTime(td.Enddatetime),
+			Ended:   time.UnixMilli(td.Enddatetime),
 			Females: int(c.TotalFemales.IntPart()),
 			ID:      td.Globalid,
 			Males:   int(c.TotalMales),

sql/trapcount_by_location_id.bob.go

@@ -22,7 +22,7 @@ import (
 //go:embed trapcount_by_location_id.bob.sql
 var formattedQueries_trapcount_by_location_id string
 
-var trapCountByLocationIDSQL = formattedQueries_trapcount_by_location_id[159:580]
+var trapCountByLocationIDSQL = formattedQueries_trapcount_by_location_id[159:591]
 
 type TrapCountByLocationIDQuery = orm.ModQuery[*dialect.SelectQuery, trapCountByLocationID, TrapCountByLocationIDRow, []TrapCountByLocationIDRow, trapCountByLocationIDTransformer]
 
@@ -59,7 +59,7 @@ func TrapCountByLocationID(OrganizationID int32, LocID []string) *TrapCountByLoc
 			q.AppendSelect(expressionTypArgs.subExpr(12, 223))
 			q.SetTable(expressionTypArgs.subExpr(234, 318))
 			q.AppendWhere(expressionTypArgs.subExpr(330, 379))
-			q.AppendGroup(expressionTypArgs.subExpr(394, 421))
+			q.AppendGroup(expressionTypArgs.subExpr(394, 432))
 		}),
 	}
 }

sql/trapcount_by_location_id.bob.sql

@@ -16,4 +16,4 @@ WHERE
     td.organization_id = $1
     AND td.loc_id IN ($2)
 GROUP BY 
-    td.globalid, td.enddatetime;
+    td.globalid, td.loc_id, td.enddatetime;

sql/trapcount_by_location_id.sql

@@ -1,6 +1,6 @@
 -- TrapCountByLocationID
 SELECT 
-    td.loc_id AS trapdata_globalid,
+    td.globalid AS trapdata_globalid,
     td.enddatetime AS trapdata_enddate,
     COALESCE(SUM(sa.females), 0) AS total_females,
     COALESCE(SUM(sa.males), 0) AS total_males,
@@ -13,5 +13,5 @@ WHERE
     td.organization_id = $1
     AND td.loc_id IN ($2)
 GROUP BY 
-    td.globalid, td.enddatetime;
+    td.globalid, td.loc_id, td.enddatetime;
 

sql/trapdata_by_location_id_recent.bob.go

@@ -0,0 +1,108 @@
+// Code generated by BobGen psql v0.41.1. DO NOT EDIT.
+// This file is meant to be re-generated in place and/or deleted at any time.
+
+package sql
+
+import (
+	"context"
+	_ "embed"
+	"io"
+	"iter"
+
+	"github.com/stephenafamo/bob"
+	"github.com/stephenafamo/bob/dialect/psql"
+	"github.com/stephenafamo/bob/dialect/psql/dialect"
+	"github.com/stephenafamo/bob/expr"
+	"github.com/stephenafamo/bob/orm"
+	"github.com/stephenafamo/scan"
+)
+
+//go:embed trapdata_by_location_id_recent.bob.sql
+var formattedQueries_trapdata_by_location_id_recent string
+
+var trapDataByLocationIDRecentSQL = formattedQueries_trapdata_by_location_id_recent[164:454]
+
+type TrapDataByLocationIDRecentQuery = orm.ModQuery[*dialect.SelectQuery, trapDataByLocationIDRecent, TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow, trapDataByLocationIDRecentTransformer]
+
+func TrapDataByLocationIDRecent(OrganizationID int32, LocID []string) *TrapDataByLocationIDRecentQuery {
+	var expressionTypArgs trapDataByLocationIDRecent
+
+	expressionTypArgs.OrganizationID = psql.Arg(OrganizationID)
+	expressionTypArgs.LocID = expr.ToArgs(LocID...)
+
+	return &TrapDataByLocationIDRecentQuery{
+		Query: orm.Query[trapDataByLocationIDRecent, TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow, trapDataByLocationIDRecentTransformer]{
+			ExecQuery: orm.ExecQuery[trapDataByLocationIDRecent]{
+				BaseQuery: bob.BaseQuery[trapDataByLocationIDRecent]{
+					Expression: expressionTypArgs,
+					Dialect:    dialect.Dialect,
+					QueryType:  bob.QueryTypeSelect,
+				},
+			},
+			Scanner: func(context.Context, []string) (func(*scan.Row) (any, error), func(any) (TrapDataByLocationIDRecentRow, error)) {
+				return func(row *scan.Row) (any, error) {
+						var t TrapDataByLocationIDRecentRow
+						row.ScheduleScanByIndex(0, &t.Enddatetime)
+						row.ScheduleScanByIndex(1, &t.Globalid)
+						row.ScheduleScanByIndex(2, &t.LocID)
+						return &t, nil
+					}, func(v any) (TrapDataByLocationIDRecentRow, error) {
+						return *(v.(*TrapDataByLocationIDRecentRow)), nil
+					}
+			},
+		},
+		Mod: bob.ModFunc[*dialect.SelectQuery](func(q *dialect.SelectQuery) {
+			q.AppendSelect(expressionTypArgs.subExpr(7, 36))
+			q.SetTable(expressionTypArgs.subExpr(42, 244))
+			q.AppendWhere(expressionTypArgs.subExpr(251, 264))
+			q.CombinedOrder.AppendOrder(expressionTypArgs.subExpr(274, 290))
+		}),
+	}
+}
+
+type TrapDataByLocationIDRecentRow = struct {
+	Enddatetime int64  `db:"enddatetime"`
+	Globalid    string `db:"globalid"`
+	LocID       string `db:"loc_id"`
+}
+
+type trapDataByLocationIDRecentTransformer = bob.SliceTransformer[TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow]
+
+type trapDataByLocationIDRecent struct {
+	OrganizationID bob.Expression
+	LocID          bob.Expression
+}
+
+func (o trapDataByLocationIDRecent) args() iter.Seq[orm.ArgWithPosition] {
+	return func(yield func(arg orm.ArgWithPosition) bool) {
+		if !yield(orm.ArgWithPosition{
+			Name:       "organizationID",
+			Start:      207,
+			Stop:       209,
+			Expression: o.OrganizationID,
+		}) {
+			return
+		}
+
+		if !yield(orm.ArgWithPosition{
+			Name:       "locID",
+			Start:      227,
+			Stop:       229,
+			Expression: o.LocID,
+		}) {
+			return
+		}
+	}
+}
+
+func (o trapDataByLocationIDRecent) raw(from, to int) string {
+	return trapDataByLocationIDRecentSQL[from:to]
+}
+
+func (o trapDataByLocationIDRecent) subExpr(from, to int) bob.Expression {
+	return orm.ArgsToExpression(trapDataByLocationIDRecentSQL, from, to, o.args())
+}
+
+func (o trapDataByLocationIDRecent) WriteSQL(ctx context.Context, w io.Writer, d bob.Dialect, start int) ([]any, error) {
+	return o.subExpr(0, len(trapDataByLocationIDRecentSQL)).WriteSQL(ctx, w, d, start)
+}

sql/trapdata_by_location_id_recent.bob.sql

@@ -0,0 +1,15 @@
+-- Code generated by BobGen psql v0.41.1. DO NOT EDIT.
+-- This file is meant to be re-generated in place and/or deleted at any time.
+
+-- TrapDataByLocationIDRecent
+SELECT enddatetime, globalid, loc_id
+FROM (
+	SELECT enddatetime, globalid, loc_id, ROW_NUMBER()
+	OVER (PARTITION BY loc_id ORDER BY enddatetime DESC) as row_num
+	FROM fs_trapdata
+	WHERE 
+		organization_id = $1 AND
+		loc_id IN ($2)
+) ranked_data
+WHERE row_num <= 10
+ORDER BY enddatetime DESC;

sql/trapdata_by_location_id_recent.bob_test.go

@@ -0,0 +1,103 @@
+// Code generated by BobGen psql v0.41.1. DO NOT EDIT.
+// This file is meant to be re-generated in place and/or deleted at any time.
+
+package sql
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stephenafamo/bob"
+	"github.com/stephenafamo/bob/dialect/psql"
+	testutils "github.com/stephenafamo/bob/test/utils"
+)
+
+func TestTrapDataByLocationIDRecent(t *testing.T) {
+	t.Run("Base", func(t *testing.T) {
+		var sb strings.Builder
+
+		query := TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)})
+
+		if _, err := query.WriteQuery(t.Context(), &sb, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff(trapDataByLocationIDRecentSQL, sb.String()); diff != "" {
+			t.Fatalf("unexpected result (-got +want):\n%s", diff)
+		}
+	})
+
+	t.Run("Mod", func(t *testing.T) {
+		var sb strings.Builder
+
+		query := TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)})
+
+		if _, err := psql.Select(query).WriteQuery(t.Context(), &sb, 1); err != nil {
+			t.Fatal(err)
+		}
+
+		queryDiff, err := testutils.QueryDiff(trapDataByLocationIDRecentSQL, sb.String(), formatQuery)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if queryDiff != "" {
+			fmt.Println(sb.String())
+			t.Fatalf("unexpected result (-got +want):\n%s", queryDiff)
+		}
+	})
+
+	t.Run("Scanning", func(t *testing.T) {
+		if testDB == nil {
+			t.Skip("skipping test, no DSN provided")
+		}
+
+		ctxTx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+
+		tx, err := testDB.Begin(ctxTx)
+		if err != nil {
+			t.Fatalf("Error starting transaction: %v", err)
+		}
+
+		defer func() {
+			if err := tx.Rollback(ctxTx); err != nil {
+				t.Fatalf("Error rolling back transaction: %v", err)
+			}
+		}()
+
+		query, args, err := bob.Build(ctxTx, psql.Select(TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)})))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rows, err := tx.QueryContext(ctxTx, query, args...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rows.Close()
+
+		columns, err := rows.Columns()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(columns) != 3 {
+			t.Fatalf("expected %d columns, got %d", 3, len(columns))
+		}
+
+		if columns[0] != "enddatetime" {
+			t.Fatalf("expected column %d to be %s, got %s", 0, "enddatetime", columns[0])
+		}
+
+		if columns[1] != "globalid" {
+			t.Fatalf("expected column %d to be %s, got %s", 1, "globalid", columns[1])
+		}
+
+		if columns[2] != "loc_id" {
+			t.Fatalf("expected column %d to be %s, got %s", 2, "loc_id", columns[2])
+		}
+	})
+}

sql/trapdata_by_location_id_recent.sql

@@ -0,0 +1,12 @@
+-- TrapDataByLocationIDRecent
+SELECT enddatetime, globalid, loc_id
+FROM (
+	SELECT enddatetime, globalid, loc_id, ROW_NUMBER()
+	OVER (PARTITION BY loc_id ORDER BY enddatetime DESC) as row_num
+	FROM fs_trapdata
+	WHERE 
+		organization_id = $1 AND
+		loc_id IN ($2)
+) ranked_data
+WHERE row_num <= 10
+ORDER BY enddatetime DESC;