Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Only get the latest 10 trap collections

Browse source 
Otherwise the page gets really swamped
This commit is contained in:
Eli Ribble 2025-11-21 16:28:03 +00:00
parent d3b9d34bd2
commit b7318ae973
No known key found for this signature in database
12 changed files with 304 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitmodules vendored
View file

@ -4,3 +4,6 @@
[submodule "go-geojson2h3"]
path = go-geojson2h3
url = git@github.com:Gleipnir-Technology/go-geojson2h3.git
[submodule "bob"]
path = bob
url = git@github.com:Gleipnir-Technology/bob.git

1
bob Submodule

@ -0,0 +1 @@
Subproject commit 96da65fd88a50ae532079e8ea69746183f4af3a1

10
database.go
View file

@ -1,7 +1,6 @@
package main
import (
"bytes"
"context"
"database/sql"
"embed"
@ -197,12 +196,3 @@ func updateSummaryTables(ctx context.Context, org *models.Organization) {
}
}
}
func insertQueryToString(query bob.BaseQuery[*dialect.InsertQuery]) string {
buf := new(bytes.Buffer)
_, err := query.WriteQuery(context.TODO(), buf, 0)
if err != nil {
return fmt.Sprintf("Failed to write query: %v", err)
}
return buf.String()
}

60
html.go
View file

@ -512,6 +512,9 @@ func htmlSource(w http.ResponseWriter, r *http.Request, user *models.User, id st
}
cadence, deltas := calculateCadenceVariance(treatment_times)
for i, treatment := range treatments {
if i >= len(deltas) {
break
}
treatment.CadenceDelta = deltas[i]
treatments[i] = treatment
}
@ -782,12 +785,57 @@ func trapsBySource(ctx context.Context, org *models.Organization, sourceID strin
location_ids = append(location_ids, location.TrapLocationGlobalid)
args = append(args, psql.Arg(location.TrapLocationGlobalid))
}
trap_data, err := org.FSTrapdata(
sm.Where(
models.FSTrapdata.Columns.LocID.In(args...),
),
sm.OrderBy("enddatetime"),
).All(ctx, PGInstance.BobDB)
/*
trap_data, err := org.FSTrapdata(
sm.Where(
models.FSTrapdata.Columns.LocID.In(args...),
),
sm.OrderBy("enddatetime"),
).All(ctx, PGInstance.BobDB)
*/
/*
query := org.FSTrapdata(
sm.From(
psql.Select(
sm.From(psql.F("ROW_NUMBER")(
fm.Over(
wm.PartitionBy(models.FSTrapdata.Columns.LocID),
wm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
),
)).As("row_num"),
sm.Where(models.FSTrapdata.Columns.LocID.In(args...))),
),
sm.Where(psql.Quote("row_num").LTE(psql.Arg(10))),
sm.OrderBy(models.FSTrapdata.Columns.LocID),
sm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
)
*/
/*
query := psql.Select(
sm.From(
psql.Select(
sm.Columns(
models.FSTrapdata.Columns.Globalid,
psql.F("ROW_NUMBER")(
fm.Over(
wm.PartitionBy(models.FSTrapdata.Columns.LocID),
wm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
),
).As("row_num"),
sm.From(models.FSTrapdata.Name()),
),
sm.Where(models.FSTrapdata.Columns.LocID.In(args...))),
),
sm.Where(psql.Quote("row_num").LTE(psql.Arg(10))),
sm.OrderBy(models.FSTrapdata.Columns.LocID),
sm.OrderBy(models.FSTrapdata.Columns.Enddatetime).Desc(),
)
log.Info().Str("trapdata", queryToString(query)).Msg("Getting trap data")
trap_data, err := query.Exec(ctx, PGInstance.BobDB)
*/
trap_data, err := sql.TrapDataByLocationIDRecent(org.ID, location_ids).All(ctx, PGInstance.BobDB)
if err != nil {
return nil, fmt.Errorf("Failed to query trap data: %w", err)
}

9
model_conversion.go
View file

@ -162,7 +162,7 @@ type Treatment struct {
Product string
}
func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data models.FSTrapdatumSlice, counts []sql.TrapCountByLocationIDRow) ([]TrapNearby, error) {
func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data []sql.TrapDataByLocationIDRecentRow, counts []sql.TrapCountByLocationIDRow) ([]TrapNearby, error) {
results := make([]TrapNearby, 0)
count_by_trap_data_id := make(map[string]*sql.TrapCountByLocationIDRow)
for _, c := range counts {
@ -174,12 +174,9 @@ func toTemplateTraps(locations []sql.TrapLocationBySourceIDRow, trap_data models
if !ok {
return results, errors.New(fmt.Sprintf("Failed to find trap count for %s", td.Globalid))
}
if td.LocID.IsNull() {
return results, errors.New("Got a trap data with no location ID")
}
loc_id := td.LocID.MustGet()
loc_id := td.LocID
count := &TrapCount{
Ended: fsToTime(td.Enddatetime),
Ended: time.UnixMilli(td.Enddatetime),
Females: int(c.TotalFemales.IntPart()),
ID: td.Globalid,
Males: int(c.TotalMales),

4
sql/trapcount_by_location_id.bob.go
View file

@ -22,7 +22,7 @@ import (
//go:embed trapcount_by_location_id.bob.sql
var formattedQueries_trapcount_by_location_id string
var trapCountByLocationIDSQL = formattedQueries_trapcount_by_location_id[159:580]
var trapCountByLocationIDSQL = formattedQueries_trapcount_by_location_id[159:591]
type TrapCountByLocationIDQuery = orm.ModQuery[*dialect.SelectQuery, trapCountByLocationID, TrapCountByLocationIDRow, []TrapCountByLocationIDRow, trapCountByLocationIDTransformer]
@ -59,7 +59,7 @@ func TrapCountByLocationID(OrganizationID int32, LocID []string) *TrapCountByLoc
q.AppendSelect(expressionTypArgs.subExpr(12, 223))
q.SetTable(expressionTypArgs.subExpr(234, 318))
q.AppendWhere(expressionTypArgs.subExpr(330, 379))
q.AppendGroup(expressionTypArgs.subExpr(394, 421))
q.AppendGroup(expressionTypArgs.subExpr(394, 432))
}),
}
}

2
sql/trapcount_by_location_id.bob.sql
View file

@ -16,4 +16,4 @@ WHERE
td.organization_id = $1
AND td.loc_id IN ($2)
GROUP BY
td.globalid, td.enddatetime;
td.globalid, td.loc_id, td.enddatetime;

4
sql/trapcount_by_location_id.sql
View file

@ -1,6 +1,6 @@
-- TrapCountByLocationID
SELECT
td.loc_id AS trapdata_globalid,
td.globalid AS trapdata_globalid,
td.enddatetime AS trapdata_enddate,
COALESCE(SUM(sa.females), 0) AS total_females,
COALESCE(SUM(sa.males), 0) AS total_males,
@ -13,5 +13,5 @@ WHERE
td.organization_id = $1
AND td.loc_id IN ($2)
GROUP BY
td.globalid, td.enddatetime;
td.globalid, td.loc_id, td.enddatetime;

108
sql/trapdata_by_location_id_recent.bob.go Normal file
View file

@ -0,0 +1,108 @@
// Code generated by BobGen psql v0.41.1. DO NOT EDIT.
// This file is meant to be re-generated in place and/or deleted at any time.
package sql
import (
"context"
_ "embed"
"io"
"iter"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/psql"
"github.com/stephenafamo/bob/dialect/psql/dialect"
"github.com/stephenafamo/bob/expr"
"github.com/stephenafamo/bob/orm"
"github.com/stephenafamo/scan"
)
//go:embed trapdata_by_location_id_recent.bob.sql
var formattedQueries_trapdata_by_location_id_recent string
var trapDataByLocationIDRecentSQL = formattedQueries_trapdata_by_location_id_recent[164:454]
type TrapDataByLocationIDRecentQuery = orm.ModQuery[*dialect.SelectQuery, trapDataByLocationIDRecent, TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow, trapDataByLocationIDRecentTransformer]
func TrapDataByLocationIDRecent(OrganizationID int32, LocID []string) *TrapDataByLocationIDRecentQuery {
var expressionTypArgs trapDataByLocationIDRecent
expressionTypArgs.OrganizationID = psql.Arg(OrganizationID)
expressionTypArgs.LocID = expr.ToArgs(LocID...)
return &TrapDataByLocationIDRecentQuery{
Query: orm.Query[trapDataByLocationIDRecent, TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow, trapDataByLocationIDRecentTransformer]{
ExecQuery: orm.ExecQuery[trapDataByLocationIDRecent]{
BaseQuery: bob.BaseQuery[trapDataByLocationIDRecent]{
Expression: expressionTypArgs,
Dialect: dialect.Dialect,
QueryType: bob.QueryTypeSelect,
},
},
Scanner: func(context.Context, []string) (func(*scan.Row) (any, error), func(any) (TrapDataByLocationIDRecentRow, error)) {
return func(row *scan.Row) (any, error) {
var t TrapDataByLocationIDRecentRow
row.ScheduleScanByIndex(0, &t.Enddatetime)
row.ScheduleScanByIndex(1, &t.Globalid)
row.ScheduleScanByIndex(2, &t.LocID)
return &t, nil
}, func(v any) (TrapDataByLocationIDRecentRow, error) {
return *(v.(*TrapDataByLocationIDRecentRow)), nil
}
},
},
Mod: bob.ModFunc[*dialect.SelectQuery](func(q *dialect.SelectQuery) {
q.AppendSelect(expressionTypArgs.subExpr(7, 36))
q.SetTable(expressionTypArgs.subExpr(42, 244))
q.AppendWhere(expressionTypArgs.subExpr(251, 264))
q.CombinedOrder.AppendOrder(expressionTypArgs.subExpr(274, 290))
}),
}
}
type TrapDataByLocationIDRecentRow = struct {
Enddatetime int64 `db:"enddatetime"`
Globalid string `db:"globalid"`
LocID string `db:"loc_id"`
}
type trapDataByLocationIDRecentTransformer = bob.SliceTransformer[TrapDataByLocationIDRecentRow, []TrapDataByLocationIDRecentRow]
type trapDataByLocationIDRecent struct {
OrganizationID bob.Expression
LocID bob.Expression
}
func (o trapDataByLocationIDRecent) args() iter.Seq[orm.ArgWithPosition] {
return func(yield func(arg orm.ArgWithPosition) bool) {
if !yield(orm.ArgWithPosition{
Name: "organizationID",
Start: 207,
Stop: 209,
Expression: o.OrganizationID,
}) {
return
}
if !yield(orm.ArgWithPosition{
Name: "locID",
Start: 227,
Stop: 229,
Expression: o.LocID,
}) {
return
}
}
}
func (o trapDataByLocationIDRecent) raw(from, to int) string {
return trapDataByLocationIDRecentSQL[from:to]
}
func (o trapDataByLocationIDRecent) subExpr(from, to int) bob.Expression {
return orm.ArgsToExpression(trapDataByLocationIDRecentSQL, from, to, o.args())
}
func (o trapDataByLocationIDRecent) WriteSQL(ctx context.Context, w io.Writer, d bob.Dialect, start int) ([]any, error) {
return o.subExpr(0, len(trapDataByLocationIDRecentSQL)).WriteSQL(ctx, w, d, start)
}

15
sql/trapdata_by_location_id_recent.bob.sql Normal file
View file

@ -0,0 +1,15 @@
-- Code generated by BobGen psql v0.41.1. DO NOT EDIT.
-- This file is meant to be re-generated in place and/or deleted at any time.
-- TrapDataByLocationIDRecent
SELECT enddatetime, globalid, loc_id
FROM (
SELECT enddatetime, globalid, loc_id, ROW_NUMBER()
OVER (PARTITION BY loc_id ORDER BY enddatetime DESC) as row_num
FROM fs_trapdata
WHERE
organization_id = $1 AND
loc_id IN ($2)
) ranked_data
WHERE row_num <= 10
ORDER BY enddatetime DESC;

103
sql/trapdata_by_location_id_recent.bob_test.go Normal file
View file

@ -0,0 +1,103 @@
// Code generated by BobGen psql v0.41.1. DO NOT EDIT.
// This file is meant to be re-generated in place and/or deleted at any time.
package sql
import (
"context"
"fmt"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/psql"
testutils "github.com/stephenafamo/bob/test/utils"
)
func TestTrapDataByLocationIDRecent(t *testing.T) {
t.Run("Base", func(t *testing.T) {
var sb strings.Builder
query := TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)})
if _, err := query.WriteQuery(t.Context(), &sb, 1); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(trapDataByLocationIDRecentSQL, sb.String()); diff != "" {
t.Fatalf("unexpected result (-got +want):\n%s", diff)
}
})
t.Run("Mod", func(t *testing.T) {
var sb strings.Builder
query := TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)})
if _, err := psql.Select(query).WriteQuery(t.Context(), &sb, 1); err != nil {
t.Fatal(err)
}
queryDiff, err := testutils.QueryDiff(trapDataByLocationIDRecentSQL, sb.String(), formatQuery)
if err != nil {
t.Fatal(err)
}
if queryDiff != "" {
fmt.Println(sb.String())
t.Fatalf("unexpected result (-got +want):\n%s", queryDiff)
}
})
t.Run("Scanning", func(t *testing.T) {
if testDB == nil {
t.Skip("skipping test, no DSN provided")
}
ctxTx, cancel := context.WithCancel(t.Context())
defer cancel()
tx, err := testDB.Begin(ctxTx)
if err != nil {
t.Fatalf("Error starting transaction: %v", err)
}
defer func() {
if err := tx.Rollback(ctxTx); err != nil {
t.Fatalf("Error rolling back transaction: %v", err)
}
}()
query, args, err := bob.Build(ctxTx, psql.Select(TrapDataByLocationIDRecent(random_int32(nil), []string{random_string(nil)})))
if err != nil {
t.Fatal(err)
}
rows, err := tx.QueryContext(ctxTx, query, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
columns, err := rows.Columns()
if err != nil {
t.Fatal(err)
}
if len(columns) != 3 {
t.Fatalf("expected %d columns, got %d", 3, len(columns))
}
if columns[0] != "enddatetime" {
t.Fatalf("expected column %d to be %s, got %s", 0, "enddatetime", columns[0])
}
if columns[1] != "globalid" {
t.Fatalf("expected column %d to be %s, got %s", 1, "globalid", columns[1])
}
if columns[2] != "loc_id" {
t.Fatalf("expected column %d to be %s, got %s", 2, "loc_id", columns[2])
}
})
}

12
sql/trapdata_by_location_id_recent.sql Normal file
View file

@ -0,0 +1,12 @@
-- TrapDataByLocationIDRecent
SELECT enddatetime, globalid, loc_id
FROM (
SELECT enddatetime, globalid, loc_id, ROW_NUMBER()
OVER (PARTITION BY loc_id ORDER BY enddatetime DESC) as row_num
FROM fs_trapdata
WHERE
organization_id = $1 AND
loc_id IN ($2)
) ranked_data
WHERE row_num <= 10
ORDER BY enddatetime DESC;