Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Remove now-extraneous latitude/longitude generated columns

Browse source 
Now that we can pull out the geometry directly into a go object we don't
need these and they complicate our insertions
This commit is contained in:
Eli Ribble 2026-05-07 16:38:42 +00:00
parent 34a136eba5
commit 7a361a330d
No known key found for this signature in database
40 changed files with 426 additions and 464 deletions
Download patch file Download diff file Expand all files Collapse all files

54
platform/address.go
View file

@ -2,63 +2,39 @@ package platform
import (
"context"
"errors"
"fmt"
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
querypublic "github.com/Gleipnir-Technology/nidus-sync/db/query/public"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/stephenafamo/scan"
)
type Address = types.Address
func AddressFromComplianceReportRequestID(ctx context.Context, public_id string) (*types.Address, error) {
row, err := bob.One(ctx, db.PGInstance.BobDB, psql.Select(
sm.Columns(
models.Addresses.Columns.Country,
models.Addresses.Columns.Gid,
models.Addresses.Columns.ID,
models.Addresses.Columns.Locality,
models.Addresses.Columns.LocationLatitude.As("location.latitude"),
models.Addresses.Columns.LocationLongitude.As("location.longitude"),
models.Addresses.Columns.Number,
models.Addresses.Columns.PostalCode,
models.Addresses.Columns.Region,
models.Addresses.Columns.Street,
models.Addresses.Columns.Unit,
),
//sm.From(models.Addresses.NameAs()),
sm.From(models.ComplianceReportRequests.NameAs()),
sm.InnerJoin(models.Leads.NameAs()).On(
models.ComplianceReportRequests.Columns.LeadID.EQ(models.Leads.Columns.ID)),
sm.InnerJoin(models.Sites.NameAs()).On(
models.Leads.Columns.SiteID.EQ(models.Sites.Columns.ID)),
sm.InnerJoin(models.Addresses.NameAs()).On(
models.Sites.Columns.AddressID.EQ(models.Addresses.Columns.ID)),
sm.Where(models.ComplianceReportRequests.Columns.PublicID.EQ(psql.Arg(public_id))),
), scan.StructMapper[*types.Address]())
row, err := querypublic.AddressFromComplianceReportRequestID(ctx, db.PGInstance.PGXPool, public_id)
if err != nil {
if errors.Is(err, db.ErrNoRows) {
return nil, nil
}
return nil, fmt.Errorf("query address from compliance report request: %w", err)
}
return row, nil
result := types.AddressFromModel(row)
return &result, nil
}
func AddressLocation(ctx context.Context, address types.Address) (*types.Location, error) {
row, err := bob.One(ctx, db.PGInstance.BobDB, psql.Select(
sm.Columns(
models.Addresses.Columns.LocationLatitude.As("latitude"),
models.Addresses.Columns.LocationLongitude.As("longitude"),
),
sm.From(models.Addresses.NameAs()),
sm.Where(models.Addresses.Columns.ID.EQ(psql.Arg(address.ID))),
), scan.StructMapper[*types.Location]())
address_id := int64(*address.ID)
addr, err := querypublic.AddressFromID(ctx, db.PGInstance.PGXPool, address_id)
if err != nil {
return nil, fmt.Errorf("query address: %w", err)
}
return row, nil
l, err := types.LocationFromGeom(addr.Location)
if err != nil {
return nil, fmt.Errorf("location from geom: %w", err)
}
return &l, nil
}
func AddressInsert(ctx context.Context) (*types.Address, error) {

19
platform/background/background.go
View file

@ -6,8 +6,11 @@ import (
"time"
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
"github.com/Gleipnir-Technology/nidus-sync/db/gen/nidus-sync/public/model"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
query "github.com/Gleipnir-Technology/nidus-sync/db/query/public"
"github.com/aarondl/opt/omit"
//"github.com/rs/zerolog/log"
)
@ -15,8 +18,8 @@ import (
func NewAudioTranscode(ctx context.Context, txn bob.Executor, audio_id int32) error {
return newJob(ctx, txn, enums.JobtypeAudioTranscode, audio_id)
}
func NewComplianceMailer(ctx context.Context, txn bob.Executor, compliance_report_request_id int32) error {
return newJob(ctx, txn, enums.JobtypeComplianceMailerSend, compliance_report_request_id)
func NewComplianceMailer(ctx context.Context, txn db.Ex, compliance_report_request_id int32) error {
return newJob2(ctx, txn, model.Jobtype_ComplianceMailerSend, compliance_report_request_id)
}
func NewCSVCommit(ctx context.Context, txn bob.Executor, csv_id int32) error {
return newJob(ctx, txn, enums.JobtypeCSVCommit, csv_id)
@ -48,3 +51,15 @@ func newJob(ctx context.Context, txn bob.Executor, t enums.Jobtype, id int32) er
}
return nil
}
func newJob2(ctx context.Context, txn db.Ex, t model.Jobtype, id int32) error {
job := model.Job{
Created: time.Now(),
Type: t,
RowID: id,
}
_, err := query.JobInsert(ctx, txn, job)
if err != nil {
return fmt.Errorf("insert job: %w", err)
}
return nil
}

53
platform/compliance.go
View file

@ -10,51 +10,45 @@ import (
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/nidus-sync/db"
modelpublic "github.com/Gleipnir-Technology/nidus-sync/db/gen/nidus-sync/public/model"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
querypublic "github.com/Gleipnir-Technology/nidus-sync/db/query/public"
"github.com/Gleipnir-Technology/nidus-sync/platform/background"
"github.com/Gleipnir-Technology/nidus-sync/platform/event"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/aarondl/opt/omit"
"github.com/aarondl/opt/omitnull"
//"github.com/rs/zerolog/log"
"github.com/stephenafamo/scan"
)
func ComplianceRequestMailerCreate(ctx context.Context, user User, site_id int32) (int32, error) {
txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
func ComplianceRequestMailerCreate(ctx context.Context, user User, site_id int64) (int32, error) {
txn, err := db.BeginTxn(ctx)
if err != nil {
return 0, fmt.Errorf("start txn: %w", err)
}
defer txn.Rollback(ctx)
site, err := models.FindSite(ctx, txn, site_id)
site, err := querypublic.SiteFromIDForOrg(ctx, txn, site_id, int64(user.Organization.ID))
if err != nil {
return 0, fmt.Errorf("find site: %w", err)
}
if site.OrganizationID != user.Organization.ID {
return 0, fmt.Errorf("permission denied")
}
address, err := models.FindAddress(ctx, txn, site.AddressID)
address, err := querypublic.AddressFromID(ctx, txn, int64(site.AddressID))
if err != nil {
return 0, fmt.Errorf("find address %d: %w", site.AddressID, err)
}
if address.PostalCode == "" {
return 0, fmt.Errorf("address %d does not have a postal code", address.ID)
}
features, err := models.Features.Query(
models.SelectWhere.Features.SiteID.EQ(site.ID),
).All(ctx, txn)
features, err := querypublic.FeaturesFromSiteID(ctx, txn, int64(site.ID))
if err != nil {
return 0, fmt.Errorf("find features: %w", err)
}
feature_ids := make([]int32, len(features))
feature_ids := make([]int64, len(features))
for i, f := range features {
feature_ids[i] = f.ID
feature_ids[i] = int64(f.ID)
}
feature_pools, err := models.FeaturePools.Query(
sm.Where(
models.FeaturePools.Columns.FeatureID.EQ(psql.Any(feature_ids)),
),
).All(ctx, txn)
feature_pools, err := querypublic.FeaturePoolsFromFeatures(ctx, txn, feature_ids)
if err != nil {
return 0, fmt.Errorf("find feature pools: %w", err)
}
@ -62,24 +56,27 @@ func ComplianceRequestMailerCreate(ctx context.Context, user User, site_id int32
return 0, fmt.Errorf("wrong number of pools: %d", len(feature_pools))
}
feature_pool := feature_pools[0]
var feature *models.Feature
var feature *modelpublic.Feature
for _, f := range features {
if f.ID == feature_pool.FeatureID {
feature = f
feature = &f
}
}
if feature == nil {
return 0, fmt.Errorf("match feature %d", feature_pool.FeatureID)
}
location := types.Location{
Latitude: feature.LocationLatitude.GetOr(0),
Longitude: feature.LocationLongitude.GetOr(0),
if feature.Location == nil {
return 0, fmt.Errorf("nil location %d", feature.ID)
}
location, err := types.LocationFromGeom(*feature.Location)
if err != nil {
return 0, fmt.Errorf("location from geom: %w", err)
}
signal, err := SignalCreateFromPool(ctx, txn, user, site.ID, feature_pool.FeatureID, location)
if err != nil {
return 0, fmt.Errorf("create signal from ppol: %w", err)
}
lead_id, err := leadCreate(ctx, txn, user, *signal, site.ID, &location)
lead, err := leadCreate(ctx, txn, user, signal.ID, site.ID, &location)
if err != nil {
return 0, fmt.Errorf("create lead from ppol: %w", err)
}
@ -87,14 +84,14 @@ func ComplianceRequestMailerCreate(ctx context.Context, user User, site_id int32
if err != nil {
return 0, fmt.Errorf("create public id: %w", err)
}
setter := models.ComplianceReportRequestSetter{
Created: omit.From(time.Now()),
Creator: omit.From(int32(user.ID)),
setter := modelpublic.ComplianceReportRequest{
Created: time.Now(),
Creator: int32(user.ID),
// ID
PublicID: omit.From(public_id),
LeadID: omitnull.From(*lead_id),
PublicID: public_id,
LeadID: &lead.ID,
}
req, err := models.ComplianceReportRequests.Insert(&setter).One(ctx, txn)
req, err := querypublic.ComplianceReportRequestInsert(ctx, txn, setter)
if err != nil {
return 0, fmt.Errorf("create compliance report request: %w", err)
}

40
platform/lead.go
View file

@ -9,44 +9,44 @@ import (
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
"github.com/Gleipnir-Technology/nidus-sync/db/gen/nidus-sync/public/model"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
query "github.com/Gleipnir-Technology/nidus-sync/db/query/public"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/aarondl/opt/omit"
"github.com/aarondl/opt/omitnull"
//"github.com/rs/zerolog/log"
"github.com/stephenafamo/scan"
)
// Create a lead from the given signal and site
func LeadCreate(ctx context.Context, user User, signal_id int32, site_id int32, pool_location *types.Location) (*int32, error) {
txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
func LeadCreate(ctx context.Context, user User, signal_id int32, site_id int32, pool_location *types.Location) (model.Lead, error) {
txn, err := db.BeginTxn(ctx)
if err != nil {
return nil, fmt.Errorf("start transaction: %w", err)
return model.Lead{}, fmt.Errorf("start transaction: %w", err)
}
defer txn.Rollback(ctx)
lead_id, err := leadCreate(ctx, txn, user, signal_id, site_id, pool_location)
lead, err := leadCreate(ctx, txn, user, signal_id, site_id, pool_location)
if err != nil {
return nil, fmt.Errorf("inner leadcreate: %w", err)
return model.Lead{}, fmt.Errorf("inner leadcreate: %w", err)
}
txn.Commit(ctx)
return lead_id, nil
return lead, nil
}
func leadCreate(ctx context.Context, txn bob.Executor, user User, signal_id int32, site_id int32, pool_location *types.Location) (*int32, error) {
lead, err := models.Leads.Insert(&models.LeadSetter{
Created: omit.From(time.Now()),
Creator: omit.From(int32(user.ID)),
func leadCreate(ctx context.Context, txn db.Ex, user User, signal_id int32, site_id int32, pool_location *types.Location) (model.Lead, error) {
lead := model.Lead{
Created: time.Now(),
Creator: int32(user.ID),
// ID
OrganizationID: omit.From(int32(user.Organization.ID)),
SiteID: omitnull.From(site_id),
Type: omit.From(enums.LeadtypeGreenPool),
}).One(ctx, txn)
if err != nil {
return nil, fmt.Errorf("failed to create lead: %w", err)
OrganizationID: int32(user.Organization.ID),
SiteID: &site_id,
Type: model.Leadtype_GreenPool,
}
return &lead.ID, nil
lead, err := query.LeadInsert(ctx, txn, lead)
if err != nil {
return model.Lead{}, fmt.Errorf("failed to create lead: %w", err)
}
return lead, nil
}
func leadsBySiteID(ctx context.Context, site_ids []int32) (map[int32][]*types.Lead, error) {
rows, err := bob.All(ctx, db.PGInstance.BobDB, psql.Select(

108
platform/signal.go
View file

@ -9,7 +9,6 @@ import (
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/im"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/nidus-sync/db"
modelpublic "github.com/Gleipnir-Technology/nidus-sync/db/gen/nidus-sync/public/model"
@ -17,7 +16,6 @@ import (
tablepublicreport "github.com/Gleipnir-Technology/nidus-sync/db/gen/nidus-sync/publicreport/table"
querypublic "github.com/Gleipnir-Technology/nidus-sync/db/query/public"
querypublicreport "github.com/Gleipnir-Technology/nidus-sync/db/query/publicreport"
"github.com/Gleipnir-Technology/nidus-sync/geomutil"
"github.com/Gleipnir-Technology/nidus-sync/platform/event"
"github.com/Gleipnir-Technology/nidus-sync/platform/publicreport"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
@ -44,71 +42,28 @@ type _rowWithID struct {
ID int32 `db:"id"`
}
func SignalCreateFromPool(ctx context.Context, txn bob.Executor, user User, site_id int32, feature_id int32, location types.Location) (*int32, error) {
/*
setter := models.SignalSetter{
Addressed: omitnull.FromPtr[time.Time](nil),
Addressor: omitnull.FromPtr[int32](nil),
Created: omit.From(time.Now()),
Creator: omit.From(int32(user.ID)),
//ID
OrganizationID: omit.From(user.Organization.ID),
Species: omitnull.FromPtr[enums.Mosquitospecies](nil),
Type: omit.From(enums.SignaltypeFlyoverPool),
SiteID: omitnull.From(site_id),
Location: omit.From(""),
//Location:
//LocationType null.Val[string] `db:"location_type,generated" `
FeaturePoolFeatureID: omitnull.From(feature_id),
ReportID: omitnull.FromPtr[int32](nil),
}
signal, err := models.Signals.Insert(&setter).One(ctx, db.PGInstance.BobDB)
*/
query := psql.Insert(
im.Into("signal",
"addressed",
"addressor",
"created",
"creator",
"feature_pool_feature_id",
"id",
"location",
"organization_id",
"report_id",
"site_id",
"species",
"type_",
),
im.Values(
psql.Raw("NULL"),
psql.Raw("NULL"),
psql.Arg(time.Now()),
psql.Arg(user.ID),
psql.Arg(feature_id),
psql.Raw("DEFAULT"),
psql.F("ST_Point", location.Longitude, location.Latitude, 4326),
psql.Arg(user.Organization.ID),
psql.Raw("NULL"),
psql.Arg(site_id),
psql.Raw("NULL"),
psql.Arg("flyover pool"),
),
im.Returning("id"),
)
row, err := bob.One(ctx, txn, query, scan.StructMapper[_rowWithID]())
if err != nil {
return nil, fmt.Errorf("insert signal: %w", err)
func SignalCreateFromPool(ctx context.Context, txn db.Ex, user User, site_id int32, feature_id int32, location types.Location) (modelpublic.Signal, error) {
g := location.ToGeom()
signal := modelpublic.Signal{
Addressed: nil,
Addressor: nil,
Created: time.Now(),
Creator: int32(user.ID),
FeaturePoolFeatureID: &feature_id,
//ID
Location: g,
OrganizationID: user.Organization.ID,
ReportID: nil,
SiteID: &site_id,
Species: nil,
Type: modelpublic.Signaltype_FlyoverPool,
}
/*
geom_query, _ := location.GeometryQuery()
_, err = psql.Update(
um.Table(models.Signals.Name()),
um.SetCol(models.Signals.Columns.Location.String()).To(geom_query),
um.Where(models.Signals.Columns.ID.EQ(psql.Arg(row.ID))),
).Exec(ctx, txn)
*/
return &row.ID, nil
var err error
signal, err = querypublic.SignalInsert(ctx, txn, signal)
if err != nil {
return modelpublic.Signal{}, fmt.Errorf("insert signal: %w", err)
}
return signal, nil
}
// Create a lead from the given signal and site
@ -139,19 +94,18 @@ func SignalCreateFromPublicreport(ctx context.Context, user User, report_id stri
return nil, fmt.Errorf("site from address: %w", err)
}
site_id = site.ID
location = geomutil.PointFromLngLat(*address.LocationLongitude, *address.LocationLatitude)
} else if report.LocationLatitude != nil && report.LocationLongitude != nil {
lat := report.LocationLatitude
lng := report.LocationLongitude
site, err := siteFromLocation(ctx, txn, user, types.Location{
Latitude: *lat,
Longitude: *lng,
})
location = address.Location
} else if report.Location != nil {
l, err := types.LocationFromGeom(*report.Location)
if err != nil {
return nil, fmt.Errorf("report location to geom: %w", err)
}
site, err := siteFromLocation(ctx, txn, user, l)
if err != nil {
return nil, fmt.Errorf("site from address: %w", err)
}
site_id = site.ID
location = geomutil.PointFromLngLat(*lng, *lat)
location = *report.Location
} else if report.AddressRaw != "" {
// At this point we don't have an address, and we don't have GPS
// We'll try geocoding and creating an address from that.
@ -164,9 +118,7 @@ func SignalCreateFromPublicreport(ctx context.Context, user User, report_id stri
return nil, fmt.Errorf("find address from raw: %w", err)
}
site_id = site.ID
lat := address.LocationLatitude
lng := address.LocationLongitude
location = geomutil.PointFromLngLat(*lng, *lat)
location = address.Location
} else {
// We have no structured address, no GPS, no unstructued address.
// There's really nothing we can make this lead from and have it be meaningful

19
platform/types/address.go
View file

@ -10,7 +10,7 @@ import (
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/gen/nidus-sync/public/model"
//"github.com/rs/zerolog/log"
"github.com/rs/zerolog/log"
"github.com/stephenafamo/scan"
)
@ -33,15 +33,16 @@ func (a Address) String() string {
}
func AddressFromModel(m model.Address) Address {
//log.Debug().Int32("id", m.ID).Float64("lat", m.LocationLatitude.GetOr(0.0)).Float64("lng", m.LocationLongitude.GetOr(0.0)).Msg("converting address")
l, err := LocationFromGeom(m.Location)
if err != nil {
log.Error().Err(err).Int32("id", m.ID).Msg("getting location for address")
}
return Address{
Country: m.Country,
GID: m.Gid,
ID: &m.ID,
Locality: m.Locality,
Location: &Location{
Latitude: *m.LocationLatitude,
Longitude: *m.LocationLongitude,
},
Country: m.Country,
GID: m.Gid,
ID: &m.ID,
Locality: m.Locality,
Location: &l,
Number: m.Number,
PostalCode: m.PostalCode,
Raw: addressToRaw(m),

10
platform/types/location.go
View file

@ -42,6 +42,16 @@ func (l Location) ToGeom() geom.T {
func LocationFromFS(pl *models.FieldseekerPointlocation) Location {
return Location{}
}
func LocationFromGeom(g geom.T) (Location, error) {
p, err := geomutil.AsPoint(g)
if err != nil {
return Location{}, fmt.Errorf("as point: %w", err)
}
return Location{
Latitude: p.Y(),
Longitude: p.X(),
}, nil
}
func LocationDistance(l1 Location, l2 Location) float64 {
lat_delta := l1.Latitude - l2.Latitude
lng_delta := l1.Longitude - l2.Longitude