From b2d8e3ba270aac4d805172378fdb06937ce9dcd5 Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Thu, 16 Apr 2026 03:06:18 +0000 Subject: [PATCH] Move address list func to types so it can be shared with csv And stop double-geocoding all the rows. --- platform/address.go | 28 ---------------------- platform/csv/csv.go | 50 +++++++++++++++++++++++---------------- platform/types/address.go | 36 ++++++++++++++++++++++++++++ platform/upload.go | 14 +++++------ 4 files changed, 73 insertions(+), 55 deletions(-) diff --git a/platform/address.go b/platform/address.go index c89ced69..a3916392 100644 --- a/platform/address.go +++ b/platform/address.go @@ -1,35 +1,7 @@ package platform import ( - "context" - - "github.com/Gleipnir-Technology/bob" - "github.com/Gleipnir-Technology/bob/dialect/psql" - "github.com/Gleipnir-Technology/bob/dialect/psql/sm" - "github.com/Gleipnir-Technology/nidus-sync/db" "github.com/Gleipnir-Technology/nidus-sync/platform/types" - "github.com/stephenafamo/scan" ) type Address = types.Address - -func AddressList(ctx context.Context, ids []int32) ([]*types.Address, error) { - rows, err := bob.All(ctx, db.PGInstance.BobDB, psql.Select( - sm.Columns( - "COALESCE(address.country, 'usa') AS \"country\"", - "COALESCE(address.gid, '') AS \"gid\"", - "COALESCE(address.locality, '') AS \"locality\"", - "COALESCE(address.number_, '') AS \"number\"", - "COALESCE(address.postal_code, '') AS \"postal_code\"", - "COALESCE(address.region, '') AS \"region\"", - "COALESCE(address.street, '') AS \"street\"", - "COALESCE(address.unit, '') AS \"unit\"", - // This will work great, up until we add polygons to signal - "COALESCE(ST_Y(address.location_latitude), 0) AS \"location.latitude\"", - "COALESCE(ST_X(address.location_longitude), 0) AS \"location.longitude\"", - ), - sm.From("address"), - sm.Where(psql.Quote("address", "id").EQ(psql.Arg(ids))), - ), scan.StructMapper[*types.Address]()) - return rows, err -} diff --git a/platform/csv/csv.go b/platform/csv/csv.go index 7d5c0aa6..a5bcf2ef 100644 --- a/platform/csv/csv.go +++ b/platform/csv/csv.go @@ -50,33 +50,43 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error { if err != nil { return fmt.Errorf("Failed to get all rows of file %d: %w", file_id, err) } + address_ids := make([]int32, 0) + for _, r := range rows { + if r.AddressID.IsValue() { + address_ids = append(address_ids, r.AddressID.MustGet()) + } + } + addresses, err := types.AddressList(ctx, address_ids) + if err != nil { + return fmt.Errorf("get address list: %w", err) + } for _, row := range rows { - a := types.Address{ - Country: "usa", - Locality: row.AddressLocality, - Number: row.AddressNumber, - PostalCode: row.AddressPostalCode, - Region: row.AddressRegion, - Street: row.AddressStreet, - Unit: "", - } - geo, err := geocode.GeocodeStructured(ctx, org, a) - if err != nil { - //return fmt.Errorf("ensure address: %w", err) - if geo == nil || geo.Address.ID == nil { - log.Warn().Err(err).Msg("ensure address failure") - } else { - log.Warn().Err(err).Int32("address.id", *geo.Address.ID).Msg("ensure address failure") + var a *types.Address + if row.AddressID.IsValue() { + var ok bool + a, ok = addresses[row.AddressID.MustGet()] + if !ok { + log.Error().Int32("id", row.AddressID.MustGet()).Msg("address is missing") + continue + } + } else { + a = &types.Address{ + Country: "usa", + Locality: row.AddressLocality, + Number: row.AddressNumber, + PostalCode: row.AddressPostalCode, + Region: row.AddressRegion, + Street: row.AddressStreet, + Unit: "", } - continue } - parcel, err := geocode.GetParcel(ctx, txn, geo.Address) + parcel, err := geocode.GetParcel(ctx, txn, *a) if err != nil { return fmt.Errorf("get parcel: %w", err) } var site *models.Site site, err = models.Sites.Query( - models.SelectWhere.Sites.AddressID.EQ(*geo.Address.ID), + models.SelectWhere.Sites.AddressID.EQ(*a.ID), ).One(ctx, txn) if err != nil { if err.Error() != "sql: no rows in result set" { @@ -87,7 +97,7 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error { parcel_id = &(*parcel).ID } setter := models.SiteSetter{ - AddressID: omit.From(*geo.Address.ID), + AddressID: omit.From(*a.ID), Created: omit.From(time.Now()), CreatorID: omit.FromPtr(file.Committer.Ptr()), FileID: omitnull.From(file_id), diff --git a/platform/types/address.go b/platform/types/address.go index 1d0d7354..68077204 100644 --- a/platform/types/address.go +++ b/platform/types/address.go @@ -1,10 +1,17 @@ package types import ( + "context" "fmt" + "github.com/Gleipnir-Technology/bob" + "github.com/Gleipnir-Technology/bob/dialect/psql" + "github.com/Gleipnir-Technology/bob/dialect/psql/sm" + "github.com/Gleipnir-Technology/nidus-sync/db" + "github.com/Gleipnir-Technology/nidus-sync/db/models" //"github.com/rs/zerolog/log" + "github.com/stephenafamo/scan" ) type Address struct { @@ -43,3 +50,32 @@ func AddressFromModel(m *models.Address) Address { Unit: m.Unit, } } +func AddressList(ctx context.Context, ids []int32) (map[int32]*Address, error) { + rows, err := bob.All(ctx, db.PGInstance.BobDB, psql.Select( + sm.Columns( + "COALESCE(address.country, 'usa') AS \"country\"", + "COALESCE(address.gid, '') AS \"gid\"", + "COALESCE(address.id, '') AS \"id\"", + "COALESCE(address.locality, '') AS \"locality\"", + "COALESCE(address.number_, '') AS \"number\"", + "COALESCE(address.postal_code, '') AS \"postal_code\"", + "COALESCE(address.region, '') AS \"region\"", + "COALESCE(address.street, '') AS \"street\"", + "COALESCE(address.unit, '') AS \"unit\"", + // This will work great, up until we add polygons to signal + "COALESCE(address.location_latitude, 0) AS \"location.latitude\"", + "COALESCE(address.location_longitude, 0) AS \"location.longitude\"", + ), + sm.From("address"), + sm.Where(psql.Quote("address", "id").EQ(psql.Any(ids))), + ), scan.StructMapper[*Address]()) + if err != nil { + return nil, fmt.Errorf("query addresses: %w", err) + } + addresses_by_id := make(map[int32]*Address, len(rows)) + for _, a := range rows { + addresses_by_id[*a.ID] = a + } + + return addresses_by_id, err +} diff --git a/platform/upload.go b/platform/upload.go index 08c5f7b5..6e1d1e6c 100644 --- a/platform/upload.go +++ b/platform/upload.go @@ -205,15 +205,10 @@ func getUploadDetailPool(ctx context.Context, file *models.FileuploadFile) (*Upl address_ids = append(address_ids, r.AddressID.MustGet()) } } - addresses, err := AddressList(ctx, address_ids) + addresses, err := types.AddressList(ctx, address_ids) if err != nil { return nil, fmt.Errorf("get address list: %w", err) } - addresses_by_id := make(map[int32]*types.Address, len(address_ids)) - for _, a := range addresses { - addresses_by_id[*a.ID] = a - } - pools := make([]UploadPoolRow, 0) count_existing := 0 count_new := 0 @@ -239,7 +234,12 @@ func getUploadDetailPool(ctx context.Context, file *models.FileuploadFile) (*Upl } var address *types.Address if r.AddressID.IsValue() { - address = addresses_by_id[r.AddressID.MustGet()] + var ok bool + address, ok = addresses[r.AddressID.MustGet()] + if !ok { + log.Error().Int32("id", r.AddressID.MustGet()).Msg("address missing") + continue + } } else { address = &types.Address{ Country: "usa",