Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move address list func to types so it can be shared with csv

Browse source 
And stop double-geocoding all the rows.
This commit is contained in:
Eli Ribble 2026-04-16 03:06:18 +00:00
parent 5bf93c3dfd
commit b2d8e3ba27
No known key found for this signature in database
4 changed files with 73 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

28
platform/address.go
View file

@ -1,35 +1,7 @@
package platform
import (
"context"
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/stephenafamo/scan"
)
type Address = types.Address
func AddressList(ctx context.Context, ids []int32) ([]*types.Address, error) {
rows, err := bob.All(ctx, db.PGInstance.BobDB, psql.Select(
sm.Columns(
"COALESCE(address.country, 'usa') AS \"country\"",
"COALESCE(address.gid, '') AS \"gid\"",
"COALESCE(address.locality, '') AS \"locality\"",
"COALESCE(address.number_, '') AS \"number\"",
"COALESCE(address.postal_code, '') AS \"postal_code\"",
"COALESCE(address.region, '') AS \"region\"",
"COALESCE(address.street, '') AS \"street\"",
"COALESCE(address.unit, '') AS \"unit\"",
// This will work great, up until we add polygons to signal
"COALESCE(ST_Y(address.location_latitude), 0) AS \"location.latitude\"",
"COALESCE(ST_X(address.location_longitude), 0) AS \"location.longitude\"",
),
sm.From("address"),
sm.Where(psql.Quote("address", "id").EQ(psql.Arg(ids))),
), scan.StructMapper[*types.Address]())
return rows, err
}

50
platform/csv/csv.go
View file

@ -50,33 +50,43 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error {
if err != nil {
return fmt.Errorf("Failed to get all rows of file %d: %w", file_id, err)
}
address_ids := make([]int32, 0)
for _, r := range rows {
if r.AddressID.IsValue() {
address_ids = append(address_ids, r.AddressID.MustGet())
}
}
addresses, err := types.AddressList(ctx, address_ids)
if err != nil {
return fmt.Errorf("get address list: %w", err)
}
for _, row := range rows {
a := types.Address{
Country: "usa",
Locality: row.AddressLocality,
Number: row.AddressNumber,
PostalCode: row.AddressPostalCode,
Region: row.AddressRegion,
Street: row.AddressStreet,
Unit: "",
}
geo, err := geocode.GeocodeStructured(ctx, org, a)
if err != nil {
//return fmt.Errorf("ensure address: %w", err)
if geo == nil || geo.Address.ID == nil {
log.Warn().Err(err).Msg("ensure address failure")
} else {
log.Warn().Err(err).Int32("address.id", *geo.Address.ID).Msg("ensure address failure")
var a *types.Address
if row.AddressID.IsValue() {
var ok bool
a, ok = addresses[row.AddressID.MustGet()]
if !ok {
log.Error().Int32("id", row.AddressID.MustGet()).Msg("address is missing")
continue
}
} else {
a = &types.Address{
Country: "usa",
Locality: row.AddressLocality,
Number: row.AddressNumber,
PostalCode: row.AddressPostalCode,
Region: row.AddressRegion,
Street: row.AddressStreet,
Unit: "",
}
continue
}
parcel, err := geocode.GetParcel(ctx, txn, geo.Address)
parcel, err := geocode.GetParcel(ctx, txn, *a)
if err != nil {
return fmt.Errorf("get parcel: %w", err)
}
var site *models.Site
site, err = models.Sites.Query(
models.SelectWhere.Sites.AddressID.EQ(*geo.Address.ID),
models.SelectWhere.Sites.AddressID.EQ(*a.ID),
).One(ctx, txn)
if err != nil {
if err.Error() != "sql: no rows in result set" {
@ -87,7 +97,7 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error {
parcel_id = &(*parcel).ID
}
setter := models.SiteSetter{
AddressID: omit.From(*geo.Address.ID),
AddressID: omit.From(*a.ID),
Created: omit.From(time.Now()),
CreatorID: omit.FromPtr(file.Committer.Ptr()),
FileID: omitnull.From(file_id),

36
platform/types/address.go
View file

@ -1,10 +1,17 @@
package types
import (
"context"
"fmt"
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
//"github.com/rs/zerolog/log"
"github.com/stephenafamo/scan"
)
type Address struct {
@ -43,3 +50,32 @@ func AddressFromModel(m *models.Address) Address {
Unit: m.Unit,
}
}
func AddressList(ctx context.Context, ids []int32) (map[int32]*Address, error) {
rows, err := bob.All(ctx, db.PGInstance.BobDB, psql.Select(
sm.Columns(
"COALESCE(address.country, 'usa') AS \"country\"",
"COALESCE(address.gid, '') AS \"gid\"",
"COALESCE(address.id, '') AS \"id\"",
"COALESCE(address.locality, '') AS \"locality\"",
"COALESCE(address.number_, '') AS \"number\"",
"COALESCE(address.postal_code, '') AS \"postal_code\"",
"COALESCE(address.region, '') AS \"region\"",
"COALESCE(address.street, '') AS \"street\"",
"COALESCE(address.unit, '') AS \"unit\"",
// This will work great, up until we add polygons to signal
"COALESCE(address.location_latitude, 0) AS \"location.latitude\"",
"COALESCE(address.location_longitude, 0) AS \"location.longitude\"",
),
sm.From("address"),
sm.Where(psql.Quote("address", "id").EQ(psql.Any(ids))),
), scan.StructMapper[*Address]())
if err != nil {
return nil, fmt.Errorf("query addresses: %w", err)
}
addresses_by_id := make(map[int32]*Address, len(rows))
for _, a := range rows {
addresses_by_id[*a.ID] = a
}
return addresses_by_id, err
}

14
platform/upload.go
View file

@ -205,15 +205,10 @@ func getUploadDetailPool(ctx context.Context, file *models.FileuploadFile) (*Upl
address_ids = append(address_ids, r.AddressID.MustGet())
}
}
addresses, err := AddressList(ctx, address_ids)
addresses, err := types.AddressList(ctx, address_ids)
if err != nil {
return nil, fmt.Errorf("get address list: %w", err)
}
addresses_by_id := make(map[int32]*types.Address, len(address_ids))
for _, a := range addresses {
addresses_by_id[*a.ID] = a
}
pools := make([]UploadPoolRow, 0)
count_existing := 0
count_new := 0
@ -239,7 +234,12 @@ func getUploadDetailPool(ctx context.Context, file *models.FileuploadFile) (*Upl
}
var address *types.Address
if r.AddressID.IsValue() {
address = addresses_by_id[r.AddressID.MustGet()]
var ok bool
address, ok = addresses[r.AddressID.MustGet()]
if !ok {
log.Error().Int32("id", r.AddressID.MustGet()).Msg("address missing")
continue
}
} else {
address = &types.Address{
Country: "usa",