Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Store addresses on every geocode

Browse source
This commit is contained in:
Eli Ribble 2026-04-10 22:32:40 +00:00
parent e04b86218d
commit 730f40956f
No known key found for this signature in database
12 changed files with 223 additions and 229 deletions
Download patch file Download diff file Expand all files Collapse all files

9
db/dberrors/address.bob.go
View file

@ -10,8 +10,17 @@ var AddressErrors = &addressErrors{
columns: []string{"id"},
s: "address_pkey",
},
ErrUniqueAddressGidUnique: &UniqueConstraintError{
schema: "",
table: "address",
columns: []string{"gid"},
s: "address_gid_unique",
},
}
type addressErrors struct {
ErrUniqueAddressPkey *UniqueConstraintError
ErrUniqueAddressGidUnique *UniqueConstraintError
}

42
db/dbinfo/address.bob.go
View file

@ -17,7 +17,7 @@ var Addresses = Table[
Columns: addressColumns{
Country: column{
Name: "country",
DBType: "public.countrytype",
DBType: "text",
Default: "",
Comment: "",
Nullable: false,
@ -160,6 +160,23 @@ var Addresses = Table[
Where: "",
Include: []string{},
},
AddressGidUnique: index{
Type: "btree",
Name: "address_gid_unique",
Columns: []indexColumn{
{
Name: "gid",
Desc: null.FromCond(false, true),
IsExpression: false,
},
},
Unique: true,
Comment: "",
NullsFirst: []bool{false},
NullsDistinct: false,
Where: "",
Include: []string{},
},
IdxAddressGeom: index{
Type: "gist",
Name: "idx_address_geom",
@ -184,6 +201,14 @@ var Addresses = Table[
Comment: "",
},
Uniques: addressUniques{
AddressGidUnique: constraint{
Name: "address_gid_unique",
Columns: []string{"gid"},
Comment: "",
},
},
Comment: "",
}
@ -211,13 +236,14 @@ func (c addressColumns) AsSlice() []column {
}
type addressIndexes struct {
AddressPkey index
IdxAddressGeom index
AddressPkey index
AddressGidUnique index
IdxAddressGeom index
}
func (i addressIndexes) AsSlice() []index {
return []index{
i.AddressPkey, i.IdxAddressGeom,
i.AddressPkey, i.AddressGidUnique, i.IdxAddressGeom,
}
}
@ -227,10 +253,14 @@ func (f addressForeignKeys) AsSlice() []foreignKey {
return []foreignKey{}
}
type addressUniques struct{}
type addressUniques struct {
AddressGidUnique constraint
}
func (u addressUniques) AsSlice() []constraint {
return []constraint{}
return []constraint{
u.AddressGidUnique,
}
}
type addressChecks struct{}

70
db/enums/enums.bob.go
View file

@ -846,76 +846,6 @@ func (e *CommsTextorigin) Scan(value any) error {
return nil
}
// Enum values for Countrytype
const (
CountrytypeUsa Countrytype = "usa"
)
func AllCountrytype() []Countrytype {
return []Countrytype{
CountrytypeUsa,
}
}
type Countrytype string
func (e Countrytype) String() string {
return string(e)
}
func (e Countrytype) Valid() bool {
switch e {
case CountrytypeUsa:
return true
default:
return false
}
}
// useful when testing in other packages
func (e Countrytype) All() []Countrytype {
return AllCountrytype()
}
func (e Countrytype) MarshalText() ([]byte, error) {
return []byte(e), nil
}
func (e *Countrytype) UnmarshalText(text []byte) error {
return e.Scan(text)
}
func (e Countrytype) MarshalBinary() ([]byte, error) {
return []byte(e), nil
}
func (e *Countrytype) UnmarshalBinary(data []byte) error {
return e.Scan(data)
}
func (e Countrytype) Value() (driver.Value, error) {
return string(e), nil
}
func (e *Countrytype) Scan(value any) error {
switch x := value.(type) {
case string:
*e = Countrytype(x)
case []byte:
*e = Countrytype(x)
case nil:
return fmt.Errorf("cannot nil into Countrytype")
default:
return fmt.Errorf("cannot scan type %T: %v", value, value)
}
if !e.Valid() {
return fmt.Errorf("invalid Countrytype value: %s", *e)
}
return nil
}
// Enum values for FileuploadCsvtype
const (
FileuploadCsvtypePoollist FileuploadCsvtype = "PoolList"

8
db/migrations/00129_address_country_text.sql Normal file
View file

@ -0,0 +1,8 @@
-- +goose Up
ALTER TABLE address
ALTER COLUMN country
TYPE TEXT
USING country::TEXT;
DROP TYPE CountryType;
-- +goose Down

7
db/migrations/00130_address_gid_unique.sql Normal file
View file

@ -0,0 +1,7 @@
-- +goose Up
UPDATE address
SET gid = gen_random_uuid()
WHERE gid = '';
ALTER TABLE address ADD CONSTRAINT address_gid_unique UNIQUE (gid);
-- +goose Down
ALTER TABLE address DROP CONSTRAINT address_gid_unique;

31
db/models/address.bob.go
View file

@ -18,7 +18,6 @@ import (
"github.com/Gleipnir-Technology/bob/expr"
"github.com/Gleipnir-Technology/bob/orm"
"github.com/Gleipnir-Technology/bob/types/pgtypes"
enums "github.com/Gleipnir-Technology/nidus-sync/db/enums"
"github.com/aarondl/opt/null"
"github.com/aarondl/opt/omit"
"github.com/aarondl/opt/omitnull"
@ -26,7 +25,7 @@ import (
// Address is an object representing the database table.
type Address struct {
Country enums.Countrytype `db:"country" `
Country string `db:"country" `
Created time.Time `db:"created" `
Location string `db:"location" `
H3cell string `db:"h3cell" `
@ -118,18 +117,18 @@ func (addressColumns) AliasedAs(alias string) addressColumns {
// All values are optional, and do not have to be set
// Generated columns are not included
type AddressSetter struct {
Country omit.Val[enums.Countrytype] `db:"country" `
Created omit.Val[time.Time] `db:"created" `
Location omit.Val[string] `db:"location" `
H3cell omit.Val[string] `db:"h3cell" `
ID omit.Val[int32] `db:"id,pk" `
Locality omit.Val[string] `db:"locality" `
PostalCode omit.Val[string] `db:"postal_code" `
Street omit.Val[string] `db:"street" `
Unit omit.Val[string] `db:"unit" `
Region omit.Val[string] `db:"region" `
Number omit.Val[string] `db:"number_" `
Gid omit.Val[string] `db:"gid" `
Country omit.Val[string] `db:"country" `
Created omit.Val[time.Time] `db:"created" `
Location omit.Val[string] `db:"location" `
H3cell omit.Val[string] `db:"h3cell" `
ID omit.Val[int32] `db:"id,pk" `
Locality omit.Val[string] `db:"locality" `
PostalCode omit.Val[string] `db:"postal_code" `
Street omit.Val[string] `db:"street" `
Unit omit.Val[string] `db:"unit" `
Region omit.Val[string] `db:"region" `
Number omit.Val[string] `db:"number_" `
Gid omit.Val[string] `db:"gid" `
}
func (s AddressSetter) SetColumns() []string {
@ -1151,7 +1150,7 @@ func (address0 *Address) AttachSite(ctx context.Context, exec bob.Executor, site
}
type addressWhere[Q psql.Filterable] struct {
Country psql.WhereMod[Q, enums.Countrytype]
Country psql.WhereMod[Q, string]
Created psql.WhereMod[Q, time.Time]
Location psql.WhereMod[Q, string]
H3cell psql.WhereMod[Q, string]
@ -1173,7 +1172,7 @@ func (addressWhere[Q]) AliasedAs(alias string) addressWhere[Q] {
func buildAddressWhere[Q psql.Filterable](cols addressColumns) addressWhere[Q] {
return addressWhere[Q]{
Country: psql.Where[Q, enums.Countrytype](cols.Country),
Country: psql.Where[Q, string](cols.Country),
Created: psql.Where[Q, time.Time](cols.Created),
Location: psql.Where[Q, string](cols.Location),
H3cell: psql.Where[Q, string](cols.H3cell),

12
platform/csv/csv.go
View file

@ -60,23 +60,23 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error {
Street: row.AddressStreet,
Unit: "",
}
address, err := geocode.EnsureAddressWithGeocode(ctx, txn, org, a)
geo, err := geocode.GeocodeStructured(ctx, org, a)
if err != nil {
//return fmt.Errorf("ensure address: %w", err)
if address == nil {
if geo == nil || geo.Address.ID == nil {
log.Warn().Err(err).Msg("ensure address failure")
} else {
log.Warn().Err(err).Int32("address.id", address.ID).Msg("ensure address failure")
log.Warn().Err(err).Int32("address.id", *geo.Address.ID).Msg("ensure address failure")
}
continue
}
parcel, err := geocode.GetParcel(ctx, txn, address)
parcel, err := geocode.GetParcel(ctx, txn, geo.Address)
if err != nil {
return fmt.Errorf("get parcel: %w", err)
}
var site *models.Site
site, err = models.Sites.Query(
models.SelectWhere.Sites.AddressID.EQ(address.ID),
models.SelectWhere.Sites.AddressID.EQ(*geo.Address.ID),
).One(ctx, txn)
if err != nil {
if err.Error() != "sql: no rows in result set" {
@ -87,7 +87,7 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error {
parcel_id = &(*parcel).ID
}
setter := models.SiteSetter{
AddressID: omit.From(address.ID),
AddressID: omit.From(*geo.Address.ID),
Created: omit.From(time.Now()),
CreatorID: omit.FromPtr(file.Committer.Ptr()),
FileID: omitnull.From(file_id),

125
platform/geocode/address.go Normal file
View file

@ -0,0 +1,125 @@
package geocode
import (
"context"
"fmt"
"time"
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/im"
//bobtypes "github.com/Gleipnir-Technology/bob/types"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
"github.com/Gleipnir-Technology/nidus-sync/h3utils"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/Gleipnir-Technology/nidus-sync/stadia"
"github.com/rs/zerolog/log"
"github.com/stephenafamo/scan"
)
// Ensure the provided address exists. If it doesn't add it to the database.
func EnsureAddress(ctx context.Context, txn bob.Executor, a types.Address, l types.Location) (*models.Address, error) {
address, err := models.Addresses.Query(
models.SelectWhere.Addresses.Country.EQ(a.Country),
models.SelectWhere.Addresses.Locality.EQ(a.Locality),
models.SelectWhere.Addresses.Number.EQ(a.Number),
models.SelectWhere.Addresses.PostalCode.EQ(a.PostalCode),
models.SelectWhere.Addresses.Region.EQ(a.Region),
models.SelectWhere.Addresses.Street.EQ(a.Street),
models.SelectWhere.Addresses.Unit.EQ(a.Unit),
).One(ctx, txn)
if err == nil {
return address, nil
}
id, err := insertAddress(ctx, txn, a, l)
if err != nil {
return nil, fmt.Errorf("insert address: %w", err)
}
return &models.Address{
Country: a.Country,
Created: time.Now(),
Gid: a.GID,
H3cell: "",
ID: *id,
Locality: a.Locality,
Location: "",
PostalCode: a.PostalCode,
Street: a.Street,
Unit: a.Unit,
Region: a.Region,
Number: a.Number,
}, nil
}
func ensureAddressFromFeature(ctx context.Context, txn bob.Executor, feature stadia.GeocodeFeature) (int32, error) {
if feature.Geometry.Type != "Point" {
return 0, fmt.Errorf("Can't hanlde stadia geometry %s", feature.Geometry.Type)
}
lat := feature.Geometry.Coordinates[1]
lng := feature.Geometry.Coordinates[0]
cell, err := h3utils.GetCell(lng, lat, 15)
if err != nil {
return 0, fmt.Errorf("failed to convert lat %f lng %f to h3 cell", lat, lng)
}
type _row struct {
ID int32 `db:"id"`
}
row, err := bob.One(ctx, txn, psql.Insert(
im.Into("address", "country", "created", "gid", "h3cell", "id", "locality", "location", "number_", "postal_code", "region", "street", "unit"),
im.Values(
psql.Arg(feature.CountryCode()),
psql.Arg(time.Now()),
psql.Arg(feature.Properties.GID),
psql.Arg(cell.String()),
psql.Raw("DEFAULT"),
psql.Arg(feature.Locality()),
psql.F("ST_Point", lng, lat, 4326),
psql.Arg(feature.Number()),
psql.Arg(feature.PostalCode()),
psql.Arg(feature.Region()),
psql.Arg(feature.Street()),
psql.Raw("''"),
),
im.Returning("id"),
), scan.StructMapper[_row]())
log.Info().Int32("id", row.ID).Msg("inserted address")
if err != nil {
return 0, fmt.Errorf("insert: %w", err)
}
return row.ID, nil
}
func insertAddress(ctx context.Context, txn bob.Executor, address types.Address, location types.Location) (*int32, error) {
cell, err := h3utils.GetCell(location.Longitude, location.Latitude, 15)
if err != nil {
return nil, fmt.Errorf("failed to convert lat %f lng %f to h3 cell", location.Longitude, location.Latitude)
}
type _row struct {
ID int32 `db:"id"`
}
row, err := bob.One(ctx, txn, psql.Insert(
im.Into("address", "country", "created", "gid", "h3cell", "id", "locality", "location", "number_", "postal_code", "region", "street", "unit"),
im.Values(
psql.Arg(address.Country),
psql.Arg(time.Now()),
psql.Arg(address.GID),
psql.Arg(cell),
psql.Raw("DEFAULT"),
psql.Arg(address.Locality),
psql.F("ST_Point", location.Longitude, location.Latitude, 4326),
psql.Arg(address.Number),
psql.Arg(address.PostalCode),
psql.Arg(address.Region),
psql.Arg(address.Street),
psql.Raw("''"),
),
im.Returning("id"),
), scan.StructMapper[_row]())
if err != nil {
return nil, fmt.Errorf("insert: %w", err)
}
return &row.ID, nil
}
func insertAddresses(ctx context.Context, txn bob.Executor, features []stadia.GeocodeFeature) error {
return nil
}

6
platform/geocode/by_gid.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/h3utils"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/Gleipnir-Technology/nidus-sync/stadia"
@ -29,10 +30,15 @@ func ByGID(ctx context.Context, gid string) (*GeocodeResult, error) {
if err != nil {
return nil, fmt.Errorf("latlngtocell: %w", err)
}
id, err := ensureAddressFromFeature(ctx, db.PGInstance.BobDB, feature)
if err != nil {
return nil, fmt.Errorf("insert address: %w", err)
}
return &GeocodeResult{
Address: types.Address{
Country: feature.Properties.Context.ISO3166A3,
GID: feature.Properties.GID,
ID: &id,
Locality: feature.Properties.Context.WhosOnFirst.Locality.Name,
Number: feature.Properties.AddressComponents.Number,
PostalCode: feature.Properties.AddressComponents.PostalCode,

133
platform/geocode/geocode.go
View file

@ -9,7 +9,6 @@ import (
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/im"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
bobtypes "github.com/Gleipnir-Technology/bob/types"
"github.com/Gleipnir-Technology/nidus-sync/db"
@ -18,7 +17,6 @@ import (
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
"github.com/Gleipnir-Technology/nidus-sync/stadia"
"github.com/aarondl/opt/omit"
"github.com/stephenafamo/scan"
//"github.com/rs/zerolog/log"
"github.com/uber/h3-go/v4"
"resty.dev/v3"
@ -52,127 +50,6 @@ func restyMiddleware(rclient *resty.Client, response *resty.Response) error {
return nil
}
// Ensure the provided address exists. If it doesn't add it to the database.
func EnsureAddress(ctx context.Context, txn bob.Executor, a types.Address, l types.Location) (*models.Address, error) {
address, err := models.Addresses.Query(
models.SelectWhere.Addresses.Country.EQ(a.CountryEnum()),
models.SelectWhere.Addresses.Locality.EQ(a.Locality),
models.SelectWhere.Addresses.Number.EQ(a.Number),
models.SelectWhere.Addresses.PostalCode.EQ(a.PostalCode),
models.SelectWhere.Addresses.Region.EQ(a.Region),
models.SelectWhere.Addresses.Street.EQ(a.Street),
models.SelectWhere.Addresses.Unit.EQ(a.Unit),
).One(ctx, txn)
if err == nil {
return address, nil
}
cell, err := h3utils.GetCell(l.Longitude, l.Latitude, 15)
if err != nil {
return nil, fmt.Errorf("failed to convert lat %f lng %f to h3 cell", l.Longitude, l.Latitude)
}
type _row struct {
ID int32 `db:"id"`
}
created := time.Now()
row, err := bob.One(ctx, txn, psql.Insert(
im.Into("address", "country", "created", "gid", "h3cell", "id", "locality", "location", "number_", "postal_code", "region", "street", "unit"),
im.Values(
psql.Arg(a.CountryEnum()),
psql.Arg(created),
psql.Arg(a.GID),
psql.Arg(cell),
psql.Raw("DEFAULT"),
psql.Arg(a.Locality),
psql.F("ST_Point", l.Longitude, l.Latitude, 4326),
psql.Arg(a.Number),
psql.Arg(a.PostalCode),
psql.Arg(a.Region),
psql.Arg(a.Street),
psql.Raw("''"),
),
im.Returning("id"),
), scan.StructMapper[_row]())
if err != nil {
return nil, fmt.Errorf("insert: %w", err)
}
return &models.Address{
Country: a.CountryEnum(),
Created: created,
Gid: a.GID,
H3cell: "",
ID: row.ID,
Locality: a.Locality,
Location: "",
PostalCode: a.PostalCode,
Street: a.Street,
Unit: a.Unit,
Region: a.Region,
Number: a.Number,
}, nil
}
// Either get an address that matches, or create a new address. Either way, return an address
// This will make a call to a structured geocode service, so it's slow.
func EnsureAddressWithGeocode(ctx context.Context, txn bob.Executor, org *models.Organization, a types.Address) (*models.Address, error) {
address, err := models.Addresses.Query(
models.SelectWhere.Addresses.Country.EQ(a.CountryEnum()),
models.SelectWhere.Addresses.Locality.EQ(a.Locality),
models.SelectWhere.Addresses.Number.EQ(a.Number),
models.SelectWhere.Addresses.PostalCode.EQ(a.PostalCode),
models.SelectWhere.Addresses.Region.EQ(a.Region),
models.SelectWhere.Addresses.Street.EQ(a.Street),
models.SelectWhere.Addresses.Unit.EQ(a.Unit),
).One(ctx, txn)
if err == nil {
return address, nil
}
// Geocode
geo, err := GeocodeStructured(ctx, org, a)
if err != nil {
return nil, fmt.Errorf("geocode: %w", err)
}
type _row struct {
ID int32 `db:"id"`
}
created := time.Now()
row, err := bob.One(ctx, txn, psql.Insert(
im.Into("address", "country", "created", "gid", "h3cell", "id", "locality", "location", "number_", "postal_code", "region", "street", "unit"),
im.Values(
psql.Arg(geo.Address.Country),
psql.Arg(created),
psql.Arg(geo.Address.GID),
psql.Arg(geo.Cell),
psql.Raw("DEFAULT"),
psql.Arg(geo.Address.Locality),
psql.F("ST_Point", geo.Location.Longitude, geo.Location.Latitude, 4326),
psql.Arg(geo.Address.Number),
psql.Arg(geo.Address.PostalCode),
psql.Arg(geo.Address.Region),
psql.Arg(geo.Address.Street),
psql.Raw("''"),
),
im.Returning("id"),
), scan.StructMapper[_row]())
if err != nil {
return nil, fmt.Errorf("insert: %w", err)
}
return &models.Address{
Country: geo.Address.CountryEnum(),
Created: created,
Gid: geo.Address.GID,
H3cell: "",
ID: row.ID,
Locality: geo.Address.Locality,
Location: "",
PostalCode: geo.Address.PostalCode,
Street: geo.Address.Street,
Unit: geo.Address.Unit,
Region: geo.Address.Region,
Number: geo.Address.Number,
}, nil
}
func GeocodeRaw(ctx context.Context, org *models.Organization, address string) (*GeocodeResult, error) {
req := stadia.RequestGeocodeRaw{
Text: address,
@ -182,6 +59,7 @@ func GeocodeRaw(ctx context.Context, org *models.Organization, address string) (
if err != nil {
return nil, fmt.Errorf("client raw geocode failure on %s: %w", address, err)
}
insertAddresses(ctx, db.PGInstance.BobDB, resp.Features)
return toGeocodeResult(*resp, address)
}
func GeocodeStructured(ctx context.Context, org *models.Organization, a types.Address) (*GeocodeResult, error) {
@ -198,6 +76,7 @@ func GeocodeStructured(ctx context.Context, org *models.Organization, a types.Ad
if err != nil {
return nil, fmt.Errorf("client structured geocode failure on %s: %w", a.String(), err)
}
insertAddresses(ctx, db.PGInstance.BobDB, resp.Features)
return toGeocodeResult(*resp, a.String())
}
func ReverseGeocode(ctx context.Context, location types.Location) (*GeocodeResult, error) {
@ -209,6 +88,7 @@ func ReverseGeocode(ctx context.Context, location types.Location) (*GeocodeResul
if err != nil {
return nil, fmt.Errorf("client reverse geocode failure on %s: %w", location.String(), err)
}
insertAddresses(ctx, db.PGInstance.BobDB, resp.Features)
return toGeocodeResult(*resp, location.String())
}
@ -264,10 +144,13 @@ func toGeocodeResult(resp stadia.GeocodeResponse, address_msg string) (*GeocodeR
}
// Get the parcel for a given address, if one can be found
func GetParcel(ctx context.Context, txn bob.Executor, a *models.Address) (*models.Parcel, error) {
func GetParcel(ctx context.Context, txn bob.Executor, a types.Address) (*models.Parcel, error) {
if a.ID == nil {
return nil, fmt.Errorf("nil address ID")
}
result, err := models.Parcels.Query(
sm.InnerJoin("address").On(psql.F("ST_Contains", psql.Raw("parcel.geometry"), psql.Raw("address.location"))),
models.SelectWhere.Addresses.ID.EQ(a.ID),
models.SelectWhere.Addresses.ID.EQ(*a.ID),
).One(ctx, txn)
if err != nil {
if err.Error() == "sql: no rows in result set" {

8
platform/types/address.go
View file

@ -3,8 +3,8 @@ package types
import (
"fmt"
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
"github.com/rs/zerolog/log"
)
type Address struct {
@ -24,12 +24,10 @@ type Address struct {
func (a Address) String() string {
return fmt.Sprintf("%s %s, %s, %s, %s, %s", a.Number, a.Street, a.Locality, a.Region, a.PostalCode, a.Country)
}
func (a Address) CountryEnum() enums.Countrytype {
return enums.CountrytypeUsa
}
func AddressFromModel(m *models.Address) Address {
log.Debug().Int32("id", m.ID).Float64("lat", m.LocationY.GetOr(0.0)).Float64("lng", m.LocationX.GetOr(0.0)).Msg("converting address")
return Address{
Country: m.Country.String(),
Country: m.Country,
GID: m.Gid,
ID: &m.ID,
Locality: m.Locality,

1
resource/compliance.go
View file

@ -12,7 +12,6 @@ import (
//"github.com/Gleipnir-Technology/nidus-sync/html"
nhttp "github.com/Gleipnir-Technology/nidus-sync/http"
"github.com/Gleipnir-Technology/nidus-sync/platform"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
//"github.com/rs/zerolog/log"
)