Push geocoding down a layer

This makes it possible to always save address information from our
geocoder.

api/routes.go

@@ -12,6 +12,7 @@ func AddRoutes(r chi.Router) {
 	r.Use(render.SetContentType(render.ContentTypeJSON))
 	r.Method("GET", "/mosquito-source", auth.NewEnsureAuth(apiMosquitoSource))
 	r.Method("GET", "/service-request", auth.NewEnsureAuth(apiServiceRequest))
+	r.Method("GET", "/signal", authenticatedHandlerJSON(listSignal))
 	r.Method("GET", "/trap-data", auth.NewEnsureAuth(apiTrapData))
 	r.Method("GET", "/client/ios", auth.NewEnsureAuth(handleClientIos))
 	r.Method("POST", "/audio/{uuid}", auth.NewEnsureAuth(apiAudioPost))

background/background.go

@@ -23,10 +23,10 @@ var waitGroup sync.WaitGroup
 func Start(ctx context.Context) {
 	newOAuthTokenChannel = make(chan struct{}, 10)
 
-	channelJobAudio = make(chan jobAudio, 100)         // Buffered channel to prevent blocking
+	channelJobAudio = make(chan jobAudio, 100)  // Buffered channel to prevent blocking
-	channelJobImportCSV = make(chan jobImportCSV, 100) // Buffered channel to prevent blocking
+	channelJobCSV = make(chan jobCSV, 100)      // Buffered channel to prevent blocking
-	channelJobEmail = make(chan email.Job, 100)        // Buffered channel to prevent blocking
+	channelJobEmail = make(chan email.Job, 100) // Buffered channel to prevent blocking
-	channelJobText = make(chan text.Job, 100)          // Buffered channel to prevent blocking
+	channelJobText = make(chan text.Job, 100)   // Buffered channel to prevent blocking
 
 	waitGroup.Add(1)
 	go func() {
@@ -49,7 +49,7 @@ func Start(ctx context.Context) {
 	waitGroup.Add(1)
 	go func() {
 		defer waitGroup.Done()
-		startWorkerCSV(ctx, channelJobImportCSV)
+		startWorkerCSV(ctx, channelJobCSV)
 	}()
 
 	waitGroup.Add(1)
@@ -97,16 +97,10 @@ func addWaitingJobs(ctx context.Context) error {
 	}
 	for _, row := range rows {
 		report_id := row.ID
-		job := jobImportCSV{
+		enqueueJobCSV(jobCSV{
-			fileID: report_id,
+			fileID:  report_id,
-			type_:  row.Type,
+			csvType: row.Type,
-		}
+		})
-		select {
-		case channelJobImportCSV <- job:
-			log.Info().Int32("report_id", report_id).Msg("CSV upload job queued")
-		default:
-			log.Warn().Int32("report_id", report_id).Msg("CSV upload job failed to queue, channel full")
-		}
 	}
 	return nil
 }

background/pool.go

@@ -1,52 +0,0 @@
-package background
-
-import (
-	"context"
-	//"fmt"
-
-	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
-	"github.com/Gleipnir-Technology/nidus-sync/platform/csv"
-	//"github.com/Gleipnir-Technology/nidus-sync/userfile"
-	//"github.com/google/uuid"
-	"github.com/rs/zerolog/log"
-)
-
-type jobImportCSV struct {
-	fileID int32
-	type_  enums.FileuploadCsvtype
-}
-
-var channelJobImportCSV chan jobImportCSV
-
-func ProcessUpload(file_id int32, t enums.FileuploadCsvtype) {
-	enqueueUploadJob(jobImportCSV{
-		fileID: file_id,
-		type_:  t,
-	})
-}
-
-func enqueueUploadJob(job jobImportCSV) {
-	select {
-	case channelJobImportCSV <- job:
-		log.Info().Int32("file_id", job.fileID).Msg("Enqueued csv job")
-	default:
-		log.Warn().Int32("file_id", job.fileID).Msg("csv channel is full, dropping job")
-	}
-}
-func startWorkerCSV(ctx context.Context, channelJobImport chan jobImportCSV) {
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				log.Info().Msg("CSV worker shutting down.")
-				return
-			case job := <-channelJobImport:
-				log.Info().Int32("id", job.fileID).Msg("Processing CSV job")
-				err := csv.ProcessJob(ctx, job.fileID, job.type_)
-				if err != nil {
-					log.Error().Err(err).Int32("id", job.fileID).Msg("Error processing CSV file")
-				}
-			}
-		}
-	}()
-}

background/upload.go

@@ -0,0 +1,80 @@
+package background
+
+import (
+	"context"
+	//"fmt"
+
+	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
+	"github.com/Gleipnir-Technology/nidus-sync/platform/csv"
+	//"github.com/Gleipnir-Technology/nidus-sync/userfile"
+	//"github.com/google/uuid"
+	"github.com/rs/zerolog/log"
+)
+
+type jobCSVAction = int
+
+const (
+	jobCSVActionCommit jobCSVAction = iota
+	jobCSVActionImport
+)
+
+type jobCSV struct {
+	action  jobCSVAction
+	csvType enums.FileuploadCsvtype
+	fileID  int32
+}
+
+var channelJobCSV chan jobCSV
+
+func CommitUpload(file_id int32) {
+	enqueueJobCSV(jobCSV{
+		action: jobCSVActionCommit,
+		fileID: file_id,
+	})
+}
+func ProcessUpload(file_id int32, t enums.FileuploadCsvtype) {
+	enqueueJobCSV(jobCSV{
+		csvType: t,
+		fileID:  file_id,
+	})
+}
+
+func enqueueJobCSV(job jobCSV) {
+	select {
+	case channelJobCSV <- job:
+		log.Info().Int32("file_id", job.fileID).Msg("Enqueued csv job")
+	default:
+		log.Warn().Int32("file_id", job.fileID).Msg("csv channel is full, dropping job")
+	}
+}
+func startWorkerCSV(ctx context.Context, channelJobImport chan jobCSV) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				log.Info().Msg("CSV worker shutting down.")
+				return
+			case job := <-channelJobImport:
+				log.Info().Int32("id", job.fileID).Msg("Processing CSV job")
+				switch job.action {
+				case jobCSVActionCommit:
+					err := csv.JobCommit(ctx, job.fileID)
+					if err != nil {
+						log.Error().Err(err).Int32("id", job.fileID).Msg("Error processing CSV file")
+						continue
+					}
+				case jobCSVActionImport:
+					err := csv.JobImport(ctx, job.fileID, job.csvType)
+					if err != nil {
+						log.Error().Err(err).Int32("id", job.fileID).Msg("Error processing CSV file")
+						continue
+					}
+				default:
+					log.Error().Msg("Unrecognized job action")
+					return
+				}
+				log.Info().Int32("id", job.fileID).Msg("Done processing CSV job")
+			}
+		}
+	}()
+}

db/dberrors/address.bob.go

@@ -10,17 +10,8 @@ var AddressErrors = &addressErrors{
 		columns: []string{"id"},
 		s:       "address_pkey",
 	},
-
-	ErrUniqueAddressCountryLocalityUnitNumber_StreetKey: &UniqueConstraintError{
-		schema:  "",
-		table:   "address",
-		columns: []string{"country", "locality", "unit", "number_", "street"},
-		s:       "address_country_locality_unit_number__street_key",
-	},
 }
 
 type addressErrors struct {
 	ErrUniqueAddressPkey *UniqueConstraintError
-
-	ErrUniqueAddressCountryLocalityUnitNumber_StreetKey *UniqueConstraintError
 }

db/dbinfo/address.bob.go

@@ -69,15 +69,6 @@ var Addresses = Table[
 			Generated: false,
 			AutoIncr:  false,
 		},
-		Number: column{
-			Name:      "number_",
-			DBType:    "integer",
-			Default:   "",
-			Comment:   "",
-			Nullable:  false,
-			Generated: false,
-			AutoIncr:  false,
-		},
 		PostalCode: column{
 			Name:      "postal_code",
 			DBType:    "text",
@@ -114,6 +105,15 @@ var Addresses = Table[
 			Generated: false,
 			AutoIncr:  false,
 		},
+		Number: column{
+			Name:      "number_",
+			DBType:    "text",
+			Default:   "",
+			Comment:   "",
+			Nullable:  false,
+			Generated: false,
+			AutoIncr:  false,
+		},
 	},
 	Indexes: addressIndexes{
 		AddressPkey: index{
@@ -133,43 +133,6 @@ var Addresses = Table[
 			Where:         "",
 			Include:       []string{},
 		},
-		AddressCountryLocalityUnitNumberStreetKey: index{
-			Type: "btree",
-			Name: "address_country_locality_unit_number__street_key",
-			Columns: []indexColumn{
-				{
-					Name:         "country",
-					Desc:         null.FromCond(false, true),
-					IsExpression: false,
-				},
-				{
-					Name:         "locality",
-					Desc:         null.FromCond(false, true),
-					IsExpression: false,
-				},
-				{
-					Name:         "unit",
-					Desc:         null.FromCond(false, true),
-					IsExpression: false,
-				},
-				{
-					Name:         "number_",
-					Desc:         null.FromCond(false, true),
-					IsExpression: false,
-				},
-				{
-					Name:         "street",
-					Desc:         null.FromCond(false, true),
-					IsExpression: false,
-				},
-			},
-			Unique:        true,
-			Comment:       "",
-			NullsFirst:    []bool{false, false, false, false, false},
-			NullsDistinct: false,
-			Where:         "",
-			Include:       []string{},
-		},
 		IdxAddressGeom: index{
 			Type: "gist",
 			Name: "idx_address_geom",
@@ -194,14 +157,6 @@ var Addresses = Table[
 		Comment: "",
 	},
 
-	Uniques: addressUniques{
-		AddressCountryLocalityUnitNumberStreetKey: constraint{
-			Name:    "address_country_locality_unit_number__street_key",
-			Columns: []string{"country", "locality", "unit", "number_", "street"},
-			Comment: "",
-		},
-	},
-
 	Comment: "",
 }
 
@@ -212,28 +167,27 @@ type addressColumns struct {
 	H3cell     column
 	ID         column
 	Locality   column
-	Number     column
 	PostalCode column
 	Street     column
 	Unit       column
 	Region     column
+	Number     column
 }
 
 func (c addressColumns) AsSlice() []column {
 	return []column{
-		c.Country, c.Created, c.Geom, c.H3cell, c.ID, c.Locality, c.Number, c.PostalCode, c.Street, c.Unit, c.Region,
+		c.Country, c.Created, c.Geom, c.H3cell, c.ID, c.Locality, c.PostalCode, c.Street, c.Unit, c.Region, c.Number,
 	}
 }
 
 type addressIndexes struct {
-	AddressPkey                               index
+	AddressPkey    index
-	AddressCountryLocalityUnitNumberStreetKey index
+	IdxAddressGeom index
-	IdxAddressGeom                            index
 }
 
 func (i addressIndexes) AsSlice() []index {
 	return []index{
-		i.AddressPkey, i.AddressCountryLocalityUnitNumberStreetKey, i.IdxAddressGeom,
+		i.AddressPkey, i.IdxAddressGeom,
 	}
 }
 
@@ -243,14 +197,10 @@ func (f addressForeignKeys) AsSlice() []foreignKey {
 	return []foreignKey{}
 }
 
-type addressUniques struct {
+type addressUniques struct{}
-	AddressCountryLocalityUnitNumberStreetKey constraint
-}
 
 func (u addressUniques) AsSlice() []constraint {
-	return []constraint{
+	return []constraint{}
-		u.AddressCountryLocalityUnitNumberStreetKey,
-	}
 }
 
 type addressChecks struct{}

db/enums/enums.bob.go

@@ -988,11 +988,13 @@ func (e *FileuploadCsvtype) Scan(value any) error {
 
 // Enum values for FileuploadFilestatustype
 const (
-	FileuploadFilestatustypeError     FileuploadFilestatustype = "error"
+	FileuploadFilestatustypeError      FileuploadFilestatustype = "error"
-	FileuploadFilestatustypeParsed    FileuploadFilestatustype = "parsed"
+	FileuploadFilestatustypeParsed     FileuploadFilestatustype = "parsed"
-	FileuploadFilestatustypeUploaded  FileuploadFilestatustype = "uploaded"
+	FileuploadFilestatustypeUploaded   FileuploadFilestatustype = "uploaded"
-	FileuploadFilestatustypeCommitted FileuploadFilestatustype = "committed"
+	FileuploadFilestatustypeParsing    FileuploadFilestatustype = "parsing"
-	FileuploadFilestatustypeDiscarded FileuploadFilestatustype = "discarded"
+	FileuploadFilestatustypeCommitting FileuploadFilestatustype = "committing"
+	FileuploadFilestatustypeCommitted  FileuploadFilestatustype = "committed"
+	FileuploadFilestatustypeDiscarded  FileuploadFilestatustype = "discarded"
 )
 
 func AllFileuploadFilestatustype() []FileuploadFilestatustype {
@@ -1000,6 +1002,8 @@ func AllFileuploadFilestatustype() []FileuploadFilestatustype {
 		FileuploadFilestatustypeError,
 		FileuploadFilestatustypeParsed,
 		FileuploadFilestatustypeUploaded,
+		FileuploadFilestatustypeParsing,
+		FileuploadFilestatustypeCommitting,
 		FileuploadFilestatustypeCommitted,
 		FileuploadFilestatustypeDiscarded,
 	}
@@ -1016,6 +1020,8 @@ func (e FileuploadFilestatustype) Valid() bool {
 	case FileuploadFilestatustypeError,
 		FileuploadFilestatustypeParsed,
 		FileuploadFilestatustypeUploaded,
+		FileuploadFilestatustypeParsing,
+		FileuploadFilestatustypeCommitting,
 		FileuploadFilestatustypeCommitted,
 		FileuploadFilestatustypeDiscarded:
 		return true

db/factory/address.bob.go

@@ -42,11 +42,11 @@ type AddressTemplate struct {
 	H3cell     func() string
 	ID         func() int32
 	Locality   func() string
-	Number     func() int32
 	PostalCode func() string
 	Street     func() string
 	Unit       func() string
 	Region     func() string
+	Number     func() string
 
 	r addressR
 	f *Factory
@@ -145,10 +145,6 @@ func (o AddressTemplate) BuildSetter() *models.AddressSetter {
 		val := o.Locality()
 		m.Locality = omit.From(val)
 	}
-	if o.Number != nil {
-		val := o.Number()
-		m.Number = omit.From(val)
-	}
 	if o.PostalCode != nil {
 		val := o.PostalCode()
 		m.PostalCode = omit.From(val)
@@ -165,6 +161,10 @@ func (o AddressTemplate) BuildSetter() *models.AddressSetter {
 		val := o.Region()
 		m.Region = omit.From(val)
 	}
+	if o.Number != nil {
+		val := o.Number()
+		m.Number = omit.From(val)
+	}
 
 	return m
 }
@@ -205,9 +205,6 @@ func (o AddressTemplate) Build() *models.Address {
 	if o.Locality != nil {
 		m.Locality = o.Locality()
 	}
-	if o.Number != nil {
-		m.Number = o.Number()
-	}
 	if o.PostalCode != nil {
 		m.PostalCode = o.PostalCode()
 	}
@@ -220,6 +217,9 @@ func (o AddressTemplate) Build() *models.Address {
 	if o.Region != nil {
 		m.Region = o.Region()
 	}
+	if o.Number != nil {
+		m.Number = o.Number()
+	}
 
 	o.setModelRels(m)
 
@@ -260,10 +260,6 @@ func ensureCreatableAddress(m *models.AddressSetter) {
 		val := random_string(nil)
 		m.Locality = omit.From(val)
 	}
-	if !(m.Number.IsValue()) {
-		val := random_int32(nil)
-		m.Number = omit.From(val)
-	}
 	if !(m.PostalCode.IsValue()) {
 		val := random_string(nil)
 		m.PostalCode = omit.From(val)
@@ -280,6 +276,10 @@ func ensureCreatableAddress(m *models.AddressSetter) {
 		val := random_string(nil)
 		m.Region = omit.From(val)
 	}
+	if !(m.Number.IsValue()) {
+		val := random_string(nil)
+		m.Number = omit.From(val)
+	}
 }
 
 // insertOptRels creates and inserts any optional the relationships on *models.Address
@@ -445,11 +445,11 @@ func (m addressMods) RandomizeAllColumns(f *faker.Faker) AddressMod {
 		AddressMods.RandomH3cell(f),
 		AddressMods.RandomID(f),
 		AddressMods.RandomLocality(f),
-		AddressMods.RandomNumber(f),
 		AddressMods.RandomPostalCode(f),
 		AddressMods.RandomStreet(f),
 		AddressMods.RandomUnit(f),
 		AddressMods.RandomRegion(f),
+		AddressMods.RandomNumber(f),
 	}
 }
 
@@ -639,37 +639,6 @@ func (m addressMods) RandomLocality(f *faker.Faker) AddressMod {
 	})
 }
 
-// Set the model columns to this value
-func (m addressMods) Number(val int32) AddressMod {
-	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
-		o.Number = func() int32 { return val }
-	})
-}
-
-// Set the Column from the function
-func (m addressMods) NumberFunc(f func() int32) AddressMod {
-	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
-		o.Number = f
-	})
-}
-
-// Clear any values for the column
-func (m addressMods) UnsetNumber() AddressMod {
-	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
-		o.Number = nil
-	})
-}
-
-// Generates a random value for the column using the given faker
-// if faker is nil, a default faker is used
-func (m addressMods) RandomNumber(f *faker.Faker) AddressMod {
-	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
-		o.Number = func() int32 {
-			return random_int32(f)
-		}
-	})
-}
-
 // Set the model columns to this value
 func (m addressMods) PostalCode(val string) AddressMod {
 	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
@@ -794,6 +763,37 @@ func (m addressMods) RandomRegion(f *faker.Faker) AddressMod {
 	})
 }
 
+// Set the model columns to this value
+func (m addressMods) Number(val string) AddressMod {
+	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
+		o.Number = func() string { return val }
+	})
+}
+
+// Set the Column from the function
+func (m addressMods) NumberFunc(f func() string) AddressMod {
+	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
+		o.Number = f
+	})
+}
+
+// Clear any values for the column
+func (m addressMods) UnsetNumber() AddressMod {
+	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
+		o.Number = nil
+	})
+}
+
+// Generates a random value for the column using the given faker
+// if faker is nil, a default faker is used
+func (m addressMods) RandomNumber(f *faker.Faker) AddressMod {
+	return AddressModFunc(func(_ context.Context, o *AddressTemplate) {
+		o.Number = func() string {
+			return random_string(f)
+		}
+	})
+}
+
 func (m addressMods) WithParentsCascading() AddressMod {
 	return AddressModFunc(func(ctx context.Context, o *AddressTemplate) {
 		if isDone, _ := addressWithParentsCascadingCtx.Value(ctx); isDone {

db/factory/bobfactory_main.bob.go

@@ -143,11 +143,11 @@ func (f *Factory) FromExistingAddress(m *models.Address) *AddressTemplate {
 	o.H3cell = func() string { return m.H3cell }
 	o.ID = func() int32 { return m.ID }
 	o.Locality = func() string { return m.Locality }
-	o.Number = func() int32 { return m.Number }
 	o.PostalCode = func() string { return m.PostalCode }
 	o.Street = func() string { return m.Street }
 	o.Unit = func() string { return m.Unit }
 	o.Region = func() string { return m.Region }
+	o.Number = func() string { return m.Number }
 
 	ctx := context.Background()
 	if len(m.R.Mailers) > 0 {

db/migrations/00081_file_status_committing.sql

@@ -0,0 +1,4 @@
+-- +goose Up
+ALTER TYPE fileupload.FileStatusType ADD VALUE 'parsing' AFTER 'uploaded';
+ALTER TYPE fileupload.FileStatusType ADD VALUE 'committing' AFTER 'parsing';
+-- +goose Down

db/migrations/00082_address_number_text.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+ALTER TABLE address DROP COLUMN number_;
+ALTER TABLE address ADD COLUMN number_ TEXT;
+UPDATE address SET number_ = '';
+ALTER TABLE address ALTER COLUMN number_ SET NOT NULL;
+-- +goose Down
+ALTER TABLE address DROP COLUMN number_;
+ALTER TABLE address ADD COLUMN number_ INTEGER;
+UPDATE address SET number_ = 0;
+ALTER TABLE address ALTER COLUMN number_ SET NOT NULL;

db/models/address.bob.go

@@ -31,11 +31,11 @@ type Address struct {
 	H3cell     string            `db:"h3cell" `
 	ID         int32             `db:"id,pk" `
 	Locality   string            `db:"locality" `
-	Number     int32             `db:"number_" `
 	PostalCode string            `db:"postal_code" `
 	Street     string            `db:"street" `
 	Unit       string            `db:"unit" `
 	Region     string            `db:"region" `
+	Number     string            `db:"number_" `
 
 	R addressR `db:"-" `
 
@@ -62,7 +62,7 @@ type addressR struct {
 func buildAddressColumns(alias string) addressColumns {
 	return addressColumns{
 		ColumnsExpr: expr.NewColumnsExpr(
-			"country", "created", "geom", "h3cell", "id", "locality", "number_", "postal_code", "street", "unit", "region",
+			"country", "created", "geom", "h3cell", "id", "locality", "postal_code", "street", "unit", "region", "number_",
 		).WithParent("address"),
 		tableAlias: alias,
 		Country:    psql.Quote(alias, "country"),
@@ -71,11 +71,11 @@ func buildAddressColumns(alias string) addressColumns {
 		H3cell:     psql.Quote(alias, "h3cell"),
 		ID:         psql.Quote(alias, "id"),
 		Locality:   psql.Quote(alias, "locality"),
-		Number:     psql.Quote(alias, "number_"),
 		PostalCode: psql.Quote(alias, "postal_code"),
 		Street:     psql.Quote(alias, "street"),
 		Unit:       psql.Quote(alias, "unit"),
 		Region:     psql.Quote(alias, "region"),
+		Number:     psql.Quote(alias, "number_"),
 	}
 }
 
@@ -88,11 +88,11 @@ type addressColumns struct {
 	H3cell     psql.Expression
 	ID         psql.Expression
 	Locality   psql.Expression
-	Number     psql.Expression
 	PostalCode psql.Expression
 	Street     psql.Expression
 	Unit       psql.Expression
 	Region     psql.Expression
+	Number     psql.Expression
 }
 
 func (c addressColumns) Alias() string {
@@ -113,11 +113,11 @@ type AddressSetter struct {
 	H3cell     omit.Val[string]            `db:"h3cell" `
 	ID         omit.Val[int32]             `db:"id,pk" `
 	Locality   omit.Val[string]            `db:"locality" `
-	Number     omit.Val[int32]             `db:"number_" `
 	PostalCode omit.Val[string]            `db:"postal_code" `
 	Street     omit.Val[string]            `db:"street" `
 	Unit       omit.Val[string]            `db:"unit" `
 	Region     omit.Val[string]            `db:"region" `
+	Number     omit.Val[string]            `db:"number_" `
 }
 
 func (s AddressSetter) SetColumns() []string {
@@ -140,9 +140,6 @@ func (s AddressSetter) SetColumns() []string {
 	if s.Locality.IsValue() {
 		vals = append(vals, "locality")
 	}
-	if s.Number.IsValue() {
-		vals = append(vals, "number_")
-	}
 	if s.PostalCode.IsValue() {
 		vals = append(vals, "postal_code")
 	}
@@ -155,6 +152,9 @@ func (s AddressSetter) SetColumns() []string {
 	if s.Region.IsValue() {
 		vals = append(vals, "region")
 	}
+	if s.Number.IsValue() {
+		vals = append(vals, "number_")
+	}
 	return vals
 }
 
@@ -177,9 +177,6 @@ func (s AddressSetter) Overwrite(t *Address) {
 	if s.Locality.IsValue() {
 		t.Locality = s.Locality.MustGet()
 	}
-	if s.Number.IsValue() {
-		t.Number = s.Number.MustGet()
-	}
 	if s.PostalCode.IsValue() {
 		t.PostalCode = s.PostalCode.MustGet()
 	}
@@ -192,6 +189,9 @@ func (s AddressSetter) Overwrite(t *Address) {
 	if s.Region.IsValue() {
 		t.Region = s.Region.MustGet()
 	}
+	if s.Number.IsValue() {
+		t.Number = s.Number.MustGet()
+	}
 }
 
 func (s *AddressSetter) Apply(q *dialect.InsertQuery) {
@@ -237,32 +237,32 @@ func (s *AddressSetter) Apply(q *dialect.InsertQuery) {
 			vals[5] = psql.Raw("DEFAULT")
 		}
 
-		if s.Number.IsValue() {
+		if s.PostalCode.IsValue() {
-			vals[6] = psql.Arg(s.Number.MustGet())
+			vals[6] = psql.Arg(s.PostalCode.MustGet())
 		} else {
 			vals[6] = psql.Raw("DEFAULT")
 		}
 
-		if s.PostalCode.IsValue() {
+		if s.Street.IsValue() {
-			vals[7] = psql.Arg(s.PostalCode.MustGet())
+			vals[7] = psql.Arg(s.Street.MustGet())
 		} else {
 			vals[7] = psql.Raw("DEFAULT")
 		}
 
-		if s.Street.IsValue() {
+		if s.Unit.IsValue() {
-			vals[8] = psql.Arg(s.Street.MustGet())
+			vals[8] = psql.Arg(s.Unit.MustGet())
 		} else {
 			vals[8] = psql.Raw("DEFAULT")
 		}
 
-		if s.Unit.IsValue() {
+		if s.Region.IsValue() {
-			vals[9] = psql.Arg(s.Unit.MustGet())
+			vals[9] = psql.Arg(s.Region.MustGet())
 		} else {
 			vals[9] = psql.Raw("DEFAULT")
 		}
 
-		if s.Region.IsValue() {
+		if s.Number.IsValue() {
-			vals[10] = psql.Arg(s.Region.MustGet())
+			vals[10] = psql.Arg(s.Number.MustGet())
 		} else {
 			vals[10] = psql.Raw("DEFAULT")
 		}
@@ -320,13 +320,6 @@ func (s AddressSetter) Expressions(prefix ...string) []bob.Expression {
 		}})
 	}
 
-	if s.Number.IsValue() {
-		exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{
-			psql.Quote(append(prefix, "number_")...),
-			psql.Arg(s.Number),
-		}})
-	}
-
 	if s.PostalCode.IsValue() {
 		exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{
 			psql.Quote(append(prefix, "postal_code")...),
@@ -355,6 +348,13 @@ func (s AddressSetter) Expressions(prefix ...string) []bob.Expression {
 		}})
 	}
 
+	if s.Number.IsValue() {
+		exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{
+			psql.Quote(append(prefix, "number_")...),
+			psql.Arg(s.Number),
+		}})
+	}
+
 	return exprs
 }
 
@@ -850,11 +850,11 @@ type addressWhere[Q psql.Filterable] struct {
 	H3cell     psql.WhereMod[Q, string]
 	ID         psql.WhereMod[Q, int32]
 	Locality   psql.WhereMod[Q, string]
-	Number     psql.WhereMod[Q, int32]
 	PostalCode psql.WhereMod[Q, string]
 	Street     psql.WhereMod[Q, string]
 	Unit       psql.WhereMod[Q, string]
 	Region     psql.WhereMod[Q, string]
+	Number     psql.WhereMod[Q, string]
 }
 
 func (addressWhere[Q]) AliasedAs(alias string) addressWhere[Q] {
@@ -869,11 +869,11 @@ func buildAddressWhere[Q psql.Filterable](cols addressColumns) addressWhere[Q] {
 		H3cell:     psql.Where[Q, string](cols.H3cell),
 		ID:         psql.Where[Q, int32](cols.ID),
 		Locality:   psql.Where[Q, string](cols.Locality),
-		Number:     psql.Where[Q, int32](cols.Number),
 		PostalCode: psql.Where[Q, string](cols.PostalCode),
 		Street:     psql.Where[Q, string](cols.Street),
 		Unit:       psql.Where[Q, string](cols.Unit),
 		Region:     psql.Where[Q, string](cols.Region),
+		Number:     psql.Where[Q, string](cols.Number),
 	}
 }
 

platform/csv/csv.go

@@ -17,6 +17,7 @@ import (
 	"github.com/Gleipnir-Technology/nidus-sync/db"
 	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
+	"github.com/Gleipnir-Technology/nidus-sync/platform/geocode"
 	//"github.com/Gleipnir-Technology/nidus-sync/h3utils"
 	//"github.com/Gleipnir-Technology/nidus-sync/platform/geom"
 	//"github.com/Gleipnir-Technology/nidus-sync/platform/text"
@@ -30,18 +31,71 @@ import (
 type csvParserFunc[T any] = func(context.Context, bob.Tx, *models.FileuploadFile, *models.FileuploadCSV) ([]T, error)
 type csvProcessorFunc[T any] = func(context.Context, bob.Tx, *models.FileuploadFile, *models.FileuploadCSV, []T) error
 
-func ProcessJob(ctx context.Context, file_id int32, type_ enums.FileuploadCsvtype) error {
+func JobCommit(ctx context.Context, file_id int32) error {
+	txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("Failed to start transaction: %w", err)
+	}
+
+	f, err := models.FindFileuploadFile(ctx, txn, file_id)
+	if err != nil {
+		return fmt.Errorf("Failed to get csv file %d from DB: %w", file_id, err)
+	}
+	org, err := models.FindOrganization(ctx, txn, f.OrganizationID)
+	if err != nil {
+		return fmt.Errorf("Failed to get org %d from DB: %w", f.OrganizationID, err)
+	}
+
+	rows, err := models.FileuploadPools.Query(
+		models.SelectWhere.FileuploadPools.CSVFile.EQ(file_id),
+	).All(ctx, txn)
+	if err != nil {
+		return fmt.Errorf("Failed to get all rows of file %d: %w", file_id, err)
+	}
+	for _, row := range rows {
+		a := geocode.Address{
+			Country:    enums.CountrytypeUsa,
+			Locality:   row.AddressLocality,
+			Number:     row.AddressNumber,
+			PostalCode: row.AddressPostalCode,
+			Region:     row.AddressRegion,
+			Street:     row.AddressStreet,
+			Unit:       "",
+		}
+		address, err := geocode.EnsureAddress(ctx, txn, org, a)
+		if err != nil {
+			return fmt.Errorf("ensure address: %w", err)
+		}
+		log.Info().Int32("id", address.ID).Msg("made address")
+	}
+	return nil
+}
+func JobImport(ctx context.Context, file_id int32, type_ enums.FileuploadCsvtype) error {
 	var err error
 	switch type_ {
 	case enums.FileuploadCsvtypePoollist:
-		err = processCSV(ctx, file_id, parseCSVPoollist, processCSVPoollist)
+		err = importCSV(ctx, file_id, parseCSVPoollist, processCSVPoollist)
 	case enums.FileuploadCsvtypeFlyover:
-		err = processCSV(ctx, file_id, parseCSVFlyover, processCSVFlyover)
+		err = importCSV(ctx, file_id, parseCSVFlyover, processCSVFlyover)
+	}
+	if err != nil {
+		psql.Update(
+			um.Table("fileupload.csv"),
+			um.SetCol("status").ToArg("error"),
+			um.Where(psql.Quote("file_id").EQ(psql.Arg(file_id))),
+		).Exec(ctx, db.PGInstance.BobDB)
 	}
 	return err
 }
 
-func processCSV[T any](ctx context.Context, file_id int32, parser csvParserFunc[T], processor csvProcessorFunc[T]) error {
+func importCSV[T any](ctx context.Context, file_id int32, parser csvParserFunc[T], processor csvProcessorFunc[T]) error {
+	// Not done in the transaction so the state shows up immediately
+	_, err := psql.Update(
+		um.Table("fileupload.csv"),
+		um.SetCol("status").ToArg("processing"),
+		um.Where(psql.Quote("file_id").EQ(psql.Arg(file_id))),
+	).Exec(ctx, db.PGInstance.BobDB)
+
 	file, c, err := loadFileAndCSV(ctx, file_id)
 	if err != nil {
 		return fmt.Errorf("load file and csv: %w", err)

platform/csv/pool.go

@@ -16,7 +16,7 @@ import (
 	"github.com/Gleipnir-Technology/nidus-sync/db"
 	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
-	"github.com/Gleipnir-Technology/nidus-sync/h3utils"
+	"github.com/Gleipnir-Technology/nidus-sync/platform/geocode"
 	"github.com/Gleipnir-Technology/nidus-sync/platform/geom"
 	"github.com/Gleipnir-Technology/nidus-sync/platform/text"
 	"github.com/Gleipnir-Technology/nidus-sync/stadia"
@@ -130,41 +130,22 @@ type jobGeocode struct {
 	pool      *models.FileuploadPool
 }
 
-func geocode(ctx context.Context, txn bob.Tx, client *stadia.StadiaMaps, job *jobGeocode) error {
+func geocodePool(ctx context.Context, txn bob.Tx, client *stadia.StadiaMaps, job *jobGeocode) error {
 	pool := job.pool
-	sublog := log.With().
+	a := geocode.Address{
-		Str("pool.address_postal", pool.AddressPostalCode).
+		Number:     pool.AddressNumber,
-		Str("pool.address_street", pool.AddressStreet).
+		Locality:   pool.AddressLocality,
-		Str("pool.postal", pool.AddressPostalCode).
+		PostalCode: pool.AddressPostalCode,
-		Logger()
+		Street:     pool.AddressStreet,
-	req := stadia.StructuredGeocodeRequest{
-		Address:    &pool.AddressStreet,
-		Locality:   &pool.AddressLocality,
-		PostalCode: &pool.AddressPostalCode,
 	}
-	maybeAddServiceArea(&req, job.org)
+	address, err := geocode.Geocode(ctx, job.org, a)
-	resp, err := client.StructuredGeocode(ctx, req)
 	if err != nil {
-		return fmt.Errorf("client structured geocode failure on %s, %s, %s: %w", pool.AddressStreet, pool.AddressLocality, pool.AddressPostalCode, err)
+		addError(ctx, txn, job.csv, job.rownumber, 0, err.Error())
 	}
-	if len(resp.Features) > 1 {
+	geom_query := geom.PostgisPointQuery(address.Longitude, address.Latitude)
-		sublog.Warn().Int("len", len(resp.Features)).Msg("More than one feature")
-		addError(ctx, txn, job.csv, job.rownumber, 0, "The address provided matched more than one location")
-	}
-	feature := resp.Features[0]
-	if feature.Geometry.Type != "Point" {
-		return fmt.Errorf("wrong type %s from %s %s", feature.Geometry.Type, pool.AddressStreet, pool.AddressPostalCode)
-	}
-	longitude := feature.Geometry.Coordinates[0]
-	latitude := feature.Geometry.Coordinates[1]
-	cell, err := h3utils.GetCell(longitude, latitude, 15)
-	if err != nil {
-		return fmt.Errorf("failed to convert lat %f lng %f to h3 cell", longitude, latitude)
-	}
-	geom_query := geom.PostgisPointQuery(longitude, latitude)
 	_, err = psql.Update(
 		um.Table("fileupload.pool"),
-		um.SetCol("h3cell").ToArg(cell),
+		um.SetCol("h3cell").ToArg(address.Cell),
 		um.SetCol("geom").To(geom_query),
 		um.Where(psql.Quote("id").EQ(psql.Arg(pool.ID))),
 	).Exec(ctx, txn)
@@ -318,31 +299,6 @@ func processCSVPoollist(ctx context.Context, txn bob.Tx, file *models.Fileupload
 	return nil
 }
 
-func maybeAddServiceArea(req *stadia.StructuredGeocodeRequest, org *models.Organization) {
-	/*
-		if org.ServiceAreaXmax.IsNull() ||
-			org.ServiceAreaYmax.IsNull() ||
-			org.ServiceAreaXmin.IsNull() ||
-			org.ServiceAreaYmin.IsNull() {
-			return
-		}
-		xmax := org.ServiceAreaXmax.MustGet()
-		ymax := org.ServiceAreaYmax.MustGet()
-		xmin := org.ServiceAreaXmin.MustGet()
-		ymin := org.ServiceAreaYmin.MustGet()
-		req.BoundaryRectMaxLon = &xmax
-		req.BoundaryRectMaxLat = &ymax
-		req.BoundaryRectMinLon = &xmin
-		req.BoundaryRectMinLat = &ymin
-	*/
-	if org.ServiceAreaCentroidX.IsNull() || org.ServiceAreaCentroidY.IsNull() {
-		return
-	}
-	centroid_x := org.ServiceAreaCentroidX.MustGet()
-	centroid_y := org.ServiceAreaCentroidY.MustGet()
-	req.FocusPointLat = &centroid_y
-	req.FocusPointLng = &centroid_x
-}
 func parseHeaders(row []string) ([]headerPoolEnum, []string) {
 	result_enums := make([]headerPoolEnum, 0)
 	result_names := make([]string, 0)
@@ -414,7 +370,7 @@ func worker(ctx context.Context, txn bob.Tx, client *stadia.StadiaMaps, jobs <-c
 	defer wg.Done()
 
 	for job := range jobs {
-		err := geocode(ctx, txn, client, job)
+		err := geocodePool(ctx, txn, client, job)
 
 		if err != nil {
 			errors <- err

platform/geocode/geocode.go

@@ -0,0 +1,187 @@
+package geocode
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/Gleipnir-Technology/bob"
+	"github.com/Gleipnir-Technology/bob/dialect/psql"
+	"github.com/Gleipnir-Technology/bob/dialect/psql/im"
+	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
+	"github.com/Gleipnir-Technology/nidus-sync/db/models"
+	"github.com/Gleipnir-Technology/nidus-sync/h3utils"
+	"github.com/Gleipnir-Technology/nidus-sync/stadia"
+	"github.com/stephenafamo/scan"
+	//"github.com/rs/zerolog/log"
+	"github.com/uber/h3-go/v4"
+)
+
+type Address struct {
+	Country    enums.Countrytype
+	Locality   string
+	Number     string
+	PostalCode string
+	Region     string
+	Street     string
+	Unit       string
+}
+type GeocodeResult struct {
+	Address   Address
+	Cell      h3.Cell
+	Longitude float64
+	Latitude  float64
+}
+
+func (a Address) String() string {
+	return fmt.Sprintf("%s %s, %s, %s, %s, %s", a.Number, a.Street, a.Locality, a.Region, a.PostalCode, a.Country)
+}
+
+var client *stadia.StadiaMaps
+
+// Either get an address that matches, or create a new address. Either way, return an address
+// This will make a call to a structured geocode service, so it's slow.
+func EnsureAddress(ctx context.Context, txn bob.Tx, org *models.Organization, a Address) (*models.Address, error) {
+	address, err := models.Addresses.Query(
+		models.SelectWhere.Addresses.Country.EQ(a.Country),
+		models.SelectWhere.Addresses.Locality.EQ(a.Locality),
+		models.SelectWhere.Addresses.Number.EQ(a.Number),
+		models.SelectWhere.Addresses.PostalCode.EQ(a.PostalCode),
+		models.SelectWhere.Addresses.Region.EQ(a.Region),
+		models.SelectWhere.Addresses.Street.EQ(a.Street),
+		models.SelectWhere.Addresses.Unit.EQ(a.Unit),
+	).One(ctx, txn)
+	if err == nil {
+		return address, nil
+	}
+	// Geocode
+	geo, err := Geocode(ctx, org, a)
+	if err != nil {
+		return nil, fmt.Errorf("geocode: %w", err)
+	}
+
+	type _row struct {
+		ID int32 `db:"id"`
+	}
+	created := time.Now()
+	row, err := bob.One(ctx, txn, psql.Insert(
+		im.Into("address", "country", "created", "geom", "h3cell", "id", "locality", "number", "postal_code", "region", "street", "unit"),
+		im.Values(
+			psql.Arg(geo.Address.Country),
+			psql.Arg(created),
+			psql.F("ST_Point", geo.Longitude, geo.Latitude, 4326),
+			psql.Arg(geo.Cell),
+			psql.Raw("DEFAULT"),
+			psql.Arg(geo.Address.Locality),
+			psql.Arg(geo.Address.Number),
+			psql.Arg(geo.Address.PostalCode),
+			psql.Arg(geo.Address.Region),
+			psql.Arg(geo.Address.Street),
+			psql.Raw("''"),
+		),
+		im.Returning("id"),
+	), scan.StructMapper[_row]())
+	if err != nil {
+		return nil, fmt.Errorf("insert: %w", err)
+	}
+
+	return &models.Address{
+		Country:    geo.Address.Country,
+		Created:    created,
+		Geom:       "",
+		H3cell:     "",
+		ID:         row.ID,
+		Locality:   geo.Address.Locality,
+		PostalCode: geo.Address.PostalCode,
+		Street:     geo.Address.Street,
+		Unit:       geo.Address.Unit,
+		Region:     geo.Address.Region,
+		Number:     geo.Address.Number,
+	}, nil
+}
+
+func Geocode(ctx context.Context, org *models.Organization, a Address) (GeocodeResult, error) {
+	street := fmt.Sprintf("%s %s", a.Number, a.Street)
+	country_s := a.Country.String()
+	/*
+		sublog := log.With().
+			Str("street", street).
+			Str("country", country).
+			Str("locality", a.Locality).
+			Str("postal", a.PostalCode).
+			Str("region", a.Region).
+			Logger()
+	*/
+	req := stadia.StructuredGeocodeRequest{
+		Address:    &street,
+		Country:    &country_s,
+		Locality:   &a.Locality,
+		PostalCode: &a.PostalCode,
+		Region:     &a.Region,
+	}
+	maybeAddServiceArea(&req, org)
+	resp, err := client.StructuredGeocode(ctx, req)
+	if err != nil {
+		return GeocodeResult{}, fmt.Errorf("client structured geocode failure on %s: %w", a.String(), err)
+	}
+	if len(resp.Features) > 1 {
+		return GeocodeResult{}, fmt.Errorf("%s matched more than one location", a.String())
+	}
+	feature := resp.Features[0]
+	if feature.Geometry.Type != "Point" {
+		return GeocodeResult{}, fmt.Errorf("wrong type %s from %s", feature.Geometry.Type, a.String())
+	}
+	longitude := feature.Geometry.Coordinates[0]
+	latitude := feature.Geometry.Coordinates[1]
+	cell, err := h3utils.GetCell(longitude, latitude, 15)
+	if err != nil {
+		return GeocodeResult{}, fmt.Errorf("failed to convert lat %f lng %f to h3 cell", longitude, latitude)
+	}
+	var country enums.Countrytype
+	country_s = strings.ToLower(feature.Properties.CountryA)
+	err = country.Scan(country_s)
+	if err != nil {
+		return GeocodeResult{}, fmt.Errorf("failed to scan country '%s': %w", country_s, err)
+	}
+	return GeocodeResult{
+		Address: Address{
+			Country:    country,
+			Locality:   feature.Properties.Locality,
+			Number:     feature.Properties.HouseNumber,
+			PostalCode: feature.Properties.PostalCode,
+			Region:     feature.Properties.Region,
+			Street:     feature.Properties.Street,
+			Unit:       "",
+		},
+		Cell:      cell,
+		Longitude: feature.Geometry.Coordinates[0],
+		Latitude:  feature.Geometry.Coordinates[1],
+	}, nil
+}
+
+func maybeAddServiceArea(req *stadia.StructuredGeocodeRequest, org *models.Organization) {
+	if org.ServiceAreaXmax.IsNull() ||
+		org.ServiceAreaYmax.IsNull() ||
+		org.ServiceAreaXmin.IsNull() ||
+		org.ServiceAreaYmin.IsNull() {
+		return
+	}
+	xmax := org.ServiceAreaXmax.MustGet()
+	ymax := org.ServiceAreaYmax.MustGet()
+	xmin := org.ServiceAreaXmin.MustGet()
+	ymin := org.ServiceAreaYmin.MustGet()
+	req.BoundaryRectMaxLon = &xmax
+	req.BoundaryRectMaxLat = &ymax
+	req.BoundaryRectMinLon = &xmin
+	req.BoundaryRectMinLat = &ymin
+
+	if org.ServiceAreaCentroidX.IsNull() || org.ServiceAreaCentroidY.IsNull() {
+		return
+	}
+	centroid_x := org.ServiceAreaCentroidX.MustGet()
+	centroid_y := org.ServiceAreaCentroidY.MustGet()
+
+	req.FocusPointLat = &centroid_y
+	req.FocusPointLng = &centroid_x
+}

platform/organization.go

@@ -0,0 +1,6 @@
+package platform
+
+type Organization struct {
+	ID   int
+	Name string
+}

platform/pool.go

@@ -9,14 +9,9 @@ import (
 	"github.com/Gleipnir-Technology/bob"
 	"github.com/Gleipnir-Technology/bob/dialect/psql"
 	"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
-	"github.com/Gleipnir-Technology/nidus-sync/background"
 	"github.com/Gleipnir-Technology/nidus-sync/db"
 	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
-	"github.com/Gleipnir-Technology/nidus-sync/userfile"
-	"github.com/aarondl/opt/omit"
-	"github.com/aarondl/opt/omitnull"
-	"github.com/rs/zerolog/log"
 	"github.com/stephenafamo/scan"
 )
 
@@ -52,43 +47,6 @@ type Upload struct {
 	Status  string    `db:"status"`
 }
 
-func NewUpload(ctx context.Context, u *models.User, upload userfile.FileUpload, t enums.FileuploadCsvtype) (Upload, error) {
-	txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
-	if err != nil {
-		return Upload{}, fmt.Errorf("Failed to begin transaction: %w", err)
-	}
-	defer txn.Rollback(ctx)
-
-	file, err := models.FileuploadFiles.Insert(&models.FileuploadFileSetter{
-		ContentType:    omit.From(upload.ContentType),
-		Created:        omit.From(time.Now()),
-		CreatorID:      omit.From(u.ID),
-		Deleted:        omitnull.FromPtr[time.Time](nil),
-		Name:           omit.From(upload.Name),
-		OrganizationID: omit.From(u.OrganizationID),
-		Status:         omit.From(enums.FileuploadFilestatustypeUploaded),
-		SizeBytes:      omit.From(int32(upload.SizeBytes)),
-		FileUUID:       omit.From(upload.UUID),
-	}).One(ctx, txn)
-	if err != nil {
-		return Upload{}, fmt.Errorf("Failed to create file upload: %w", err)
-	}
-	_, err = models.FileuploadCSVS.Insert(&models.FileuploadCSVSetter{
-		Committed: omitnull.FromPtr[time.Time](nil),
-		FileID:    omit.From(file.ID),
-		Rowcount:  omit.From(int32(0)),
-		Type:      omit.From(t),
-	}).One(ctx, txn)
-	if err != nil {
-		return Upload{}, fmt.Errorf("Failed to create csv: %w", err)
-	}
-	log.Info().Int32("id", file.ID).Msg("Created new pool CSV upload")
-	txn.Commit(ctx)
-	background.ProcessUpload(file.ID, t)
-	return Upload{
-		ID: file.ID,
-	}, nil
-}
 func GetUploadDetail(ctx context.Context, organization_id int32, file_id int32) (UploadPoolDetail, error) {
 	file, err := models.FindFileuploadFile(ctx, db.PGInstance.BobDB, file_id)
 	if err != nil {

platform/upload.go

@@ -9,8 +9,14 @@ import (
 	"github.com/Gleipnir-Technology/bob/dialect/psql"
 	"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
 	"github.com/Gleipnir-Technology/bob/dialect/psql/um"
+	"github.com/Gleipnir-Technology/nidus-sync/background"
 	"github.com/Gleipnir-Technology/nidus-sync/db"
+	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
+	"github.com/Gleipnir-Technology/nidus-sync/userfile"
+	"github.com/aarondl/opt/omit"
+	"github.com/aarondl/opt/omitnull"
+	"github.com/rs/zerolog/log"
 	"github.com/stephenafamo/scan"
 )
 
@@ -35,8 +41,55 @@ type UploadSummary struct {
 	Type        string    `db:"type"`
 }
 
+func NewUpload(ctx context.Context, u *models.User, upload userfile.FileUpload, t enums.FileuploadCsvtype) (Upload, error) {
+	txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
+	if err != nil {
+		return Upload{}, fmt.Errorf("Failed to begin transaction: %w", err)
+	}
+	defer txn.Rollback(ctx)
+
+	file, err := models.FileuploadFiles.Insert(&models.FileuploadFileSetter{
+		ContentType:    omit.From(upload.ContentType),
+		Created:        omit.From(time.Now()),
+		CreatorID:      omit.From(u.ID),
+		Deleted:        omitnull.FromPtr[time.Time](nil),
+		Name:           omit.From(upload.Name),
+		OrganizationID: omit.From(u.OrganizationID),
+		Status:         omit.From(enums.FileuploadFilestatustypeUploaded),
+		SizeBytes:      omit.From(int32(upload.SizeBytes)),
+		FileUUID:       omit.From(upload.UUID),
+	}).One(ctx, txn)
+	if err != nil {
+		return Upload{}, fmt.Errorf("Failed to create file upload: %w", err)
+	}
+	_, err = models.FileuploadCSVS.Insert(&models.FileuploadCSVSetter{
+		Committed: omitnull.FromPtr[time.Time](nil),
+		FileID:    omit.From(file.ID),
+		Rowcount:  omit.From(int32(0)),
+		Type:      omit.From(t),
+	}).One(ctx, txn)
+	if err != nil {
+		return Upload{}, fmt.Errorf("Failed to create csv: %w", err)
+	}
+	log.Info().Int32("id", file.ID).Msg("Created new pool CSV upload")
+	txn.Commit(ctx)
+	background.ProcessUpload(file.ID, t)
+	return Upload{
+		ID: file.ID,
+	}, nil
+}
 func UploadCommit(ctx context.Context, org *models.Organization, file_id int32) error {
-	return nil
+	// Create addresses for each row
+	// Create sites for each row
+	// Create pools for each row
+	_, err := psql.Update(
+		um.Table(models.FileuploadFiles.Alias()),
+		um.SetCol("status").ToArg("committed"),
+		um.Where(psql.Quote("id").EQ(psql.Arg(file_id))),
+		um.Where(psql.Quote("organization_id").EQ(psql.Arg(org.ID))),
+	).Exec(ctx, db.PGInstance.BobDB)
+	background.CommitUpload(file_id)
+	return err
 }
 func UploadDiscard(ctx context.Context, org *models.Organization, file_id int32) error {
 	_, err := psql.Update(

platform/user.go

@@ -0,0 +1,14 @@
+package platform
+
+import (
+	"github.com/Gleipnir-Technology/nidus-sync/notification"
+)
+
+type User struct {
+	DisplayName   string
+	Initials      string
+	Notifications []notification.Notification
+	Organization  Organization
+	Role          string
+	Username      string
+}

stadia/response_type.go

@@ -33,19 +33,32 @@ type GeocodeGeometry struct {
 
 // GeocodeProperties contains the properties of a geocoding result
 type GeocodeProperties struct {
-	GID                   string                 `json:"gid"`
+	Addendum    map[string]interface{} `json:"addendum,omitempty"`
-	Layer                 string                 `json:"layer"`
+	Accuracy    string                 `json:"accuracy"`     // 'point'
-	Sources               []GeocodeSource        `json:"sources"`
+	Confidence  float64                `json:"confidence"`   // 1
-	Precision             string                 `json:"precision"`
+	Country     string                 `json:"country"`      // 'United States'
-	Name                  string                 `json:"name"`
+	CountryA    string                 `json:"country_a"`    // 'USA'
-	FormattedAddressLines []string               `json:"formatted_address_lines"`
+	CountryCode string                 `json:"country_code"` // 'US'
-	FormattedAddressLine  string                 `json:"formatted_address_line"`
+	CountryGID  string                 `json:"country_gid"`  // 'whosonfirst:country:85633793'
-	CoarseLocation        string                 `json:"coarse_location"`
+	County      string                 `json:"county"`       // "Tulare County"
-	AddressComponents     AddressComponents      `json:"address_components,omitempty"`
+	CountyA     string                 `json:"county_a"`     // 'TL'
-	Context               GeocodeContext         `json:"context,omitempty"`
+	CountyGID   string                 `json:"county_gid"`   // 'whosonfirst:county:102082895'
-	Confidence            float64                `json:"confidence,omitempty"`
+	GID         string                 `json:"gid"`          // 'openaddresses:address:us/ca/tulare-addresses-county:fe9dfab3d45c4550'
-	Distance              float64                `json:"distance,omitempty"`
+	HouseNumber string                 `json:"housenumber"`  // '1234'
-	Addendum              map[string]interface{} `json:"addendum,omitempty"`
+	ID          string                 `json:"id"`           // us/ca/tulare-addresses-county:fe9dfab3d45c4550
+	Label       string                 `json:"label"`        // 1234 Main St, Dinuba, CA, USA
+	Layer       string                 `json:"layer"`        // 'address'
+	Locality    string                 `json:"locality"`     // 'Dinuba'
+	LocalityGID string                 `json:"locality_gid"` // 'whosonfirst:locality:85922491'
+	MatchType   string                 `json:"match_type"`   // 'exact'
+	Name        string                 `json:"name"`         // '1234 Main St'
+	PostalCode  string                 `json:"postalcode"`   // '93618'
+	Region      string                 `json:"region"`       // 'California'
+	RegionA     string                 `json:"region_a"`     // 'CA'
+	RegionGID   string                 `json:"region_gid"`   // 'whosonfirst:region:85688637'
+	Source      string                 `json:"source"`       // 'openaddresses'
+	SourceID    string                 `json:"source"`       // 'us/ca/tulare-addresses-county:fe9dfab3d45c4550'
+	Street      string                 `json:"street"`       // 'Main Street'
 }
 
 // GeocodeSource represents a source of geocoding data
@@ -53,34 +66,3 @@ type GeocodeSource struct {
 	Source   string `json:"source"`
 	SourceID string `json:"source_id"`
 }
-
-// AddressComponents represents the structured components of an address
-type AddressComponents struct {
-	Number     string `json:"number,omitempty"`
-	Street     string `json:"street,omitempty"`
-	Unit       string `json:"unit,omitempty"`
-	PostalCode string `json:"postal_code,omitempty"`
-}
-
-// GeocodeContext represents the geographic context of a result
-type GeocodeContext struct {
-	WhosOnFirst WhosOnFirstContext `json:"whosonfirst,omitempty"`
-	ISO3166A2   string             `json:"iso_3166_a2,omitempty"`
-	ISO3166A3   string             `json:"iso_3166_a3,omitempty"`
-}
-
-// WhosOnFirstContext contains geographic hierarchy information
-type WhosOnFirstContext struct {
-	Country       *ContextPlace `json:"country,omitempty"`
-	Region        *ContextPlace `json:"region,omitempty"`
-	County        *ContextPlace `json:"county,omitempty"`
-	Locality      *ContextPlace `json:"locality,omitempty"`
-	Neighbourhood *ContextPlace `json:"neighbourhood,omitempty"`
-	Borough       *ContextPlace `json:"borough,omitempty"`
-}
-
-// ContextPlace represents a place in the geographic hierarchy
-type ContextPlace struct {
-	GID  string `json:"gid"`
-	Name string `json:"name"`
-}

stadia/stadia.go

@@ -2,9 +2,9 @@ package stadia
 
 import (
 	"crypto/tls"
+	"github.com/rs/zerolog/log"
 	"os"
 	"resty.dev/v3"
-	//"github.com/rs/zerolog/log"
 )
 
 type StadiaMaps struct {
@@ -20,6 +20,7 @@ func NewStadiaMaps(api_key string) *StadiaMaps {
 	//r := resty.New().SetDebug(true)
 	r := resty.New()
 	if os.Getenv("STADIA_INSECURE_SKIP_VERIFY") != "" {
+		log.Warn().Msg("Using insecure TLS verification settings")
 		r.SetTLSClientConfig(&tls.Config{
 			InsecureSkipVerify: true,
 		})

sync/dash.go

@@ -15,6 +15,7 @@ import (
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
 	"github.com/Gleipnir-Technology/nidus-sync/html"
 	nhttp "github.com/Gleipnir-Technology/nidus-sync/http"
+	"github.com/Gleipnir-Technology/nidus-sync/platform"
 	"github.com/go-chi/chi/v5"
 	"github.com/google/uuid"
 )
@@ -30,12 +31,12 @@ type contentSource struct {
 	Treatments  []Treatment
 	//TreatmentCadence TreatmentCadence
 	TreatmentModels []TreatmentModel
-	User            User
+	User            platform.User
 }
 type contentTrap struct {
 	MapData ComponentMap
 	Trap    Trap
-	User    User
+	User    platform.User
 }
 type contentDashboard struct {
 	CountTraps           int
@@ -49,7 +50,7 @@ type contentDashboard struct {
 }
 
 type contentLayoutTest struct {
-	User User
+	User platform.User
 }
 type ContentDistrict struct {
 	MapboxToken string
@@ -103,7 +104,7 @@ func getSource(ctx context.Context, r *http.Request, org *models.Organization, u
 	if err != nil {
 		return nil, nhttp.NewError("globalid is not a UUID: %w", nil)
 	}
-	userContent, err := contentForUser(r.Context(), user)
+	userContent, err := auth.ContentForUser(r.Context(), user)
 	if err != nil {
 		return nil, nhttp.NewError("Failed to get user content: %w", err)
 	}
@@ -172,7 +173,7 @@ func getTrap(ctx context.Context, r *http.Request, org *models.Organization, use
 	if err != nil {
 		return nil, nhttp.NewError("globalid is not a UUID: %w", nil)
 	}
-	userContent, err := contentForUser(r.Context(), user)
+	userContent, err := auth.ContentForUser(r.Context(), user)
 	if err != nil {
 		return nil, nhttp.NewError("Failed to get user content: %w", err)
 	}
@@ -254,7 +255,7 @@ func dashboard(ctx context.Context, w http.ResponseWriter, org *models.Organizat
 		},
 		RecentRequests: requests,
 	}
-	userContent, err := contentForUser(ctx, user)
+	userContent, err := auth.ContentForUser(ctx, user)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return

sync/handler.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
 	"github.com/Gleipnir-Technology/nidus-sync/html"
 	nhttp "github.com/Gleipnir-Technology/nidus-sync/http"
+	"github.com/Gleipnir-Technology/nidus-sync/platform"
 	"github.com/rs/zerolog/log"
 )
 
@@ -19,14 +20,14 @@ type contentAuthenticated[T any] struct {
 	Config       html.ContentConfig
 	Organization *models.Organization
 	URL          html.ContentURL
-	User         User
+	User         platform.User
 }
 
 // w http.ResponseWriter, r *http.Request, u *models.User) {
 func authenticatedHandler[T any](f handlerFunctionGet[T]) http.Handler {
 	return auth.NewEnsureAuth(func(w http.ResponseWriter, r *http.Request, u *models.User) {
 		ctx := r.Context()
-		userContent, err := contentForUser(ctx, u)
+		userContent, err := auth.ContentForUser(ctx, u)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return

sync/types.go

@@ -3,7 +3,6 @@ package sync
 import (
 	"time"
 
-	"github.com/Gleipnir-Technology/nidus-sync/notification"
 	"github.com/google/uuid"
 	"github.com/uber/h3-go/v4"
 )
@@ -48,9 +47,6 @@ type ContentReportDetail struct {
 }
 type ContentReportDiagnostic struct {
 }
-type ContentDashboardLoading struct {
-	User User
-}
 
 type Inspection struct {
 	Action     string
@@ -63,20 +59,8 @@ type Link struct {
 	Href  string
 	Title string
 }
-type Organization struct {
-	ID   int
-	Name string
-}
 type ServiceRequestSummary struct {
 	Date     time.Time
 	Location string
 	Status   string
 }
-type User struct {
-	DisplayName   string
-	Initials      string
-	Notifications []notification.Notification
-	Organization  Organization
-	Role          string
-	Username      string
-}

sync/upload.go

@@ -85,7 +85,7 @@ func postUploadCommit(ctx context.Context, r *http.Request, org *models.Organiza
 	}
 	err = platform.UploadCommit(ctx, org, int32(file_id_))
 	if err != nil {
-		return "", nhttp.NewError("Failed to mark discarded: %w", err)
+		return "", nhttp.NewError("Failed to mark committed: %w", err)
 	}
 	return "/configuration/upload", nil
 }

sync/utils.go

@@ -12,7 +12,6 @@ import (
 	"github.com/Gleipnir-Technology/nidus-sync/db"
 	"github.com/Gleipnir-Technology/nidus-sync/db/models"
 	"github.com/Gleipnir-Technology/nidus-sync/db/sql"
-	"github.com/Gleipnir-Technology/nidus-sync/notification"
 	"github.com/google/uuid"
 	"github.com/uber/h3-go/v4"
 )
@@ -77,41 +76,6 @@ func sourceByGlobalId(ctx context.Context, org *models.Organization, id uuid.UUI
 	return toTemplateBreedingSource(row), nil
 }
 
-func extractInitials(name string) string {
-	parts := strings.Fields(name)
-	var initials strings.Builder
-
-	for _, part := range parts {
-		if len(part) > 0 {
-			initials.WriteString(strings.ToUpper(string(part[0])))
-		}
-	}
-
-	return initials.String()
-}
-
-func contentForUser(ctx context.Context, user *models.User) (User, error) {
-	notifications, err := notification.ForUser(ctx, user)
-	if err != nil {
-		return User{}, err
-	}
-	org := user.R.Organization
-	var organization Organization
-	if org != nil {
-		organization.ID = int(org.ID)
-		organization.Name = org.Name
-	}
-	return User{
-		DisplayName:   user.DisplayName,
-		Initials:      extractInitials(user.DisplayName),
-		Notifications: notifications,
-		Organization:  organization,
-		Role:          user.Role.String(),
-		Username:      user.Username,
-	}, nil
-
-}
-
 func trapsBySource(ctx context.Context, org *models.Organization, sourceID uuid.UUID) ([]TrapNearby, error) {
 	locations, err := sql.TrapLocationBySourceID(org.ID, sourceID).All(ctx, db.PGInstance.BobDB)
 	if err != nil {