Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix up upload by ID

Browse source 
Show the street number as well as the rest of the address, emit an event
when the upload is processed, actually check if pools are existing, etc.
This commit is contained in:
Eli Ribble 2026-03-19 15:31:04 +00:00
parent 97c9269215
commit 786a6c16a3
No known key found for this signature in database
5 changed files with 68 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

10
html/template/sync/upload-by-id.html
View file

@ -128,7 +128,7 @@ document.addEventListener('DOMContentLoaded', onLoad);
<strong>Error:</strong>{{ .Message }}
</div>
{{ end }}
{{ if eq .C.Upload.Status "uploaded" }}
{{ if (or (eq .C.Upload.Status "uploaded") (eq .C.Upload.Status "parsing")) }}
<div class="alert alert-info" role="alert">
<i class="bi bi-exclamation-triangle me-2"></i>
<strong>Working:</strong> File is still processing... refresh this
@ -147,6 +147,7 @@ document.addEventListener('DOMContentLoaded', onLoad);
<thead class="table-light">
<tr class="header">
<th></th>
<th>Number</th>
<th>Street</th>
<th>City</th>
<th>Post</th>
@ -169,9 +170,10 @@ document.addEventListener('DOMContentLoaded', onLoad);
{{ end }}
</td>
<td>{{ .Street }}</td>
<td>{{ .Locality }}</td>
<td>{{ .PostalCode }}</td>
<td>{{ .Address.Number }}</td>
<td>{{ .Address.Street }}</td>
<td>{{ .Address.Locality }}</td>
<td>{{ .Address.PostalCode }}</td>
<td>
<span class="badge status {{ .Status }}"
>{{ .Status|title }}</span

10
platform/csv/csv.go
View file

@ -17,6 +17,7 @@ import (
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
"github.com/Gleipnir-Technology/nidus-sync/platform/event"
"github.com/Gleipnir-Technology/nidus-sync/platform/geocode"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
//"github.com/Gleipnir-Technology/nidus-sync/h3utils"
@ -33,6 +34,7 @@ type csvParserFunc[T any] = func(context.Context, bob.Tx, *models.FileuploadFile
type csvProcessorFunc[T any] = func(context.Context, bob.Tx, *models.FileuploadFile, *models.FileuploadCSV, []T) error
func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error {
log.Debug().Int32("file_id", file_id).Msg("begin job commit")
file, err := models.FindFileuploadFile(ctx, txn, file_id)
if err != nil {
return fmt.Errorf("Failed to get csv file %d from DB: %w", file_id, err)
@ -166,9 +168,16 @@ func JobCommit(ctx context.Context, txn bob.Executor, file_id int32) error {
if err != nil {
return fmt.Errorf("update file status to committed: %w", err)
}
event.Updated(event.TypeFileCSV, file.OrganizationID, strconv.Itoa(int(file.ID)))
return nil
}
func JobImport(ctx context.Context, txn bob.Executor, file_id int32) error {
file, err := models.FileuploadFiles.Query(
models.SelectWhere.FileuploadFiles.ID.EQ(file_id),
).One(ctx, txn)
if err != nil {
return fmt.Errorf("find file: %w", err)
}
csv, err := models.FileuploadCSVS.Query(
models.SelectWhere.FileuploadCSVS.FileID.EQ(file_id),
).One(ctx, txn)
@ -192,6 +201,7 @@ func JobImport(ctx context.Context, txn bob.Executor, file_id int32) error {
log.Error().Err(err).Msg("Failed to set upload to error status")
}
}
event.Updated(event.TypeFileCSV, file.OrganizationID, strconv.Itoa(int(file.ID)))
return nil
}

31
platform/csv/flyover.go
View file

@ -144,6 +144,7 @@ func insertFlyover(ctx context.Context, txn bob.Tx, file *models.FileuploadFile,
setter := models.FileuploadPoolSetter{
// required fields
//AddressLocality: omit.From(),
//AddressNumber: omit.From(),
//AddressPostalCode: omit.From(),
//AddressRegion: omit.From(),
//AddressStreet: omit.From(),
@ -156,8 +157,9 @@ func insertFlyover(ctx context.Context, txn bob.Tx, file *models.FileuploadFile,
Geom: omitnull.FromPtr[string](nil),
H3cell: omitnull.FromPtr[string](nil),
// ID - generated
IsInDistrict: omit.From(false),
IsNew: omit.From(true),
IsInDistrict: omit.From(false),
// Calculated after we gather the address data
//IsNew: omit.From(true),
LineNumber: omit.From(line_number),
Notes: omit.From(""),
PropertyOwnerName: omit.From(""),
@ -207,6 +209,11 @@ func insertFlyover(ctx context.Context, txn bob.Tx, file *models.FileuploadFile,
}
}
setter.Tags = omit.From(db.ConvertToPGData(map[string]string{}))
is_existing, err := hasExistingPool(ctx, txn, &setter)
if err != nil {
return nil, fmt.Errorf("has existing pool: %w", err)
}
setter.IsNew = omit.From(!is_existing)
flyover, err := models.FileuploadPools.Insert(&setter).One(ctx, txn)
if err != nil {
return nil, fmt.Errorf("Failed to create flyover: %w", err)
@ -239,6 +246,26 @@ func insertFlyover(ctx context.Context, txn bob.Tx, file *models.FileuploadFile,
}
return flyover, nil
}
func hasExistingPool(ctx context.Context, txn bob.Executor, setter *models.FileuploadPoolSetter) (bool, error) {
exists, err := models.Addresses.Query(
models.SelectWhere.Addresses.Locality.EQ(setter.AddressLocality.GetOr("")),
models.SelectWhere.Addresses.Number.EQ(setter.AddressNumber.GetOr("")),
models.SelectWhere.Addresses.PostalCode.EQ(setter.AddressPostalCode.GetOr("")),
//models.SelectWhere.Addresses.Region.EQ(setter.AddressRegion.GetOr("")),
models.SelectWhere.Addresses.Street.EQ(setter.AddressStreet.GetOr("")),
).Exists(ctx, txn)
if err != nil {
return false, fmt.Errorf("query address: %w", err)
}
log.Debug().
Str("number", setter.AddressNumber.GetOr("")).
Str("postal_code", setter.AddressPostalCode.GetOr("")).
Str("region", setter.AddressRegion.GetOr("")).
Str("street", setter.AddressStreet.GetOr("")).
Str("locality", setter.AddressLocality.GetOr("")).
Bool("exists", exists).Msg("checking pool exists")
return exists, nil
}
func insertPoollistRow(ctx context.Context, txn bob.Tx, file *models.FileuploadFile, c *models.FileuploadCSV, line_number int32, header_types []headerFlyoverEnum, header_names []string, row []string) (*models.FileuploadPool, error) {
tags := make(map[string]string, 0)
// Start with a setter with default values, comment out the required fields to ensure they're set

5
platform/event/event.go
View file

@ -72,6 +72,7 @@ type ResourceType int
const (
TypeUnknown = iota
TypeFileCSV
TypeRMONuisance
TypeRMOReport
TypeRMOWater
@ -104,6 +105,8 @@ func Send(env Envelope) {
}
func resourceString(t ResourceType) string {
switch t {
case TypeFileCSV:
return "sync:filecsv"
case TypeRMONuisance:
return "rmo:nuisance"
case TypeRMOReport:
@ -116,6 +119,8 @@ func resourceString(t ResourceType) string {
}
func makeURI(t ResourceType, id string) string {
switch t {
case TypeFileCSV:
return config.MakeURLNidus("/upload/%s", id)
case TypeRMONuisance:
return config.MakeURLReport("/report/%s", id)
case TypeRMOWater:

34
platform/pool.go
View file

@ -9,6 +9,7 @@ import (
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
"github.com/Gleipnir-Technology/nidus-sync/platform/types"
)
type UploadPoolDetail struct {
@ -28,14 +29,11 @@ type UploadPoolError struct {
Message string
}
type UploadPoolRow struct {
Condition string
Errors []UploadPoolError
Locality string
PostalCode string
Region string
Status string
Street string
Tags map[string]string
Address types.Address
Condition string
Errors []UploadPoolError
Status string
Tags map[string]string
}
type Upload struct {
Created time.Time `db:"created"`
@ -97,14 +95,18 @@ func getUploadPoollistDetail(ctx context.Context, file *models.FileuploadFile) (
errors = []UploadPoolError{}
}
pools = append(pools, UploadPoolRow{
Condition: r.Condition.String(),
Errors: errors,
Locality: r.AddressLocality,
PostalCode: r.AddressPostalCode,
Region: r.AddressRegion,
Status: status,
Street: r.AddressStreet,
Tags: tags,
Address: types.Address{
Country: "usa",
Locality: r.AddressLocality,
Number: r.AddressNumber,
PostalCode: r.AddressPostalCode,
Region: r.AddressRegion,
Street: r.AddressStreet,
},
Condition: r.Condition.String(),
Errors: errors,
Status: status,
Tags: tags,
})
}
return UploadPoolDetail{