From a65f1e0776b56c77d102dff75a5fc8c1bb8afc24 Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Mon, 16 Feb 2026 17:59:18 +0000 Subject: [PATCH] Fix up attaching errors to rows --- db/dberrors/fileupload.pool.bob.go | 2 +- db/dbinfo/fileupload.pool.bob.go | 57 ++-- db/factory/bobfactory_main.bob.go | 6 +- db/factory/fileupload.pool.bob.go | 284 +++++++++--------- db/migrations/00062_csv_pool_phone_fix.sql | 5 - .../00062_fileupload_pool_rebuild.sql | 44 +++ db/migrations/00063_pool_empty_status.sql | 2 - db/migrations/00064_csv_pool_geom_srid.sql | 6 - db/migrations/00065_pool_tags.sql | 4 - db/models/fileupload.pool.bob.go | 168 +++++------ platform/csv/pool.go | 33 +- platform/pool.go | 18 +- scss/sync/pool-by-id.scss | 6 +- tools/delete-all-pool-uploads.sql | 8 + 14 files changed, 338 insertions(+), 305 deletions(-) delete mode 100644 db/migrations/00062_csv_pool_phone_fix.sql create mode 100644 db/migrations/00062_fileupload_pool_rebuild.sql delete mode 100644 db/migrations/00063_pool_empty_status.sql delete mode 100644 db/migrations/00064_csv_pool_geom_srid.sql delete mode 100644 db/migrations/00065_pool_tags.sql create mode 100644 tools/delete-all-pool-uploads.sql diff --git a/db/dberrors/fileupload.pool.bob.go b/db/dberrors/fileupload.pool.bob.go index 457407fa..7d8b2fd2 100644 --- a/db/dberrors/fileupload.pool.bob.go +++ b/db/dberrors/fileupload.pool.bob.go @@ -7,7 +7,7 @@ var FileuploadPoolErrors = &fileuploadPoolErrors{ ErrUniquePoolPkey: &UniqueConstraintError{ schema: "fileupload", table: "pool", - columns: []string{"id", "version"}, + columns: []string{"id"}, s: "pool_pkey", }, } diff --git a/db/dbinfo/fileupload.pool.bob.go b/db/dbinfo/fileupload.pool.bob.go index f7243be8..81020468 100644 --- a/db/dbinfo/fileupload.pool.bob.go +++ b/db/dbinfo/fileupload.pool.bob.go @@ -96,6 +96,15 @@ var FileuploadPools = Table[ Generated: false, AutoIncr: false, }, + Geom: column{ + Name: "geom", + DBType: "geometry", + Default: "NULL", + Comment: "", + Nullable: true, + Generated: false, + AutoIncr: false, + }, H3cell: column{ Name: "h3cell", DBType: "h3index", @@ -159,27 +168,18 @@ var FileuploadPools = Table[ Generated: false, AutoIncr: false, }, - ResidentOwned: column{ - Name: "resident_owned", - DBType: "boolean", + PropertyOwnerPhoneE164: column{ + Name: "property_owner_phone_e164", + DBType: "text", Default: "NULL", Comment: "", Nullable: true, Generated: false, AutoIncr: false, }, - Version: column{ - Name: "version", - DBType: "integer", - Default: "", - Comment: "", - Nullable: false, - Generated: false, - AutoIncr: false, - }, - PropertyOwnerPhoneE164: column{ - Name: "property_owner_phone_e164", - DBType: "text", + ResidentOwned: column{ + Name: "resident_owned", + DBType: "boolean", Default: "NULL", Comment: "", Nullable: true, @@ -195,12 +195,12 @@ var FileuploadPools = Table[ Generated: false, AutoIncr: false, }, - Geom: column{ - Name: "geom", - DBType: "geometry", - Default: "NULL", + LineNumber: column{ + Name: "line_number", + DBType: "integer", + Default: "", Comment: "", - Nullable: true, + Nullable: false, Generated: false, AutoIncr: false, }, @@ -224,15 +224,10 @@ var FileuploadPools = Table[ Desc: null.FromCond(false, true), IsExpression: false, }, - { - Name: "version", - Desc: null.FromCond(false, true), - IsExpression: false, - }, }, Unique: true, Comment: "", - NullsFirst: []bool{false, false}, + NullsFirst: []bool{false}, NullsDistinct: false, Where: "", Include: []string{}, @@ -240,7 +235,7 @@ var FileuploadPools = Table[ }, PrimaryKey: &constraint{ Name: "pool_pkey", - Columns: []string{"id", "version"}, + Columns: []string{"id"}, Comment: "", }, ForeignKeys: fileuploadPoolForeignKeys{ @@ -304,6 +299,7 @@ type fileuploadPoolColumns struct { CreatorID column CSVFile column Deleted column + Geom column H3cell column ID column IsInDistrict column @@ -311,17 +307,16 @@ type fileuploadPoolColumns struct { Notes column OrganizationID column PropertyOwnerName column - ResidentOwned column - Version column PropertyOwnerPhoneE164 column + ResidentOwned column ResidentPhoneE164 column - Geom column + LineNumber column Tags column } func (c fileuploadPoolColumns) AsSlice() []column { return []column{ - c.AddressCity, c.AddressPostalCode, c.AddressStreet, c.Committed, c.Condition, c.Created, c.CreatorID, c.CSVFile, c.Deleted, c.H3cell, c.ID, c.IsInDistrict, c.IsNew, c.Notes, c.OrganizationID, c.PropertyOwnerName, c.ResidentOwned, c.Version, c.PropertyOwnerPhoneE164, c.ResidentPhoneE164, c.Geom, c.Tags, + c.AddressCity, c.AddressPostalCode, c.AddressStreet, c.Committed, c.Condition, c.Created, c.CreatorID, c.CSVFile, c.Deleted, c.Geom, c.H3cell, c.ID, c.IsInDistrict, c.IsNew, c.Notes, c.OrganizationID, c.PropertyOwnerName, c.PropertyOwnerPhoneE164, c.ResidentOwned, c.ResidentPhoneE164, c.LineNumber, c.Tags, } } diff --git a/db/factory/bobfactory_main.bob.go b/db/factory/bobfactory_main.bob.go index 5d97a44d..4c72bfc5 100644 --- a/db/factory/bobfactory_main.bob.go +++ b/db/factory/bobfactory_main.bob.go @@ -2425,6 +2425,7 @@ func (f *Factory) FromExistingFileuploadPool(m *models.FileuploadPool) *Fileuplo o.CreatorID = func() int32 { return m.CreatorID } o.CSVFile = func() int32 { return m.CSVFile } o.Deleted = func() null.Val[time.Time] { return m.Deleted } + o.Geom = func() null.Val[string] { return m.Geom } o.H3cell = func() null.Val[string] { return m.H3cell } o.ID = func() int32 { return m.ID } o.IsInDistrict = func() bool { return m.IsInDistrict } @@ -2432,11 +2433,10 @@ func (f *Factory) FromExistingFileuploadPool(m *models.FileuploadPool) *Fileuplo o.Notes = func() string { return m.Notes } o.OrganizationID = func() int32 { return m.OrganizationID } o.PropertyOwnerName = func() string { return m.PropertyOwnerName } - o.ResidentOwned = func() null.Val[bool] { return m.ResidentOwned } - o.Version = func() int32 { return m.Version } o.PropertyOwnerPhoneE164 = func() null.Val[string] { return m.PropertyOwnerPhoneE164 } + o.ResidentOwned = func() null.Val[bool] { return m.ResidentOwned } o.ResidentPhoneE164 = func() null.Val[string] { return m.ResidentPhoneE164 } - o.Geom = func() null.Val[string] { return m.Geom } + o.LineNumber = func() int32 { return m.LineNumber } o.Tags = func() pgtypes.HStore { return m.Tags } ctx := context.Background() diff --git a/db/factory/fileupload.pool.bob.go b/db/factory/fileupload.pool.bob.go index edec6589..021d4806 100644 --- a/db/factory/fileupload.pool.bob.go +++ b/db/factory/fileupload.pool.bob.go @@ -48,6 +48,7 @@ type FileuploadPoolTemplate struct { CreatorID func() int32 CSVFile func() int32 Deleted func() null.Val[time.Time] + Geom func() null.Val[string] H3cell func() null.Val[string] ID func() int32 IsInDistrict func() bool @@ -55,11 +56,10 @@ type FileuploadPoolTemplate struct { Notes func() string OrganizationID func() int32 PropertyOwnerName func() string - ResidentOwned func() null.Val[bool] - Version func() int32 PropertyOwnerPhoneE164 func() null.Val[string] + ResidentOwned func() null.Val[bool] ResidentPhoneE164 func() null.Val[string] - Geom func() null.Val[string] + LineNumber func() int32 Tags func() pgtypes.HStore r fileuploadPoolR @@ -179,6 +179,10 @@ func (o FileuploadPoolTemplate) BuildSetter() *models.FileuploadPoolSetter { val := o.Deleted() m.Deleted = omitnull.FromNull(val) } + if o.Geom != nil { + val := o.Geom() + m.Geom = omitnull.FromNull(val) + } if o.H3cell != nil { val := o.H3cell() m.H3cell = omitnull.FromNull(val) @@ -207,25 +211,21 @@ func (o FileuploadPoolTemplate) BuildSetter() *models.FileuploadPoolSetter { val := o.PropertyOwnerName() m.PropertyOwnerName = omit.From(val) } - if o.ResidentOwned != nil { - val := o.ResidentOwned() - m.ResidentOwned = omitnull.FromNull(val) - } - if o.Version != nil { - val := o.Version() - m.Version = omit.From(val) - } if o.PropertyOwnerPhoneE164 != nil { val := o.PropertyOwnerPhoneE164() m.PropertyOwnerPhoneE164 = omitnull.FromNull(val) } + if o.ResidentOwned != nil { + val := o.ResidentOwned() + m.ResidentOwned = omitnull.FromNull(val) + } if o.ResidentPhoneE164 != nil { val := o.ResidentPhoneE164() m.ResidentPhoneE164 = omitnull.FromNull(val) } - if o.Geom != nil { - val := o.Geom() - m.Geom = omitnull.FromNull(val) + if o.LineNumber != nil { + val := o.LineNumber() + m.LineNumber = omit.From(val) } if o.Tags != nil { val := o.Tags() @@ -280,6 +280,9 @@ func (o FileuploadPoolTemplate) Build() *models.FileuploadPool { if o.Deleted != nil { m.Deleted = o.Deleted() } + if o.Geom != nil { + m.Geom = o.Geom() + } if o.H3cell != nil { m.H3cell = o.H3cell() } @@ -301,20 +304,17 @@ func (o FileuploadPoolTemplate) Build() *models.FileuploadPool { if o.PropertyOwnerName != nil { m.PropertyOwnerName = o.PropertyOwnerName() } - if o.ResidentOwned != nil { - m.ResidentOwned = o.ResidentOwned() - } - if o.Version != nil { - m.Version = o.Version() - } if o.PropertyOwnerPhoneE164 != nil { m.PropertyOwnerPhoneE164 = o.PropertyOwnerPhoneE164() } + if o.ResidentOwned != nil { + m.ResidentOwned = o.ResidentOwned() + } if o.ResidentPhoneE164 != nil { m.ResidentPhoneE164 = o.ResidentPhoneE164() } - if o.Geom != nil { - m.Geom = o.Geom() + if o.LineNumber != nil { + m.LineNumber = o.LineNumber() } if o.Tags != nil { m.Tags = o.Tags() @@ -391,9 +391,9 @@ func ensureCreatableFileuploadPool(m *models.FileuploadPoolSetter) { val := random_string(nil) m.PropertyOwnerName = omit.From(val) } - if !(m.Version.IsValue()) { + if !(m.LineNumber.IsValue()) { val := random_int32(nil) - m.Version = omit.From(val) + m.LineNumber = omit.From(val) } if !(m.Tags.IsValue()) { val := random_pgtypes_HStore(nil) @@ -601,6 +601,7 @@ func (m fileuploadPoolMods) RandomizeAllColumns(f *faker.Faker) FileuploadPoolMo FileuploadPoolMods.RandomCreatorID(f), FileuploadPoolMods.RandomCSVFile(f), FileuploadPoolMods.RandomDeleted(f), + FileuploadPoolMods.RandomGeom(f), FileuploadPoolMods.RandomH3cell(f), FileuploadPoolMods.RandomID(f), FileuploadPoolMods.RandomIsInDistrict(f), @@ -608,11 +609,10 @@ func (m fileuploadPoolMods) RandomizeAllColumns(f *faker.Faker) FileuploadPoolMo FileuploadPoolMods.RandomNotes(f), FileuploadPoolMods.RandomOrganizationID(f), FileuploadPoolMods.RandomPropertyOwnerName(f), - FileuploadPoolMods.RandomResidentOwned(f), - FileuploadPoolMods.RandomVersion(f), FileuploadPoolMods.RandomPropertyOwnerPhoneE164(f), + FileuploadPoolMods.RandomResidentOwned(f), FileuploadPoolMods.RandomResidentPhoneE164(f), - FileuploadPoolMods.RandomGeom(f), + FileuploadPoolMods.RandomLineNumber(f), FileuploadPoolMods.RandomTags(f), } } @@ -918,6 +918,59 @@ func (m fileuploadPoolMods) RandomDeletedNotNull(f *faker.Faker) FileuploadPoolM }) } +// Set the model columns to this value +func (m fileuploadPoolMods) Geom(val null.Val[string]) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.Geom = func() null.Val[string] { return val } + }) +} + +// Set the Column from the function +func (m fileuploadPoolMods) GeomFunc(f func() null.Val[string]) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.Geom = f + }) +} + +// Clear any values for the column +func (m fileuploadPoolMods) UnsetGeom() FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.Geom = nil + }) +} + +// Generates a random value for the column using the given faker +// if faker is nil, a default faker is used +// The generated value is sometimes null +func (m fileuploadPoolMods) RandomGeom(f *faker.Faker) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.Geom = func() null.Val[string] { + if f == nil { + f = &defaultFaker + } + + val := random_string(f) + return null.From(val) + } + }) +} + +// Generates a random value for the column using the given faker +// if faker is nil, a default faker is used +// The generated value is never null +func (m fileuploadPoolMods) RandomGeomNotNull(f *faker.Faker) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.Geom = func() null.Val[string] { + if f == nil { + f = &defaultFaker + } + + val := random_string(f) + return null.From(val) + } + }) +} + // Set the model columns to this value func (m fileuploadPoolMods) H3cell(val null.Val[string]) FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { @@ -1157,90 +1210,6 @@ func (m fileuploadPoolMods) RandomPropertyOwnerName(f *faker.Faker) FileuploadPo }) } -// Set the model columns to this value -func (m fileuploadPoolMods) ResidentOwned(val null.Val[bool]) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.ResidentOwned = func() null.Val[bool] { return val } - }) -} - -// Set the Column from the function -func (m fileuploadPoolMods) ResidentOwnedFunc(f func() null.Val[bool]) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.ResidentOwned = f - }) -} - -// Clear any values for the column -func (m fileuploadPoolMods) UnsetResidentOwned() FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.ResidentOwned = nil - }) -} - -// Generates a random value for the column using the given faker -// if faker is nil, a default faker is used -// The generated value is sometimes null -func (m fileuploadPoolMods) RandomResidentOwned(f *faker.Faker) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.ResidentOwned = func() null.Val[bool] { - if f == nil { - f = &defaultFaker - } - - val := random_bool(f) - return null.From(val) - } - }) -} - -// Generates a random value for the column using the given faker -// if faker is nil, a default faker is used -// The generated value is never null -func (m fileuploadPoolMods) RandomResidentOwnedNotNull(f *faker.Faker) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.ResidentOwned = func() null.Val[bool] { - if f == nil { - f = &defaultFaker - } - - val := random_bool(f) - return null.From(val) - } - }) -} - -// Set the model columns to this value -func (m fileuploadPoolMods) Version(val int32) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Version = func() int32 { return val } - }) -} - -// Set the Column from the function -func (m fileuploadPoolMods) VersionFunc(f func() int32) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Version = f - }) -} - -// Clear any values for the column -func (m fileuploadPoolMods) UnsetVersion() FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Version = nil - }) -} - -// Generates a random value for the column using the given faker -// if faker is nil, a default faker is used -func (m fileuploadPoolMods) RandomVersion(f *faker.Faker) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Version = func() int32 { - return random_int32(f) - } - }) -} - // Set the model columns to this value func (m fileuploadPoolMods) PropertyOwnerPhoneE164(val null.Val[string]) FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { @@ -1294,6 +1263,59 @@ func (m fileuploadPoolMods) RandomPropertyOwnerPhoneE164NotNull(f *faker.Faker) }) } +// Set the model columns to this value +func (m fileuploadPoolMods) ResidentOwned(val null.Val[bool]) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.ResidentOwned = func() null.Val[bool] { return val } + }) +} + +// Set the Column from the function +func (m fileuploadPoolMods) ResidentOwnedFunc(f func() null.Val[bool]) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.ResidentOwned = f + }) +} + +// Clear any values for the column +func (m fileuploadPoolMods) UnsetResidentOwned() FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.ResidentOwned = nil + }) +} + +// Generates a random value for the column using the given faker +// if faker is nil, a default faker is used +// The generated value is sometimes null +func (m fileuploadPoolMods) RandomResidentOwned(f *faker.Faker) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.ResidentOwned = func() null.Val[bool] { + if f == nil { + f = &defaultFaker + } + + val := random_bool(f) + return null.From(val) + } + }) +} + +// Generates a random value for the column using the given faker +// if faker is nil, a default faker is used +// The generated value is never null +func (m fileuploadPoolMods) RandomResidentOwnedNotNull(f *faker.Faker) FileuploadPoolMod { + return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { + o.ResidentOwned = func() null.Val[bool] { + if f == nil { + f = &defaultFaker + } + + val := random_bool(f) + return null.From(val) + } + }) +} + // Set the model columns to this value func (m fileuploadPoolMods) ResidentPhoneE164(val null.Val[string]) FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { @@ -1348,54 +1370,32 @@ func (m fileuploadPoolMods) RandomResidentPhoneE164NotNull(f *faker.Faker) Fileu } // Set the model columns to this value -func (m fileuploadPoolMods) Geom(val null.Val[string]) FileuploadPoolMod { +func (m fileuploadPoolMods) LineNumber(val int32) FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Geom = func() null.Val[string] { return val } + o.LineNumber = func() int32 { return val } }) } // Set the Column from the function -func (m fileuploadPoolMods) GeomFunc(f func() null.Val[string]) FileuploadPoolMod { +func (m fileuploadPoolMods) LineNumberFunc(f func() int32) FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Geom = f + o.LineNumber = f }) } // Clear any values for the column -func (m fileuploadPoolMods) UnsetGeom() FileuploadPoolMod { +func (m fileuploadPoolMods) UnsetLineNumber() FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Geom = nil + o.LineNumber = nil }) } // Generates a random value for the column using the given faker // if faker is nil, a default faker is used -// The generated value is sometimes null -func (m fileuploadPoolMods) RandomGeom(f *faker.Faker) FileuploadPoolMod { +func (m fileuploadPoolMods) RandomLineNumber(f *faker.Faker) FileuploadPoolMod { return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Geom = func() null.Val[string] { - if f == nil { - f = &defaultFaker - } - - val := random_string(f) - return null.From(val) - } - }) -} - -// Generates a random value for the column using the given faker -// if faker is nil, a default faker is used -// The generated value is never null -func (m fileuploadPoolMods) RandomGeomNotNull(f *faker.Faker) FileuploadPoolMod { - return FileuploadPoolModFunc(func(_ context.Context, o *FileuploadPoolTemplate) { - o.Geom = func() null.Val[string] { - if f == nil { - f = &defaultFaker - } - - val := random_string(f) - return null.From(val) + o.LineNumber = func() int32 { + return random_int32(f) } }) } diff --git a/db/migrations/00062_csv_pool_phone_fix.sql b/db/migrations/00062_csv_pool_phone_fix.sql deleted file mode 100644 index 4366ad61..00000000 --- a/db/migrations/00062_csv_pool_phone_fix.sql +++ /dev/null @@ -1,5 +0,0 @@ --- +goose Up -ALTER TABLE fileupload.pool DROP COLUMN property_owner_phone; -ALTER TABLE fileupload.pool DROP COLUMN resident_phone; -ALTER TABLE fileupload.pool ADD COLUMN property_owner_phone_e164 TEXT REFERENCES comms.phone(e164); -ALTER TABLE fileupload.pool ADD COLUMN resident_phone_e164 TEXT REFERENCES comms.phone(e164); diff --git a/db/migrations/00062_fileupload_pool_rebuild.sql b/db/migrations/00062_fileupload_pool_rebuild.sql new file mode 100644 index 00000000..fe2df67f --- /dev/null +++ b/db/migrations/00062_fileupload_pool_rebuild.sql @@ -0,0 +1,44 @@ +-- +goose Up +DROP TABLE fileupload.pool; + +CREATE TABLE fileupload.pool ( + address_city TEXT NOT NULL, + address_postal_code TEXT NOT NULL, + address_street TEXT NOT NULL, + committed BOOLEAN NOT NULL, -- Whether or not its just proposed before a CSV file is committed + condition fileupload.PoolConditionType NOT NULL, + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + creator_id INTEGER REFERENCES user_(id) NOT NULL, + csv_file INTEGER REFERENCES fileupload.csv(file_id) NOT NULL, + deleted TIMESTAMP WITHOUT TIME ZONE, + geom geometry(Point, 4326), + h3cell h3index, + id SERIAL, + is_in_district BOOLEAN NOT NULL, -- Whether or not the pool is within the district + is_new BOOLEAN NOT NULL, -- Whether or not we already have a pool in the system for this row + notes TEXT NOT NULL, + organization_id INTEGER REFERENCES organization(id) NOT NULL, + property_owner_name TEXT NOT NULL, + property_owner_phone_e164 TEXT REFERENCES comms.phone(e164), + resident_owned BOOLEAN, + resident_phone_e164 TEXT REFERENCES comms.phone(e164), + line_number INTEGER NOT NULL, + tags HSTORE NOT NULL, + PRIMARY KEY (id) +); + +-- migration 62 +-- ALTER TABLE fileupload.pool ADD COLUMN property_owner_phone_e164 TEXT REFERENCES comms.phone(e164); +-- ALTER TABLE fileupload.pool ADD COLUMN resident_phone_e164 TEXT REFERENCES comms.phone(e164); + +-- migration 64 +-- ALTER TABLE fileupload.pool DROP COLUMN geom; +-- ALTER TABLE fileupload.pool ADD COLUMN geom geometry(Point, 4326); + +-- migration 65 +-- ALTER TABLE fileupload.pool ADD COLUMN tags HSTORE NOT NULL; + +-- migration 66 +-- ALTER TABLE fileupload.pool ADD COLUMN row_number INTEGER NOT NULL; + +-- +goose Down diff --git a/db/migrations/00063_pool_empty_status.sql b/db/migrations/00063_pool_empty_status.sql deleted file mode 100644 index 3d5dfbfc..00000000 --- a/db/migrations/00063_pool_empty_status.sql +++ /dev/null @@ -1,2 +0,0 @@ --- +goose Up -ALTER TYPE fileupload.PoolConditionType ADD VALUE 'empty' AFTER 'blue'; diff --git a/db/migrations/00064_csv_pool_geom_srid.sql b/db/migrations/00064_csv_pool_geom_srid.sql deleted file mode 100644 index d1501924..00000000 --- a/db/migrations/00064_csv_pool_geom_srid.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -ALTER TABLE fileupload.pool DROP COLUMN geom; -ALTER TABLE fileupload.pool ADD COLUMN geom geometry(Point, 4326); --- +goose Down -ALTER TABLE fileupload.pool DROP COLUMN geom; -ALTER TABLE fileupload.pool ADD COLUMN geom geometry(Point, 3857); diff --git a/db/migrations/00065_pool_tags.sql b/db/migrations/00065_pool_tags.sql deleted file mode 100644 index b4d73da8..00000000 --- a/db/migrations/00065_pool_tags.sql +++ /dev/null @@ -1,4 +0,0 @@ --- +goose Up -ALTER TABLE fileupload.pool ADD COLUMN tags HSTORE NOT NULL; --- +goose Down -ALTER TABLE fileupload.pool DROP COLUMN tags; diff --git a/db/models/fileupload.pool.bob.go b/db/models/fileupload.pool.bob.go index 73732bfc..3a207249 100644 --- a/db/models/fileupload.pool.bob.go +++ b/db/models/fileupload.pool.bob.go @@ -36,6 +36,7 @@ type FileuploadPool struct { CreatorID int32 `db:"creator_id" ` CSVFile int32 `db:"csv_file" ` Deleted null.Val[time.Time] `db:"deleted" ` + Geom null.Val[string] `db:"geom" ` H3cell null.Val[string] `db:"h3cell" ` ID int32 `db:"id,pk" ` IsInDistrict bool `db:"is_in_district" ` @@ -43,11 +44,10 @@ type FileuploadPool struct { Notes string `db:"notes" ` OrganizationID int32 `db:"organization_id" ` PropertyOwnerName string `db:"property_owner_name" ` - ResidentOwned null.Val[bool] `db:"resident_owned" ` - Version int32 `db:"version,pk" ` PropertyOwnerPhoneE164 null.Val[string] `db:"property_owner_phone_e164" ` + ResidentOwned null.Val[bool] `db:"resident_owned" ` ResidentPhoneE164 null.Val[string] `db:"resident_phone_e164" ` - Geom null.Val[string] `db:"geom" ` + LineNumber int32 `db:"line_number" ` Tags pgtypes.HStore `db:"tags" ` R fileuploadPoolR `db:"-" ` @@ -75,7 +75,7 @@ type fileuploadPoolR struct { func buildFileuploadPoolColumns(alias string) fileuploadPoolColumns { return fileuploadPoolColumns{ ColumnsExpr: expr.NewColumnsExpr( - "address_city", "address_postal_code", "address_street", "committed", "condition", "created", "creator_id", "csv_file", "deleted", "h3cell", "id", "is_in_district", "is_new", "notes", "organization_id", "property_owner_name", "resident_owned", "version", "property_owner_phone_e164", "resident_phone_e164", "geom", "tags", + "address_city", "address_postal_code", "address_street", "committed", "condition", "created", "creator_id", "csv_file", "deleted", "geom", "h3cell", "id", "is_in_district", "is_new", "notes", "organization_id", "property_owner_name", "property_owner_phone_e164", "resident_owned", "resident_phone_e164", "line_number", "tags", ).WithParent("fileupload.pool"), tableAlias: alias, AddressCity: psql.Quote(alias, "address_city"), @@ -87,6 +87,7 @@ func buildFileuploadPoolColumns(alias string) fileuploadPoolColumns { CreatorID: psql.Quote(alias, "creator_id"), CSVFile: psql.Quote(alias, "csv_file"), Deleted: psql.Quote(alias, "deleted"), + Geom: psql.Quote(alias, "geom"), H3cell: psql.Quote(alias, "h3cell"), ID: psql.Quote(alias, "id"), IsInDistrict: psql.Quote(alias, "is_in_district"), @@ -94,11 +95,10 @@ func buildFileuploadPoolColumns(alias string) fileuploadPoolColumns { Notes: psql.Quote(alias, "notes"), OrganizationID: psql.Quote(alias, "organization_id"), PropertyOwnerName: psql.Quote(alias, "property_owner_name"), - ResidentOwned: psql.Quote(alias, "resident_owned"), - Version: psql.Quote(alias, "version"), PropertyOwnerPhoneE164: psql.Quote(alias, "property_owner_phone_e164"), + ResidentOwned: psql.Quote(alias, "resident_owned"), ResidentPhoneE164: psql.Quote(alias, "resident_phone_e164"), - Geom: psql.Quote(alias, "geom"), + LineNumber: psql.Quote(alias, "line_number"), Tags: psql.Quote(alias, "tags"), } } @@ -115,6 +115,7 @@ type fileuploadPoolColumns struct { CreatorID psql.Expression CSVFile psql.Expression Deleted psql.Expression + Geom psql.Expression H3cell psql.Expression ID psql.Expression IsInDistrict psql.Expression @@ -122,11 +123,10 @@ type fileuploadPoolColumns struct { Notes psql.Expression OrganizationID psql.Expression PropertyOwnerName psql.Expression - ResidentOwned psql.Expression - Version psql.Expression PropertyOwnerPhoneE164 psql.Expression + ResidentOwned psql.Expression ResidentPhoneE164 psql.Expression - Geom psql.Expression + LineNumber psql.Expression Tags psql.Expression } @@ -151,6 +151,7 @@ type FileuploadPoolSetter struct { CreatorID omit.Val[int32] `db:"creator_id" ` CSVFile omit.Val[int32] `db:"csv_file" ` Deleted omitnull.Val[time.Time] `db:"deleted" ` + Geom omitnull.Val[string] `db:"geom" ` H3cell omitnull.Val[string] `db:"h3cell" ` ID omit.Val[int32] `db:"id,pk" ` IsInDistrict omit.Val[bool] `db:"is_in_district" ` @@ -158,11 +159,10 @@ type FileuploadPoolSetter struct { Notes omit.Val[string] `db:"notes" ` OrganizationID omit.Val[int32] `db:"organization_id" ` PropertyOwnerName omit.Val[string] `db:"property_owner_name" ` - ResidentOwned omitnull.Val[bool] `db:"resident_owned" ` - Version omit.Val[int32] `db:"version,pk" ` PropertyOwnerPhoneE164 omitnull.Val[string] `db:"property_owner_phone_e164" ` + ResidentOwned omitnull.Val[bool] `db:"resident_owned" ` ResidentPhoneE164 omitnull.Val[string] `db:"resident_phone_e164" ` - Geom omitnull.Val[string] `db:"geom" ` + LineNumber omit.Val[int32] `db:"line_number" ` Tags omit.Val[pgtypes.HStore] `db:"tags" ` } @@ -195,6 +195,9 @@ func (s FileuploadPoolSetter) SetColumns() []string { if !s.Deleted.IsUnset() { vals = append(vals, "deleted") } + if !s.Geom.IsUnset() { + vals = append(vals, "geom") + } if !s.H3cell.IsUnset() { vals = append(vals, "h3cell") } @@ -216,20 +219,17 @@ func (s FileuploadPoolSetter) SetColumns() []string { if s.PropertyOwnerName.IsValue() { vals = append(vals, "property_owner_name") } - if !s.ResidentOwned.IsUnset() { - vals = append(vals, "resident_owned") - } - if s.Version.IsValue() { - vals = append(vals, "version") - } if !s.PropertyOwnerPhoneE164.IsUnset() { vals = append(vals, "property_owner_phone_e164") } + if !s.ResidentOwned.IsUnset() { + vals = append(vals, "resident_owned") + } if !s.ResidentPhoneE164.IsUnset() { vals = append(vals, "resident_phone_e164") } - if !s.Geom.IsUnset() { - vals = append(vals, "geom") + if s.LineNumber.IsValue() { + vals = append(vals, "line_number") } if s.Tags.IsValue() { vals = append(vals, "tags") @@ -265,6 +265,9 @@ func (s FileuploadPoolSetter) Overwrite(t *FileuploadPool) { if !s.Deleted.IsUnset() { t.Deleted = s.Deleted.MustGetNull() } + if !s.Geom.IsUnset() { + t.Geom = s.Geom.MustGetNull() + } if !s.H3cell.IsUnset() { t.H3cell = s.H3cell.MustGetNull() } @@ -286,20 +289,17 @@ func (s FileuploadPoolSetter) Overwrite(t *FileuploadPool) { if s.PropertyOwnerName.IsValue() { t.PropertyOwnerName = s.PropertyOwnerName.MustGet() } - if !s.ResidentOwned.IsUnset() { - t.ResidentOwned = s.ResidentOwned.MustGetNull() - } - if s.Version.IsValue() { - t.Version = s.Version.MustGet() - } if !s.PropertyOwnerPhoneE164.IsUnset() { t.PropertyOwnerPhoneE164 = s.PropertyOwnerPhoneE164.MustGetNull() } + if !s.ResidentOwned.IsUnset() { + t.ResidentOwned = s.ResidentOwned.MustGetNull() + } if !s.ResidentPhoneE164.IsUnset() { t.ResidentPhoneE164 = s.ResidentPhoneE164.MustGetNull() } - if !s.Geom.IsUnset() { - t.Geom = s.Geom.MustGetNull() + if s.LineNumber.IsValue() { + t.LineNumber = s.LineNumber.MustGet() } if s.Tags.IsValue() { t.Tags = s.Tags.MustGet() @@ -367,62 +367,62 @@ func (s *FileuploadPoolSetter) Apply(q *dialect.InsertQuery) { vals[8] = psql.Raw("DEFAULT") } - if !s.H3cell.IsUnset() { - vals[9] = psql.Arg(s.H3cell.MustGetNull()) + if !s.Geom.IsUnset() { + vals[9] = psql.Arg(s.Geom.MustGetNull()) } else { vals[9] = psql.Raw("DEFAULT") } - if s.ID.IsValue() { - vals[10] = psql.Arg(s.ID.MustGet()) + if !s.H3cell.IsUnset() { + vals[10] = psql.Arg(s.H3cell.MustGetNull()) } else { vals[10] = psql.Raw("DEFAULT") } - if s.IsInDistrict.IsValue() { - vals[11] = psql.Arg(s.IsInDistrict.MustGet()) + if s.ID.IsValue() { + vals[11] = psql.Arg(s.ID.MustGet()) } else { vals[11] = psql.Raw("DEFAULT") } - if s.IsNew.IsValue() { - vals[12] = psql.Arg(s.IsNew.MustGet()) + if s.IsInDistrict.IsValue() { + vals[12] = psql.Arg(s.IsInDistrict.MustGet()) } else { vals[12] = psql.Raw("DEFAULT") } - if s.Notes.IsValue() { - vals[13] = psql.Arg(s.Notes.MustGet()) + if s.IsNew.IsValue() { + vals[13] = psql.Arg(s.IsNew.MustGet()) } else { vals[13] = psql.Raw("DEFAULT") } - if s.OrganizationID.IsValue() { - vals[14] = psql.Arg(s.OrganizationID.MustGet()) + if s.Notes.IsValue() { + vals[14] = psql.Arg(s.Notes.MustGet()) } else { vals[14] = psql.Raw("DEFAULT") } - if s.PropertyOwnerName.IsValue() { - vals[15] = psql.Arg(s.PropertyOwnerName.MustGet()) + if s.OrganizationID.IsValue() { + vals[15] = psql.Arg(s.OrganizationID.MustGet()) } else { vals[15] = psql.Raw("DEFAULT") } - if !s.ResidentOwned.IsUnset() { - vals[16] = psql.Arg(s.ResidentOwned.MustGetNull()) + if s.PropertyOwnerName.IsValue() { + vals[16] = psql.Arg(s.PropertyOwnerName.MustGet()) } else { vals[16] = psql.Raw("DEFAULT") } - if s.Version.IsValue() { - vals[17] = psql.Arg(s.Version.MustGet()) + if !s.PropertyOwnerPhoneE164.IsUnset() { + vals[17] = psql.Arg(s.PropertyOwnerPhoneE164.MustGetNull()) } else { vals[17] = psql.Raw("DEFAULT") } - if !s.PropertyOwnerPhoneE164.IsUnset() { - vals[18] = psql.Arg(s.PropertyOwnerPhoneE164.MustGetNull()) + if !s.ResidentOwned.IsUnset() { + vals[18] = psql.Arg(s.ResidentOwned.MustGetNull()) } else { vals[18] = psql.Raw("DEFAULT") } @@ -433,8 +433,8 @@ func (s *FileuploadPoolSetter) Apply(q *dialect.InsertQuery) { vals[19] = psql.Raw("DEFAULT") } - if !s.Geom.IsUnset() { - vals[20] = psql.Arg(s.Geom.MustGetNull()) + if s.LineNumber.IsValue() { + vals[20] = psql.Arg(s.LineNumber.MustGet()) } else { vals[20] = psql.Raw("DEFAULT") } @@ -519,6 +519,13 @@ func (s FileuploadPoolSetter) Expressions(prefix ...string) []bob.Expression { }}) } + if !s.Geom.IsUnset() { + exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ + psql.Quote(append(prefix, "geom")...), + psql.Arg(s.Geom), + }}) + } + if !s.H3cell.IsUnset() { exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ psql.Quote(append(prefix, "h3cell")...), @@ -568,20 +575,6 @@ func (s FileuploadPoolSetter) Expressions(prefix ...string) []bob.Expression { }}) } - if !s.ResidentOwned.IsUnset() { - exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ - psql.Quote(append(prefix, "resident_owned")...), - psql.Arg(s.ResidentOwned), - }}) - } - - if s.Version.IsValue() { - exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ - psql.Quote(append(prefix, "version")...), - psql.Arg(s.Version), - }}) - } - if !s.PropertyOwnerPhoneE164.IsUnset() { exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ psql.Quote(append(prefix, "property_owner_phone_e164")...), @@ -589,6 +582,13 @@ func (s FileuploadPoolSetter) Expressions(prefix ...string) []bob.Expression { }}) } + if !s.ResidentOwned.IsUnset() { + exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ + psql.Quote(append(prefix, "resident_owned")...), + psql.Arg(s.ResidentOwned), + }}) + } + if !s.ResidentPhoneE164.IsUnset() { exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ psql.Quote(append(prefix, "resident_phone_e164")...), @@ -596,10 +596,10 @@ func (s FileuploadPoolSetter) Expressions(prefix ...string) []bob.Expression { }}) } - if !s.Geom.IsUnset() { + if s.LineNumber.IsValue() { exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ - psql.Quote(append(prefix, "geom")...), - psql.Arg(s.Geom), + psql.Quote(append(prefix, "line_number")...), + psql.Arg(s.LineNumber), }}) } @@ -615,26 +615,23 @@ func (s FileuploadPoolSetter) Expressions(prefix ...string) []bob.Expression { // FindFileuploadPool retrieves a single record by primary key // If cols is empty Find will return all columns. -func FindFileuploadPool(ctx context.Context, exec bob.Executor, IDPK int32, VersionPK int32, cols ...string) (*FileuploadPool, error) { +func FindFileuploadPool(ctx context.Context, exec bob.Executor, IDPK int32, cols ...string) (*FileuploadPool, error) { if len(cols) == 0 { return FileuploadPools.Query( sm.Where(FileuploadPools.Columns.ID.EQ(psql.Arg(IDPK))), - sm.Where(FileuploadPools.Columns.Version.EQ(psql.Arg(VersionPK))), ).One(ctx, exec) } return FileuploadPools.Query( sm.Where(FileuploadPools.Columns.ID.EQ(psql.Arg(IDPK))), - sm.Where(FileuploadPools.Columns.Version.EQ(psql.Arg(VersionPK))), sm.Columns(FileuploadPools.Columns.Only(cols...)), ).One(ctx, exec) } // FileuploadPoolExists checks the presence of a single record by primary key -func FileuploadPoolExists(ctx context.Context, exec bob.Executor, IDPK int32, VersionPK int32) (bool, error) { +func FileuploadPoolExists(ctx context.Context, exec bob.Executor, IDPK int32) (bool, error) { return FileuploadPools.Query( sm.Where(FileuploadPools.Columns.ID.EQ(psql.Arg(IDPK))), - sm.Where(FileuploadPools.Columns.Version.EQ(psql.Arg(VersionPK))), ).Exists(ctx, exec) } @@ -658,14 +655,11 @@ func (o *FileuploadPool) AfterQueryHook(ctx context.Context, exec bob.Executor, // primaryKeyVals returns the primary key values of the FileuploadPool func (o *FileuploadPool) primaryKeyVals() bob.Expression { - return psql.ArgGroup( - o.ID, - o.Version, - ) + return psql.Arg(o.ID) } func (o *FileuploadPool) pkEQ() dialect.Expression { - return psql.Group(psql.Quote("fileupload.pool", "id"), psql.Quote("fileupload.pool", "version")).EQ(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { + return psql.Quote("fileupload.pool", "id").EQ(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { return o.primaryKeyVals().WriteSQL(ctx, w, d, start) })) } @@ -693,7 +687,6 @@ func (o *FileuploadPool) Delete(ctx context.Context, exec bob.Executor) error { func (o *FileuploadPool) Reload(ctx context.Context, exec bob.Executor) error { o2, err := FileuploadPools.Query( sm.Where(FileuploadPools.Columns.ID.EQ(psql.Arg(o.ID))), - sm.Where(FileuploadPools.Columns.Version.EQ(psql.Arg(o.Version))), ).One(ctx, exec) if err != nil { return err @@ -727,7 +720,7 @@ func (o FileuploadPoolSlice) pkIN() dialect.Expression { return psql.Raw("NULL") } - return psql.Group(psql.Quote("fileupload.pool", "id"), psql.Quote("fileupload.pool", "version")).In(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { + return psql.Quote("fileupload.pool", "id").In(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { pkPairs := make([]bob.Expression, len(o)) for i, row := range o { pkPairs[i] = row.primaryKeyVals() @@ -745,9 +738,6 @@ func (o FileuploadPoolSlice) copyMatchingRows(from ...*FileuploadPool) { if new.ID != old.ID { continue } - if new.Version != old.Version { - continue - } new.R = old.R o[i] = new break @@ -1216,6 +1206,7 @@ type fileuploadPoolWhere[Q psql.Filterable] struct { CreatorID psql.WhereMod[Q, int32] CSVFile psql.WhereMod[Q, int32] Deleted psql.WhereNullMod[Q, time.Time] + Geom psql.WhereNullMod[Q, string] H3cell psql.WhereNullMod[Q, string] ID psql.WhereMod[Q, int32] IsInDistrict psql.WhereMod[Q, bool] @@ -1223,11 +1214,10 @@ type fileuploadPoolWhere[Q psql.Filterable] struct { Notes psql.WhereMod[Q, string] OrganizationID psql.WhereMod[Q, int32] PropertyOwnerName psql.WhereMod[Q, string] - ResidentOwned psql.WhereNullMod[Q, bool] - Version psql.WhereMod[Q, int32] PropertyOwnerPhoneE164 psql.WhereNullMod[Q, string] + ResidentOwned psql.WhereNullMod[Q, bool] ResidentPhoneE164 psql.WhereNullMod[Q, string] - Geom psql.WhereNullMod[Q, string] + LineNumber psql.WhereMod[Q, int32] Tags psql.WhereMod[Q, pgtypes.HStore] } @@ -1246,6 +1236,7 @@ func buildFileuploadPoolWhere[Q psql.Filterable](cols fileuploadPoolColumns) fil CreatorID: psql.Where[Q, int32](cols.CreatorID), CSVFile: psql.Where[Q, int32](cols.CSVFile), Deleted: psql.WhereNull[Q, time.Time](cols.Deleted), + Geom: psql.WhereNull[Q, string](cols.Geom), H3cell: psql.WhereNull[Q, string](cols.H3cell), ID: psql.Where[Q, int32](cols.ID), IsInDistrict: psql.Where[Q, bool](cols.IsInDistrict), @@ -1253,11 +1244,10 @@ func buildFileuploadPoolWhere[Q psql.Filterable](cols fileuploadPoolColumns) fil Notes: psql.Where[Q, string](cols.Notes), OrganizationID: psql.Where[Q, int32](cols.OrganizationID), PropertyOwnerName: psql.Where[Q, string](cols.PropertyOwnerName), - ResidentOwned: psql.WhereNull[Q, bool](cols.ResidentOwned), - Version: psql.Where[Q, int32](cols.Version), PropertyOwnerPhoneE164: psql.WhereNull[Q, string](cols.PropertyOwnerPhoneE164), + ResidentOwned: psql.WhereNull[Q, bool](cols.ResidentOwned), ResidentPhoneE164: psql.WhereNull[Q, string](cols.ResidentPhoneE164), - Geom: psql.WhereNull[Q, string](cols.Geom), + LineNumber: psql.Where[Q, int32](cols.LineNumber), Tags: psql.Where[Q, pgtypes.HStore](cols.Tags), } } diff --git a/platform/csv/pool.go b/platform/csv/pool.go index ae0f07f5..45449e85 100644 --- a/platform/csv/pool.go +++ b/platform/csv/pool.go @@ -106,28 +106,35 @@ func bulkGeocode(ctx context.Context, txn bob.Tx, file models.FileuploadFile, po log.Info().Int("len", len(responses)).Msg("bulk query response") for i, resp := range responses { pool := pools[i] + sublog := log.With(). + Int("row", i). + Str("pool.address_postal", pool.AddressPostalCode). + Str("pool.address_street", pool.AddressStreet). + Str("pool.postal", pool.AddressPostalCode). + Logger() + if resp.Status != 200 { - log.Info().Int("row", i).Int("status", resp.Status).Str("pool.address", pool.AddressStreet).Str("pool.postal", pool.AddressPostalCode).Str("msg", resp.Message).Msg("Non-200 status on geocode request") + sublog.Info().Int("status", resp.Status).Str("msg", resp.Message).Msg("Non-200 status on geocode request") continue } if resp.Response == nil { - log.Info().Int("row", i).Str("pool.address", pool.AddressStreet).Str("pool.postal", pool.AddressPostalCode).Str("msg", resp.Message).Msg("nil response on geocode") + sublog.Info().Str("msg", resp.Message).Msg("nil response on geocode") continue } if len(resp.Response.Features) > 1 { - log.Warn().Int("row", i).Int("len", len(resp.Response.Features)).Msg("too many features") + sublog.Warn().Int("len", len(resp.Response.Features)).Msg("too many features") continue } feature := resp.Response.Features[0] if feature.Geometry.Type != "Point" { - log.Warn().Int("row", i).Str("type", feature.Geometry.Type).Msg("wrong type") + sublog.Warn().Str("type", feature.Geometry.Type).Msg("wrong type") continue } longitude := feature.Geometry.Coordinates[0] latitude := feature.Geometry.Coordinates[1] cell, err := h3utils.GetCell(longitude, latitude, 15) if err != nil { - log.Warn().Err(err).Int("row", i).Float64("lng", longitude).Float64("lat", latitude).Msg("failed to convert h3 cell") + sublog.Warn().Err(err).Float64("lng", longitude).Float64("lat", latitude).Msg("failed to convert h3 cell") continue } geom_query := geom.PostgisPointQuery(longitude, latitude) @@ -184,7 +191,8 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* }) return pools, nil } - row_number := 0 + // Start at 2 because the header is line 1, not line 0 + line_number := int32(2) for { row, err := reader.Read() if err != nil { @@ -214,6 +222,7 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* // ID - generated IsInDistrict: omit.From(false), IsNew: omit.From(true), + LineNumber: omit.From(line_number), Notes: omit.From(""), OrganizationID: omit.From(file.OrganizationID), PropertyOwnerName: omit.From(""), @@ -221,7 +230,6 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* ResidentOwned: omitnull.FromPtr[bool](nil), ResidentPhoneE164: omitnull.FromPtr[string](nil), //Tags: convertToPGData(tags), - Version: omit.From(int32(0)), } for i, col := range row { hdr_t := header_types[i] @@ -239,7 +247,8 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* var condition enums.FileuploadPoolconditiontype err := condition.Scan(strings.ToLower(col)) if err != nil { - addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not a pool condition that we recognize. It should be one of %s", col, poolConditionValidValues())) + addError(ctx, txn, c, int32(line_number), int32(i), fmt.Sprintf("'%s' is not a pool condition that we recognize. It should be one of %s", col, poolConditionValidValues())) + setter.Condition = omit.From(enums.FileuploadPoolconditiontypeUnknown) continue } setter.Condition = omit.From(condition) @@ -250,7 +259,7 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* case headerPropertyOwnerPhone: phone, err := text.ParsePhoneNumber(col) if err != nil { - addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not a phone number that we recognize. Ideally it should be of the form '+12223334444'", col)) + addError(ctx, txn, c, int32(line_number), int32(i), fmt.Sprintf("'%s' is not a phone number that we recognize. Ideally it should be of the form '+12223334444'", col)) continue } text.EnsureInDB(ctx, txn, *phone) @@ -258,14 +267,14 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* case headerResidentOwned: boolValue, err := parseBool(col) if err != nil { - addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not something that we recognize as a true/false value. Please use either 'true' or 'false'", col)) + addError(ctx, txn, c, int32(line_number), int32(i), fmt.Sprintf("'%s' is not something that we recognize as a true/false value. Please use either 'true' or 'false'", col)) continue } setter.ResidentOwned = omitnull.From(boolValue) case headerResidentPhone: phone, err := text.ParsePhoneNumber(col) if err != nil { - addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not a phone number that we recognize. Ideally it should be of the form '+12223334444'", col)) + addError(ctx, txn, c, int32(line_number), int32(i), fmt.Sprintf("'%s' is not a phone number that we recognize. Ideally it should be of the form '+12223334444'", col)) continue } text.EnsureInDB(ctx, txn, *phone) @@ -281,7 +290,7 @@ func parseFile(ctx context.Context, txn bob.Tx, file models.FileuploadFile) ([]* return pools, fmt.Errorf("Failed to create pool: %w", err) } pools = append(pools, pool) - row_number = row_number + 1 + line_number = line_number + 1 } } func addError(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, row_number int32, column_number int32, msg string) error { diff --git a/platform/pool.go b/platform/pool.go index c6a59772..f376dad2 100644 --- a/platform/pool.go +++ b/platform/pool.go @@ -105,7 +105,7 @@ func GetUploadPoolDetail(ctx context.Context, organization_id int32, file_id int return UploadPoolDetail{}, fmt.Errorf("Failed to lookup errors in csv %d: %w", file_id, err) } file_errors := make([]UploadPoolError, 0) - errors_by_row := make(map[int32][]UploadPoolError, 0) + errors_by_line := make(map[int32][]UploadPoolError, 0) for _, row := range error_rows { e := UploadPoolError{ Column: uint(row.Col), @@ -115,13 +115,14 @@ func GetUploadPoolDetail(ctx context.Context, organization_id int32, file_id int if row.Line == 0 { file_errors = append(file_errors, e) } else { - by_row, ok := errors_by_row[row.Line] + log.Info().Int32("line", row.Line).Msg("Found error") + by_line, ok := errors_by_line[row.Line] if !ok { - errors_by_row[row.Line] = []UploadPoolError{e} + errors_by_line[row.Line] = []UploadPoolError{e} continue } - by_row = append(by_row, e) - errors_by_row[row.Line] = by_row + by_line = append(by_line, e) + errors_by_line[row.Line] = by_line } } @@ -137,7 +138,7 @@ func GetUploadPoolDetail(ctx context.Context, organization_id int32, file_id int count_new := 0 count_outside := 0 status := "unknown" - for i, r := range pool_rows { + for _, r := range pool_rows { if r.IsNew { count_new = count_new + 1 status = "new" @@ -149,9 +150,12 @@ func GetUploadPoolDetail(ctx context.Context, organization_id int32, file_id int status = "existing" } tags := db.ConvertFromPGData(r.Tags) - errors, ok := errors_by_row[int32(i)] + // add 2 here because our file lines are 1-indexed and we skip the header line, but we are ranging 0-indexed + errors, ok := errors_by_line[r.LineNumber] if !ok { errors = []UploadPoolError{} + } else { + log.Info().Int32("line", r.LineNumber).Int32("id", r.ID).Msg("Found errors in errors_by_line") } pools = append(pools, UploadPoolRow{ City: r.AddressCity, diff --git a/scss/sync/pool-by-id.scss b/scss/sync/pool-by-id.scss index 1acd725b..f113fd1b 100644 --- a/scss/sync/pool-by-id.scss +++ b/scss/sync/pool-by-id.scss @@ -5,9 +5,9 @@ .summary-card:hover { transform: translateY(-5px); } -.warning-row { - background-color: rgba(255, 193, 7, 0.15) !important; -} .status-badge { font-size: 0.85rem; } +tr.has-error { + background-color: rgba(255, 193, 7, 0.15) !important; +} diff --git a/tools/delete-all-pool-uploads.sql b/tools/delete-all-pool-uploads.sql new file mode 100644 index 00000000..4c107b62 --- /dev/null +++ b/tools/delete-all-pool-uploads.sql @@ -0,0 +1,8 @@ +BEGIN TRANSACTION; + DELETE FROM fileupload.pool; + DELETE FROM fileupload.error_csv; + DELETE FROM fileupload.csv; + DELETE FROM fileupload.error_file; + DELETE FROM fileupload.file; +COMMIT; +