Do file upload, show list of uploads, do initial processing.
This commit is contained in:
parent
8d4195a024
commit
135ad2b73e
45 changed files with 7126 additions and 1464 deletions
|
|
@ -5,31 +5,93 @@ import (
|
|||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Gleipnir-Technology/bob"
|
||||
"github.com/Gleipnir-Technology/nidus-sync/db"
|
||||
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
|
||||
"github.com/Gleipnir-Technology/nidus-sync/db/models"
|
||||
"github.com/Gleipnir-Technology/nidus-sync/userfile"
|
||||
"github.com/aarondl/opt/omit"
|
||||
"github.com/aarondl/opt/omitnull"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type headerPoolEnum int
|
||||
|
||||
const (
|
||||
headerAddressCity = iota
|
||||
headerAddressPostalCode
|
||||
headerAddressStreet
|
||||
headerCondition
|
||||
headerNotes
|
||||
headerPropertyOwnerName
|
||||
headerPropertyOwnerPhone
|
||||
headerResidentOwned
|
||||
headerResidentPhone
|
||||
headerTag
|
||||
)
|
||||
|
||||
func (e headerPoolEnum) String() string {
|
||||
switch e {
|
||||
case headerAddressCity:
|
||||
return "City"
|
||||
case headerAddressPostalCode:
|
||||
return "Postal Code"
|
||||
case headerAddressStreet:
|
||||
return "Street Address"
|
||||
case headerCondition:
|
||||
return "Condition"
|
||||
case headerNotes:
|
||||
return "Notes"
|
||||
case headerPropertyOwnerName:
|
||||
return "Property Owner Name"
|
||||
case headerPropertyOwnerPhone:
|
||||
return "Property Owner Phone"
|
||||
case headerResidentOwned:
|
||||
return "Resident Owned"
|
||||
case headerResidentPhone:
|
||||
return "Resident Phone"
|
||||
default:
|
||||
return "bad programmer"
|
||||
}
|
||||
}
|
||||
func ProcessJob(ctx context.Context, file_id int32) error {
|
||||
file, err := models.FindFileuploadFile(ctx, db.PGInstance.BobDB, file_id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get file %d from DB: %w", file_id, err)
|
||||
}
|
||||
c, err := models.FindFileuploadCSV(ctx, db.PGInstance.BobDB, file_id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get file %d from DB: %w", file_id, err)
|
||||
}
|
||||
r, err := userfile.NewFileReader(userfile.CollectionCSV, file.FileUUID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get filereader for %d: %w", file_id, err)
|
||||
}
|
||||
reader := csv.NewReader(r)
|
||||
header, err := reader.Read()
|
||||
h, err := reader.Read()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to read header of CSV for file %d: %w", file_id, err)
|
||||
}
|
||||
err = validateHeader(header)
|
||||
txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
addImportError(file, err)
|
||||
return fmt.Errorf("Failed to start transaction: %w", err)
|
||||
}
|
||||
defer txn.Rollback(ctx)
|
||||
headers := parseHeaders(h)
|
||||
missing_headers := missingRequiredHeaders(headers)
|
||||
for _, mh := range missing_headers {
|
||||
errorMissingHeader(ctx, txn, c, mh)
|
||||
file.Update(ctx, txn, &models.FileuploadFileSetter{
|
||||
Status: omit.From(enums.FileuploadFilestatustypeError),
|
||||
})
|
||||
txn.Commit(ctx)
|
||||
return nil
|
||||
}
|
||||
row_number := 0
|
||||
for {
|
||||
row, err := reader.Read()
|
||||
if err != nil {
|
||||
|
|
@ -38,14 +100,172 @@ func ProcessJob(ctx context.Context, file_id int32) error {
|
|||
}
|
||||
return fmt.Errorf("Failed to read all CSV records for file %d: %w", file_id, err)
|
||||
}
|
||||
log.Debug().Strs("row", row).Msg("Line")
|
||||
setter := models.PoolSetter{
|
||||
// required fields
|
||||
//AddressCity: omit.From(),
|
||||
//AddressPostalCode: omit.From(),
|
||||
//AddressStreet: omit.From(),
|
||||
Created: omit.From(time.Now()),
|
||||
CreatorID: omit.From(file.CreatorID),
|
||||
Committed: omit.From(false),
|
||||
Condition: omit.From(enums.PoolconditiontypeUnknown),
|
||||
Deleted: omitnull.FromPtr[time.Time](nil),
|
||||
// ID - generated
|
||||
Notes: omit.From(""),
|
||||
OrganizationID: omit.From(file.OrganizationID),
|
||||
PropertyOwnerName: omit.From(""),
|
||||
PropertyOwnerPhone: omitnull.FromPtr[string](nil),
|
||||
ResidentOwned: omitnull.FromPtr[bool](nil),
|
||||
ResidentPhone: omitnull.FromPtr[string](nil),
|
||||
Version: omit.From(int32(0)),
|
||||
}
|
||||
for i, col := range row {
|
||||
hdr := headers[i]
|
||||
switch hdr {
|
||||
case headerAddressCity:
|
||||
setter.AddressCity = omit.From(col)
|
||||
case headerAddressPostalCode:
|
||||
setter.AddressPostalCode = omit.From(col)
|
||||
case headerAddressStreet:
|
||||
setter.AddressStreet = omit.From(col)
|
||||
case headerCondition:
|
||||
var condition enums.Poolconditiontype
|
||||
err := condition.Scan(strings.ToLower(col))
|
||||
if err != nil {
|
||||
addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not a pool condition that we recognize. It should be one of %s", poolConditionValidValues()))
|
||||
continue
|
||||
}
|
||||
setter.Condition = omit.From(condition)
|
||||
case headerNotes:
|
||||
setter.Notes = omit.From(col)
|
||||
case headerPropertyOwnerName:
|
||||
setter.PropertyOwnerName = omit.From(col)
|
||||
case headerPropertyOwnerPhone:
|
||||
setter.PropertyOwnerPhone = omitnull.From(col)
|
||||
case headerResidentOwned:
|
||||
boolValue, err := parseBool(col)
|
||||
if err != nil {
|
||||
addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not something that we recognize as a true/false value. Please use either 'true' or 'false'", col))
|
||||
continue
|
||||
}
|
||||
setter.ResidentOwned = omitnull.From(boolValue)
|
||||
case headerResidentPhone:
|
||||
setter.ResidentPhone = omitnull.From(col)
|
||||
}
|
||||
}
|
||||
_, err = models.Pools.Insert(&setter).Exec(ctx, txn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create pool: %w", err)
|
||||
}
|
||||
row_number = row_number + 1
|
||||
}
|
||||
file.Update(ctx, txn, &models.FileuploadFileSetter{
|
||||
Status: omit.From(enums.FileuploadFilestatustypeParsed),
|
||||
})
|
||||
txn.Commit(ctx)
|
||||
return nil
|
||||
}
|
||||
func addError(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, row_number int32, column_number int32, msg string) error {
|
||||
r, err := models.FileuploadErrorCSVS.Insert(&models.FileuploadErrorCSVSetter{
|
||||
Col: omit.From(column_number),
|
||||
CSVFileID: omit.From(c.FileID),
|
||||
// ID
|
||||
Line: omit.From(row_number),
|
||||
Message: omit.From(msg),
|
||||
}).One(ctx, txn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to add error: %w", err)
|
||||
}
|
||||
log.Info().Int32("id", r.ID).Int32("file_id", c.FileID).Str("msg", msg).Msg("Created CSV file error")
|
||||
return nil
|
||||
}
|
||||
|
||||
func addImportError(file *models.FileuploadFile, err error) {
|
||||
log.Debug().Err(err).Int32("file_id", file.ID).Msg("Fake add import error")
|
||||
}
|
||||
func validateHeader(row []string) error {
|
||||
return nil
|
||||
func parseBool(s string) (bool, error) {
|
||||
sl := strings.ToLower(s)
|
||||
boolValue, err := strconv.ParseBool(sl)
|
||||
if err != nil {
|
||||
// Handle some of the stuff that strconv doesn't handle
|
||||
switch sl {
|
||||
case "yes":
|
||||
return true, nil
|
||||
case "no":
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("unrecognized '%s'", sl)
|
||||
}
|
||||
|
||||
}
|
||||
return boolValue, err
|
||||
}
|
||||
|
||||
func errorMissingHeader(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, h headerPoolEnum) error {
|
||||
msg := fmt.Sprintf("The file is missing the '%s' header", h.String())
|
||||
return addError(ctx, txn, c, 0, 0, msg)
|
||||
}
|
||||
func parseHeaders(row []string) []headerPoolEnum {
|
||||
results := make([]headerPoolEnum, 0)
|
||||
for _, h := range row {
|
||||
ht := strings.TrimSpace(h)
|
||||
hl := strings.ToLower(ht)
|
||||
log.Debug().Str("header", hl).Msg("Saw CSV header")
|
||||
var type_ headerPoolEnum
|
||||
switch hl {
|
||||
case "city":
|
||||
type_ = headerAddressCity
|
||||
case "zip":
|
||||
case "postal code":
|
||||
type_ = headerAddressPostalCode
|
||||
case "street address":
|
||||
type_ = headerAddressStreet
|
||||
case "condition":
|
||||
case "pool condition":
|
||||
type_ = headerCondition
|
||||
case "notes":
|
||||
type_ = headerNotes
|
||||
case "property owner":
|
||||
case "property owner name":
|
||||
type_ = headerPropertyOwnerName
|
||||
case "property owner phone":
|
||||
type_ = headerPropertyOwnerPhone
|
||||
case "resident owned":
|
||||
type_ = headerResidentOwned
|
||||
case "resident phone":
|
||||
case "resident phone number":
|
||||
type_ = headerResidentPhone
|
||||
default:
|
||||
type_ = headerTag
|
||||
}
|
||||
results = append(results, type_)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
func missingRequiredHeaders(headers []headerPoolEnum) []headerPoolEnum {
|
||||
results := make([]headerPoolEnum, 0)
|
||||
for _, rh := range []headerPoolEnum{headerAddressCity, headerAddressPostalCode, headerAddressStreet} {
|
||||
present := false
|
||||
for _, h := range headers {
|
||||
if h == rh {
|
||||
present = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !present {
|
||||
results = append(results, rh)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
func poolConditionValidValues() string {
|
||||
var b strings.Builder
|
||||
for i, cond := range enums.AllPoolconditiontype() {
|
||||
if i == 0 {
|
||||
fmt.Fprintf(&b, "'%s'", cond)
|
||||
} else {
|
||||
fmt.Fprintf(&b, ", '%s'", cond)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue