271 lines
7.6 KiB
Go
271 lines
7.6 KiB
Go
package csv
|
|
|
|
import (
|
|
"context"
|
|
"encoding/csv"
|
|
"fmt"
|
|
"io"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Gleipnir-Technology/bob"
|
|
"github.com/Gleipnir-Technology/nidus-sync/db"
|
|
"github.com/Gleipnir-Technology/nidus-sync/db/enums"
|
|
"github.com/Gleipnir-Technology/nidus-sync/db/models"
|
|
"github.com/Gleipnir-Technology/nidus-sync/userfile"
|
|
"github.com/aarondl/opt/omit"
|
|
"github.com/aarondl/opt/omitnull"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
type headerPoolEnum int
|
|
|
|
const (
|
|
headerAddressCity = iota
|
|
headerAddressPostalCode
|
|
headerAddressStreet
|
|
headerCondition
|
|
headerNotes
|
|
headerPropertyOwnerName
|
|
headerPropertyOwnerPhone
|
|
headerResidentOwned
|
|
headerResidentPhone
|
|
headerTag
|
|
)
|
|
|
|
func (e headerPoolEnum) String() string {
|
|
switch e {
|
|
case headerAddressCity:
|
|
return "City"
|
|
case headerAddressPostalCode:
|
|
return "Postal Code"
|
|
case headerAddressStreet:
|
|
return "Street Address"
|
|
case headerCondition:
|
|
return "Condition"
|
|
case headerNotes:
|
|
return "Notes"
|
|
case headerPropertyOwnerName:
|
|
return "Property Owner Name"
|
|
case headerPropertyOwnerPhone:
|
|
return "Property Owner Phone"
|
|
case headerResidentOwned:
|
|
return "Resident Owned"
|
|
case headerResidentPhone:
|
|
return "Resident Phone"
|
|
default:
|
|
return "bad programmer"
|
|
}
|
|
}
|
|
func ProcessJob(ctx context.Context, file_id int32) error {
|
|
file, err := models.FindFileuploadFile(ctx, db.PGInstance.BobDB, file_id)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to get file %d from DB: %w", file_id, err)
|
|
}
|
|
c, err := models.FindFileuploadCSV(ctx, db.PGInstance.BobDB, file_id)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to get file %d from DB: %w", file_id, err)
|
|
}
|
|
r, err := userfile.NewFileReader(userfile.CollectionCSV, file.FileUUID)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to get filereader for %d: %w", file_id, err)
|
|
}
|
|
reader := csv.NewReader(r)
|
|
h, err := reader.Read()
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to read header of CSV for file %d: %w", file_id, err)
|
|
}
|
|
txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to start transaction: %w", err)
|
|
}
|
|
defer txn.Rollback(ctx)
|
|
headers := parseHeaders(h)
|
|
missing_headers := missingRequiredHeaders(headers)
|
|
for _, mh := range missing_headers {
|
|
errorMissingHeader(ctx, txn, c, mh)
|
|
file.Update(ctx, txn, &models.FileuploadFileSetter{
|
|
Status: omit.From(enums.FileuploadFilestatustypeError),
|
|
})
|
|
txn.Commit(ctx)
|
|
return nil
|
|
}
|
|
row_number := 0
|
|
for {
|
|
row, err := reader.Read()
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("Failed to read all CSV records for file %d: %w", file_id, err)
|
|
}
|
|
setter := models.PoolSetter{
|
|
// required fields
|
|
//AddressCity: omit.From(),
|
|
//AddressPostalCode: omit.From(),
|
|
//AddressStreet: omit.From(),
|
|
Created: omit.From(time.Now()),
|
|
CreatorID: omit.From(file.CreatorID),
|
|
Committed: omit.From(false),
|
|
Condition: omit.From(enums.PoolconditiontypeUnknown),
|
|
Deleted: omitnull.FromPtr[time.Time](nil),
|
|
// ID - generated
|
|
Notes: omit.From(""),
|
|
OrganizationID: omit.From(file.OrganizationID),
|
|
PropertyOwnerName: omit.From(""),
|
|
PropertyOwnerPhone: omitnull.FromPtr[string](nil),
|
|
ResidentOwned: omitnull.FromPtr[bool](nil),
|
|
ResidentPhone: omitnull.FromPtr[string](nil),
|
|
Version: omit.From(int32(0)),
|
|
}
|
|
for i, col := range row {
|
|
hdr := headers[i]
|
|
switch hdr {
|
|
case headerAddressCity:
|
|
setter.AddressCity = omit.From(col)
|
|
case headerAddressPostalCode:
|
|
setter.AddressPostalCode = omit.From(col)
|
|
case headerAddressStreet:
|
|
setter.AddressStreet = omit.From(col)
|
|
case headerCondition:
|
|
var condition enums.Poolconditiontype
|
|
err := condition.Scan(strings.ToLower(col))
|
|
if err != nil {
|
|
addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not a pool condition that we recognize. It should be one of %s", poolConditionValidValues()))
|
|
continue
|
|
}
|
|
setter.Condition = omit.From(condition)
|
|
case headerNotes:
|
|
setter.Notes = omit.From(col)
|
|
case headerPropertyOwnerName:
|
|
setter.PropertyOwnerName = omit.From(col)
|
|
case headerPropertyOwnerPhone:
|
|
setter.PropertyOwnerPhone = omitnull.From(col)
|
|
case headerResidentOwned:
|
|
boolValue, err := parseBool(col)
|
|
if err != nil {
|
|
addError(ctx, txn, c, int32(row_number), int32(i), fmt.Sprintf("'%s' is not something that we recognize as a true/false value. Please use either 'true' or 'false'", col))
|
|
continue
|
|
}
|
|
setter.ResidentOwned = omitnull.From(boolValue)
|
|
case headerResidentPhone:
|
|
setter.ResidentPhone = omitnull.From(col)
|
|
}
|
|
}
|
|
_, err = models.Pools.Insert(&setter).Exec(ctx, txn)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to create pool: %w", err)
|
|
}
|
|
row_number = row_number + 1
|
|
}
|
|
file.Update(ctx, txn, &models.FileuploadFileSetter{
|
|
Status: omit.From(enums.FileuploadFilestatustypeParsed),
|
|
})
|
|
txn.Commit(ctx)
|
|
return nil
|
|
}
|
|
func addError(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, row_number int32, column_number int32, msg string) error {
|
|
r, err := models.FileuploadErrorCSVS.Insert(&models.FileuploadErrorCSVSetter{
|
|
Col: omit.From(column_number),
|
|
CSVFileID: omit.From(c.FileID),
|
|
// ID
|
|
Line: omit.From(row_number),
|
|
Message: omit.From(msg),
|
|
}).One(ctx, txn)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to add error: %w", err)
|
|
}
|
|
log.Info().Int32("id", r.ID).Int32("file_id", c.FileID).Str("msg", msg).Msg("Created CSV file error")
|
|
return nil
|
|
}
|
|
func addImportError(file *models.FileuploadFile, err error) {
|
|
log.Debug().Err(err).Int32("file_id", file.ID).Msg("Fake add import error")
|
|
}
|
|
func parseBool(s string) (bool, error) {
|
|
sl := strings.ToLower(s)
|
|
boolValue, err := strconv.ParseBool(sl)
|
|
if err != nil {
|
|
// Handle some of the stuff that strconv doesn't handle
|
|
switch sl {
|
|
case "yes":
|
|
return true, nil
|
|
case "no":
|
|
return false, nil
|
|
default:
|
|
return false, fmt.Errorf("unrecognized '%s'", sl)
|
|
}
|
|
|
|
}
|
|
return boolValue, err
|
|
}
|
|
|
|
func errorMissingHeader(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, h headerPoolEnum) error {
|
|
msg := fmt.Sprintf("The file is missing the '%s' header", h.String())
|
|
return addError(ctx, txn, c, 0, 0, msg)
|
|
}
|
|
func parseHeaders(row []string) []headerPoolEnum {
|
|
results := make([]headerPoolEnum, 0)
|
|
for _, h := range row {
|
|
ht := strings.TrimSpace(h)
|
|
hl := strings.ToLower(ht)
|
|
log.Debug().Str("header", hl).Msg("Saw CSV header")
|
|
var type_ headerPoolEnum
|
|
switch hl {
|
|
case "city":
|
|
type_ = headerAddressCity
|
|
case "zip":
|
|
case "postal code":
|
|
type_ = headerAddressPostalCode
|
|
case "street address":
|
|
type_ = headerAddressStreet
|
|
case "condition":
|
|
case "pool condition":
|
|
type_ = headerCondition
|
|
case "notes":
|
|
type_ = headerNotes
|
|
case "property owner":
|
|
case "property owner name":
|
|
type_ = headerPropertyOwnerName
|
|
case "property owner phone":
|
|
type_ = headerPropertyOwnerPhone
|
|
case "resident owned":
|
|
type_ = headerResidentOwned
|
|
case "resident phone":
|
|
case "resident phone number":
|
|
type_ = headerResidentPhone
|
|
default:
|
|
type_ = headerTag
|
|
}
|
|
results = append(results, type_)
|
|
}
|
|
|
|
return results
|
|
}
|
|
func missingRequiredHeaders(headers []headerPoolEnum) []headerPoolEnum {
|
|
results := make([]headerPoolEnum, 0)
|
|
for _, rh := range []headerPoolEnum{headerAddressCity, headerAddressPostalCode, headerAddressStreet} {
|
|
present := false
|
|
for _, h := range headers {
|
|
if h == rh {
|
|
present = true
|
|
break
|
|
}
|
|
}
|
|
if !present {
|
|
results = append(results, rh)
|
|
}
|
|
}
|
|
return results
|
|
}
|
|
func poolConditionValidValues() string {
|
|
var b strings.Builder
|
|
for i, cond := range enums.AllPoolconditiontype() {
|
|
if i == 0 {
|
|
fmt.Fprintf(&b, "'%s'", cond)
|
|
} else {
|
|
fmt.Fprintf(&b, ", '%s'", cond)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|