Split out ability to upload flyover data from pool uploads

Tons of changes here, all in the name of quickly getting to where I can
create test compliance letters.

platform/csv/csv.go

@@ -0,0 +1,128 @@
+package csv
+
+import (
+	"context"
+	//"encoding/csv"
+	"fmt"
+	//"io"
+	"strconv"
+	"strings"
+	//"sync"
+	//"time"
+
+	"github.com/Gleipnir-Technology/bob"
+	"github.com/Gleipnir-Technology/bob/dialect/psql"
+	"github.com/Gleipnir-Technology/bob/dialect/psql/um"
+	//"github.com/Gleipnir-Technology/nidus-sync/config"
+	"github.com/Gleipnir-Technology/nidus-sync/db"
+	"github.com/Gleipnir-Technology/nidus-sync/db/enums"
+	"github.com/Gleipnir-Technology/nidus-sync/db/models"
+	//"github.com/Gleipnir-Technology/nidus-sync/h3utils"
+	//"github.com/Gleipnir-Technology/nidus-sync/platform/geom"
+	//"github.com/Gleipnir-Technology/nidus-sync/platform/text"
+	//"github.com/Gleipnir-Technology/nidus-sync/stadia"
+	//"github.com/Gleipnir-Technology/nidus-sync/userfile"
+	"github.com/aarondl/opt/omit"
+	//"github.com/aarondl/opt/omitnull"
+	"github.com/rs/zerolog/log"
+)
+
+type csvParserFunc[T any] = func(context.Context, bob.Tx, *models.FileuploadFile, *models.FileuploadCSV) ([]T, error)
+type csvProcessorFunc[T any] = func(context.Context, bob.Tx, *models.FileuploadFile, *models.FileuploadCSV, []T) error
+
+func ProcessJob(ctx context.Context, file_id int32, type_ enums.FileuploadCsvtype) error {
+	var err error
+	switch type_ {
+	case enums.FileuploadCsvtypePoollist:
+		err = processCSV(ctx, file_id, parseCSVPoollist, processCSVPoollist)
+	case enums.FileuploadCsvtypeFlyover:
+		err = processCSV(ctx, file_id, parseCSVFlyover, processCSVFlyover)
+	}
+	return err
+}
+
+func processCSV[T any](ctx context.Context, file_id int32, parser csvParserFunc[T], processor csvProcessorFunc[T]) error {
+	file, c, err := loadFileAndCSV(ctx, file_id)
+	if err != nil {
+		return fmt.Errorf("load file and csv: %w", err)
+	}
+	txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("Failed to start transaction: %w", err)
+	}
+	defer txn.Rollback(ctx)
+	parsed, err := parser(ctx, txn, file, c)
+	if err != nil {
+		return fmt.Errorf("parse file: %w", err)
+	}
+	_, err = psql.Update(
+		um.Table("fileupload.csv"),
+		um.SetCol("rowcount").ToArg(len(parsed)),
+		um.Where(psql.Quote("file_id").EQ(psql.Arg(file_id))),
+	).Exec(ctx, txn)
+	if err != nil {
+		return fmt.Errorf("update csv row: %w", err)
+	}
+	err = processor(ctx, txn, file, c, parsed)
+	if err != nil {
+		return fmt.Errorf("process parsed file: %w", err)
+	}
+
+	file.Update(ctx, txn, &models.FileuploadFileSetter{
+		Status: omit.From(enums.FileuploadFilestatustypeParsed),
+	})
+	log.Info().Int32("file.ID", file.ID).Msg("Set file to parsed")
+	txn.Commit(ctx)
+	return nil
+}
+func loadFileAndCSV(ctx context.Context, file_id int32) (*models.FileuploadFile, *models.FileuploadCSV, error) {
+	file, err := models.FindFileuploadFile(ctx, db.PGInstance.BobDB, file_id)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to get file %d from DB: %w", file_id, err)
+	}
+	c, err := models.FindFileuploadCSV(ctx, db.PGInstance.BobDB, file.ID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to get csv file %d from DB: %w", file.ID, err)
+	}
+	return file, c, nil
+}
+
+func addError(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, row_number int32, column_number int32, msg string) error {
+	r, err := models.FileuploadErrorCSVS.Insert(&models.FileuploadErrorCSVSetter{
+		Col:       omit.From(column_number),
+		CSVFileID: omit.From(c.FileID),
+		// ID
+		Line:    omit.From(row_number),
+		Message: omit.From(msg),
+	}).One(ctx, txn)
+	if err != nil {
+		return fmt.Errorf("Failed to add error: %w", err)
+	}
+	log.Info().Int32("id", r.ID).Int32("file_id", c.FileID).Str("msg", msg).Int32("row", row_number).Int32("col", column_number).Msg("Created CSV file error")
+	return nil
+}
+func addImportError(file *models.FileuploadFile, err error) {
+	log.Debug().Err(err).Int32("file_id", file.ID).Msg("Fake add import error")
+}
+func parseBool(s string) (bool, error) {
+	sl := strings.ToLower(s)
+	boolValue, err := strconv.ParseBool(sl)
+	if err != nil {
+		// Handle some of the stuff that strconv doesn't handle
+		switch sl {
+		case "yes":
+			return true, nil
+		case "no":
+			return false, nil
+		default:
+			return false, fmt.Errorf("unrecognized '%s'", sl)
+		}
+
+	}
+	return boolValue, err
+}
+
+func errorMissingHeader(ctx context.Context, txn bob.Tx, c *models.FileuploadCSV, h headerPoolEnum) error {
+	msg := fmt.Sprintf("The file is missing the '%s' header", h.String())
+	return addError(ctx, txn, c, 0, 0, msg)
+}