Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rework userfile yet again

Browse source 
I'm settling on the idea that strings should never be returned from the
userfile system. Instead, indicate which collection you want and pass
objects across.
This commit is contained in:
Eli Ribble 2026-02-08 04:36:12 +00:00
parent c2d84b8734
commit fdd783c19c
No known key found for this signature in database
10 changed files with 198 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api/api.go
View file

@ -68,7 +68,7 @@ func apiAudioContentPost(w http.ResponseWriter, r *http.Request, u *models.User)
http.Error(w, "Failed to parse image UUID", http.StatusBadRequest)
return
}
err = userfile.AudioFileContentWrite(audioUUID, r.Body)
err = userfile.FileContentWrite(r.Body, userfile.CollectionAudioRaw, audioUUID)
if err != nil {
log.Printf("Failed to write content file: %v", err)
http.Error(w, "failed to write content file", http.StatusInternalServerError)

7
background/pool.go
View file

@ -4,6 +4,7 @@ import (
"context"
//"fmt"
"github.com/Gleipnir-Technology/nidus-sync/platform/csv"
//"github.com/Gleipnir-Technology/nidus-sync/userfile"
//"github.com/google/uuid"
"github.com/rs/zerolog/log"
@ -16,10 +17,6 @@ type jobImportCSVPool struct {
var channelJobImportCSVPool chan jobImportCSVPool
func processCSVJob(file_id int32) error {
log.Debug().Int32("file_id", file_id).Msg("Fake processing CSV job")
return nil
}
func startWorkerCSV(ctx context.Context, channelJobImport chan jobImportCSVPool) {
go func() {
for {
@ -29,7 +26,7 @@ func startWorkerCSV(ctx context.Context, channelJobImport chan jobImportCSVPool)
return
case job := <-channelJobImport:
log.Info().Int32("id", job.fileID).Msg("Processing CSV job")
err := processCSVJob(job.fileID)
err := csv.ProcessJob(job.fileID)
if err != nil {
log.Error().Err(err).Int32("id", job.fileID).Msg("Error processing CSV file")
}

13
platform/csv/pool.go Normal file
View file

@ -0,0 +1,13 @@
package csv
import (
//"encoding/csv"
//"github.com/Gleipnir-Technology/nidus-sync/platform/csv"
//"github.com/Gleipnir-Technology/nidus-sync/userfile"
//"github.com/rs/zerolog/log"
)
func ProcessJob(file_id int32) error {
//userfile.NewFileReader("csv"
return nil
}

2
sync/pool.go
View file

@ -62,7 +62,7 @@ func postPoolUpload(w http.ResponseWriter, r *http.Request, u *models.User) {
respondError(w, "Failed to parse form", err, http.StatusBadRequest)
return
}
uploads, err := userfile.SaveFileUpload(r, "csvfile", "pool", "csv")
uploads, err := userfile.SaveFileUpload(r, "csvfile", userfile.CollectionCSV)
if err != nil {
respondError(w, "Failed to extract image uploads", err, http.StatusInternalServerError)
return

6
userfile/audio.go
View file

@ -13,7 +13,7 @@ import (
func NormalizeAudio(audioUUID uuid.UUID) error {
//source := AudioFileContentPathRaw(audioUUID.String())
source := fileContentPath("user", audioUUID, "m4a")
source := fileContentPath(CollectionAudioRaw, audioUUID)
_, err := os.Stat(source)
if errors.Is(err, os.ErrNotExist) {
log.Warn().Str("source", source).Msg("file doesn't exist, skipping normalization")
@ -47,7 +47,7 @@ func TranscodeToOgg(audioUUID uuid.UUID) error {
}
log.Info().Str("source", source).Msg("Transcoding to ogg")
//destination := userfile.AudioFileContentPathOgg(audioUUID.String())
destination := fileContentPath("user", audioUUID, "ogg")
destination := fileContentPath(CollectionAudioTranscoded, audioUUID)
// Use "ffmpeg" directly, assuming it's in the system PATH
cmd := exec.Command("ffmpeg", "-i", source, "-vn", "-acodec", "libvorbis", destination)
out, err := cmd.CombinedOutput()
@ -65,5 +65,5 @@ func TranscodeToOgg(audioUUID uuid.UUID) error {
func fileContentPathAudioNormalized(u uuid.UUID) string {
//destination := AudioFileContentPathNormalized(audioUUID.String())
return fileContentPath("user", u, "normalized.m4a")
return fileContentPath(CollectionAudioNormalized, u)
}

95
userfile/base.go Normal file
View file

@ -0,0 +1,95 @@
package userfile
import (
"fmt"
"io"
"net/http"
"os"
"github.com/Gleipnir-Technology/nidus-sync/config"
"github.com/google/uuid"
//"github.com/rs/zerolog/log"
)
func audioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error {
return nil
}
var collectionToExtension map[Collection]string = map[Collection]string{
CollectionAudioRaw: "raw",
CollectionAudioTranscoded: "ogg",
CollectionCSV: "csv",
CollectionLogo: "png",
CollectionPublicImage: "img",
CollectionImageRaw: "raw",
}
var collectionToSubdir map[Collection]string = map[Collection]string{
CollectionAudioRaw: "audio-raw",
CollectionAudioTranscoded: "audio-transcoded",
CollectionCSV: "csv",
CollectionLogo: "logo",
CollectionPublicImage: "public-image",
CollectionImageRaw: "image-raw",
}
func fileContentPath(collection Collection, uid uuid.UUID) string {
subdir, ok := collectionToSubdir[collection]
if !ok {
panic(fmt.Sprintf("No subdir for collection %d", int(collection)))
}
extension, ok := collectionToExtension[collection]
return fmt.Sprintf("%s/%s/%s.%s", config.FilesDirectory, subdir, uid.String(), extension)
}
/*
func fileContentWrite(body io.Reader, subdir string, uid uuid.UUID, extension string) error {
// Create file in configured directory
filepath := fileContentPath(subdir, uid, extension)
dst, err := os.Create(filepath)
if err != nil {
log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create file")
return fmt.Errorf("Failed to create file at %s: %v", filepath, err)
}
defer dst.Close()
// Copy rest of request body to file
_, err = io.Copy(dst, body)
if err != nil {
return fmt.Errorf("Unable to save content of %s: %v", filepath, err)
}
return nil
}
*/
func writeFileContent(w http.ResponseWriter, image_path string) {
// Open the file
file, err := os.Open(image_path)
if err != nil {
if os.IsNotExist(err) {
http.Error(w, "Image not found", http.StatusNotFound)
} else {
http.Error(w, "Failed to retrieve image", http.StatusInternalServerError)
}
return
}
defer file.Close()
// Get file info for Content-Length header
fileInfo, err := file.Stat()
if err != nil {
http.Error(w, "Failed to get image information", http.StatusInternalServerError)
return
}
// Set appropriate headers
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size()))
// Copy file contents to response writer
_, err = io.Copy(w, file)
if err != nil {
// Note: At this point, we've already started writing the response,
// so we can't change the status code anymore. The best we can do
// is log the error and abandon the connection.
return
}
}

13
userfile/enum.go Normal file
View file

@ -0,0 +1,13 @@
package userfile
type Collection int
const (
CollectionAudioRaw Collection = iota
CollectionAudioNormalized
CollectionAudioTranscoded
CollectionCSV
CollectionImageRaw
CollectionLogo
CollectionPublicImage
)

60
userfile/image.go Normal file
View file

@ -0,0 +1,60 @@
package userfile
import (
"fmt"
"io"
"net/http"
"os"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
)
func ImageFileContentWrite(uid uuid.UUID, body io.Reader) error {
filepath := fileContentPath(CollectionImageRaw, uid)
// Create file in configured directory
dst, err := os.Create(filepath)
if err != nil {
return fmt.Errorf("Failed to create image file %s: %w", filepath, err)
}
defer dst.Close()
// Copy rest of request body to file
_, err = io.Copy(dst, body)
if err != nil {
return fmt.Errorf("Unable to save file %s: %w", filepath, err)
}
return nil
}
func ImageFileContentWriteLogo(w http.ResponseWriter, uid uuid.UUID) {
//image_path := imageFileContentPathLogoPng(uid.String())
image_path := fileContentPath(CollectionLogo, uid)
writeFileContent(w, image_path)
}
func PublicImageFileContentWrite(uid uuid.UUID, body io.Reader) error {
// Create file in configured directory
//filepath := PublicImageFileContentPathRaw(uid.String())
filepath := fileContentPath(CollectionPublicImage, uid)
dst, err := os.Create(filepath)
if err != nil {
log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create public image file")
return fmt.Errorf("Failed to create public image file at %s: %v", filepath, err)
}
defer dst.Close()
// Copy rest of request body to file
_, err = io.Copy(dst, body)
if err != nil {
return fmt.Errorf("Unable to save file to create audio file at %s: %v", filepath, err)
}
log.Info().Str("filepath", filepath).Msg("Saved public report image file content")
return nil
}
func PublicImageFileToResponse(w http.ResponseWriter, uid uuid.UUID) {
//image_path := PublicImageFileContentPathRaw(uid)
image_path := fileContentPath(CollectionPublicImage, uid)
writeFileContent(w, image_path)
}

12
userfile/upload.go
View file

@ -18,7 +18,7 @@ type FileUpload struct {
UUID uuid.UUID
}
func SaveFileUpload(r *http.Request, name string, subdir string, extension string) ([]FileUpload, error) {
func SaveFileUpload(r *http.Request, name string, collection Collection) ([]FileUpload, error) {
results := make([]FileUpload, 0)
for n, fheaders := range r.MultipartForm.File {
log.Debug().Str("n", n).Msg("looking at header")
@ -26,7 +26,7 @@ func SaveFileUpload(r *http.Request, name string, subdir string, extension strin
continue
}
for _, headers := range fheaders {
f, err := saveFileUpload(headers, subdir, extension)
f, err := saveFileUpload(headers, collection)
if err != nil {
return results, fmt.Errorf("Failed to extract photo upload: %w", err)
}
@ -35,11 +35,11 @@ func SaveFileUpload(r *http.Request, name string, subdir string, extension strin
}
return results, nil
}
func saveFileUploads(r *http.Request, subdir string, extension string) ([]FileUpload, error) {
func saveFileUploads(r *http.Request, collection Collection) ([]FileUpload, error) {
results := make([]FileUpload, 0)
for name, fheaders := range r.MultipartForm.File {
for _, headers := range fheaders {
upload, err := saveFileUpload(headers, subdir, extension)
upload, err := saveFileUpload(headers, collection)
if err != nil {
return results, fmt.Errorf("Failed to save upload '%s': %w", name, err)
}
@ -48,7 +48,7 @@ func saveFileUploads(r *http.Request, subdir string, extension string) ([]FileUp
}
return results, nil
}
func saveFileUpload(headers *multipart.FileHeader, subdir string, extension string) (upload FileUpload, err error) {
func saveFileUpload(headers *multipart.FileHeader, collection Collection) (upload FileUpload, err error) {
file, err := headers.Open()
if err != nil {
return upload, fmt.Errorf("Failed to open header: %w", err)
@ -62,7 +62,7 @@ func saveFileUpload(headers *multipart.FileHeader, subdir string, extension stri
if err != nil {
return upload, fmt.Errorf("Failed to create uuid", err)
}
err = fileContentWrite(bytes.NewReader(file_bytes), subdir, u, extension)
err = FileContentWrite(bytes.NewReader(file_bytes), collection, u)
if err != nil {
return upload, fmt.Errorf("Failed to write file to disk: %w", err)
}

148
userfile/userfile.go
View file

@ -3,34 +3,17 @@ package userfile
import (
"fmt"
"io"
"net/http"
//"net/http"
"os"
"github.com/Gleipnir-Technology/nidus-sync/config"
//"github.com/Gleipnir-Technology/nidus-sync/config"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
)
/*
func AudioFileContentPathRaw(audioUUID string) string {
return fmt.Sprintf("%s/%s.m4a", config.FilesDirectoryUser, audioUUID)
}
func AudioFileContentPathMp3(audioUUID string) string {
return fmt.Sprintf("%s/%s.mp3", config.FilesDirectoryUser, audioUUID)
}
func AudioFileContentPathNormalized(audioUUID string) string {
return fmt.Sprintf("%s/%s-normalized.m4a", config.FilesDirectoryUser, audioUUID)
}
func AudioFileContentPathOgg(audioUUID string) string {
return fmt.Sprintf("%s/%s.ogg", config.FilesDirectoryUser, audioUUID)
}
*/
func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error {
func FileContentWrite(body io.Reader, collection Collection, uid uuid.UUID) error {
// Create file in configured directory
filepath := fileContentPath("user", audioUUID, "m4a")
filepath := fileContentPath(collection, uid)
dst, err := os.Create(filepath)
if err != nil {
log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create audio file")
@ -46,126 +29,3 @@ func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error {
log.Info().Str("filepath", filepath).Msg("Save audio file content")
return nil
}
/*
func ImageFileContentPathRawUser(uid string) string {
return imageFileContentPath(config.FilesDirectoryUser, uid, "raw")
}
func imageFileContentPathLogoPng(uid string) string {
return imageFileContentPath(config.FilesDirectoryLogo, uid, "png")
}
func imageFileContentPath(dir string, uid string, ext string) string {
return fmt.Sprintf("%s/%s.%s", dir, uid, ext)
}
*/
func ImageFileContentWrite(uid uuid.UUID, body io.Reader) error {
filepath := fileContentPath("user", uid, "raw")
// Create file in configured directory
dst, err := os.Create(filepath)
if err != nil {
return fmt.Errorf("Failed to create image file %s: %w", filepath, err)
}
defer dst.Close()
// Copy rest of request body to file
_, err = io.Copy(dst, body)
if err != nil {
return fmt.Errorf("Unable to save file %s: %w", filepath, err)
}
return nil
}
func ImageFileContentWriteLogo(w http.ResponseWriter, uid uuid.UUID) {
//image_path := imageFileContentPathLogoPng(uid.String())
image_path := fileContentPath("logo", uid, "png")
writeFileContent(w, image_path)
}
func PublicImageFileContentWrite(uid uuid.UUID, body io.Reader) error {
// Create file in configured directory
//filepath := PublicImageFileContentPathRaw(uid.String())
filepath := fileContentPath("public", uid, "raw")
dst, err := os.Create(filepath)
if err != nil {
log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create public image file")
return fmt.Errorf("Failed to create public image file at %s: %v", filepath, err)
}
defer dst.Close()
// Copy rest of request body to file
_, err = io.Copy(dst, body)
if err != nil {
return fmt.Errorf("Unable to save file to create audio file at %s: %v", filepath, err)
}
log.Info().Str("filepath", filepath).Msg("Saved public report image file content")
return nil
}
/*
func PublicImageFileContentPathRaw(uid string) string {
return fmt.Sprintf("%s/%s.raw", config.FilesDirectoryPublic, uid)
}
*/
func PublicImageFileToResponse(w http.ResponseWriter, uid uuid.UUID) {
//image_path := PublicImageFileContentPathRaw(uid)
image_path := fileContentPath("public", uid, "raw")
writeFileContent(w, image_path)
}
func fileContentPath(subdir string, uid uuid.UUID, extension string) string {
return fmt.Sprintf("%s/%s/%s.%s", config.FilesDirectory, subdir, uid.String(), extension)
}
func fileContentWrite(body io.Reader, subdir string, uid uuid.UUID, extension string) error {
// Create file in configured directory
filepath := fileContentPath(subdir, uid, extension)
dst, err := os.Create(filepath)
if err != nil {
log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create file")
return fmt.Errorf("Failed to create file at %s: %v", filepath, err)
}
defer dst.Close()
// Copy rest of request body to file
_, err = io.Copy(dst, body)
if err != nil {
return fmt.Errorf("Unable to save content of %s: %v", filepath, err)
}
return nil
}
func writeFileContent(w http.ResponseWriter, image_path string) {
// Open the file
file, err := os.Open(image_path)
if err != nil {
if os.IsNotExist(err) {
http.Error(w, "Image not found", http.StatusNotFound)
} else {
http.Error(w, "Failed to retrieve image", http.StatusInternalServerError)
}
return
}
defer file.Close()
// Get file info for Content-Length header
fileInfo, err := file.Stat()
if err != nil {
http.Error(w, "Failed to get image information", http.StatusInternalServerError)
return
}
// Set appropriate headers
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size()))
// Copy file contents to response writer
_, err = io.Copy(w, file)
if err != nil {
// Note: At this point, we've already started writing the response,
// so we can't change the status code anymore. The best we can do
// is log the error and abandon the connection.
return
}
}