diff --git a/api/api.go b/api/api.go index eb3ec2d3..5a25aff2 100644 --- a/api/api.go +++ b/api/api.go @@ -68,7 +68,7 @@ func apiAudioContentPost(w http.ResponseWriter, r *http.Request, u *models.User) http.Error(w, "Failed to parse image UUID", http.StatusBadRequest) return } - err = userfile.AudioFileContentWrite(audioUUID, r.Body) + err = userfile.FileContentWrite(r.Body, userfile.CollectionAudioRaw, audioUUID) if err != nil { log.Printf("Failed to write content file: %v", err) http.Error(w, "failed to write content file", http.StatusInternalServerError) diff --git a/background/pool.go b/background/pool.go index 220c626f..f5105d8d 100644 --- a/background/pool.go +++ b/background/pool.go @@ -4,6 +4,7 @@ import ( "context" //"fmt" + "github.com/Gleipnir-Technology/nidus-sync/platform/csv" //"github.com/Gleipnir-Technology/nidus-sync/userfile" //"github.com/google/uuid" "github.com/rs/zerolog/log" @@ -16,10 +17,6 @@ type jobImportCSVPool struct { var channelJobImportCSVPool chan jobImportCSVPool -func processCSVJob(file_id int32) error { - log.Debug().Int32("file_id", file_id).Msg("Fake processing CSV job") - return nil -} func startWorkerCSV(ctx context.Context, channelJobImport chan jobImportCSVPool) { go func() { for { @@ -29,7 +26,7 @@ func startWorkerCSV(ctx context.Context, channelJobImport chan jobImportCSVPool) return case job := <-channelJobImport: log.Info().Int32("id", job.fileID).Msg("Processing CSV job") - err := processCSVJob(job.fileID) + err := csv.ProcessJob(job.fileID) if err != nil { log.Error().Err(err).Int32("id", job.fileID).Msg("Error processing CSV file") } diff --git a/platform/csv/pool.go b/platform/csv/pool.go new file mode 100644 index 00000000..08693313 --- /dev/null +++ b/platform/csv/pool.go @@ -0,0 +1,13 @@ +package csv + +import ( +//"encoding/csv" +//"github.com/Gleipnir-Technology/nidus-sync/platform/csv" +//"github.com/Gleipnir-Technology/nidus-sync/userfile" +//"github.com/rs/zerolog/log" +) + +func ProcessJob(file_id int32) error { + //userfile.NewFileReader("csv" + return nil +} diff --git a/sync/pool.go b/sync/pool.go index 36411851..7ab22389 100644 --- a/sync/pool.go +++ b/sync/pool.go @@ -62,7 +62,7 @@ func postPoolUpload(w http.ResponseWriter, r *http.Request, u *models.User) { respondError(w, "Failed to parse form", err, http.StatusBadRequest) return } - uploads, err := userfile.SaveFileUpload(r, "csvfile", "pool", "csv") + uploads, err := userfile.SaveFileUpload(r, "csvfile", userfile.CollectionCSV) if err != nil { respondError(w, "Failed to extract image uploads", err, http.StatusInternalServerError) return diff --git a/userfile/audio.go b/userfile/audio.go index 6d972444..60e1b07c 100644 --- a/userfile/audio.go +++ b/userfile/audio.go @@ -13,7 +13,7 @@ import ( func NormalizeAudio(audioUUID uuid.UUID) error { //source := AudioFileContentPathRaw(audioUUID.String()) - source := fileContentPath("user", audioUUID, "m4a") + source := fileContentPath(CollectionAudioRaw, audioUUID) _, err := os.Stat(source) if errors.Is(err, os.ErrNotExist) { log.Warn().Str("source", source).Msg("file doesn't exist, skipping normalization") @@ -47,7 +47,7 @@ func TranscodeToOgg(audioUUID uuid.UUID) error { } log.Info().Str("source", source).Msg("Transcoding to ogg") //destination := userfile.AudioFileContentPathOgg(audioUUID.String()) - destination := fileContentPath("user", audioUUID, "ogg") + destination := fileContentPath(CollectionAudioTranscoded, audioUUID) // Use "ffmpeg" directly, assuming it's in the system PATH cmd := exec.Command("ffmpeg", "-i", source, "-vn", "-acodec", "libvorbis", destination) out, err := cmd.CombinedOutput() @@ -65,5 +65,5 @@ func TranscodeToOgg(audioUUID uuid.UUID) error { func fileContentPathAudioNormalized(u uuid.UUID) string { //destination := AudioFileContentPathNormalized(audioUUID.String()) - return fileContentPath("user", u, "normalized.m4a") + return fileContentPath(CollectionAudioNormalized, u) } diff --git a/userfile/base.go b/userfile/base.go new file mode 100644 index 00000000..0d7a9437 --- /dev/null +++ b/userfile/base.go @@ -0,0 +1,95 @@ +package userfile + +import ( + "fmt" + "io" + "net/http" + "os" + + "github.com/Gleipnir-Technology/nidus-sync/config" + "github.com/google/uuid" + //"github.com/rs/zerolog/log" +) + +func audioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error { + return nil +} + +var collectionToExtension map[Collection]string = map[Collection]string{ + CollectionAudioRaw: "raw", + CollectionAudioTranscoded: "ogg", + CollectionCSV: "csv", + CollectionLogo: "png", + CollectionPublicImage: "img", + CollectionImageRaw: "raw", +} +var collectionToSubdir map[Collection]string = map[Collection]string{ + CollectionAudioRaw: "audio-raw", + CollectionAudioTranscoded: "audio-transcoded", + CollectionCSV: "csv", + CollectionLogo: "logo", + CollectionPublicImage: "public-image", + CollectionImageRaw: "image-raw", +} + +func fileContentPath(collection Collection, uid uuid.UUID) string { + subdir, ok := collectionToSubdir[collection] + if !ok { + panic(fmt.Sprintf("No subdir for collection %d", int(collection))) + } + extension, ok := collectionToExtension[collection] + return fmt.Sprintf("%s/%s/%s.%s", config.FilesDirectory, subdir, uid.String(), extension) +} + +/* + func fileContentWrite(body io.Reader, subdir string, uid uuid.UUID, extension string) error { + // Create file in configured directory + filepath := fileContentPath(subdir, uid, extension) + dst, err := os.Create(filepath) + if err != nil { + log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create file") + return fmt.Errorf("Failed to create file at %s: %v", filepath, err) + } + defer dst.Close() + + // Copy rest of request body to file + _, err = io.Copy(dst, body) + if err != nil { + return fmt.Errorf("Unable to save content of %s: %v", filepath, err) + } + return nil + } +*/ +func writeFileContent(w http.ResponseWriter, image_path string) { + // Open the file + file, err := os.Open(image_path) + if err != nil { + if os.IsNotExist(err) { + http.Error(w, "Image not found", http.StatusNotFound) + } else { + http.Error(w, "Failed to retrieve image", http.StatusInternalServerError) + } + return + } + defer file.Close() + + // Get file info for Content-Length header + fileInfo, err := file.Stat() + if err != nil { + http.Error(w, "Failed to get image information", http.StatusInternalServerError) + return + } + + // Set appropriate headers + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size())) + + // Copy file contents to response writer + _, err = io.Copy(w, file) + if err != nil { + // Note: At this point, we've already started writing the response, + // so we can't change the status code anymore. The best we can do + // is log the error and abandon the connection. + return + } +} diff --git a/userfile/enum.go b/userfile/enum.go new file mode 100644 index 00000000..130a44c6 --- /dev/null +++ b/userfile/enum.go @@ -0,0 +1,13 @@ +package userfile + +type Collection int + +const ( + CollectionAudioRaw Collection = iota + CollectionAudioNormalized + CollectionAudioTranscoded + CollectionCSV + CollectionImageRaw + CollectionLogo + CollectionPublicImage +) diff --git a/userfile/image.go b/userfile/image.go new file mode 100644 index 00000000..ff775174 --- /dev/null +++ b/userfile/image.go @@ -0,0 +1,60 @@ +package userfile + +import ( + "fmt" + "io" + "net/http" + "os" + + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +func ImageFileContentWrite(uid uuid.UUID, body io.Reader) error { + filepath := fileContentPath(CollectionImageRaw, uid) + + // Create file in configured directory + dst, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("Failed to create image file %s: %w", filepath, err) + } + defer dst.Close() + + // Copy rest of request body to file + _, err = io.Copy(dst, body) + if err != nil { + return fmt.Errorf("Unable to save file %s: %w", filepath, err) + } + return nil +} +func ImageFileContentWriteLogo(w http.ResponseWriter, uid uuid.UUID) { + //image_path := imageFileContentPathLogoPng(uid.String()) + image_path := fileContentPath(CollectionLogo, uid) + writeFileContent(w, image_path) +} + +func PublicImageFileContentWrite(uid uuid.UUID, body io.Reader) error { + // Create file in configured directory + //filepath := PublicImageFileContentPathRaw(uid.String()) + filepath := fileContentPath(CollectionPublicImage, uid) + dst, err := os.Create(filepath) + if err != nil { + log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create public image file") + return fmt.Errorf("Failed to create public image file at %s: %v", filepath, err) + } + defer dst.Close() + + // Copy rest of request body to file + _, err = io.Copy(dst, body) + if err != nil { + return fmt.Errorf("Unable to save file to create audio file at %s: %v", filepath, err) + } + log.Info().Str("filepath", filepath).Msg("Saved public report image file content") + return nil +} + +func PublicImageFileToResponse(w http.ResponseWriter, uid uuid.UUID) { + //image_path := PublicImageFileContentPathRaw(uid) + image_path := fileContentPath(CollectionPublicImage, uid) + writeFileContent(w, image_path) +} diff --git a/userfile/upload.go b/userfile/upload.go index caa21db9..bbb507cb 100644 --- a/userfile/upload.go +++ b/userfile/upload.go @@ -18,7 +18,7 @@ type FileUpload struct { UUID uuid.UUID } -func SaveFileUpload(r *http.Request, name string, subdir string, extension string) ([]FileUpload, error) { +func SaveFileUpload(r *http.Request, name string, collection Collection) ([]FileUpload, error) { results := make([]FileUpload, 0) for n, fheaders := range r.MultipartForm.File { log.Debug().Str("n", n).Msg("looking at header") @@ -26,7 +26,7 @@ func SaveFileUpload(r *http.Request, name string, subdir string, extension strin continue } for _, headers := range fheaders { - f, err := saveFileUpload(headers, subdir, extension) + f, err := saveFileUpload(headers, collection) if err != nil { return results, fmt.Errorf("Failed to extract photo upload: %w", err) } @@ -35,11 +35,11 @@ func SaveFileUpload(r *http.Request, name string, subdir string, extension strin } return results, nil } -func saveFileUploads(r *http.Request, subdir string, extension string) ([]FileUpload, error) { +func saveFileUploads(r *http.Request, collection Collection) ([]FileUpload, error) { results := make([]FileUpload, 0) for name, fheaders := range r.MultipartForm.File { for _, headers := range fheaders { - upload, err := saveFileUpload(headers, subdir, extension) + upload, err := saveFileUpload(headers, collection) if err != nil { return results, fmt.Errorf("Failed to save upload '%s': %w", name, err) } @@ -48,7 +48,7 @@ func saveFileUploads(r *http.Request, subdir string, extension string) ([]FileUp } return results, nil } -func saveFileUpload(headers *multipart.FileHeader, subdir string, extension string) (upload FileUpload, err error) { +func saveFileUpload(headers *multipart.FileHeader, collection Collection) (upload FileUpload, err error) { file, err := headers.Open() if err != nil { return upload, fmt.Errorf("Failed to open header: %w", err) @@ -62,7 +62,7 @@ func saveFileUpload(headers *multipart.FileHeader, subdir string, extension stri if err != nil { return upload, fmt.Errorf("Failed to create uuid", err) } - err = fileContentWrite(bytes.NewReader(file_bytes), subdir, u, extension) + err = FileContentWrite(bytes.NewReader(file_bytes), collection, u) if err != nil { return upload, fmt.Errorf("Failed to write file to disk: %w", err) } diff --git a/userfile/userfile.go b/userfile/userfile.go index adf7d232..ee63dcdd 100644 --- a/userfile/userfile.go +++ b/userfile/userfile.go @@ -3,34 +3,17 @@ package userfile import ( "fmt" "io" - "net/http" + //"net/http" "os" - "github.com/Gleipnir-Technology/nidus-sync/config" + //"github.com/Gleipnir-Technology/nidus-sync/config" "github.com/google/uuid" "github.com/rs/zerolog/log" ) -/* - func AudioFileContentPathRaw(audioUUID string) string { - return fmt.Sprintf("%s/%s.m4a", config.FilesDirectoryUser, audioUUID) - } - - func AudioFileContentPathMp3(audioUUID string) string { - return fmt.Sprintf("%s/%s.mp3", config.FilesDirectoryUser, audioUUID) - } - - func AudioFileContentPathNormalized(audioUUID string) string { - return fmt.Sprintf("%s/%s-normalized.m4a", config.FilesDirectoryUser, audioUUID) - } - - func AudioFileContentPathOgg(audioUUID string) string { - return fmt.Sprintf("%s/%s.ogg", config.FilesDirectoryUser, audioUUID) - } -*/ -func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error { +func FileContentWrite(body io.Reader, collection Collection, uid uuid.UUID) error { // Create file in configured directory - filepath := fileContentPath("user", audioUUID, "m4a") + filepath := fileContentPath(collection, uid) dst, err := os.Create(filepath) if err != nil { log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create audio file") @@ -46,126 +29,3 @@ func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error { log.Info().Str("filepath", filepath).Msg("Save audio file content") return nil } - -/* - func ImageFileContentPathRawUser(uid string) string { - return imageFileContentPath(config.FilesDirectoryUser, uid, "raw") - } - - func imageFileContentPathLogoPng(uid string) string { - return imageFileContentPath(config.FilesDirectoryLogo, uid, "png") - } - - func imageFileContentPath(dir string, uid string, ext string) string { - return fmt.Sprintf("%s/%s.%s", dir, uid, ext) - } -*/ -func ImageFileContentWrite(uid uuid.UUID, body io.Reader) error { - filepath := fileContentPath("user", uid, "raw") - - // Create file in configured directory - dst, err := os.Create(filepath) - if err != nil { - return fmt.Errorf("Failed to create image file %s: %w", filepath, err) - } - defer dst.Close() - - // Copy rest of request body to file - _, err = io.Copy(dst, body) - if err != nil { - return fmt.Errorf("Unable to save file %s: %w", filepath, err) - } - return nil -} -func ImageFileContentWriteLogo(w http.ResponseWriter, uid uuid.UUID) { - //image_path := imageFileContentPathLogoPng(uid.String()) - image_path := fileContentPath("logo", uid, "png") - writeFileContent(w, image_path) -} - -func PublicImageFileContentWrite(uid uuid.UUID, body io.Reader) error { - // Create file in configured directory - //filepath := PublicImageFileContentPathRaw(uid.String()) - filepath := fileContentPath("public", uid, "raw") - dst, err := os.Create(filepath) - if err != nil { - log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create public image file") - return fmt.Errorf("Failed to create public image file at %s: %v", filepath, err) - } - defer dst.Close() - - // Copy rest of request body to file - _, err = io.Copy(dst, body) - if err != nil { - return fmt.Errorf("Unable to save file to create audio file at %s: %v", filepath, err) - } - log.Info().Str("filepath", filepath).Msg("Saved public report image file content") - return nil -} - -/* -func PublicImageFileContentPathRaw(uid string) string { - return fmt.Sprintf("%s/%s.raw", config.FilesDirectoryPublic, uid) -} -*/ - -func PublicImageFileToResponse(w http.ResponseWriter, uid uuid.UUID) { - //image_path := PublicImageFileContentPathRaw(uid) - image_path := fileContentPath("public", uid, "raw") - writeFileContent(w, image_path) -} - -func fileContentPath(subdir string, uid uuid.UUID, extension string) string { - return fmt.Sprintf("%s/%s/%s.%s", config.FilesDirectory, subdir, uid.String(), extension) -} -func fileContentWrite(body io.Reader, subdir string, uid uuid.UUID, extension string) error { - // Create file in configured directory - filepath := fileContentPath(subdir, uid, extension) - dst, err := os.Create(filepath) - if err != nil { - log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create file") - return fmt.Errorf("Failed to create file at %s: %v", filepath, err) - } - defer dst.Close() - - // Copy rest of request body to file - _, err = io.Copy(dst, body) - if err != nil { - return fmt.Errorf("Unable to save content of %s: %v", filepath, err) - } - return nil -} - -func writeFileContent(w http.ResponseWriter, image_path string) { - // Open the file - file, err := os.Open(image_path) - if err != nil { - if os.IsNotExist(err) { - http.Error(w, "Image not found", http.StatusNotFound) - } else { - http.Error(w, "Failed to retrieve image", http.StatusInternalServerError) - } - return - } - defer file.Close() - - // Get file info for Content-Length header - fileInfo, err := file.Stat() - if err != nil { - http.Error(w, "Failed to get image information", http.StatusInternalServerError) - return - } - - // Set appropriate headers - w.Header().Set("Content-Type", "image/png") - w.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size())) - - // Copy file contents to response writer - _, err = io.Copy(w, file) - if err != nil { - // Note: At this point, we've already started writing the response, - // so we can't change the status code anymore. The best we can do - // is log the error and abandon the connection. - return - } -}