diff --git a/background/audio.go b/background/audio.go index ddd2b4e6..0cb1e555 100644 --- a/background/audio.go +++ b/background/audio.go @@ -2,12 +2,8 @@ package background import ( "context" - "errors" "fmt" - "os" - "os/exec" - "github.com/Gleipnir-Technology/nidus-sync/db" "github.com/Gleipnir-Technology/nidus-sync/userfile" "github.com/google/uuid" "github.com/rs/zerolog/log" @@ -55,39 +51,15 @@ func enqueueAudioJob(job jobAudio) { } } -func normalizeAudio(audioUUID uuid.UUID) error { - source := userfile.AudioFileContentPathRaw(audioUUID.String()) - _, err := os.Stat(source) - if errors.Is(err, os.ErrNotExist) { - log.Warn().Str("source", source).Msg("file doesn't exist, skipping normalization") - return nil - } - log.Info().Str("sourcce", source).Msg("Normalizing") - destination := userfile.AudioFileContentPathNormalized(audioUUID.String()) - // Use "ffmpeg" directly, assuming it's in the system PATH - cmd := exec.Command("ffmpeg", "-i", source, "-filter:a", "loudnorm", destination) - out, err := cmd.CombinedOutput() - if err != nil { - log.Printf("FFmpeg output for normalization: %s", out) - return fmt.Errorf("ffmpeg normalization failed: %v", err) - } - err = db.NoteAudioNormalized(audioUUID.String()) - if err != nil { - return fmt.Errorf("failed to update database for normalized audio %s: %v", audioUUID, err) - } - log.Info().Str("destination", destination).Msg("Normalized audio") - return nil -} - func processAudioFile(audioUUID uuid.UUID) error { // Normalize audio - err := normalizeAudio(audioUUID) + err := userfile.NormalizeAudio(audioUUID) if err != nil { return fmt.Errorf("failed to normalize audio %s: %v", audioUUID, err) } // Transcode to OGG - err = transcodeToOgg(audioUUID) + err = userfile.TranscodeToOgg(audioUUID) if err != nil { return fmt.Errorf("failed to transcode audio %s to OGG: %v", audioUUID, err) } @@ -97,27 +69,3 @@ func processAudioFile(audioUUID uuid.UUID) error { }) return nil } - -func transcodeToOgg(audioUUID uuid.UUID) error { - source := userfile.AudioFileContentPathNormalized(audioUUID.String()) - _, err := os.Stat(source) - if errors.Is(err, os.ErrNotExist) { - log.Warn().Str("source", source).Msg("file doesn't exist, skipping OGG transcoding") - return nil - } - log.Info().Str("source", source).Msg("Transcoding to ogg") - destination := userfile.AudioFileContentPathOgg(audioUUID.String()) - // Use "ffmpeg" directly, assuming it's in the system PATH - cmd := exec.Command("ffmpeg", "-i", source, "-vn", "-acodec", "libvorbis", destination) - out, err := cmd.CombinedOutput() - if err != nil { - log.Error().Err(err).Bytes("out", out).Msg("FFmpeg output for OGG transcoding") - return fmt.Errorf("ffmpeg OGG transcoding failed: %v", err) - } - err = db.NoteAudioTranscodedToOgg(audioUUID.String()) - if err != nil { - return fmt.Errorf("failed to update database for OGG transcoded audio %s: %v", audioUUID, err) - } - log.Info().Str("destination", destination).Msg("Transcoded audio") - return nil -} diff --git a/background/label_studio.go b/background/label_studio.go index fa8c34f5..80f6f17b 100644 --- a/background/label_studio.go +++ b/background/label_studio.go @@ -130,7 +130,7 @@ func processLabelTask(ctx context.Context, minioClient *minio.Client, minioBucke func createTask(client *labelstudio.Client, project *labelstudio.Project, minioClient *minio.Client, bucket string, customer string, note *models.NoteAudio) error { audioRef := fmt.Sprintf("s3://%s/%s-normalized.m4a", bucket, note.UUID) - audioFile := fmt.Sprintf("%s/%s-normalized.m4a", config.FilesDirectoryUser, note.UUID) + audioFile := fmt.Sprintf("%s/user/%s-normalized.m4a", config.FilesDirectory, note.UUID) uploadPath := fmt.Sprintf("%s-normalized.m4a", note.UUID) if !minioClient.ObjectExists(bucket, uploadPath) { diff --git a/config/config.go b/config/config.go index 978db9d7..e17f301e 100644 --- a/config/config.go +++ b/config/config.go @@ -16,9 +16,7 @@ var ( DomainNidus string DomainTegola string Environment string - FilesDirectoryLogo string - FilesDirectoryPublic string - FilesDirectoryUser string + FilesDirectory string FieldseekerSchemaDirectory string ForwardEmailAPIToken string ForwardEmailReportAddress string @@ -100,17 +98,9 @@ func Parse() (err error) { if FieldseekerSchemaDirectory == "" { return fmt.Errorf("You must specify a non-empty FIELDSEEKER_SCHEMA_DIRECTORY") } - FilesDirectoryLogo = os.Getenv("FILES_DIRECTORY_LOGO") - if FilesDirectoryLogo == "" { - return fmt.Errorf("You must specify a non-empty FILES_DIRECTORY_LOGO") - } - FilesDirectoryPublic = os.Getenv("FILES_DIRECTORY_PUBLIC") - if FilesDirectoryPublic == "" { - return fmt.Errorf("You must specify a non-empty FILES_DIRECTORY_PUBLIC") - } - FilesDirectoryUser = os.Getenv("FILES_DIRECTORY_USER") - if FilesDirectoryUser == "" { - return fmt.Errorf("You must specify a non-empty FILES_DIRECTORY_USER") + FilesDirectory = os.Getenv("FILES_DIRECTORY") + if FilesDirectory == "" { + return fmt.Errorf("You must specify a non-empty FILES_DIRECTORY") } ForwardEmailReportAddress = os.Getenv("FORWARDEMAIL_REPORT_ADDRESS") if ForwardEmailReportAddress == "" { diff --git a/sync/pool.go b/sync/pool.go index 642584f5..6a9d2a76 100644 --- a/sync/pool.go +++ b/sync/pool.go @@ -3,6 +3,7 @@ package sync import ( "github.com/Gleipnir-Technology/nidus-sync/db/models" "github.com/Gleipnir-Technology/nidus-sync/html" + "github.com/Gleipnir-Technology/nidus-sync/userfile" "net/http" ) diff --git a/userfile/audio.go b/userfile/audio.go new file mode 100644 index 00000000..6d972444 --- /dev/null +++ b/userfile/audio.go @@ -0,0 +1,69 @@ +package userfile + +import ( + "errors" + "fmt" + "os" + "os/exec" + + "github.com/Gleipnir-Technology/nidus-sync/db" + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +func NormalizeAudio(audioUUID uuid.UUID) error { + //source := AudioFileContentPathRaw(audioUUID.String()) + source := fileContentPath("user", audioUUID, "m4a") + _, err := os.Stat(source) + if errors.Is(err, os.ErrNotExist) { + log.Warn().Str("source", source).Msg("file doesn't exist, skipping normalization") + return nil + } + log.Info().Str("sourcce", source).Msg("Normalizing") + //destination := AudioFileContentPathNormalized(audioUUID.String()) + destination := fileContentPathAudioNormalized(audioUUID) + // Use "ffmpeg" directly, assuming it's in the system PATH + cmd := exec.Command("ffmpeg", "-i", source, "-filter:a", "loudnorm", destination) + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("FFmpeg output for normalization: %s", out) + return fmt.Errorf("ffmpeg normalization failed: %v", err) + } + err = db.NoteAudioNormalized(audioUUID.String()) + if err != nil { + return fmt.Errorf("failed to update database for normalized audio %s: %v", audioUUID, err) + } + log.Info().Str("destination", destination).Msg("Normalized audio") + return nil +} + +func TranscodeToOgg(audioUUID uuid.UUID) error { + //source := AudioFileContentPathNormalized(audioUUID.String()) + source := fileContentPathAudioNormalized(audioUUID) + _, err := os.Stat(source) + if errors.Is(err, os.ErrNotExist) { + log.Warn().Str("source", source).Msg("file doesn't exist, skipping OGG transcoding") + return nil + } + log.Info().Str("source", source).Msg("Transcoding to ogg") + //destination := userfile.AudioFileContentPathOgg(audioUUID.String()) + destination := fileContentPath("user", audioUUID, "ogg") + // Use "ffmpeg" directly, assuming it's in the system PATH + cmd := exec.Command("ffmpeg", "-i", source, "-vn", "-acodec", "libvorbis", destination) + out, err := cmd.CombinedOutput() + if err != nil { + log.Error().Err(err).Bytes("out", out).Msg("FFmpeg output for OGG transcoding") + return fmt.Errorf("ffmpeg OGG transcoding failed: %v", err) + } + err = db.NoteAudioTranscodedToOgg(audioUUID.String()) + if err != nil { + return fmt.Errorf("failed to update database for OGG transcoded audio %s: %v", audioUUID, err) + } + log.Info().Str("destination", destination).Msg("Transcoded audio") + return nil +} + +func fileContentPathAudioNormalized(u uuid.UUID) string { + //destination := AudioFileContentPathNormalized(audioUUID.String()) + return fileContentPath("user", u, "normalized.m4a") +} diff --git a/userfile/upload.go b/userfile/upload.go new file mode 100644 index 00000000..1e8a577c --- /dev/null +++ b/userfile/upload.go @@ -0,0 +1,77 @@ +package userfile + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +type FileUpload struct { + ContentType string + + UploadFilesize int + UploadFilename string + UUID uuid.UUID +} + +func SaveFileUpload(r *http.Request, name string, subdir string, extension string) ([]FileUpload, error) { + results := make([]FileUpload, 0) + for n, fheaders := range r.MultipartForm.File { + log.Debug().Str("n", n).Msg("looking at header") + if n != name { + continue + } + for _, headers := range fheaders { + f, err := saveFileUpload(headers, subdir, extension) + if err != nil { + return results, fmt.Errorf("Failed to extract photo upload: %w", err) + } + results = append(results, f) + } + } + return results, nil +} +func saveFileUploads(r *http.Request, subdir string, extension string) ([]FileUpload, error) { + results := make([]FileUpload, 0) + for name, fheaders := range r.MultipartForm.File { + for _, headers := range fheaders { + upload, err := saveFileUpload(headers, subdir, extension) + if err != nil { + return results, fmt.Errorf("Failed to save upload '%s': %w", name, err) + } + results = append(results, upload) + } + } + return results, nil +} +func saveFileUpload(headers *multipart.FileHeader, subdir string, extension string) (upload FileUpload, err error) { + file, err := headers.Open() + if err != nil { + return upload, fmt.Errorf("Failed to open header: %w", err) + } + defer file.Close() + + file_bytes, err := io.ReadAll(file) + content_type := http.DetectContentType(file_bytes) + + u, err := uuid.NewUUID() + if err != nil { + return upload, fmt.Errorf("Failed to create uuid", err) + } + err = fileContentWrite(bytes.NewReader(file_bytes), subdir, u, extension) + if err != nil { + return upload, fmt.Errorf("Failed to write file to disk: %w", err) + } + log.Info().Int("size", len(file_bytes)).Str("uploaded_filename", headers.Filename).Str("content-type", content_type).Str("uuid", u.String()).Msg("Saved an uploaded file to disk") + return FileUpload{ + ContentType: content_type, + UploadFilename: headers.Filename, + UploadFilesize: len(file_bytes), + UUID: u, + }, nil +} diff --git a/userfile/userfile.go b/userfile/userfile.go index fc9f20d6..adf7d232 100644 --- a/userfile/userfile.go +++ b/userfile/userfile.go @@ -11,21 +11,26 @@ import ( "github.com/rs/zerolog/log" ) -func AudioFileContentPathRaw(audioUUID string) string { - return fmt.Sprintf("%s/%s.m4a", config.FilesDirectoryUser, audioUUID) -} -func AudioFileContentPathMp3(audioUUID string) string { - return fmt.Sprintf("%s/%s.mp3", config.FilesDirectoryUser, audioUUID) -} -func AudioFileContentPathNormalized(audioUUID string) string { - return fmt.Sprintf("%s/%s-normalized.m4a", config.FilesDirectoryUser, audioUUID) -} -func AudioFileContentPathOgg(audioUUID string) string { - return fmt.Sprintf("%s/%s.ogg", config.FilesDirectoryUser, audioUUID) -} +/* + func AudioFileContentPathRaw(audioUUID string) string { + return fmt.Sprintf("%s/%s.m4a", config.FilesDirectoryUser, audioUUID) + } + + func AudioFileContentPathMp3(audioUUID string) string { + return fmt.Sprintf("%s/%s.mp3", config.FilesDirectoryUser, audioUUID) + } + + func AudioFileContentPathNormalized(audioUUID string) string { + return fmt.Sprintf("%s/%s-normalized.m4a", config.FilesDirectoryUser, audioUUID) + } + + func AudioFileContentPathOgg(audioUUID string) string { + return fmt.Sprintf("%s/%s.ogg", config.FilesDirectoryUser, audioUUID) + } +*/ func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error { // Create file in configured directory - filepath := AudioFileContentPathRaw(audioUUID.String()) + filepath := fileContentPath("user", audioUUID, "m4a") dst, err := os.Create(filepath) if err != nil { log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create audio file") @@ -41,17 +46,22 @@ func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error { log.Info().Str("filepath", filepath).Msg("Save audio file content") return nil } -func ImageFileContentPathRawUser(uid string) string { - return imageFileContentPath(config.FilesDirectoryUser, uid, "raw") -} -func imageFileContentPathLogoPng(uid string) string { - return imageFileContentPath(config.FilesDirectoryLogo, uid, "png") -} -func imageFileContentPath(dir string, uid string, ext string) string { - return fmt.Sprintf("%s/%s.%s", dir, uid, ext) -} + +/* + func ImageFileContentPathRawUser(uid string) string { + return imageFileContentPath(config.FilesDirectoryUser, uid, "raw") + } + + func imageFileContentPathLogoPng(uid string) string { + return imageFileContentPath(config.FilesDirectoryLogo, uid, "png") + } + + func imageFileContentPath(dir string, uid string, ext string) string { + return fmt.Sprintf("%s/%s.%s", dir, uid, ext) + } +*/ func ImageFileContentWrite(uid uuid.UUID, body io.Reader) error { - filepath := ImageFileContentPathRawUser(uid.String()) + filepath := fileContentPath("user", uid, "raw") // Create file in configured directory dst, err := os.Create(filepath) @@ -68,13 +78,15 @@ func ImageFileContentWrite(uid uuid.UUID, body io.Reader) error { return nil } func ImageFileContentWriteLogo(w http.ResponseWriter, uid uuid.UUID) { - image_path := imageFileContentPathLogoPng(uid.String()) + //image_path := imageFileContentPathLogoPng(uid.String()) + image_path := fileContentPath("logo", uid, "png") writeFileContent(w, image_path) } func PublicImageFileContentWrite(uid uuid.UUID, body io.Reader) error { // Create file in configured directory - filepath := PublicImageFileContentPathRaw(uid.String()) + //filepath := PublicImageFileContentPathRaw(uid.String()) + filepath := fileContentPath("public", uid, "raw") dst, err := os.Create(filepath) if err != nil { log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create public image file") @@ -91,15 +103,39 @@ func PublicImageFileContentWrite(uid uuid.UUID, body io.Reader) error { return nil } +/* func PublicImageFileContentPathRaw(uid string) string { return fmt.Sprintf("%s/%s.raw", config.FilesDirectoryPublic, uid) } +*/ -func PublicImageFileToResponse(w http.ResponseWriter, uid string) { - image_path := PublicImageFileContentPathRaw(uid) +func PublicImageFileToResponse(w http.ResponseWriter, uid uuid.UUID) { + //image_path := PublicImageFileContentPathRaw(uid) + image_path := fileContentPath("public", uid, "raw") writeFileContent(w, image_path) } +func fileContentPath(subdir string, uid uuid.UUID, extension string) string { + return fmt.Sprintf("%s/%s/%s.%s", config.FilesDirectory, subdir, uid.String(), extension) +} +func fileContentWrite(body io.Reader, subdir string, uid uuid.UUID, extension string) error { + // Create file in configured directory + filepath := fileContentPath(subdir, uid, extension) + dst, err := os.Create(filepath) + if err != nil { + log.Error().Err(err).Str("filepath", filepath).Msg("Failed to create file") + return fmt.Errorf("Failed to create file at %s: %v", filepath, err) + } + defer dst.Close() + + // Copy rest of request body to file + _, err = io.Copy(dst, body) + if err != nil { + return fmt.Errorf("Unable to save content of %s: %v", filepath, err) + } + return nil +} + func writeFileContent(w http.ResponseWriter, image_path string) { // Open the file file, err := os.Open(image_path)