From 8e325b7c773a588c170598043adfc9659a751c30 Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Tue, 16 Dec 2025 16:37:53 +0000 Subject: [PATCH] WIP migration of API from fieldseeker-sync --- api/api.go | 375 ++++++++ api/endpoint.go | 26 + api/types.go | 875 ++++++++++++++++++ auth.go => auth/auth.go | 119 +-- .../content_negotiation.go | 2 +- auth/session.go | 18 + db/geo.go | 32 + db/migrations/00018_nidus_notes.sql | 89 ++ db/migrations/goose.sh | 1 + db/query.go | 46 + endpoint.go | 17 +- go.mod | 23 +- go.sum | 45 +- label-studio/client.go | 129 +++ label-studio/import_tasks.go | 78 ++ label-studio/list_tasks.go | 143 +++ label-studio/projects.go | 126 +++ label-studio/tasks_annotation.go | 75 ++ label-studio/tasks_draft.go | 106 +++ label-studio/tasks_update.go | 183 ++++ label-studio/users.go | 62 ++ main.go | 25 +- minio/client.go | 77 ++ queue/audio_processing.go | 121 +++ queue/label_studio.go | 226 +++++ userfile/userfile.go | 43 + 26 files changed, 2960 insertions(+), 102 deletions(-) create mode 100644 api/api.go create mode 100644 api/endpoint.go create mode 100644 api/types.go rename auth.go => auth/auth.go (92%) rename content_negotiation.go => auth/content_negotiation.go (99%) create mode 100644 auth/session.go create mode 100644 db/geo.go create mode 100644 db/migrations/00018_nidus_notes.sql create mode 100644 db/migrations/goose.sh create mode 100644 db/query.go create mode 100644 label-studio/client.go create mode 100644 label-studio/import_tasks.go create mode 100644 label-studio/list_tasks.go create mode 100644 label-studio/projects.go create mode 100644 label-studio/tasks_annotation.go create mode 100644 label-studio/tasks_draft.go create mode 100644 label-studio/tasks_update.go create mode 100644 label-studio/users.go create mode 100644 minio/client.go create mode 100644 queue/audio_processing.go create mode 100644 queue/label_studio.go create mode 100644 userfile/userfile.go diff --git a/api/api.go b/api/api.go new file mode 100644 index 00000000..c3bffecc --- /dev/null +++ b/api/api.go @@ -0,0 +1,375 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "time" + + "github.com/Gleipnir-Technology/nidus-sync/db" + "github.com/Gleipnir-Technology/nidus-sync/db/models" + "github.com/Gleipnir-Technology/nidus-sync/queue" + "github.com/Gleipnir-Technology/nidus-sync/userfile" + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +func apiAudioPost(w http.ResponseWriter, r *http.Request, u *models.User) { + id := chi.URLParam(r, "uuid") + noteUUID, err := uuid.Parse(id) + if err != nil { + http.Error(w, "Failed to decode the uuid", http.StatusBadRequest) + return + } + + var payload NoteAudioPayload + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read the payload", http.StatusBadRequest) + return + } + if err := json.Unmarshal(body, &payload); err != nil { + log.Error().Err(err).Msg("Audio note POST JSON decode error") + output, err := os.OpenFile("/tmp/request.body", os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + log.Info().Msg("Failed to open temp request.bady") + } + defer output.Close() + output.Write(body) + log.Info().Msg("Wrote request to /tmp/request.body") + + http.Error(w, "Failed to decode the payload", http.StatusBadRequest) + return + } + if err := db.NoteAudioCreate(context.Background(), noteUUID, db.NoteAudio{}, u.ID); err != nil { + render.Render(w, r, errRender(err)) + return + } + w.WriteHeader(http.StatusAccepted) +} + +func apiAudioContentPost(w http.ResponseWriter, r *http.Request, u *models.User) { + u_str := chi.URLParam(r, "uuid") + audioUUID, err := uuid.Parse(u_str) + if err != nil { + http.Error(w, "Failed to parse image UUID", http.StatusBadRequest) + return + } + err = userfile.AudioFileContentWrite(audioUUID, r.Body) + if err != nil { + log.Printf("Failed to write content file: %v", err) + http.Error(w, "failed to write content file", http.StatusInternalServerError) + } + + queue.EnqueueAudioJob(queue.AudioJob{AudioUUID: audioUUID}) + w.WriteHeader(http.StatusOK) +} + +func apiClientIos(w http.ResponseWriter, r *http.Request, u *models.User) { + query := db.NewGeoQuery() + query.Limit = 0 + sources, err := db.MosquitoSourceQuery(&query) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + requests, err := db.ServiceRequestQuery(&query) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + traps, err := db.TrapDataQuery(&query) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + + response := NewResponseClientIos(sources, requests, traps) + if err := render.Render(w, r, response); err != nil { + render.Render(w, r, errRender(err)) + return + } +} + +func apiClientIosNotePut(w http.ResponseWriter, r *http.Request, u *models.User) { + id := chi.URLParam(r, "uuid") + noteUUID, err := uuid.Parse(id) + if err != nil { + http.Error(w, "Failed to decode the uuid", http.StatusBadRequest) + return + } + var payload NidusNotePayload + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read the payload", http.StatusBadRequest) + return + } + if err := json.Unmarshal(body, &payload); err != nil { + log.Error().Err(err).Msg("Note PUT JSON decode error") + output, err := os.OpenFile("/tmp/request.body", os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + log.Info().Msg("Failed to open temp request.bady") + } + defer output.Close() + output.Write(body) + log.Info().Msg("Wrote request to /tmp/request.body") + + http.Error(w, "Failed to decode the payload", http.StatusBadRequest) + return + } + if err := db.NoteUpdate(context.Background(), noteUUID, db.NidusNotePayload{}); err != nil { + render.Render(w, r, errRender(err)) + return + } + w.WriteHeader(http.StatusAccepted) +} + +func apiImagePost(w http.ResponseWriter, r *http.Request, u *models.User) { + id := chi.URLParam(r, "uuid") + noteUUID, err := uuid.Parse(id) + if err != nil { + http.Error(w, "Failed to decode the uuid", http.StatusBadRequest) + return + } + + var payload NoteImagePayload + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read the payload", http.StatusBadRequest) + return + } + if err := json.Unmarshal(body, &payload); err != nil { + log.Error().Err(err).Msg("Image note POST JSON decode error") + output, err := os.OpenFile("/tmp/request.body", os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + log.Info().Msg("Failed to open temp request.bady") + } + defer output.Close() + output.Write(body) + log.Info().Msg("Wrote request to /tmp/request.body") + + http.Error(w, "Failed to decode the payload", http.StatusBadRequest) + return + } + err = db.NoteImageCreate(context.Background(), noteUUID, db.NoteImage{}, u.ID) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + w.WriteHeader(http.StatusAccepted) +} + +func apiImageContentPost(w http.ResponseWriter, r *http.Request, u *models.User) { + u_str := chi.URLParam(r, "uuid") + imageUUID, err := uuid.Parse(u_str) + if err != nil { + log.Error().Err(err).Msg("Failed to parse image UUID") + http.Error(w, "Failed to parse image UUID", http.StatusBadRequest) + } + // Read first 8 bytes to check PNG signature + filepath := fmt.Sprintf("%s/%s.photo", userfile.UserFilesDirectory, imageUUID.String()) + + // Create file in configured directory + dst, err := os.Create(filepath) + if err != nil { + log.Printf("Failed to create image file %s: %v", filepath, err) + http.Error(w, "Unable to create file", http.StatusInternalServerError) + return + } + defer dst.Close() + + // Copy rest of request body to file + _, err = io.Copy(dst, r.Body) + if err != nil { + http.Error(w, "Unable to save file", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + log.Printf("Saved image file %s\n", imageUUID) + fmt.Fprintf(w, "PNG uploaded successfully to %s", filepath) +} + +func apiMosquitoSource(w http.ResponseWriter, r *http.Request, u *models.User) { + bounds, err := parseBounds(r) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + + query := db.NewGeoQuery() + query.Bounds = *bounds + query.Limit = 100 + sources, err := db.MosquitoSourceQuery(&query) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + + data := []render.Renderer{} + for _, s := range sources { + data = append(data, NewResponseMosquitoSource(s)) + } + if err := render.RenderList(w, r, data); err != nil { + render.Render(w, r, errRender(err)) + } +} + +func apiTrapData(w http.ResponseWriter, r *http.Request, u *models.User) { + bounds, err := parseBounds(r) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + + query := db.NewGeoQuery() + query.Bounds = *bounds + query.Limit = 100 + trap_data, err := db.TrapDataQuery(&query) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + + data := []render.Renderer{} + for _, td := range trap_data { + data = append(data, NewResponseTrapDatum(td)) + } + if err := render.RenderList(w, r, data); err != nil { + render.Render(w, r, errRender(err)) + } +} + +func apiServiceRequest(w http.ResponseWriter, r *http.Request, u *models.User) { + bounds, err := parseBounds(r) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + query := db.NewGeoQuery() + query.Bounds = *bounds + query.Limit = 100 + requests, err := db.ServiceRequestQuery(&query) + if err != nil { + render.Render(w, r, errRender(err)) + return + } + + data := []render.Renderer{} + for _, sr := range requests { + data = append(data, NewResponseServiceRequest(sr)) + } + if err := render.RenderList(w, r, data); err != nil { + render.Render(w, r, errRender(err)) + } +} + +func parseBounds(r *http.Request) (*db.GeoBounds, error) { + err := r.ParseForm() + if err != nil { + return nil, err + } + + east := r.FormValue("east") + north := r.FormValue("north") + south := r.FormValue("south") + west := r.FormValue("west") + + bounds := db.GeoBounds{} + + var temp float64 + temp, err = strconv.ParseFloat(east, 64) + if err != nil { + return nil, err + } + bounds.East = temp + temp, err = strconv.ParseFloat(north, 64) + if err != nil { + return nil, err + } + bounds.North = temp + temp, err = strconv.ParseFloat(south, 64) + if err != nil { + return nil, err + } + bounds.South = temp + temp, err = strconv.ParseFloat(west, 64) + if err != nil { + return nil, err + } + bounds.West = temp + return &bounds, nil +} + +func errRender(err error) render.Renderer { + log.Error().Err(err).Msg("Rendering error") + return &ResponseErr{ + Error: err, + HTTPStatusCode: 500, + StatusText: "Error rendering response", + ErrorText: err.Error(), + } +} + +func webhookFieldseeker(w http.ResponseWriter, r *http.Request) { + // Create or open the log file + file, err := os.OpenFile("webhook/request.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + log.Printf("Error opening log file: %v", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + defer file.Close() + + // Write timestamp + timestamp := time.Now().Format("2006-01-02 15:04:05") + fmt.Fprintf(file, "\n=== Request logged at %s ===\n", timestamp) + + // Write request line + fmt.Fprintf(file, "%s %s %s\n", r.Method, r.RequestURI, r.Proto) + + // Write all headers + fmt.Fprintf(file, "\nHeaders:\n") + for name, values := range r.Header { + for _, value := range values { + fmt.Fprintf(file, "%s: %s\n", name, value) + } + } + + // Write body + fmt.Fprintf(file, "\nBody:\n") + body, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("Error reading request body: %v", err) + fmt.Fprintf(file, "Error reading body: %v\n", err) + } else { + file.Write(body) + if len(body) == 0 { + fmt.Fprintf(file, "(empty body)") + } + } + + fmt.Fprintf(file, "\n=== End of request ===\n\n") + + // Extract the crc_token value for the signature portion + + // Respond with 204 No Content + w.WriteHeader(http.StatusNoContent) +} + +func parseTime(x string) time.Time { + created_epoch, err := strconv.ParseInt(x, 10, 64) + if err != nil { + log.Error().Err(err).Msg("Unable to convert inspection timestamp") + } + created := time.UnixMilli(created_epoch) + return created +} + diff --git a/api/endpoint.go b/api/endpoint.go new file mode 100644 index 00000000..ab413554 --- /dev/null +++ b/api/endpoint.go @@ -0,0 +1,26 @@ +package api + +import ( + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" + + "github.com/Gleipnir-Technology/nidus-sync/auth" +) + +func AddRoutes(r chi.Router) { + // Authenticated endpoints + r.Use(render.SetContentType(render.ContentTypeJSON)) + r.Method("GET", "/mosquito-source", auth.NewEnsureAuth(apiMosquitoSource)) + r.Method("GET", "/service-request", auth.NewEnsureAuth(apiServiceRequest)) + r.Method("GET", "/trap-data", auth.NewEnsureAuth(apiTrapData)) + r.Method("GET", "/client/ios", auth.NewEnsureAuth(apiClientIos)) + r.Method("PUT", "/client/ios/note/{uuid}", auth.NewEnsureAuth(apiClientIosNotePut)) + r.Method("POST", "/audio/{uuid}", auth.NewEnsureAuth(apiAudioPost)) + r.Method("POST", "/audio/{uuid}/content", auth.NewEnsureAuth(apiAudioContentPost)) + r.Method("POST", "/image/{uuid}", auth.NewEnsureAuth(apiImagePost)) + r.Method("POST", "/image/{uuid}/content", auth.NewEnsureAuth(apiImageContentPost)) + + // Unauthenticated endpoints + r.Get("/webhook/fieldseeker", webhookFieldseeker) + r.Post("/webhook/fieldseeker", webhookFieldseeker) +} diff --git a/api/types.go b/api/types.go new file mode 100644 index 00000000..1ad1c4ca --- /dev/null +++ b/api/types.go @@ -0,0 +1,875 @@ +package api + +import ( + "net/http" + "sort" + "time" + + "github.com/Gleipnir-Technology/nidus-sync/db" + "github.com/go-chi/render" + "github.com/google/uuid" +) + +type H3Cell uint64 + +type hasCreated interface { + getCreated() string +} + +type FS_Geometry struct { + X float64 `db:"X"` + Y float64 `db:"Y"` +} + +func (geo FS_Geometry) Latitude() float64 { + return geo.Y +} +func (geo FS_Geometry) Longitude() float64 { + return geo.X +} + +type FS_InspectionSample struct { + Geometry FS_Geometry `db:"geometry"` + CreationDate string `db:"creationdate"` + Creator string `db:"creator"` + EditDate string `db:"editdate"` + Editor string `db:"editor"` + IDByTech string `db:"idbytech"` + InspectionID string `db:"insp_id"` + Processed int `db:"processed"` + SampleID string `db:"sampleid"` +} + +type FS_MosquitoInspection struct { + ActionTaken *string `db:"actiontaken"` + Comments *string `db:"comments"` + Condition *string `db:"sitecond"` + EndDateTime string `db:"enddatetime"` + FieldTech *string `db:"fieldtech"` + GlobalID string `db:"globalid"` + LocationName *string `db:"locationname"` + PointLocationID string `db:"pointlocid"` + SiteCond *string `db:"sitecond"` + Zone *string `db:"zone"` +} + +type FS_PointLocation struct { + Access *string `db:"accessdesc"` + Active *int `db:"active"` + Comments *string `db:"comments"` + CreationDate *int64 `db:"creationdate"` + Description *string `db:"description"` + Geometry FS_Geometry `db:"geometry"` + GlobalID string `db:"globalid"` + Habitat *string `db:"habitat"` + Inspections MosquitoInspectionSlice + LastInspectDate *int64 `db:"lastinspectdate"` + Name *string `db:"name"` + NextActionDateScheduled *int64 `db:"nextactiondatescheduled"` + Treatments []MosquitoTreatment + UseType *string `db:"usetype"` + WaterOrigin *string `db:"waterorigin"` + Zone *string `db:"zone"` +} + +type FS_ServiceRequest struct { + AssignedTech *string `db:"assignedtech"` + CreationDate *int64 `db:"creationdate"` + City *string `db:"reqcity"` + Dog *int `db:"dog"` + Geometry FS_Geometry `db:"geometry"` + GlobalID string `db:"globalid"` + Priority *string `db:"priority"` + RecDateTime *int64 `db:"recdatetime"` + ReqAddr1 *string `db:"reqaddr1"` + ReqTarget *string `db:"reqtarget"` + ReqZip *string `db:"reqzip"` + Source *string `db:"source"` + Spanish *int `db:"spanish"` + Status *string `db:"status"` +} +type FS_TrapLocation struct { + Access *string `db:"accessdesc"` + CreationDate *int64 `db:"creationdate"` + Description *string `db:"description"` + Geometry FS_Geometry `db:"geometry"` + GlobalID string `db:"globalid"` + ObjectID int `db:"objectid"` + Name *string `db:"name"` +} + +type FS_Treatment struct { + Comments *string `db:"comments"` + EndDateTime *int64 `db:"enddatetime"` + FieldTech *string `db:"fieldtech"` + GlobalID string `db:"globalid"` + Habitat *string `db:"habitat"` + PointLocationID string `db:"pointlocid"` + Product *string `db:"product"` + Quantity float64 `db:"qty"` + QuantityUnit *string `db:"qtyunit"` + SiteCondition *string `db:"sitecond"` + TreatAcres *float64 `db:"treatacres"` + TreatHectares *float64 `db:"treathectares"` +} + +/* +type User struct { + DisplayName string `db:"display_name"` + ID int `db:"id"` + PasswordHashType string `db:"password_hash_type"` + PasswordHash string `db:"password_hash"` + Username string `db:"username"` +} +*/ + +type Bounds struct { + East float64 + North float64 + South float64 + West float64 +} + +func NewBounds() Bounds { + return Bounds{ + East: 180, + North: 180, + South: -180, + West: -180, + } +} + +type MosquitoInspection struct { + data *FS_MosquitoInspection +} + +func (mi MosquitoInspection) ActionTaken() string { + if mi.data.ActionTaken == nil { + return "" + } + return *mi.data.ActionTaken +} + +func (mi MosquitoInspection) Comments() string { + if mi.data.Comments == nil { + return "" + } + return *mi.data.Comments +} + +func (mi MosquitoInspection) Condition() string { + if mi.data.Condition == nil { + return "" + } + return *mi.data.Condition +} + +func (mi MosquitoInspection) Created() time.Time { + return parseTime(mi.data.EndDateTime) +} + +func (mi MosquitoInspection) FieldTechnician() string { + if mi.data.FieldTech == nil { + return "" + } + return *mi.data.FieldTech +} + +func (mi MosquitoInspection) ID() string { + return mi.data.GlobalID +} + +func (mi MosquitoInspection) LocationName() string { + if mi.data.LocationName == nil { + return "" + } + return *mi.data.LocationName +} + +func (mi MosquitoInspection) SiteCondition() string { + if mi.data.SiteCond == nil { + return "" + } + return *mi.data.SiteCond +} + +func NewMosquitoInspections(inspections []*FS_MosquitoInspection) []MosquitoInspection { + results := make([]MosquitoInspection, 0) + for _, t := range inspections { + results = append(results, MosquitoInspection{data: t}) + } + MosquitoInspectionSlice(results).Sort() + + return results +} + +type MosquitoInspectionSlice []MosquitoInspection +type ByCreatedMI []MosquitoInspection + +func (a ByCreatedMI) Len() int { return len(a) } +func (a ByCreatedMI) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByCreatedMI) Less(i, j int) bool { return a[i].Created().After(a[j].Created()) } + +func (inspections MosquitoInspectionSlice) Sort() { + sort.Sort(ByCreatedMI(inspections)) +} + +type MosquitoSource struct { + location *FS_PointLocation + Inspections []MosquitoInspection + Treatments []MosquitoTreatment +} + +func (s MosquitoSource) Access() string { + if s.location.Access == nil { + return "" + } + return *s.location.Access +} + +func (s MosquitoSource) Active() *bool { + var result bool + if s.location.Active == nil { + return nil + } else if *s.location.Active == 0 { + result = false + } else { + result = true + } + return &result +} + +func (s MosquitoSource) Comments() string { + if s.location.Comments == nil { + return "" + } + return *s.location.Comments +} + +func (s MosquitoSource) Created() time.Time { + if s.location.CreationDate == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*s.location.CreationDate) +} + +func (s MosquitoSource) Description() string { + if s.location.Description == nil { + return "" + } + return *s.location.Description +} + +func (s MosquitoSource) ID() uuid.UUID { + return uuid.MustParse(s.location.GlobalID) +} +func (s MosquitoSource) Habitat() string { + if s.location.Habitat == nil { + return "" + } + return *s.location.Habitat +} + +func (s MosquitoSource) LastInspectionDate() time.Time { + if s.location.LastInspectDate == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*s.location.LastInspectDate) +} + +func (s MosquitoSource) Location() LatLong { + return s.location.Geometry +} + +func (s MosquitoSource) Name() string { + if s.location.Name == nil { + return "" + } + return *s.location.Name +} + +func (s MosquitoSource) NextActionDateScheduled() time.Time { + if s.location.NextActionDateScheduled == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*s.location.NextActionDateScheduled) +} + +func (s MosquitoSource) UseType() string { + if s.location.UseType == nil { + return "" + } + return *s.location.UseType +} +func (s MosquitoSource) WaterOrigin() string { + if s.location.WaterOrigin == nil { + return "" + } + return *s.location.WaterOrigin +} +func (s MosquitoSource) Zone() string { + if s.location.Zone == nil { + return "" + } + return *s.location.Zone +} +func NewMosquitoSource(location *FS_PointLocation, inspections []*FS_MosquitoInspection, treatments []*FS_Treatment) MosquitoSource { + return MosquitoSource{ + location: location, + Inspections: NewMosquitoInspections(inspections), + Treatments: NewMosquitoTreatments(treatments), + } +} + +type MosquitoTreatment struct { + data *FS_Treatment +} + +func (t MosquitoTreatment) Comments() string { + if t.data.Comments == nil { + return "" + } + return *t.data.Comments +} +func (t MosquitoTreatment) Created() time.Time { + if t.data.EndDateTime == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*t.data.EndDateTime) +} +func (t MosquitoTreatment) FieldTechnician() string { + if t.data.FieldTech == nil { + return "" + } + return *t.data.FieldTech +} +func (mi MosquitoTreatment) ID() string { + return mi.data.GlobalID +} +func (t MosquitoTreatment) Habitat() string { + if t.data.Habitat == nil { + return "" + } + return *t.data.Habitat +} +func (t MosquitoTreatment) Product() string { + if t.data.Product == nil { + return "" + } + return *t.data.Product +} +func (t MosquitoTreatment) Quantity() float64 { + return t.data.Quantity +} +func (t MosquitoTreatment) QuantityUnit() string { + if t.data.QuantityUnit == nil { + return "" + } + return *t.data.QuantityUnit +} +func (t MosquitoTreatment) SiteCondition() string { + if t.data.SiteCondition == nil { + return "" + } + return *t.data.SiteCondition +} +func (t MosquitoTreatment) TreatAcres() float64 { + if t.data.TreatAcres == nil { + return 0 + } + return *t.data.TreatAcres +} +func (t MosquitoTreatment) TreatHectares() float64 { + if t.data.TreatHectares == nil { + return 0 + } + return *t.data.TreatHectares +} +func NewMosquitoTreatments(treatments []*FS_Treatment) []MosquitoTreatment { + results := make([]MosquitoTreatment, 0) + for _, t := range treatments { + results = append(results, MosquitoTreatment{data: t}) + } + MosquitoTreatmentSlice(results).Sort() + return results +} + +type MosquitoTreatmentSlice []MosquitoTreatment +type ByCreatedMT []MosquitoTreatment + +func (a ByCreatedMT) Len() int { return len(a) } +func (a ByCreatedMT) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByCreatedMT) Less(i, j int) bool { return a[i].Created().After(a[j].Created()) } + +func (inspections MosquitoTreatmentSlice) Sort() { + sort.Sort(ByCreatedMT(inspections)) +} + +type LatLong interface { + Latitude() float64 + Longitude() float64 +} + +type ServiceRequest struct { + data *FS_ServiceRequest +} + +func (sr ServiceRequest) Address() string { + if sr.data.ReqAddr1 == nil { + return "" + } + return *sr.data.ReqAddr1 +} +func (sr ServiceRequest) AssignedTechnician() string { + if sr.data.AssignedTech == nil { + return "" + } + return *sr.data.AssignedTech +} +func (sr ServiceRequest) City() string { + if sr.data.City == nil { + return "" + } + return *sr.data.City +} +func (sr ServiceRequest) Created() time.Time { + if sr.data.CreationDate == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*sr.data.CreationDate) +} +func (sr ServiceRequest) HasDog() *bool { + var result bool + if sr.data.Dog == nil { + return nil + } else if *sr.data.Dog == 0 { + result = false + } else { + result = true + } + return &result +} +func (sr ServiceRequest) HasSpanishSpeaker() *bool { + var result bool + if sr.data.Spanish == nil { + return nil + } else if *sr.data.Spanish == 0 { + result = false + } else { + result = true + } + return &result +} +func (sr ServiceRequest) ID() uuid.UUID { + return uuid.MustParse(sr.data.GlobalID) +} +func (sr ServiceRequest) Location() LatLong { + return sr.data.Geometry +} +func (sr ServiceRequest) Priority() string { + if sr.data.Priority == nil { + return "" + } + return *sr.data.Priority +} +func (sr ServiceRequest) RecDateTime() time.Time { + if sr.data.RecDateTime == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*sr.data.RecDateTime) +} +func (sr ServiceRequest) Status() string { + if sr.data.Status == nil { + return "" + } + return *sr.data.Status +} +func (sr ServiceRequest) Source() string { + if sr.data.Source == nil { + return "" + } + return *sr.data.Source +} +func (sr ServiceRequest) Target() string { + if sr.data.ReqTarget == nil { + return "" + } + return *sr.data.ReqTarget +} +func (sr ServiceRequest) UseType() string { + return "" +} +func (sr ServiceRequest) WaterOrigin() string { + return "" +} +func (sr ServiceRequest) Zip() string { + if sr.data.ReqZip == nil { + return "" + } + return *sr.data.ReqZip +} +func NewServiceRequest(data *FS_ServiceRequest) ServiceRequest { + return ServiceRequest{data: data} +} + +type TrapData struct { + data *FS_TrapLocation +} + +func (tl TrapData) Access() string { + if tl.data.Access == nil { + return "" + } + return *tl.data.Access +} +func (tl TrapData) Created() time.Time { + if tl.data.CreationDate == nil { + return time.UnixMilli(0) + } + return time.UnixMilli(*tl.data.CreationDate) +} +func (tl TrapData) Description() string { + if tl.data.Description == nil { + return "" + } + return *tl.data.Description +} +func (tl TrapData) ID() uuid.UUID { + return uuid.MustParse(tl.data.GlobalID) +} +func (tl TrapData) Location() LatLong { + return tl.data.Geometry +} +func (tl TrapData) Name() string { + if tl.data.Name == nil { + return "" + } + return *tl.data.Name +} +func NewTrapData(data *FS_TrapLocation) TrapData { + return TrapData{data: data} +} + +type Location struct { + Latitude float64 + Longitude float64 +} + +type NoteImagePayload struct { + UUID string `json:"uuid"` + Cell H3Cell `json:"cell"` + Created time.Time `json:"created"` +} + +type NoteAudio struct { + UUID string `db:"uuid"` + Breadcrumbs []NoteAudioBreadcrumbPayload + Created time.Time `db:"created"` + Creator int `db:"creator"` + Deleted *time.Time `db:"deleted"` + Duration int `db:"duration"` + IsAudioNormalized bool `db:"is_audio_normalized"` + IsTranscodedeToOgg bool `db:"is_transcoded_to_ogg"` + Transcription *string `db:"transcription"` + TranscriptionUserEdited bool `db:"transcription_user_edited"` + Version int `db:"version"` +} + +type NoteAudioPayload struct { + UUID string `json:"uuid"` + Breadcrumbs []NoteAudioBreadcrumbPayload `json:"breadcrumbs"` + Created time.Time `json:"created"` + Duration int `json:"duration"` + Transcription *string `json:"transcription"` + TranscriptionUserEdited bool `json:"transcriptionUserEdited"` + Version int `json:"version"` +} + +type ResponseMosquitoSource struct { + Access string `json:"access"` + Active *bool `json:"active"` + Comments string `json:"comments"` + Created string `json:"created"` + Description string `json:"description"` + ID string `json:"id"` + LastInspectionDate string `json:"last_inspection_date"` + Location ResponseLocation `json:"location"` + Habitat string `json:"habitat"` + Inspections []ResponseMosquitoInspection `json:"inspections"` + Name string `json:"name"` + NextActionDateScheduled string `json:"next_action_date_scheduled"` + Treatments []ResponseMosquitoTreatment `json:"treatments"` + UseType string `json:"use_type"` + WaterOrigin string `json:"water_origin"` + Zone string `json:"zone"` +} + + +type NoteAudioBreadcrumbPayload struct { + Cell H3Cell `json:"cell"` + Created time.Time `json:"created"` + ManuallySelected bool `json:"manuallySelected"` +} + +type NidusNotePayload struct { + UUID string `json:"uuid"` + Timestamp time.Time `json:"timestamp"` + Images []string `json:"images"` + Location Location `json:"location"` + Text string `json:"text"` +} + + +// ResponseErr renderer type for handling all sorts of errors. +type ResponseClientIos struct { + MosquitoSources []ResponseMosquitoSource `json:"sources"` + ServiceRequests []ResponseServiceRequest `json:"requests"` + TrapData []ResponseTrapData `json:"traps"` +} + +func (i ResponseClientIos) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} +func NewResponseClientIos(sources []db.MosquitoSource, requests []db.ServiceRequest, traps []db.TrapData) ResponseClientIos { + return ResponseClientIos{ + MosquitoSources: NewResponseMosquitoSources(sources), + ServiceRequests: NewResponseServiceRequests(requests), + TrapData: NewResponseTrapData(traps), + } +} + +// In the best case scenario, the excellent github.com/pkg/errors package +// helps reveal information on the error, setting it on Err, and in the Render() +// method, using it to set the application-specific error code in AppCode. +type ResponseErr struct { + Error error `json:"-"` // low-level runtime error + HTTPStatusCode int `json:"-"` // http response status code + + StatusText string `json:"status"` // user-level status message + AppCode int64 `json:"code,omitempty"` // application-specific error code + ErrorText string `json:"error,omitempty"` // application-level error message, for debugging +} + +func (e *ResponseErr) Render(w http.ResponseWriter, r *http.Request) error { + render.Status(r, e.HTTPStatusCode) + return nil +} + +type ResponseLocation struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} + +func (rtd ResponseLocation) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func NewResponseLocation(l LatLong) ResponseLocation { + return ResponseLocation{ + Latitude: l.Latitude(), + Longitude: l.Longitude(), + } +} + +type ResponseMosquitoInspection struct { + ActionTaken string `json:"action_taken"` + Comments string `json:"comments"` + Condition string `json:"condition"` + Created string `json:"created"` + EndDateTime string `json:"end_date_time"` + FieldTechnician string `json:"field_technician"` + ID string `json:"id"` + LocationName string `json:"location_name"` + SiteCondition string `json:"site_condition"` +} + +func (rtd ResponseMosquitoInspection) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} +func NewResponseMosquitoInspection(i MosquitoInspection) ResponseMosquitoInspection { + return ResponseMosquitoInspection{ + ActionTaken: i.ActionTaken(), + Comments: i.Comments(), + Condition: i.Condition(), + Created: i.Created().Format("2006-01-02T15:04:05.000Z"), + ID: i.ID(), + LocationName: i.LocationName(), + SiteCondition: i.SiteCondition(), + } +} +func NewResponseMosquitoInspections(inspections []MosquitoInspection) []ResponseMosquitoInspection { + results := make([]ResponseMosquitoInspection, 0) + for _, i := range inspections { + results = append(results, NewResponseMosquitoInspection(i)) + } + return results +} + +func (rtd ResponseMosquitoSource) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func NewResponseMosquitoSource(ms db.MosquitoSource) ResponseMosquitoSource { + + return ResponseMosquitoSource{ + /* + Active: ms.Active(), + Access: ms.Access(), + Comments: ms.Comments(), + Created: ms.Created().Format("2006-01-02T15:04:05.000Z"), + Description: ms.Description(), + ID: ms.ID().String(), + LastInspectionDate: ms.LastInspectionDate().Format("2006-01-02T15:04:05.000Z"), + Location: NewResponseLocation(ms.Location()), + Habitat: ms.Habitat(), + Inspections: NewResponseMosquitoInspections(ms.Inspections), + Name: ms.Name(), + NextActionDateScheduled: ms.NextActionDateScheduled().Format("2006-01-02T15:04:05.000Z"), + Treatments: NewResponseMosquitoTreatments(ms.Treatments), + UseType: ms.UseType(), + WaterOrigin: ms.WaterOrigin(), + Zone: ms.Zone(), + */ + } +} +func NewResponseMosquitoSources(sources []db.MosquitoSource) []ResponseMosquitoSource { + results := make([]ResponseMosquitoSource, 0) + for _, i := range sources { + results = append(results, NewResponseMosquitoSource(i)) + } + return results +} + +type ResponseMosquitoTreatment struct { + Comments string `json:"comments"` + Created string `json:"created"` + EndDateTime string `json:"end_date_time"` + FieldTechnician string `json:"field_technician"` + Habitat string `json:"habitat"` + ID string `json:"id"` + Product string `json:"product"` + Quantity float64 `json:"quantity"` + QuantityUnit string `json:"quantity_unit"` + SiteCondition string `json:"site_condition"` + TreatAcres float64 `json:"treat_acres"` + TreatHectares float64 `json:"treat_hectares"` +} + +func (rtd ResponseMosquitoTreatment) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} +func NewResponseMosquitoTreatment(i db.MosquitoTreatment) ResponseMosquitoTreatment { + return ResponseMosquitoTreatment{ + /* + Comments: i.Comments(), + Created: i.Created().Format("2006-01-02T15:04:05.000Z"), + FieldTechnician: i.FieldTechnician(), + Habitat: i.Habitat(), + ID: i.ID(), + Product: i.Product(), + Quantity: i.Quantity(), + QuantityUnit: i.QuantityUnit(), + SiteCondition: i.SiteCondition(), + TreatAcres: i.TreatAcres(), + TreatHectares: i.TreatHectares(), + */ + } +} +func NewResponseMosquitoTreatments(treatments []db.MosquitoTreatment) []ResponseMosquitoTreatment { + results := make([]ResponseMosquitoTreatment, 0) + for _, i := range treatments { + results = append(results, NewResponseMosquitoTreatment(i)) + } + return results +} + +type ResponseNote struct { + CategoryName string `json:"categoryName"` + Content string `json:"content"` + + ID string `json:"id"` + Location ResponseLocation `json:"location"` + Timestamp string `json:"timestamp"` +} + +func (rtd ResponseNote) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +type ResponseServiceRequest struct { + Address string `json:"address"` + AssignedTechnician string `json:"assigned_technician"` + City string `json:"city"` + Created string `json:"created"` + HasDog *bool `json:"has_dog"` + HasSpanishSpeaker *bool `json:"has_spanish_speaker"` + ID string `json:"id"` + Location ResponseLocation `json:"location"` + Priority string `json:"priority"` + RecordedDate string `json:"recorded_date"` + Source string `json:"source"` + Status string `json:"status"` + Target string `json:"target"` + Zip string `json:"zip"` +} + +func (srr ResponseServiceRequest) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func NewResponseServiceRequest(sr db.ServiceRequest) ResponseServiceRequest { + return ResponseServiceRequest{ + /* + Address: sr.Address(), + AssignedTechnician: sr.AssignedTechnician(), + City: sr.City(), + Created: sr.Created().Format("2006-01-02T15:04:05.000Z"), + HasDog: sr.HasDog(), + HasSpanishSpeaker: sr.HasSpanishSpeaker(), + ID: sr.ID().String(), + Location: NewResponseLocation(sr.Location()), + Priority: sr.Priority(), + Status: sr.Status(), + Source: sr.Source(), + Target: sr.Target(), + Zip: sr.Zip(), + */ + } +} +func NewResponseServiceRequests(requests []db.ServiceRequest) []ResponseServiceRequest { + results := make([]ResponseServiceRequest, 0) + for _, i := range requests { + results = append(results, NewResponseServiceRequest(i)) + } + return results +} + +type ResponseTrapData struct { + Created string `json:"created"` + Description string `json:"description"` + ID string `json:"id"` + Location ResponseLocation `json:"location"` + Name string `json:"name"` +} + +func (rtd ResponseTrapData) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} +func NewResponseTrapDatum(td db.TrapData) ResponseTrapData { + return ResponseTrapData{ + /* + Created: td.Created.Format("2006-01-02T15:04:05.000Z"), + Description: td.Description, + ID: td.ID.String(), + Location: NewResponseLocation(td.Location), + Name: td.Name, + */ + } +} +func NewResponseTrapData(data []db.TrapData) []ResponseTrapData { + results := make([]ResponseTrapData, 0) + for _, i := range data { + results = append(results, NewResponseTrapDatum(i)) + } + return results +} diff --git a/auth.go b/auth/auth.go similarity index 92% rename from auth.go rename to auth/auth.go index de92ad1e..641a73cd 100644 --- a/auth.go +++ b/auth/auth.go @@ -1,4 +1,4 @@ -package main +package auth import ( "context" @@ -37,6 +37,40 @@ type EnsureAuth struct { handler AuthenticatedHandler } +func AddUserSession(r *http.Request, user *models.User) { + id := strconv.Itoa(int(user.ID)) + sessionManager.Put(r.Context(), "user_id", id) + sessionManager.Put(r.Context(), "username", user.Username) + log.Info().Str("username", user.Username).Str("user_id", id).Msg("Created new user session") +} + +func GetAuthenticatedUser(r *http.Request) (*models.User, error) { + //user_id := sessionManager.GetInt(r.Context(), "user_id") + user_id_str := sessionManager.GetString(r.Context(), "user_id") + if user_id_str != "" { + user_id, err := strconv.Atoi(user_id_str) + if err != nil { + return nil, fmt.Errorf("Failed to convert user_id to int: %w", err) + } + username := sessionManager.GetString(r.Context(), "username") + log.Info().Int("user_id", user_id).Str("username", username).Msg("Current session info") + if user_id > 0 && username != "" { + return findUser(r.Context(), user_id) + } + } + // If we can't get the user from the session try to get from auth headers + username, password, ok := r.BasicAuth() + if !ok { + return nil, &NoCredentialsError{} + } + user, err := validateUser(r.Context(), username, password) + if err != nil { + return nil, err + } + AddUserSession(r, user) + return user, nil +} + func NewEnsureAuth(handlerToWrap AuthenticatedHandler) *EnsureAuth { return &EnsureAuth{handlerToWrap} } @@ -47,7 +81,7 @@ func (ea *EnsureAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { offers := []string{"application/json", "text/html"} content_type := NegotiateContent(accept, offers) - user, err := getAuthenticatedUser(r) + user, err := GetAuthenticatedUser(r) if err != nil || user == nil { var msg []byte // Separate return codes for different authentication failures @@ -75,61 +109,7 @@ func (ea *EnsureAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { ea.handler(w, r, user) } -func addUserSession(r *http.Request, user *models.User) { - id := strconv.Itoa(int(user.ID)) - sessionManager.Put(r.Context(), "user_id", id) - sessionManager.Put(r.Context(), "username", user.Username) - log.Info().Str("username", user.Username).Str("user_id", id).Msg("Created new user session") -} - -// Helper function to translate strings into solid error types for operating on -func findUser(ctx context.Context, user_id int) (*models.User, error) { - user, err := models.FindUser(ctx, db.PGInstance.BobDB, int32(user_id)) - if err != nil { - if err.Error() == "No such user" { - return nil, &NoUserError{} - } else { - LogErrorTypeInfo(err) - log.Error().Err(err).Msg("Unrecognized error. This should be updated in the findUser code") - return nil, err - } - } - return user, err -} - -func getAuthenticatedUser(r *http.Request) (*models.User, error) { - //user_id := sessionManager.GetInt(r.Context(), "user_id") - user_id_str := sessionManager.GetString(r.Context(), "user_id") - if user_id_str != "" { - user_id, err := strconv.Atoi(user_id_str) - if err != nil { - return nil, fmt.Errorf("Failed to convert user_id to int: %w", err) - } - username := sessionManager.GetString(r.Context(), "username") - log.Info().Int("user_id", user_id).Str("username", username).Msg("Current session info") - if user_id > 0 && username != "" { - return findUser(r.Context(), user_id) - } - } - // If we can't get the user from the session try to get from auth headers - username, password, ok := r.BasicAuth() - if !ok { - return nil, &NoCredentialsError{} - } - user, err := validateUser(r.Context(), username, password) - if err != nil { - return nil, err - } - addUserSession(r, user) - return user, nil -} - -func hashPassword(password string) (string, error) { - bytes, err := bcrypt.GenerateFromPassword([]byte(password), 14) - return string(bytes), err -} - -func signinUser(r *http.Request, username string, password string) (*models.User, error) { +func SigninUser(r *http.Request, username string, password string) (*models.User, error) { user, err := validateUser(r.Context(), username, password) if err != nil { return nil, err @@ -137,11 +117,11 @@ func signinUser(r *http.Request, username string, password string) (*models.User if user == nil { return nil, errors.New("No matching user") } - addUserSession(r, user) + AddUserSession(r, user) return user, nil } -func signupUser(username string, name string, password string) (*models.User, error) { +func SignupUser(username string, name string, password string) (*models.User, error) { passwordHash, err := hashPassword(password) if err != nil { return nil, fmt.Errorf("Cannot signup user: %w", err) @@ -161,6 +141,27 @@ func signupUser(username string, name string, password string) (*models.User, er return u, nil } + +// Helper function to translate strings into solid error types for operating on +func findUser(ctx context.Context, user_id int) (*models.User, error) { + user, err := models.FindUser(ctx, db.PGInstance.BobDB, int32(user_id)) + if err != nil { + if err.Error() == "No such user" { + return nil, &NoUserError{} + } else { + //LogErrorTypeInfo(err) + //log.Error().Err(err).Msg("Unrecognized error. This should be updated in the findUser code") + return nil, err + } + } + return user, err +} + +func hashPassword(password string) (string, error) { + bytes, err := bcrypt.GenerateFromPassword([]byte(password), 14) + return string(bytes), err +} + func validatePassword(password, hash string) bool { err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) return err == nil diff --git a/content_negotiation.go b/auth/content_negotiation.go similarity index 99% rename from content_negotiation.go rename to auth/content_negotiation.go index c456d407..aa9cb852 100644 --- a/content_negotiation.go +++ b/auth/content_negotiation.go @@ -1,4 +1,4 @@ -package main +package auth import ( "sort" diff --git a/auth/session.go b/auth/session.go new file mode 100644 index 00000000..f01b82ed --- /dev/null +++ b/auth/session.go @@ -0,0 +1,18 @@ +package auth + +import ( + "time" + + "github.com/alexedwards/scs/v2" + "github.com/alexedwards/scs/pgxstore" + "github.com/Gleipnir-Technology/nidus-sync/db" +) + +var sessionManager *scs.SessionManager + +func NewSessionManager() *scs.SessionManager { + sessionManager = scs.New() + sessionManager.Store = pgxstore.New(db.PGInstance.PGXPool) + sessionManager.Lifetime = 24 * time.Hour + return sessionManager +} diff --git a/db/geo.go b/db/geo.go new file mode 100644 index 00000000..4d3690ed --- /dev/null +++ b/db/geo.go @@ -0,0 +1,32 @@ +package db + +import ( +) + +type GeoBounds struct { + East float64 + North float64 + South float64 + West float64 +} + +type GeoQuery struct { + Bounds GeoBounds + Limit int +} + +func NewGeoBounds() GeoBounds { + return GeoBounds{ + East: 180, + North: 180, + South: -180, + West: -180, + } +} + +func NewGeoQuery() GeoQuery { + return GeoQuery{ + Bounds: NewGeoBounds(), + Limit: 0, + } +} diff --git a/db/migrations/00018_nidus_notes.sql b/db/migrations/00018_nidus_notes.sql new file mode 100644 index 00000000..736f8987 --- /dev/null +++ b/db/migrations/00018_nidus_notes.sql @@ -0,0 +1,89 @@ +-- +goose Up +CREATE TABLE note_audio ( + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + creator_id INTEGER REFERENCES user_ (id) NOT NULL, + deleted TIMESTAMP WITHOUT TIME ZONE, + deletor_id INTEGER REFERENCES user_ (id), + duration REAL NOT NULL, + organization_id INTEGER REFERENCES organization (id) NOT NULL, + transcription TEXT, + transcription_user_edited BOOLEAN NOT NULL, + version INTEGER NOT NULL, + uuid UUID NOT NULL, + + PRIMARY KEY(version, uuid) +); + +CREATE TABLE note_audio_breadcrumb ( + cell h3index NOT NULL, + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + manually_selected BOOLEAN NOT NULL, + note_audio_version INTEGER NOT NULL, + note_audio_uuid UUID NOT NULL, + position INTEGER NOT NULL, + + FOREIGN KEY (note_audio_version, note_audio_uuid) REFERENCES note_audio (version, uuid), + PRIMARY KEY (note_audio_version, note_audio_uuid, position) +); + +CREATE TYPE AudioDataType AS ENUM ( + 'raw', + 'raw_normalized', + 'ogg'); + +CREATE TABLE note_audio_data ( + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + note_audio_version INTEGER NOT NULL, + node_audio_uuid UUID NOT NULL, + type_ AudioDataType NOT NULL, + + FOREIGN KEY (note_audio_version, note_audio_uuid) REFERENCES note_audio (version, uuid), + PRIMARY KEY (note_audio_version, note_audio_uuid, type_) +); + +CREATE TABLE note_image ( + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + creator_id INTEGER REFERENCES user_ (id) NOT NULL, + deleted TIMESTAMP WITHOUT TIME ZONE, + deletor_id INTEGER REFERENCES user_ (id), + organization_id INTEGER REFERENCES organization (id) NOT NULL, + version INTEGER NOT NULL, + uuid UUID NOT NULL, + + PRIMARY KEY(version, uuid) +); + +CREATE TYPE ImageDataType AS ENUM ( + 'raw', + 'png', +); + +CREATE TABLE note_image_data ( + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + note_image_version INTEGER NOT NULL, + node_image_uuid UUID NOT NULL, + type_ AudioDataType NOT NULL, +); + + +CREATE TABLE note_image_breadcrumb ( + cell h3index NOT NULL, + created TIMESTAMP WITHOUT TIME ZONE NOT NULL, + manually_selected BOOLEAN NOT NULL, + note_image_version INTEGER NOT NULL, + node_image_uuid UUID NOT NULL, + position INTEGER NOT NULL, + + FOREIGN KEY (note_image_version, note_image_uuid) REFERENCES note_image (version, uuid), + PRIMARY KEY (note_image_version, note_image_uuid, position) +); + +-- +goose Down +DROP TABLE note_image_breadcrumb; +DROP TABLE note_image_data; +DROP TABLE note_image; +DROP TYPE ImageDataType +DROP TABLE note_audio_breadcrumb; +DROP TABLE note_audio_data; +DROP TABLE note_audio; +DROP TYPE AudioDataType; diff --git a/db/migrations/goose.sh b/db/migrations/goose.sh new file mode 100644 index 00000000..7aef17a7 --- /dev/null +++ b/db/migrations/goose.sh @@ -0,0 +1 @@ +GOOSE_DRIVER=postgres GOOSE_DBSTRING=dbname=nidus-sync sslmode=disable goose up diff --git a/db/query.go b/db/query.go new file mode 100644 index 00000000..0ae1e410 --- /dev/null +++ b/db/query.go @@ -0,0 +1,46 @@ +package db + +import ( + "context" + + "github.com/google/uuid" +) +type NidusNotePayload struct {} +type NoteAudio struct { + Transcription string + Version int + UUID uuid.UUID +} +type NoteImage struct {} +type MosquitoSource struct { } +type MosquitoTreatment struct { } +type ServiceRequest struct { } +type TrapData struct { } + +func MosquitoSourceQuery(q *GeoQuery) ([]MosquitoSource, error) { + return make([]MosquitoSource, 0), nil +} +func NoteAudioCreate(ctx context.Context, noteUUID uuid.UUID, payload NoteAudio, userID int32) error { + return nil +} +func NoteAudioGetLatest(ctx context.Context, uuid string) (*NoteAudio, error) { + return nil, nil +} +func NoteAudioNormalized(uuid string) error { + return nil +} +func NoteAudioTranscodedToOgg(uuid string) error { + return nil +} +func NoteImageCreate(ctx context.Context, noteUUID uuid.UUID, payload NoteImage, userID int32) error { + return nil +} +func NoteUpdate(ctx context.Context, noteUUID uuid.UUID, payload NidusNotePayload) error { + return nil +} +func ServiceRequestQuery(q *GeoQuery) ([]ServiceRequest, error) { + return make([]ServiceRequest, 0), nil +} +func TrapDataQuery(q *GeoQuery) ([]TrapData, error) { + return make([]TrapData, 0), nil +} diff --git a/endpoint.go b/endpoint.go index c5bc33dc..d3ae4a85 100644 --- a/endpoint.go +++ b/endpoint.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/Gleipnir-Technology/nidus-sync/auth" "github.com/Gleipnir-Technology/nidus-sync/db/models" "github.com/go-chi/chi/v5" "github.com/rs/zerolog/log" @@ -27,7 +28,7 @@ func getArcgisOauthCallback(w http.ResponseWriter, r *http.Request) { respondError(w, "Access code is empty", nil, http.StatusBadRequest) return } - user, err := getAuthenticatedUser(r) + user, err := auth.GetAuthenticatedUser(r) if err != nil { respondError(w, "You're not currently authenticated, which really shouldn't happen.", err, http.StatusUnauthorized) return @@ -61,7 +62,7 @@ func getFavicon(w http.ResponseWriter, r *http.Request) { } func getOAuthRefresh(w http.ResponseWriter, r *http.Request) { - user, err := getAuthenticatedUser(r) + user, err := auth.GetAuthenticatedUser(r) if err != nil { http.Redirect(w, r, "/?next=/oauth/refresh", http.StatusFound) return @@ -130,8 +131,8 @@ func getQRCodeReport(w http.ResponseWriter, r *http.Request) { } func getRoot(w http.ResponseWriter, r *http.Request) { - user, err := getAuthenticatedUser(r) - if err != nil && !errors.Is(err, &NoCredentialsError{}) { + user, err := auth.GetAuthenticatedUser(r) + if err != nil && !errors.Is(err, &auth.NoCredentialsError{}) { respondError(w, "Failed to get root", err, http.StatusInternalServerError) return } @@ -256,9 +257,9 @@ func postSignin(w http.ResponseWriter, r *http.Request) { log.Info().Str("username", username).Msg("Signin") - _, err := signinUser(r, username, password) + _, err := auth.SigninUser(r, username, password) if err != nil { - if errors.Is(err, InvalidCredentials{}) { + if errors.Is(err, auth.InvalidCredentials{}) { http.Redirect(w, r, "/?error=invalid-credentials", http.StatusFound) return } @@ -288,13 +289,13 @@ func postSignup(w http.ResponseWriter, r *http.Request) { return } - user, err := signupUser(username, name, password) + user, err := auth.SignupUser(username, name, password) if err != nil { respondError(w, "Failed to signup user", err, http.StatusInternalServerError) return } - addUserSession(r, user) + auth.AddUserSession(r, user) http.Redirect(w, r, "/", http.StatusFound) } diff --git a/go.mod b/go.mod index 420cc597..c35ea302 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/alexedwards/scs/v2 v2.9.0 github.com/alitto/pond/v2 v2.5.0 github.com/go-chi/chi/v5 v5.2.3 - github.com/go-webauthn/webauthn v0.14.0 + github.com/go-chi/render v1.0.3 github.com/gofrs/uuid/v5 v5.4.0 - github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.7.6 github.com/jaswdr/faker/v2 v2.8.1 github.com/lib/pq v1.10.9 + github.com/minio/minio-go/v7 v7.0.97 github.com/pressly/goose/v3 v3.26.0 github.com/rs/zerolog v1.34.0 github.com/shopspring/decimal v1.4.0 @@ -31,19 +31,26 @@ require ( ) require ( - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-webauthn/x v0.1.25 // indirect - github.com/google/go-tpm v0.9.5 // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mfridman/interpolate v0.0.2 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/minio/crc64nvme v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect github.com/tidwall/geoindex v1.4.4 // indirect github.com/tidwall/gjson v1.12.1 // indirect @@ -51,11 +58,13 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/rtree v1.3.1 // indirect github.com/tidwall/sjson v1.2.4 // indirect + github.com/tinylib/msgp v1.3.0 // indirect github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect - github.com/x448/float16 v0.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.43.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.29.0 // indirect google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4d325681..3be312f0 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aarondl/opt v0.0.0-20250607033636-982744e1bd65 h1:lbdPe4LBNmNDzeQFwNhEc88w90841qv737MI4+aXSYU= github.com/aarondl/opt v0.0.0-20250607033636-982744e1bd65/go.mod h1:+xKBXrTAUOvrDXO5PRwIr4E1wciHY3Glgl+6OkCXknU= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alexedwards/scs/pgxstore v0.0.0-20251002162104-209de6e426de h1:wNJVpr0ag/BL2nRGBIESdLe1qoljXIolF/qPi1gleRA= github.com/alexedwards/scs/pgxstore v0.0.0-20251002162104-209de6e426de/go.mod h1:hwveArYcjyOK66EViVgVU5Iqj7zyEsWjKXMQhDJrTLI= github.com/alexedwards/scs/v2 v2.9.0 h1:xa05mVpwTBm1iLeTMNFfAWpKUm4fXAW7CeAViqBVS90= @@ -47,33 +49,27 @@ github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0o github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= +github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-webauthn/webauthn v0.14.0 h1:ZLNPUgPcDlAeoxe+5umWG/tEeCoQIDr7gE2Zx2QnhL0= -github.com/go-webauthn/webauthn v0.14.0/go.mod h1:QZzPFH3LJ48u5uEPAu+8/nWJImoLBWM7iAH/kSVSo6k= -github.com/go-webauthn/x v0.1.25 h1:g/0noooIGcz/yCVqebcFgNnGIgBlJIccS+LYAa+0Z88= -github.com/go-webauthn/x v0.1.25/go.mod h1:ieblaPY1/BVCV0oQTsA/VAo08/TWayQuJuo5Q+XxmTY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid/v5 v5.4.0 h1:EfbpCTjqMuGyq5ZJwxqzn3Cbr2d0rUZU7v5ycAk/e/0= github.com/gofrs/uuid/v5 v5.4.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= -github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -91,11 +87,18 @@ github.com/jaswdr/faker/v2 v2.8.1 h1:2AcPgHDBXYQregFUH9LgVZKfFupc4SIquYhp29sf5wQ github.com/jaswdr/faker/v2 v2.8.1/go.mod h1:jZq+qzNQr8/P+5fHd9t3txe2GNPnthrTfohtnJ7B+68= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -111,8 +114,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= +github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= @@ -137,6 +144,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wBGdtTtBvls= github.com/pganalyze/pg_query_go/v6 v6.1.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -150,6 +159,9 @@ github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494/go.mod h1:yipyliwI08 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -204,6 +216,8 @@ github.com/tidwall/rtree v1.3.1 h1:xu3vJPKJrmGce7YJcFUCoqLrp9DTUEJBnVgdPSXHgHs= github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M= github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -214,8 +228,6 @@ github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 h1:mJdDDPblDfP github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07/go.mod h1:Ak17IJ037caFp4jpCw/iQQ7/W74Sqpb1YuKJU6HTKfM= github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -229,8 +241,6 @@ go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/ go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= -go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -247,6 +257,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -290,6 +302,7 @@ google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwl google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/label-studio/client.go b/label-studio/client.go new file mode 100644 index 00000000..176faa24 --- /dev/null +++ b/label-studio/client.go @@ -0,0 +1,129 @@ +package labelstudio + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// Client represents a Label Studio API client +type Client struct { + BaseURL string + APIKey string + AccessToken string + AccessTokenExpires time.Time + HTTPClient *http.Client +} + +// NewClient creates a new Label Studio client +func NewClient(baseURL string, apiKey string) *Client { + return &Client{ + BaseURL: baseURL, + APIKey: apiKey, + HTTPClient: &http.Client{}, + } +} + +// According to https://github.com/HumanSignal/label-studio/blob/develop/docs/source/guide/access_tokens.md +// the access tokens expire "in about 5 minutes". We'll do 4 minutes to give us a bit of margin. +var ACCESS_TOKEN_DURATION_SECONDS time.Duration = 240 * time.Second + +// GetAccessToken converts the API key into an access token +func (c *Client) GetAccessToken() error { + // Create request body + reqBody := map[string]string{ + "refresh": c.APIKey, + } + + // Marshal to JSON + jsonBody, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + // Create request + req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/token/refresh", c.BaseURL), bytes.NewBuffer(jsonBody)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + req.Header.Set("Content-Type", "application/json") + + // Send request + resp, err := c.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Check for successful response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("API returned error: %s", resp.Status) + } + + // Parse response + var result map[string]string + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("failed to parse response: %w", err) + } + + // Get access token + accessToken, ok := result["access"] + if !ok { + return fmt.Errorf("response did not contain access token") + } + + // Store access token + c.AccessToken = accessToken + c.AccessTokenExpires = time.Now().Add(ACCESS_TOKEN_DURATION_SECONDS) + return nil +} + +func (c *Client) makeRequest(method string, path string, payload []byte) (*http.Response, error) { + // Check if we have an access token, if not try to get it + if c.AccessToken == "" || time.Now().After(c.AccessTokenExpires) { + if err := c.GetAccessToken(); err != nil { + return nil, fmt.Errorf("failed to get access token: %w", err) + } + } + // Create request + url := fmt.Sprintf("%s/%s", c.BaseURL, path) + req, err := http.NewRequest(method, url, bytes.NewBuffer(payload)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.AccessToken)) + req.Header.Set("Content-Type", "application/json") + + // Send request + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + // Check for successful response + if resp.StatusCode > http.StatusBadRequest { + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("Got status code %d and failed to read response body: %v", resp.StatusCode, err) + } + bodyString := string(bodyBytes) + // Try to read error message + var errorResp map[string]interface{} + if err := json.Unmarshal(bodyBytes, &errorResp); err == nil { + return nil, fmt.Errorf("API returned JSON error %d: %v", resp.StatusCode, errorResp) + } + return nil, fmt.Errorf("API returned error status %d: %s: ", resp.Status, bodyString) + } + + + return resp, nil +} diff --git a/label-studio/import_tasks.go b/label-studio/import_tasks.go new file mode 100644 index 00000000..3e2b6bae --- /dev/null +++ b/label-studio/import_tasks.go @@ -0,0 +1,78 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" + "net/http" +) + +// TaskImportResponse represents the response from the import tasks endpoint +type TaskImportResponse struct { + // Common fields that might be returned + TaskCount int `json:"task_count,omitempty"` + Annotation map[string]interface{} `json:"annotation,omitempty"` + Task map[string]interface{} `json:"task,omitempty"` + + // For handling any other fields in the response + AdditionalProperties map[string]interface{} `json:"-"` +} + +// UnmarshalJSON custom unmarshaler for TaskImportResponse to capture all fields +func (r *TaskImportResponse) UnmarshalJSON(data []byte) error { + // First unmarshal the known fields + type Alias TaskImportResponse + aux := &struct { + *Alias + }{ + Alias: (*Alias)(r), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Then capture any additional fields + var rawMap map[string]interface{} + if err := json.Unmarshal(data, &rawMap); err != nil { + return err + } + + r.AdditionalProperties = make(map[string]interface{}) + for k, v := range rawMap { + // Skip fields we already processed + if k != "task_count" && k != "annotation" && k != "task" { + r.AdditionalProperties[k] = v + } + } + + return nil +} + +// ImportTasks imports tasks into a Label Studio project +// tasks parameter can be any data structure that can be marshalled to JSON +func (c *Client) ImportTasks(projectID int, tasks interface{}) (*TaskImportResponse, error) { + // Marshal the tasks to JSON + taskJSON, err := json.Marshal(tasks) + if err != nil { + return nil, fmt.Errorf("failed to marshal tasks: %w", err) + } + + path := fmt.Sprintf("/api/projects/%d/import", projectID) + resp, err := c.makeRequest("POST", path, taskJSON) + if err != nil { + return nil, fmt.Errorf("Failed to POST %s: %v", path, err) + } + defer resp.Body.Close() + + // Check for successful response + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + } + + // Parse response + var response TaskImportResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &response, nil +} diff --git a/label-studio/list_tasks.go b/label-studio/list_tasks.go new file mode 100644 index 00000000..bd0e66e0 --- /dev/null +++ b/label-studio/list_tasks.go @@ -0,0 +1,143 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" + "net/url" + "time" +) + +// TasksListResponse represents the response from the /api/tasks endpoint +type TasksListResponse struct { + Tasks []Task `json:"tasks"` + Total int `json:"total"` + TotalAnnotations int `json:"total_annotations"` + TotalPredictions int `json:"total_predictions"` +} + +// Task represents a single task returned by the Label Studio API +type Task struct { + Agreement string `json:"agreement"` + AgreementSelected string `json:"agreement_selected"` + Annotations json.RawMessage `json:"annotations"` + AnnotationsIDs json.RawMessage `json:"annotations_ids"` + AnnotationsResults json.RawMessage `json:"annotations_results"` + Annotators []int `json:"annotators"` + AnnotatorsCount int `json:"annotators_count"` + AvgLeadTime float64 `json:"avg_lead_time"` + CancelledAnnotations int `json:"cancelled_annotations"` + CommentAuthors []map[string]interface{} `json:"comment_authors"` + CommentAuthorsCount int `json:"comment_authors_count"` + CommentCount int `json:"comment_count"` + Comments json.RawMessage `json:"comments"` + CompletedAt string `json:"completed_at"` + CreatedAt time.Time `json:"created_at"` + Data map[string]interface{} `json:"data"` + DraftExists bool `json:"draft_exists"` + Drafts []json.RawMessage `json:"drafts"` + FileUpload string `json:"file_upload"` + GroundTruth bool `json:"ground_truth"` + ID int `json:"id"` + InnerID int `json:"inner_id"` + IsLabeled bool `json:"is_labeled"` + LastCommentUpdatedAt string `json:"last_comment_updated_at"` + Meta map[string]interface{} `json:"meta"` + Overlap int `json:"overlap"` + Predictions []json.RawMessage `json:"predictions"` + PredictionsModelVersions json.RawMessage `json:"predictions_model_versions"` + PredictionsResults json.RawMessage `json:"predictions_results"` + PredictionsScore float64 `json:"predictions_score"` + Project int `json:"project"` + ReviewTime int `json:"review_time"` + Reviewed bool `json:"reviewed"` + Reviewers []map[string]interface{} `json:"reviewers"` + ReviewersCount int `json:"reviewers_count"` + ReviewsAccepted int `json:"reviews_accepted"` + ReviewsRejected int `json:"reviews_rejected"` + StorageFilename string `json:"storage_filename"` + TotalAnnotations int `json:"total_annotations"` + TotalPredictions int `json:"total_predictions"` + UnresolvedCommentCount int `json:"unresolved_comment_count"` + UpdatedAt time.Time `json:"updated_at"` + UpdatedBy []map[string]interface{} `json:"updated_by"` +} + +// TasksListOptions represents query parameters that can be used to filter tasks +type TasksListOptions struct { + ProjectID int // Filter by project ID + Page int // Page number for pagination + PageSize int // Number of items per page + Ordering string // Field to order by (e.g., "created_at", "-created_at" for descending) + Query string // Search query for filtering tasks + IsLabeled *bool // Filter by labeled status + IsReviewed *bool // Filter by review status + GroundTruth *bool // Filter by ground truth status +} + +// ListTasks fetches the list of tasks from the Label Studio API +func (c *Client) ListTasks(options *TasksListOptions) (*TasksListResponse, error) { + + // Build URL with query parameters + path := "/api/tasks/" + if options != nil { + queryParams := url.Values{} + + // Add all the possible filter parameters + if options.ProjectID > 0 { + queryParams.Add("project", fmt.Sprintf("%d", options.ProjectID)) + } + if options.Page > 0 { + queryParams.Add("page", fmt.Sprintf("%d", options.Page)) + } + if options.PageSize > 0 { + queryParams.Add("page_size", fmt.Sprintf("%d", options.PageSize)) + } + if options.Ordering != "" { + queryParams.Add("ordering", options.Ordering) + } + if options.Query != "" { + queryParams.Add("query", options.Query) + } + if options.IsLabeled != nil { + if *options.IsLabeled { + queryParams.Add("is_labeled", "true") + } else { + queryParams.Add("is_labeled", "false") + } + } + if options.IsReviewed != nil { + if *options.IsReviewed { + queryParams.Add("reviewed", "true") + } else { + queryParams.Add("reviewed", "false") + } + } + if options.GroundTruth != nil { + if *options.GroundTruth { + queryParams.Add("ground_truth", "true") + } else { + queryParams.Add("ground_truth", "false") + } + } + + // Add query params to URL if we have any + if len(queryParams) > 0 { + path = fmt.Sprintf("%s?%s", path, queryParams.Encode()) + } + } + + // Create request + resp, err := c.makeRequest("GET", path, nil) + if err != nil { + return nil, fmt.Errorf("Failed to request %s: %v", path, err) + } + defer resp.Body.Close() + + // Parse response + var tasksResponse TasksListResponse + if err := json.NewDecoder(resp.Body).Decode(&tasksResponse); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &tasksResponse, nil +} diff --git a/label-studio/projects.go b/label-studio/projects.go new file mode 100644 index 00000000..3fa2c4b6 --- /dev/null +++ b/label-studio/projects.go @@ -0,0 +1,126 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" + "time" +) + +// ProjectsResponse represents the response from the /api/projects endpoint +type ProjectsResponse struct { + Count int `json:"count"` + Results []Project `json:"results"` + Next string `json:"next"` + Previous string `json:"previous"` +} + +// Project represents a single project returned by the Label Studio API +type Project struct { + AllowStream bool `json:"allow_stream"` + AssignmentSettings AssignmentSettings `json:"assignment_settings"` + Blueprints []Blueprint `json:"blueprints"` + ConfigHasControlTags bool `json:"config_has_control_tags"` + ConfigSuitableForBulkAnnotation bool `json:"config_suitable_for_bulk_annotation"` + CreatedAt time.Time `json:"created_at"` + DataTypes map[string]string `json:"data_types"` + DescriptionShort string `json:"description_short"` + FinishedTaskNumber int `json:"finished_task_number"` + GroundTruthNumber int `json:"ground_truth_number"` + ID int `json:"id"` + Members string `json:"members"` + MembersCount int `json:"members_count"` + NumTasksWithAnnotations int `json:"num_tasks_with_annotations"` + //ParsedLabelConfig map[string]string `json:"parsed_label_config"` + Prompts string `json:"prompts"` + QueueDone int `json:"queue_done"` + QueueLeft int `json:"queue_left"` + //QueueTotal string `json:"queue_total"` + Ready bool `json:"ready"` + Rejected int `json:"rejected"` + ReviewSettings ReviewSettings `json:"review_settings"` + ReviewTotalTasks int `json:"review_total_tasks"` + ReviewedNumber int `json:"reviewed_number"` + ReviewerQueueTotal int `json:"reviewer_queue_total"` + //SkippedAnnotationsNumber string `json:"skipped_annotations_number"` + StartTrainingOnAnnotationUpdate bool `json:"start_training_on_annotation_update"` + TaskNumber int `json:"task_number"` + //TotalAnnotationsNumber string `json:"total_annotations_number"` + TotalPredictionsNumber int `json:"total_predictions_number"` + Workspace string `json:"workspace"` + WorkspaceTitle string `json:"workspace_title"` + AnnotationLimitCount int `json:"annotation_limit_count"` + AnnotationLimitPercent string `json:"annotation_limit_percent"` + AnnotatorEvaluationMinimumScore string `json:"annotator_evaluation_minimum_score"` + AnnotatorEvaluationMinimumTasks int `json:"annotator_evaluation_minimum_tasks"` + Color string `json:"color"` + CommentClassificationConfig string `json:"comment_classification_config"` + //ControlWeights map[string]string `json:"control_weights"` + CreatedBy User `json:"created_by"` + CustomScript string `json:"custom_script"` + CustomTaskLockTtl int `json:"custom_task_lock_ttl"` + Description string `json:"description"` + DuplicationDone bool `json:"duplication_done"` + DuplicationStatus string `json:"duplication_status"` + EnableEmptyAnnotation bool `json:"enable_empty_annotation"` + EvaluatePredictionsAutomatically bool `json:"evaluate_predictions_automatically"` + ExpertInstruction string `json:"expert_instruction"` + IsDraft bool `json:"is_draft"` + IsPublished bool `json:"is_published"` + LabelConfig string `json:"label_config"` + MaximumAnnotations int `json:"maximum_annotations"` + MinAnnotationsToStartTraining int `json:"min_annotations_to_start_training"` + ModelVersion string `json:"model_version"` + Organization int `json:"organization"` + OverlapCohortPercentage int `json:"overlap_cohort_percentage"` + PauseOnFailedAnnotatorEvaluation bool `json:"pause_on_failed_annotator_evaluation"` + PinnedAt string `json:"pinned_at"` + RequireCommentOnSkip bool `json:"require_comment_on_skip"` + RevealPreannotationsInteractively bool `json:"reveal_preannotations_interactively"` + Sampling string `json:"sampling"` + ShowAnnotationHistory bool `json:"show_annotation_history"` + ShowCollabPredictions bool `json:"show_collab_predictions"` + ShowGroundTruthFirst bool `json:"show_ground_truth_first"` + ShowInstruction bool `json:"show_instruction"` + ShowOverlapFirst bool `json:"show_overlap_first"` + ShowSkipButton bool `json:"show_skip_button"` + ShowUnusedDataColumnsToAnnotators bool `json:"show_unused_data_columns_to_annotators"` + SkipQueue string `json:"skip_queue"` + Title string `json:"title"` + UsefulAnnotationNumber int `json:"useful_annotation_number"` +} + +// Blueprint represents a blueprint in a project +type Blueprint struct { + CreatedAt time.Time `json:"created_at"` + ID int `json:"id"` + ShareID string `json:"share_id"` + ShortURL string `json:"short_url"` +} + +// AssignmentSettings represents the assignment settings of a project +type AssignmentSettings struct { + ID int `json:"id"` +} + +// ReviewSettings represents the review settings of a project +type ReviewSettings struct { + ID int `json:"id"` + RequeueRejectedTasksToAnnotator bool `json:"requeue_rejected_tasks_to_annotator"` +} + +// Projects fetches the list of projects from the Label Studio API +func (c *Client) Projects() (*ProjectsResponse, error) { + resp, err := c.makeRequest("GET", "/api/projects", nil) + if err != nil { + return nil, fmt.Errorf("Failed to GET /api/projects: %w", err) + } + defer resp.Body.Close() + + // Parse response + var projects ProjectsResponse + if err := json.NewDecoder(resp.Body).Decode(&projects); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &projects, nil +} diff --git a/label-studio/tasks_annotation.go b/label-studio/tasks_annotation.go new file mode 100644 index 00000000..aa26a5cd --- /dev/null +++ b/label-studio/tasks_annotation.go @@ -0,0 +1,75 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" + "time" +) + +// AnnotationRequest represents the request body for creating a draft +type AnnotationRequest struct { + DraftID int `json:"draft_id"` + LeadTime float64 `json:"lead_time"` + ParentAnnotation *int `json:"parent_annotation,omitempty"` + ParentPrediction *int `json:"parent_prediction,omitempty"` + Project int `json:"project"` + Result []TaskResult `json:"result"` + StartedAt string `json:"started_at"` +} + +// Annotation represents a draft annotation returned by the API +type Annotation struct { + BulkCreated bool `json:"bulk_created"` + CompletedBy int `json:"completed_by"` + CreatedAgo string `json:"created_ago"` + CreatedAt string `json:"created_at"` + CreatedUsername string `json:"created_username"` + DraftCreatedAt string `json:"draft_created_at"` + GroundTruth bool `json:"ground_truth"` + ID int `json:"id"` + ImportID *string `json:"import_id"` + LastAction *string `json:"last_action"` + LastCreatedBy *string `json:"last_created_by"` + LeadTime float64 `json:"lead_time"` + ParentAnnotation *int `json:"parent_annotation,omitempty"` + ParentPrediction *int `json:"parent_prediction,omitempty"` + Project int `json:"project"` + Result []TaskResult `json:"result"` + Task int `json:"task"` + WasCancelled bool `json:"was_cancelled"` + UpdatedAt string `json:"updated_at"` + UpdatedBy int `json:"updated_by"` +} + +// NewAnnotation creates a new draft request builder +func NewAnnotationRequest(projectID int) *AnnotationRequest { + return &AnnotationRequest{ + Project: projectID, + StartedAt: time.Now().UTC().Format(time.RFC3339Nano), + } +} + +// CreateAnnotation creates a new annotation on a task +func (c *Client) CreateAnnotation(taskID int, annotation *AnnotationRequest) (*Annotation, error) { + // Marshal the annotation request to JSON + annotationJSON, err := json.Marshal(annotation) + if err != nil { + return nil, fmt.Errorf("failed to marshal annotation request: %w", err) + } + + // Create request URL with query parameter + path := fmt.Sprintf("/api/tasks/%d/annotations", taskID) + resp, err := c.makeRequest("POST", path, annotationJSON) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + defer resp.Body.Close() + + // Parse response + var createdAnnotation Annotation + if err := json.NewDecoder(resp.Body).Decode(&createdAnnotation); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &createdAnnotation, nil +} diff --git a/label-studio/tasks_draft.go b/label-studio/tasks_draft.go new file mode 100644 index 00000000..2ca95816 --- /dev/null +++ b/label-studio/tasks_draft.go @@ -0,0 +1,106 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" + "time" +) + +// DraftRequest represents the request body for creating a draft +type DraftRequest struct { + Annotation *string `json:"annotation"` + CreatedAgo string `json:"created_ago"` + CreatedAt string `json:"created_at"` + CreatedUsername string `json:"created_username"` + DraftID int `json:"draft_id"` + ID int `json:"id"` + ImportID *string `json:"import_id"` + LeadTime float64 `json:"lead_time"` + ParentAnnotation *int `json:"parent_annotation,omitempty"` + ParentPrediction *int `json:"parent_prediction,omitempty"` + Project string `json:"project"` + Result []TaskResult `json:"result"` + StartedAt string `json:"started_at"` + Task int `json:"task"` + User string `json:"user"` + WasPostponed bool `json:"was_postponed"` +} + +// Draft represents a draft annotation returned by the API +type Draft struct { + ID int `json:"id"` + TaskID int `json:"task"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + LeadTime float64 `json:"lead_time"` + Result []map[string]interface{} `json:"result"` + Annotation *int `json:"annotation,omitempty"` + User string `json:"user"` +} + +// NewDraft creates a new draft request builder +func NewDraft(projectID int) *DraftRequest { + return &DraftRequest{ + DraftID: 0, + Project: string(projectID), + StartedAt: time.Now().UTC().Format(time.RFC3339Nano), + } +} + +// SetLeadTime sets the time spent on the draft +func (d *DraftRequest) SetLeadTime(leadTime float64) *DraftRequest { + d.LeadTime = leadTime + return d +} + +// SetResult sets the annotation result +func (d *DraftRequest) SetResult(result []TaskResult) *DraftRequest { + d.Result = result + return d +} + +// SetParentPrediction sets the parent prediction ID if the draft is based on a prediction +func (d *DraftRequest) SetParentPrediction(predictionID int) *DraftRequest { + d.ParentPrediction = &predictionID + return d +} + +// SetParentAnnotation sets the parent annotation ID if the draft is based on an annotation +func (d *DraftRequest) SetParentAnnotation(annotationID int) *DraftRequest { + d.ParentAnnotation = &annotationID + return d +} + +// SetStartedAt sets the time when work on the draft started +func (d *DraftRequest) SetStartedAt(startedAt time.Time) *DraftRequest { + d.StartedAt = startedAt.UTC().Format(time.RFC3339Nano) + return d +} + +// CreateDraft creates a new draft for a task +func (c *Client) CreateDraft(taskID int, draft *DraftRequest) (*Draft, error) { + // Marshal the draft request to JSON + draftJSON, err := json.Marshal(draft) + if err != nil { + return nil, fmt.Errorf("failed to marshal draft request: %w", err) + } + + // Create request URL with query parameter + //url := fmt.Sprintf("%s/api/tasks/%d/drafts?project=%s", c.BaseURL, taskID, draft.Project) + path := fmt.Sprintf("/api/tasks/%d/drafts", taskID) + + // Create request + resp, err := c.makeRequest("POST", path, draftJSON) + if err != nil { + return nil, fmt.Errorf("failed to POST %s: %w", path, err) + } + defer resp.Body.Close() + + // Parse response + var createdDraft Draft + if err := json.NewDecoder(resp.Body).Decode(&createdDraft); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &createdDraft, nil +} diff --git a/label-studio/tasks_update.go b/label-studio/tasks_update.go new file mode 100644 index 00000000..635eda63 --- /dev/null +++ b/label-studio/tasks_update.go @@ -0,0 +1,183 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" +) + +type TaskResultValue struct { + Text []string `json:"text"` +} + +type TaskResult struct { + ID string `json:"id"` + FromName string `json:"from_name"` + Origin string `json:"origin"` + ToName string `json:"to_name"` + Type string `json:"type"` + Value TaskResultValue `json:"value"` +} + +// TaskUpdate defines fields that can be updated in a task +type TaskUpdate struct { + // Fields that can be updated + Annotations json.RawMessage `json:"annotations,omitempty"` + Data *map[string]interface{} `json:"data,omitempty"` + DraftExists *bool `json:"draft_exists,omitempty"` + Drafts json.RawMessage `json:"drafts,omitempty"` + GroundTruth *bool `json:"ground_truth,omitempty"` + IsLabeled *bool `json:"is_labeled,omitempty"` + Meta *map[string]interface{} `json:"meta,omitempty"` + Predictions json.RawMessage `json:"predictions,omitempty"` + Reviewed *bool `json:"reviewed"` + + // Internal tracking + fieldsToUpdate map[string]bool +} + +// NewTaskUpdate creates a new TaskUpdate builder +func NewTaskUpdate() *TaskUpdate { + return &TaskUpdate{ + fieldsToUpdate: make(map[string]bool), + } +} + +func (t *TaskUpdate) MarshalJSON() ([]byte, error) { + // Only include fields that are explicitly set + updateMap := make(map[string]interface{}) + + if t.fieldsToUpdate["annotations"] { + // Parse raw JSON back to interface{} to include in the map + var annotations interface{} + if err := json.Unmarshal(t.Annotations, &annotations); err != nil { + return nil, err + } + updateMap["annotations"] = annotations + } + if t.fieldsToUpdate["data"] { + updateMap["data"] = t.Data + } + if t.fieldsToUpdate["draft_exists"] { + updateMap["draft_exists"] = t.DraftExists + } + if t.fieldsToUpdate["drafts"] { + var drafts interface{} + if err := json.Unmarshal(t.Drafts, &drafts); err != nil { + return nil, err + } + updateMap["drafts"] = drafts + } + if t.fieldsToUpdate["ground_truth"] { + updateMap["ground_truth"] = t.GroundTruth + } + if t.fieldsToUpdate["is_labeled"] { + updateMap["is_labeled"] = t.IsLabeled + } + if t.fieldsToUpdate["meta"] { + updateMap["meta"] = t.Meta + } + if t.fieldsToUpdate["predictions"] { + var predictions interface{} + if err := json.Unmarshal(t.Predictions, &predictions); err != nil { + return nil, err + } + updateMap["predictions"] = predictions + } + if t.fieldsToUpdate["reviewed"] { + updateMap["reviewed"] = t.Reviewed + } + + return json.Marshal(updateMap) +} + +func (t *TaskUpdate) SetAnnotations(annotations interface{}) *TaskUpdate { + annotationsJSON, err := json.Marshal(annotations) + if err != nil { + // Handle error gracefully in a builder pattern + // Could store the error and check it later + return t + } + t.Annotations = annotationsJSON + t.fieldsToUpdate["annotations"] = true + return t +} + +func (t *TaskUpdate) SetData(data map[string]interface{}) *TaskUpdate { + t.Data = &data + t.fieldsToUpdate["data"] = true + return t +} + +func (t *TaskUpdate) SetDraftExists(draftExists bool) *TaskUpdate { + t.DraftExists = &draftExists + t.fieldsToUpdate["draft_exists"] = true + return t +} + +func (t *TaskUpdate) SetDrafts(drafts interface{}) *TaskUpdate { + draftsJSON, err := json.Marshal(drafts) + if err != nil { + return t + } + t.Drafts = draftsJSON + t.fieldsToUpdate["drafts"] = true + return t +} + +func (t *TaskUpdate) SetGroundTruth(groundTruth bool) *TaskUpdate { + t.GroundTruth = &groundTruth + t.fieldsToUpdate["ground_truth"] = true + return t +} + +func (t *TaskUpdate) SetIsLabeled(isLabeled bool) *TaskUpdate { + t.IsLabeled = &isLabeled + t.fieldsToUpdate["is_labeled"] = true + return t +} + +func (t *TaskUpdate) SetMeta(meta map[string]interface{}) *TaskUpdate { + t.Meta = &meta + t.fieldsToUpdate["meta"] = true + return t +} + +func (t *TaskUpdate) SetPredictions(predictions interface{}) *TaskUpdate { + predictionsJSON, err := json.Marshal(predictions) + if err != nil { + return t + } + t.Predictions = predictionsJSON + t.fieldsToUpdate["predictions"] = true + return t +} + +func (t *TaskUpdate) SetReviewed(isReviewed bool) *TaskUpdate { + t.Reviewed = &isReviewed + t.fieldsToUpdate["reviewed"] = true + return t +} + +func (c *Client) TaskUpdate(taskID int, update *TaskUpdate) (*Task, error) { + // Marshal the updates to JSON + updateJSON, err := json.Marshal(update) + if err != nil { + return nil, fmt.Errorf("failed to marshal updates: %w", err) + } + + // Create request + path := fmt.Sprintf("/api/tasks/%d", taskID) + resp, err := c.makeRequest("PATCH", path, updateJSON) + if err != nil { + return nil, fmt.Errorf("failed to PATCH %s: %w", path, err) + } + defer resp.Body.Close() + + // Parse response + var updatedTask Task + if err := json.NewDecoder(resp.Body).Decode(&updatedTask); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &updatedTask, nil +} diff --git a/label-studio/users.go b/label-studio/users.go new file mode 100644 index 00000000..139e804c --- /dev/null +++ b/label-studio/users.go @@ -0,0 +1,62 @@ +package labelstudio + +import ( + "encoding/json" + "fmt" + "time" +) + +// User represents a user in Label Studio +type User struct { + ID int `json:"id"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Username string `json:"username"` + Email string `json:"email"` + LastActivity time.Time `json:"last_activity"` + CustomHotkeys map[string]interface{} `json:"custom_hotkeys"` + Avatar *string `json:"avatar"` + Initials string `json:"initials"` + Phone string `json:"phone"` + ActiveOrganization int `json:"active_organization"` + ActiveOrganizationMeta struct { + Title string `json:"title"` + Email string `json:"email"` + } `json:"active_organization_meta"` + AllowNewsletters *bool `json:"allow_newsletters"` + DateJoined time.Time `json:"date_joined"` +} + +// ListUsers fetches the list of users from the Label Studio API +func (c *Client) ListUsers() ([]User, error) { + resp, err := c.makeRequest("GET", "/api/users", nil) + if err != nil { + return nil, fmt.Errorf("failed to GET /api/userls: %w", err) + } + defer resp.Body.Close() + + // Parse response + var users []User + if err := json.NewDecoder(resp.Body).Decode(&users); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return users, nil +} + +// GetUser fetches a specific user by ID +func (c *Client) GetUser(userID int) (*User, error) { + resp, err := c.makeRequest("GET", fmt.Sprintf("/api/users/%d", userID), nil) + if err != nil { + return nil, fmt.Errorf("failed to GET /api/users/%d: %w", userID, err) + } + defer resp.Body.Close() + + // Parse response + var user User + if err := json.NewDecoder(resp.Body).Decode(&user); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &user, nil +} diff --git a/main.go b/main.go index 55a3ad7d..16832e47 100644 --- a/main.go +++ b/main.go @@ -10,16 +10,16 @@ import ( "syscall" "time" + "github.com/Gleipnir-Technology/nidus-sync/api" + "github.com/Gleipnir-Technology/nidus-sync/auth" "github.com/Gleipnir-Technology/nidus-sync/db" - "github.com/alexedwards/scs/pgxstore" - "github.com/alexedwards/scs/v2" + "github.com/Gleipnir-Technology/nidus-sync/userfile" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) -var sessionManager *scs.SessionManager var BaseURL, ClientID, ClientSecret, Environment, FieldseekerSchemaDirectory, MapboxToken string @@ -70,6 +70,11 @@ func main() { log.Error().Msg("You must specify a non-empty FIELDSEEKER_SCHEMA_DIRECTORY") os.Exit(1) } + userfile.UserFilesDirectory = os.Getenv("USER_FILES_DIRECTORY") + if userfile.UserFilesDirectory == "" { + log.Error().Msg("You must specify a non-empty USER_FILES_DIRECTORY") + os.Exit(1) + } log.Info().Msg("Starting...") err := db.InitializeDatabase(context.TODO(), pg_dsn) @@ -77,14 +82,11 @@ func main() { log.Error().Str("err", err.Error()).Msg("Failed to connect to database") os.Exit(2) } - sessionManager = scs.New() - sessionManager.Store = pgxstore.New(db.PGInstance.PGXPool) - sessionManager.Lifetime = 24 * time.Hour router_logger := log.With().Logger() r := chi.NewRouter() r.Use(LoggerMiddleware(&router_logger)) - r.Use(sessionManager.LoadAndSave) + r.Use(auth.NewSessionManager().LoadAndSave) // Root is a special endpoint that is neither authenticated nor unauthenticated r.Get("/", getRoot) @@ -138,10 +140,11 @@ func main() { r.Post("/sms/{org}", postSMS) // Authenticated endpoints - r.Method("GET", "/cell/{cell}", NewEnsureAuth(getCellDetails)) - r.Method("GET", "/settings", NewEnsureAuth(getSettings)) - r.Method("GET", "/source/{globalid}", NewEnsureAuth(getSource)) - r.Method("GET", "/vector-tiles/{org_id}/{tileset_id}/{zoom}/{x}/{y}.{format}", NewEnsureAuth(getVectorTiles)) + r.Route("/api", api.AddRoutes) + r.Method("GET", "/cell/{cell}", auth.NewEnsureAuth(getCellDetails)) + r.Method("GET", "/settings", auth.NewEnsureAuth(getSettings)) + r.Method("GET", "/source/{globalid}", auth.NewEnsureAuth(getSource)) + r.Method("GET", "/vector-tiles/{org_id}/{tileset_id}/{zoom}/{x}/{y}.{format}", auth.NewEnsureAuth(getVectorTiles)) localFS := http.Dir("./static") FileServer(r, "/static", localFS, embeddedStaticFS, "static") diff --git a/minio/client.go b/minio/client.go new file mode 100644 index 00000000..cbdbdf44 --- /dev/null +++ b/minio/client.go @@ -0,0 +1,77 @@ +package minio + +import ( + "context" + "fmt" + "log" + "net/url" + "os" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Client struct { + client *minio.Client +} + +func NewClient(baseURL string, accessKeyID string, secretAccessKey string) (*Client, error) { + log.Printf("Connecting to S3 at %s", baseURL) + minioClient, err := minio.New(baseURL, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: true, + }) + if err != nil { + return nil, fmt.Errorf("Failed to connect to minio: %v", err) + } + return &Client{ + client: minioClient, + }, nil +} + +func signUrl(minioClient *minio.Client, bucketName string, filePath string) { + // Set request parameters for content-disposition. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\""+filePath+"\"") + + // Generates a presigned url which expires in a day. + presignedURL, err := minioClient.PresignedGetObject(context.Background(), bucketName, filePath, time.Second*24*60*60, reqParams) + if err != nil { + fmt.Println(err) + return + } + + fmt.Println("Successfully generated presigned URL", presignedURL) +} +func (minioClient *Client) ObjectExists(bucket string, path string) bool { + ctx := context.Background() + opts := minio.ListObjectsOptions{ + UseV1: false, + Prefix: path, + Recursive: false, + } + for object := range minioClient.client.ListObjects(ctx, bucket, opts) { + if object.Err == nil { + return true + } + log.Printf("Error getting object %s/%s: %v", bucket, path, object.Err) + } + return false +} + +func (minioClient *Client) UploadFile(bucketName string, filePath string, uploadPath string) error { + // Open the file for reading + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("Failed to open file %s to upload: %v", filePath, err) + } + defer file.Close() + + // Upload the file + _, err = minioClient.client.FPutObject(context.Background(), bucketName, uploadPath, filePath, minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("Failed to put object to bucket %s: %v", bucketName, err) + } + return nil +} diff --git a/queue/audio_processing.go b/queue/audio_processing.go new file mode 100644 index 00000000..426441c4 --- /dev/null +++ b/queue/audio_processing.go @@ -0,0 +1,121 @@ +package queue + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/exec" + + "github.com/Gleipnir-Technology/nidus-sync/db" + "github.com/Gleipnir-Technology/nidus-sync/userfile" + "github.com/google/uuid" +) + +// AudioJob represents a job to process an audio file. +type AudioJob struct { + AudioUUID uuid.UUID +} + +// audioJobChannel is the channel used to send audio processing jobs to the worker. +var audioJobChannel chan AudioJob + +// StartAudioWorker initializes the audio job channel and starts the worker goroutine. +func StartAudioWorker(ctx context.Context) { + buffer := 100 + audioJobChannel = make(chan AudioJob, buffer) // Buffered channel to prevent blocking + log.Printf("Started audio worker with buffer depth %d", buffer) + go func() { + for { + select { + case <-ctx.Done(): + log.Println("Audio worker shutting down.") + return + case job := <-audioJobChannel: + log.Printf("Processing audio job for UUID: %s", job.AudioUUID) + err := processAudioFile(job.AudioUUID) + if err != nil { + log.Printf("Error processing audio file %s: %v", job.AudioUUID, err) + } + } + } + }() +} + +// EnqueueAudioJob sends an audio processing job to the worker. +func EnqueueAudioJob(job AudioJob) { + select { + case audioJobChannel <- job: + log.Printf("Enqueued audio job for UUID: %s", job.AudioUUID) + default: + log.Printf("Audio job channel is full, dropping job for UUID: %s", job.AudioUUID) + } +} + +func processAudioFile(audioUUID uuid.UUID) error { + // Normalize audio + err := normalizeAudio(audioUUID) + if err != nil { + return fmt.Errorf("failed to normalize audio %s: %v", audioUUID, err) + } + + // Transcode to OGG + err = transcodeToOgg(audioUUID) + if err != nil { + return fmt.Errorf("failed to transcode audio %s to OGG: %v", audioUUID, err) + } + + EnqueueLabelStudioJob(LabelStudioJob{ + UUID: audioUUID, + }) + return nil +} + +func normalizeAudio(audioUUID uuid.UUID) error { + source := userfile.AudioFileContentPathRaw(audioUUID.String()) + _, err := os.Stat(source) + if errors.Is(err, os.ErrNotExist) { + log.Printf("%s doesn't exist, skipping normalization", source) + return nil + } + log.Printf("Normalizing %s", source) + destination := userfile.AudioFileContentPathNormalized(audioUUID.String()) + // Use "ffmpeg" directly, assuming it's in the system PATH + cmd := exec.Command("ffmpeg", "-i", source, "-filter:a", "loudnorm", destination) + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("FFmpeg output for normalization: %s", out) + return fmt.Errorf("ffmpeg normalization failed: %v", err) + } + err = db.NoteAudioNormalized(audioUUID.String()) + if err != nil { + return fmt.Errorf("failed to update database for normalized audio %s: %v", audioUUID, err) + } + log.Printf("Normalized audio to %s", destination) + return nil +} + +func transcodeToOgg(audioUUID uuid.UUID) error { + source := userfile.AudioFileContentPathNormalized(audioUUID.String()) + _, err := os.Stat(source) + if errors.Is(err, os.ErrNotExist) { + log.Printf("%s doesn't exist, skipping OGG transcoding", source) + return nil + } + log.Printf("Transcoding %s to ogg", source) + destination := userfile.AudioFileContentPathOgg(audioUUID.String()) + // Use "ffmpeg" directly, assuming it's in the system PATH + cmd := exec.Command("ffmpeg", "-i", source, "-vn", "-acodec", "libvorbis", destination) + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("FFmpeg output for OGG transcoding: %s", out) + return fmt.Errorf("ffmpeg OGG transcoding failed: %v", err) + } + err = db.NoteAudioTranscodedToOgg(audioUUID.String()) + if err != nil { + return fmt.Errorf("failed to update database for OGG transcoded audio %s: %v", audioUUID, err) + } + log.Printf("Transcoded audio to %s", destination) + return nil +} diff --git a/queue/label_studio.go b/queue/label_studio.go new file mode 100644 index 00000000..2a6c90dc --- /dev/null +++ b/queue/label_studio.go @@ -0,0 +1,226 @@ +package queue + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "os" + + "github.com/Gleipnir-Technology/nidus-sync/db" + "github.com/Gleipnir-Technology/nidus-sync/label-studio" + "github.com/Gleipnir-Technology/nidus-sync/minio" + "github.com/Gleipnir-Technology/nidus-sync/userfile" + "github.com/google/uuid" +) + +type LabelStudioJob struct { + UUID uuid.UUID +} + +var labelJobChannel chan LabelStudioJob + +func EnqueueLabelStudioJob(job LabelStudioJob) { + select { + case labelJobChannel <- job: + log.Printf("Enqueued label job for UUID: %s", job.UUID) + default: + log.Printf("Label job channel is full, dropping job for UUID: %s", job.UUID) + } +} + +func StartLabelStudioWorker(ctx context.Context) error { + // Initialize the minio client + minioBucket := os.Getenv("S3_BUCKET") + + labelStudioClient, err := createLabelStudioClient() + if err != nil { + return fmt.Errorf("Failed to create label studio client: %v", err) + } + // Get the project we are going to upload to + project, err := findLabelStudioProject(labelStudioClient, "Nidus Speech-to-Text Transcriptions") + if err != nil { + return errors.New(fmt.Sprintf("Failed to find the label studio project")) + } + minioClient, err := createMinioClient() + if err != nil { + return fmt.Errorf("Failed to create minio client: %v", err) + } + buffer := 100 + labelJobChannel = make(chan LabelStudioJob, buffer) // Buffered channel to prevent blocking + log.Printf("Started label studio worker with buffer depth %d", buffer) + go func() { + for { + select { + case <-ctx.Done(): + log.Println("Audio worker shutting down.") + return + case job := <-labelJobChannel: + log.Printf("Processing label job for UUID: %s", job.UUID) + err := processLabelTask(ctx, minioClient, minioBucket, labelStudioClient, project, job) + if err != nil { + log.Printf("Error processing label job for audio file %s: %v", job.UUID, err) + } + } + } + }() + return nil +} + +func createMinioClient() (*minio.Client, error) { + baseUrl := os.Getenv("S3_BASE_URL") + accessKeyID := os.Getenv("S3_ACCESS_KEY_ID") + secretAccessKey := os.Getenv("S3_SECRET_ACCESS_KEY") + + client, err := minio.NewClient(baseUrl, accessKeyID, secretAccessKey) + if err != nil { + return nil, err + } + log.Println("Created minio client") + return client, err +} + +func createLabelStudioClient() (*labelstudio.Client, error) { + // Initialize the client with your Label Studio base URL and API key + labelStudioApiKey := os.Getenv("LABEL_STUDIO_API_KEY") + labelStudioBaseUrl := os.Getenv("LABEL_STUDIO_BASE_URL") + labelStudioClient := labelstudio.NewClient(labelStudioBaseUrl, labelStudioApiKey) + log.Println("Created label studio client") + + // Get and store the access token + err := labelStudioClient.GetAccessToken() + if err != nil { + return nil, errors.New(fmt.Sprintf("Failed to get access token: %v", err)) + } + log.Println("Got label studio client access token") + + return labelStudioClient, nil +} + +func processLabelTask(ctx context.Context, minioClient *minio.Client, minioBucket string, labelStudioClient *labelstudio.Client, project *labelstudio.Project, job LabelStudioJob) error { + customer := os.Getenv("CUSTOMER") + if customer == "" { + return errors.New("You must specify a CUSTOMER env var") + } + note, err := db.NoteAudioGetLatest(ctx, job.UUID.String()) + if err != nil { + return errors.New(fmt.Sprintf("Failed to get note %s", note.UUID)) + } + + if note.Version != 1 { + return errors.New(fmt.Sprintf("Got version %d of %s", note.Version, note.UUID)) + } + task, err := findMatchingTask(labelStudioClient, project, customer, note) + if err != nil { + return errors.New(fmt.Sprintf("Failed to search for a task: %v", err)) + } + // We already have a task, nothing to do. + if task != nil { + return nil + } + + err = createTask(labelStudioClient, project, minioClient, minioBucket, customer, note) + if err != nil { + return errors.New(fmt.Sprintf("Failed to create a task: %v", err)) + } + return nil +} + +func createTask(client *labelstudio.Client, project *labelstudio.Project, minioClient *minio.Client, bucket string, customer string, note *db.NoteAudio) error { + audioRef := fmt.Sprintf("s3://%s/%s-normalized.m4a", bucket, note.UUID) + audioFile := fmt.Sprintf("%s/%s-normalized.m4a", userfile.UserFilesDirectory, note.UUID) + uploadPath := fmt.Sprintf("%s-normalized.m4a", note.UUID) + + if !minioClient.ObjectExists(bucket, uploadPath) { + err := minioClient.UploadFile(bucket, audioFile, uploadPath) + if err != nil { + return fmt.Errorf("Failed to upload audio: %v", err) + } + } + var transcription string = "" + //if note.Transcription.IsValue() { + //transcription = note.Transcription.MustGet() + //} + transcription = note.Transcription + simpleTasks := []map[string]interface{}{ + { + "data": map[string]string{ + "audio": audioRef, + "note_uuid": note.UUID.String(), + "transcription": transcription, + }, + "meta": map[string]string{ + "customer": customer, + "note_uuid": note.UUID.String(), + }, + }, + } + _, err := client.ImportTasks(project.ID, simpleTasks) + if err != nil { + log.Fatalf("Failed to import tasks: %v", err) + } + log.Printf("Created task for note audio %s", note.UUID) + return nil +} + +func findLabelStudioProject(client *labelstudio.Client, title string) (*labelstudio.Project, error) { + // Attempt to get live projects + projects, err := client.Projects() + if err != nil { + log.Fatalf("Failed to get projects: %v", err) + } + fmt.Printf("Found %d projects:\n", projects.Count) + for i, p := range projects.Results { + fmt.Printf("%d. %s (ID: %d) - Tasks: %d\n", + i+1, + p.Title, + p.ID, + p.TaskNumber) + if p.Title == title { + return &p, nil + } + } + return nil, fmt.Errorf("No such project '%s'", title) +} + +func findMatchingTask(client *labelstudio.Client, project *labelstudio.Project, customer string, note *db.NoteAudio) (*labelstudio.Task, error) { + /*meta := map[string]string{ + "customer": customer, + "note_uuid": note.UUID, + }*/ + items := []map[string]interface{}{ + {"filter": "filter:tasks:data.note_uuid", "operator": "equal", "type": "string", "value": note.UUID}, + } + filters := map[string]interface{}{ + "conjunction": "and", + "items": items, + } + query := map[string]interface{}{ + "filters": filters, + } + queryStr, err := json.Marshal(query) + if err != nil { + return nil, fmt.Errorf("Failed to marshal query JSON: %v", err) + } + // Get all tasks + options := &labelstudio.TasksListOptions{ + ProjectID: project.ID, + Query: string(queryStr), + } + tasksResponse, err := client.ListTasks(options) + if err != nil { + return nil, fmt.Errorf("Failed to get tasks: %v", err) + } + if len(tasksResponse.Tasks) == 0 { + return nil, nil + } else if len(tasksResponse.Tasks) == 1 { + return &tasksResponse.Tasks[0], nil + } else { + return nil, fmt.Errorf("Got too many tasks: %d", len(tasksResponse.Tasks)) + } + // Specify bucket name + //bucketNamePtr := flag.String("bucket", "label-studio", "The bucket to upload to") + //filePathPtr := flag.String("file", "example.txt", "The file to upload") + //flag.Parse() +} diff --git a/userfile/userfile.go b/userfile/userfile.go new file mode 100644 index 00000000..d33c05ff --- /dev/null +++ b/userfile/userfile.go @@ -0,0 +1,43 @@ +package userfile + +import ( + "fmt" + "io" + "log" + "os" + + "github.com/google/uuid" +) + +var UserFilesDirectory string + +func AudioFileContentPathRaw(audioUUID string) string { + return fmt.Sprintf("%s/%s.m4a", UserFilesDirectory, audioUUID) +} +func AudioFileContentPathMp3(audioUUID string) string { + return fmt.Sprintf("%s/%s.mp3", UserFilesDirectory, audioUUID) +} +func AudioFileContentPathNormalized(audioUUID string) string { + return fmt.Sprintf("%s/%s-normalized.m4a", UserFilesDirectory, audioUUID) +} +func AudioFileContentPathOgg(audioUUID string) string { + return fmt.Sprintf("%s/%s.ogg", UserFilesDirectory, audioUUID) +} +func AudioFileContentWrite(audioUUID uuid.UUID, body io.Reader) error { + // Create file in configured directory + filepath := AudioFileContentPathRaw(audioUUID.String()) + dst, err := os.Create(filepath) + if err != nil { + log.Printf("Failed to create audio file at %s: %v\n", filepath, err) + return fmt.Errorf("Failed to create audio file at %s: %v", filepath, err) + } + defer dst.Close() + + // Copy rest of request body to file + _, err = io.Copy(dst, body) + if err != nil { + return fmt.Errorf("Unable to save file to create audio file at %s: %v", filepath, err) + } + log.Printf("Saved audio content to %s\n", filepath) + return nil +}