diff --git a/platform/arcgis.go b/platform/arcgis.go index 22a3a416..1d9b0ecf 100644 --- a/platform/arcgis.go +++ b/platform/arcgis.go @@ -33,7 +33,6 @@ import ( "github.com/aarondl/opt/omit" "github.com/aarondl/opt/omitnull" "github.com/alitto/pond/v2" - "github.com/jackc/pgx/v5" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/twpayne/go-geom" @@ -674,236 +673,6 @@ func saveRawQuery(fssync fieldseeker.FieldSeeker, layer arcgis.LayerFeature, que } */ -func insertRowFromFeature(ctx context.Context, table string, sorted_columns []string, feature *response.Feature, org_id int32) error { - txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("Unable to start transaction") - } - defer lint.LogOnErrRollback(txn.Rollback, ctx, "rollback") - - err = insertRowFromFeatureFS(ctx, txn, table, sorted_columns, feature, org_id) - if err != nil { - return fmt.Errorf("Unable to insert FS: %w", err) - } - - err = insertRowFromFeatureHistory(ctx, txn, table, sorted_columns, feature, org_id, 1) - if err != nil { - return fmt.Errorf("Failed to insert history: %w", err) - } - - if err := txn.Commit(ctx); err != nil { - return fmt.Errorf("Failed to commit transaction: %w", err) - } - return nil -} - -func insertRowFromFeatureFS(ctx context.Context, txn bob.Tx, table string, sorted_columns []string, feature *response.Feature, org_id int32) error { - // Create the query to produce the main row - var sb strings.Builder - sb.WriteString("INSERT INTO ") - sb.WriteString(table) - sb.WriteString(" (") - for _, field := range sorted_columns { - sb.WriteString(field) - sb.WriteString(",") - } - // Specially add the geometry values since they aren't in the fields - sb.WriteString("geometry_x,geometry_y,organization_id,updated") - sb.WriteString(")\nVALUES (") - for _, field := range sorted_columns { - sb.WriteString("@") - sb.WriteString(field) - sb.WriteString(",") - } - // Specially add the geometry values since they aren't in the fields - sb.WriteString("@geometry_x,@geometry_y,@organization_id,@updated)") - - args := pgx.NamedArgs{} - for k, v := range feature.Attributes { - args[k] = v - } - // specially add geometry since it isn't in the list of attributes - //args["geometry_x"] = feature.Geometry.X - //args["geometry_y"] = feature.Geometry.Y - args["organization_id"] = org_id - args["updated"] = time.Now() - - _, err := txn.ExecContext(ctx, sb.String(), args) - if err != nil { - return fmt.Errorf("Failed to insert row into %s: %w", table, err) - } - return nil -} -func hasUpdates(row map[string]string, feature response.Feature) bool { - return false - /* - for key, value := range feature.Attributes { - rowdata := row[strings.ToLower(key)] - // We'll accept any 'nil' as represented by the empty string in the database - if value == nil { - if rowdata == "" { - continue - } else if len(rowdata) > 0 { - return true - } else { - log.Error().Msg("Looks like our original value is nil, but our row value is something non-empty with a zero length. Need a programmer to look into this.") - } - } - // check strings first, their simplest - if featureAsString, ok := value.(response.TextValue); ok { - if featureAsString.String() != rowdata { - return true - } - continue - } else if featureAsInt, ok := value.(response.Int32Value); ok { - // Previously had a nil value, now we have a real value - if rowdata == "" { - return true - } - rowAsInt, err := strconv.Atoi(rowdata) - if err != nil { - log.Error().Msg(fmt.Sprintf("Failed to convert '%s' to an int to compare against %v for %v", rowdata, featureAsInt, key)) - } - if rowAsInt != featureAsInt.V { - return true - } else { - continue - } - } else if featureAsFloat, ok := value.(Float64Value); ok { - // Previously had a nil value, now we have a real value - if rowdata == "" { - return true - } - rowAsFloat, err := strconv.ParseFloat(rowdata, 64) - if err != nil { - log.Error().Msg(fmt.Sprintf("Failed to convert '%s' to a float64 to compare against %v for %v", rowdata, featureAsFloat, key)) - } - if rowAsFloat != featureAsFloat { - return true - } else { - continue - } - } - log.Error().Str("key", key).Str("rowdata", rowdata).Msg("we've hit a point where we can't tell if we have an update or not, need a programmer to look at the above") - } - return false - */ -} -func updateRowFromFeature(ctx context.Context, table string, sorted_columns []string, feature *response.Feature, org_id int32) error { - return nil - /* - // Get the current highest version for the row in question - history_table := toHistoryTable(table) - var sb strings.Builder - sb.WriteString("SELECT MAX(version) FROM ") - sb.WriteString(history_table) - sb.WriteString(" WHERE OBJECTID=@objectid") - - args := pgx.NamedArgs{} - o := feature.Attributes["OBJECTID"].(float64) - args["objectid"] = int(o) - - var version int - if err := db.PGInstance.PGXPool.QueryRow(ctx, sb.String(), args).Scan(&version); err != nil { - return fmt.Errorf("Failed to query for version: %w", err) - } - - txn, err := db.PGInstance.BobDB.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("Unable to start transaction") - } - defer txn.Rollback(ctx) - - err = insertRowFromFeatureHistory(ctx, txn, table, sorted_columns, feature, org_id, version+1) - if err != nil { - return fmt.Errorf("Failed to insert history: %w", err) - } - err = updateRowFromFeatureFS(ctx, txn, table, sorted_columns, feature) - if err != nil { - return fmt.Errorf("Failed to update row from feature: %w", err) - } - - txn.Commit(ctx) - return nil - */ -} -func insertRowFromFeatureHistory(ctx context.Context, transaction bob.Tx, table string, sorted_columns []string, feature *response.Feature, org_id int32, version int) error { - history_table := toHistoryTable(table) - var sb strings.Builder - sb.WriteString("INSERT INTO ") - sb.WriteString(history_table) - sb.WriteString(" (") - for _, field := range sorted_columns { - sb.WriteString(field) - sb.WriteString(",") - } - // Specially add the geometry values since they aren't in the fields - sb.WriteString("created,geometry_x,geometry_y,organization_id,version") - sb.WriteString(")\nVALUES (") - for _, field := range sorted_columns { - sb.WriteString("@") - sb.WriteString(field) - sb.WriteString(",") - } - // Specially add the geometry values since they aren't in the fields - sb.WriteString("@created,@geometry_x,@geometry_y,@organization_id,@version)") - args := pgx.NamedArgs{} - for k, v := range feature.Attributes { - args[k] = v - } - args["created"] = time.Now() - args["organization_id"] = org_id - args["version"] = version - if _, err := transaction.ExecContext(ctx, sb.String(), args); err != nil { - return fmt.Errorf("Failed to insert history row into %s: %w", table, err) - } - return nil -} -func selectAllFromQueryResult(table string, sorted_columns []string) string { - var sb strings.Builder - sb.WriteString("SELECT * FROM ") - sb.WriteString(table) - sb.WriteString(" WHERE OBJECTID=ANY(@objectids)") - return sb.String() -} -func toHistoryTable(table string) string { - return "History_" + table[3:] -} - -func updateRowFromFeatureFS(ctx context.Context, transaction bob.Tx, table string, sorted_columns []string, feature *response.Feature) error { - // Create the query to produce the main row - var sb strings.Builder - sb.WriteString("UPDATE ") - sb.WriteString(table) - sb.WriteString(" SET ") - for _, field := range sorted_columns { - // OBJECTID is special as our primary key, so skip it - if field == "OBJECTID" { - continue - } - sb.WriteString(field) - sb.WriteString("=@") - sb.WriteString(field) - sb.WriteString(",") - } - // Specially add the geometry values since they aren't in the fields - sb.WriteString("geometry_x=@geometry_x,geometry_y=@geometry_y,updated=@updated WHERE OBJECTID=@OBJECTID") - - args := pgx.NamedArgs{} - for k, v := range feature.Attributes { - args[k] = v - } - // specially add geometry since it isn't in the list of attributes - //args["geometry_x"] = feature.Geometry.X - //args["geometry_y"] = feature.Geometry.Y - args["updated"] = time.Now() - - _, err := transaction.ExecContext(ctx, sb.String(), args) - if err != nil { - return fmt.Errorf("Failed to update row into %s: %w", table, err) - } - return nil -} func exportFieldseekerLayer(ctx context.Context, group pond.ResultTaskGroup[SyncStats], org *models.Organization, fssync *fieldseeker.FieldSeeker, layer response.Layer) (SyncStats, error) { var stats SyncStats diff --git a/platform/trap.go b/platform/trap.go index e51a4c5c..2aeea0c3 100644 --- a/platform/trap.go +++ b/platform/trap.go @@ -289,83 +289,6 @@ func toTemplateTrapsNearby(locations []sql.TrapLocationBySourceIDRow, trap_data return results, nil } -func toTemplateTrapData(trap_data models.FieldseekerTrapdatumSlice) ([]TrapData, error) { - var results []TrapData - for _, r := range trap_data { - if r.H3cell.IsNull() { - continue - } - cell, err := h3utils.ToCell(r.H3cell.MustGet()) - if err != nil { - log.Error().Err(err).Msg("Failed to get location for trap data") - continue - } - results = append(results, TrapData{ - // Basic Identifiers - OrganizationID: r.OrganizationID, - ObjectID: r.Objectid, - GlobalID: r.Globalid, - LocationName: r.Locationname.GetOr(""), - LocationID: r.LocID.GetOr(uuid.UUID{}), - SRID: r.Srid.GetOr(uuid.UUID{}), - Field: int64(r.Field.GetOr(0)), - - // Trap Information - TrapType: r.Traptype.GetOr(""), - TrapCondition: r.Trapcondition.GetOr(""), - TrapActivityType: r.Trapactivitytype.GetOr(""), - TrapNights: r.Trapnights.GetOr(0), - Lure: r.Lure.GetOr(""), - - // Personnel - FieldTechnician: r.Fieldtech.GetOr(""), - IdentifiedByTechnician: r.Idbytech.GetOr(""), - SortedByTechnician: r.Sortbytech.GetOr(""), - - // Timing - StartDateTime: getTimeOrNull(r.Startdatetime), - EndDateTime: getTimeOrNull(r.Enddatetime), - - // Environmental Conditions - AverageTemperature: r.Avetemp.GetOr(0), - Rainfall: r.Raingauge.GetOr(0), - WindDirection: r.Winddir.GetOr(""), - WindSpeed: r.Windspeed.GetOr(0), - SiteCondition: r.Sitecond.GetOr(""), - - // Status and Processing - Processed: fsIntToBool(r.Processed), - RecordStatus: r.Recordstatus.GetOr(0), - Reviewed: fsIntToBool(r.Reviewed), - ReviewedBy: r.Reviewedby.GetOr(""), - ReviewedDate: getTimeOrNull(r.Revieweddate), - GatewaySynced: fsIntToBool(r.Gatewaysync), - LR: fsIntToBool(r.LR), - Voltage: r.Voltage.GetOr(0), - - // Location Data - H3Cell: cell, - Zone: r.Zone.GetOr(""), - Zone2: r.Zone2.GetOr(""), - - // Vector Survey IDs - VectorSurveyTrapDataID: r.Vectorsurvtrapdataid.GetOr(""), - VectorSurveyTrapLocationID: r.Vectorsurvtraplocationid.GetOr(""), - - // Metadata - Created: getTimeOrNull(r.Creationdate), - Creator: r.Creator.GetOr(""), - CreatedByUser: r.CreatedUser.GetOr(""), - CreatedDateAlt: getTimeOrNull(r.CreatedDate), - Edited: getTimeOrNull(r.Editdate), - Editor: r.Editor.GetOr(""), - LastEditedDate: getTimeOrNull(r.LastEditedDate), - LastEditedUser: r.LastEditedUser.GetOr(""), - Comments: r.Comments.GetOr(""), - }) - } - return results, nil -} func toTreatment(rows models.FieldseekerTreatmentSlice) ([]Treatment, error) { var results []Treatment for _, r := range rows { @@ -394,16 +317,6 @@ func toTemplateInspection(rows models.FieldseekerMosquitoinspectionSlice) ([]Ins } // Helper function to convert unix timestamp to time.Time -func fsToTime(val null.Val[int64]) time.Time { - v, ok := val.Get() - if !ok { - return time.UnixMilli(0) - } - t := time.UnixMilli(v) - return t -} - -// Helper function to convert int16 to bool func fsIntToBool(val null.Val[int16]) bool { if !val.IsValue() { return false diff --git a/sync/signin.go b/sync/signin.go deleted file mode 100644 index d5103e1e..00000000 --- a/sync/signin.go +++ /dev/null @@ -1,75 +0,0 @@ -package sync - -import ( - "errors" - "net/http" - - "github.com/Gleipnir-Technology/nidus-sync/auth" - "github.com/Gleipnir-Technology/nidus-sync/config" - "github.com/Gleipnir-Technology/nidus-sync/html" - "github.com/Gleipnir-Technology/nidus-sync/platform" - "github.com/rs/zerolog/log" -) - -type contentSignin struct { - InvalidCredentials bool - Next string -} - -func getSignout(w http.ResponseWriter, r *http.Request, user platform.User) { - auth.SignoutUser(r, user) - http.Redirect(w, r, "/signin", http.StatusFound) -} - -func postSignin(w http.ResponseWriter, r *http.Request) { - if err := r.ParseForm(); err != nil { - respondError(w, "Could not parse form", err, http.StatusBadRequest) - return - } - - next := r.FormValue("next") - username := r.FormValue("username") - password := r.FormValue("password") - - log.Info().Str("username", username).Str("next", next).Msg("HTML Signin") - - _, err := auth.SigninUser(r, username, password) - if err != nil { - if errors.Is(err, auth.InvalidCredentials{}) { - http.Redirect(w, r, "/signin?error=invalid-credentials", http.StatusFound) - return - } - if errors.Is(err, auth.InvalidUsername{}) { - http.Redirect(w, r, "/signin?error=invalid-credentials", http.StatusFound) - return - } - respondError(w, "Failed to signin user", err, http.StatusInternalServerError) - return - } - if next == "" { - next = "/" - } - location := config.MakeURLNidus(next) - http.Redirect(w, r, location, http.StatusFound) -} - -type contentUnauthenticated[T any] struct { - C T - Config html.ContentConfig - URL html.ContentURL -} - -func signin(w http.ResponseWriter, errorCode string, next string) { - if next == "" { - next = "/" - } - data := contentUnauthenticated[contentSignin]{ - C: contentSignin{ - InvalidCredentials: errorCode == "invalid-credentials", - Next: next, - }, - Config: html.NewContentConfig(), - URL: html.NewContentURL(), - } - html.RenderOrError(w, "sync/signin.html", data) -} diff --git a/sync/sms.go b/sync/sms.go deleted file mode 100644 index 836c0d8e..00000000 --- a/sync/sms.go +++ /dev/null @@ -1,204 +0,0 @@ -package sync - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path" - "path/filepath" - "strings" - "time" - - "github.com/Gleipnir-Technology/nidus-sync/lint" - - "github.com/gorilla/mux" - "github.com/rs/zerolog/log" -) - -type SMSWebhookBody struct { - Data SMSWebhookData `json:"data"` -} -type SMSWebhookData struct { - ID int64 `json:"id"` - EventType string `json:"event_type"` - RecordType string `json:"record_type"` - Payload SMSMessagePayload `json:"payload"` -} - -type SMSMessagePayload struct { - ID int64 `json:"id"` - RecordType string `json:"record_type"` - From SMSContact `json:"from"` - To []SMSContact `json:"to"` - Text string `json:"text"` - ReceivedAt string `json:"received_at"` - Type string `json:"type"` - Media []MMSMedia `json:"media"` -} - -type MMSMedia struct { - URL string `json:"url"` -} - -// Contact represents a phone contact -type SMSContact struct { - PhoneNumber string `json:"phone_number"` - Status string `json:"status,omitempty"` -} - -func handleSMSMessage(data *SMSWebhookData) error { - log.Info().Int64("ID", data.ID).Str("event_type", data.EventType).Str("record_type", data.RecordType).Str("from", data.Payload.From.PhoneNumber).Str("msg", data.Payload.Text).Str("receieved", data.Payload.ReceivedAt).Msg("Got SMS Message") - - for _, media := range data.Payload.Media { - filePath, err := downloadMedia(media.URL) - if err != nil { - log.Error().Err(err).Str("filePath", filePath).Msg("Failed to download media") - continue - } - fmt.Printf("Downloaded media to: %s\n", filePath) - } - return nil -} - -// DownloadMedia downloads a media file from the given URL to a temporary location -// and returns the path to the downloaded file -func downloadMedia(mediaURL string) (string, error) { - // Make GET request to the media URL - resp, err := http.Get(mediaURL) - if err != nil { - return "", fmt.Errorf("failed to download media: %w", err) - } - defer lint.LogOnErr(resp.Body.Close, "close media response body") - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to download media: status code %d", resp.StatusCode) - } - - // Extract filename from URL or headers - filename := getFilenameFromURL(mediaURL, resp) - - // Create temporary file with proper extension - tmpDir := os.TempDir() - timestamp := time.Now().UnixNano() - tmpFilePath := filepath.Join(tmpDir, fmt.Sprintf("media_%d_%s", timestamp, filename)) - - // Create the file - out, err := os.Create(tmpFilePath) - if err != nil { - return "", fmt.Errorf("failed to create temporary file: %w", err) - } - defer lint.LogOnErr(out.Close, "close output file") - - // Write the response body to the file - _, err = io.Copy(out, resp.Body) - if err != nil { - return "", fmt.Errorf("failed to save media file: %w", err) - } - - return tmpFilePath, nil -} - -// getFilenameFromURL extracts filename from URL or Content-Disposition header -func getFilenameFromURL(mediaURL string, resp *http.Response) string { - // First try Content-Disposition header - contentDisp := resp.Header.Get("Content-Disposition") - if contentDisp != "" { - if strings.Contains(contentDisp, "filename=") { - parts := strings.Split(contentDisp, "filename=") - if len(parts) > 1 { - filename := strings.Trim(parts[1], "\"' ") - if filename != "" { - return sanitizeFilename(filename) - } - } - } - } - - // Fall back to URL path - urlPath := path.Base(mediaURL) - if urlPath != "" && urlPath != "." && urlPath != "/" { - return sanitizeFilename(urlPath) - } - - // Default to generic name with extension based on Content-Type - contentType := resp.Header.Get("Content-Type") - ext := ".bin" - - switch { - case strings.Contains(contentType, "image/jpeg"): - ext = ".jpg" - case strings.Contains(contentType, "image/png"): - ext = ".png" - case strings.Contains(contentType, "image/gif"): - ext = ".gif" - case strings.Contains(contentType, "video/mp4"): - ext = ".mp4" - case strings.Contains(contentType, "audio/mpeg"): - ext = ".mp3" - } - - return fmt.Sprintf("media%s", ext) -} - -// sanitizeFilename removes potentially unsafe characters from filename -func sanitizeFilename(name string) string { - // Replace unsafe characters with underscore - unsafe := []string{"/", "\\", "?", "%", "*", ":", "|", "\"", "<", ">"} - result := name - for _, c := range unsafe { - result = strings.ReplaceAll(result, c, "_") - } - return result -} -func postSMS(w http.ResponseWriter, r *http.Request) { - // Log all request headers - for name, values := range r.Header { - for _, value := range values { - log.Info().Str("name", name).Str("value", value).Msg("header") - } - } - - // Read the request body - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - //return nil, fmt.Errorf("failed to read request body: %w", err) - respondError(w, "Failed to read request body", err, http.StatusInternalServerError) - return - } - log.Info().Str("body", string(bodyBytes)).Msg("body") - // Close the original body - defer lint.LogOnErr(r.Body.Close, "close request body") - - // Parse JSON into webhook struct - var body SMSWebhookBody - if err := json.Unmarshal(bodyBytes, &body); err != nil { - respondError(w, "Failed to parse JSON", err, http.StatusBadRequest) - return - } - - if err := handleSMSMessage(&body.Data); err != nil { - log.Error().Err(err).Msg("Failed to handle SMS Message") - } - - w.WriteHeader(http.StatusOK) - lint.Write(w, []byte("ok")) -} -func getSMS(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - org := vars["org"] - - to := r.URL.Query().Get("error") - from := r.URL.Query().Get("error") - message := r.URL.Query().Get("error") - files := r.URL.Query().Get("error") - id := r.URL.Query().Get("error") - date := r.URL.Query().Get("error") - - log.Info().Str("org", org).Str("to", to).Str("from", from).Str("message", message).Str("files", files).Str("id", id).Str("date", date).Msg("Got SMS Message") - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-type", "text/plain") - // Signifies to Voip.ms that the callback worked. - lint.Fprintf(w, "ok") -}