From 3ccc05d4c58debcef5dcea43612a8b91fdd5c44e Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Wed, 11 Mar 2026 17:01:47 +0000 Subject: [PATCH] Save tiles to the database to make empty tile load faster --- api/compliance.go | 6 +- api/tile.go | 107 ++++++--- db/dberrors/tile.cached_image.bob.go | 6 +- db/dbinfo/tile.cached_image.bob.go | 41 ++-- db/migrations/00100_tile_empty.sql | 19 ++ db/models/arcgis.service_map.bob.go | 68 ++++++ db/models/tile.cached_image.bob.go | 333 +++++++++++++++++++++++++-- html/template/sync/review/pool.html | 8 +- platform/imagetile/imagetile.go | 65 ++++-- sync/tile.go | 4 +- 10 files changed, 559 insertions(+), 98 deletions(-) create mode 100644 db/migrations/00100_tile_empty.sql diff --git a/api/compliance.go b/api/compliance.go index 5160411e..4ef50432 100644 --- a/api/compliance.go +++ b/api/compliance.go @@ -98,10 +98,10 @@ func writeImage(ctx context.Context, w http.ResponseWriter, org *models.Organiza if err != nil { return fmt.Errorf("image at point: %w", err) } - log.Info().Int("size", len(img)).Msg("image") + log.Info().Int("size", len(img.Content)).Msg("image") w.Header().Set("Content-Type", "image/png") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img))) - _, err = io.Copy(w, bytes.NewBuffer(img)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img.Content))) + _, err = io.Copy(w, bytes.NewBuffer(img.Content)) if err != nil { return fmt.Errorf("copy bytes: %w", err) } diff --git a/api/tile.go b/api/tile.go index c7873624..087702f7 100644 --- a/api/tile.go +++ b/api/tile.go @@ -3,7 +3,6 @@ package api import ( "bytes" "context" - "errors" "fmt" "io" "net/http" @@ -11,7 +10,11 @@ import ( "path/filepath" "strconv" + "github.com/aarondl/opt/omit" + //"github.com/Gleipnir-Technology/bob" + //"github.com/Gleipnir-Technology/bob/dialect/psql" "github.com/Gleipnir-Technology/nidus-sync/config" + "github.com/Gleipnir-Technology/nidus-sync/db" "github.com/Gleipnir-Technology/nidus-sync/db/models" "github.com/Gleipnir-Technology/nidus-sync/platform/imagetile" "github.com/go-chi/chi/v5" @@ -50,49 +53,89 @@ func handleTile(ctx context.Context, w http.ResponseWriter, org *models.Organiza return fmt.Errorf("no map service ID set") } map_service_id := org.ArcgisMapServiceID.MustGet() - tile_path := fmt.Sprintf("%s/tile-cache/%s/%d/%d/%d.raw", config.FilesDirectory, map_service_id, z, y, x) - file, err := os.Open(tile_path) + tile_path := tilePath(map_service_id, z, y, x) + tile_row, err := models.TileCachedImages.Query( + models.SelectWhere.TileCachedImages.ArcgisID.EQ(map_service_id), + models.SelectWhere.TileCachedImages.X.EQ(int32(x)), + models.SelectWhere.TileCachedImages.Y.EQ(int32(y)), + models.SelectWhere.TileCachedImages.Z.EQ(int32(z)), + ).One(ctx, db.PGInstance.BobDB) if err == nil { - defer file.Close() - img, err := io.ReadAll(file) - if err != nil { - return fmt.Errorf("readall from %s: %w", tile_path, err) - } - w.Header().Set("Content-Type", "image/png") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img))) - _, err = io.Copy(w, bytes.NewBuffer(img)) - if err != nil { - return fmt.Errorf("copy bytes from %s: %w", tile_path) - } - return nil - } - content, err := imagetile.ImageAtTile(ctx, org, uint(z), uint(y), uint(x)) - if err != nil { - if errors.Is(err, imagetile.ErrNoTile) { - w.Header().Set("Content-Type", "image/png") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(content))) - _, err = io.Copy(w, bytes.NewBuffer(content)) + var tile *imagetile.TileRaster + if tile_row.IsEmpty { + tile = imagetile.TileRasterPlaceholder() + } else { + tile, err = loadTileFromDisk(tile_path) if err != nil { - return fmt.Errorf("write image file: %w", err) + return fmt.Errorf("load tile from disk: %w", err) } - return nil } + log.Debug().Uint("z", z).Uint("y", y).Uint("x", x).Bool("is empty", tile_row.IsEmpty).Msg("tile from cache") + return writeTile(w, tile) + } + if err.Error() != "sql: no rows in result set" { + return fmt.Errorf("query db: %w", err) + } + image, err := imagetile.ImageAtTile(ctx, org, uint(z), uint(y), uint(x)) + if err != nil { return fmt.Errorf("image at tile: %w", err) } + if !image.IsPlaceholder { + err = saveTileToDisk(image, tile_path) + if err != nil { + return fmt.Errorf("save tile: %w", err) + } + } + _, err = models.TileCachedImages.Insert(&models.TileCachedImageSetter{ + ArcgisID: omit.From(map_service_id), + X: omit.From(int32(x)), + Y: omit.From(int32(y)), + Z: omit.From(int32(z)), + IsEmpty: omit.From(image.IsPlaceholder), + }).One(ctx, db.PGInstance.BobDB) + if err != nil { + return fmt.Errorf("save to db: %w", err) + } + log.Debug().Uint("z", z).Uint("y", y).Uint("x", x).Bool("placeholder", image.IsPlaceholder).Msg("caching tile") + return writeTile(w, image) +} +func loadTileFromDisk(tile_path string) (*imagetile.TileRaster, error) { + file, err := os.Open(tile_path) + if err != nil { + return nil, fmt.Errorf("open: %w", err) + } + defer file.Close() + img, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("readall from %s: %w", tile_path, err) + } + return &imagetile.TileRaster{ + Content: img, + IsPlaceholder: false, + }, nil +} +func saveTileToDisk(image *imagetile.TileRaster, tile_path string) error { parent := filepath.Dir(tile_path) - err = os.MkdirAll(parent, 0750) + err := os.MkdirAll(parent, 0750) if err != nil { return fmt.Errorf("mkdirall: %w", err) } - err = os.WriteFile(tile_path, content, 0644) - if err != nil { - return fmt.Errorf("write image file: %w", err) - } - w.Header().Set("Content-Type", "image/png") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(content))) - _, err = io.Copy(w, bytes.NewBuffer(content)) + err = os.WriteFile(tile_path, image.Content, 0644) if err != nil { return fmt.Errorf("write image file: %w", err) } return nil } +func tilePath(map_service_id string, z, y, x uint) string { + return fmt.Sprintf("%s/tile-cache/%s/%d/%d/%d.raw", config.FilesDirectory, map_service_id, z, y, x) +} + +func writeTile(w http.ResponseWriter, image *imagetile.TileRaster) error { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(image.Content))) + _, err := io.Copy(w, bytes.NewBuffer(image.Content)) + if err != nil { + return fmt.Errorf("io.copy: %w", err) + } + return nil +} diff --git a/db/dberrors/tile.cached_image.bob.go b/db/dberrors/tile.cached_image.bob.go index 71befdba..ce4d642b 100644 --- a/db/dberrors/tile.cached_image.bob.go +++ b/db/dberrors/tile.cached_image.bob.go @@ -4,14 +4,14 @@ package dberrors var TileCachedImageErrors = &tileCachedImageErrors{ - ErrUniqueCachedImageArcgisIdXYZKey: &UniqueConstraintError{ + ErrUniqueCachedImagePkey: &UniqueConstraintError{ schema: "tile", table: "cached_image", columns: []string{"arcgis_id", "x", "y", "z"}, - s: "cached_image_arcgis_id_x_y_z_key", + s: "cached_image_pkey", }, } type tileCachedImageErrors struct { - ErrUniqueCachedImageArcgisIdXYZKey *UniqueConstraintError + ErrUniqueCachedImagePkey *UniqueConstraintError } diff --git a/db/dbinfo/tile.cached_image.bob.go b/db/dbinfo/tile.cached_image.bob.go index d064b1de..34a66d76 100644 --- a/db/dbinfo/tile.cached_image.bob.go +++ b/db/dbinfo/tile.cached_image.bob.go @@ -51,11 +51,20 @@ var TileCachedImages = Table[ Generated: false, AutoIncr: false, }, + IsEmpty: column{ + Name: "is_empty", + DBType: "boolean", + Default: "", + Comment: "", + Nullable: false, + Generated: false, + AutoIncr: false, + }, }, Indexes: tileCachedImageIndexes{ - CachedImageArcgisIDXYZKey: index{ + CachedImagePkey: index{ Type: "btree", - Name: "cached_image_arcgis_id_x_y_z_key", + Name: "cached_image_pkey", Columns: []indexColumn{ { Name: "arcgis_id", @@ -86,7 +95,11 @@ var TileCachedImages = Table[ Include: []string{}, }, }, - + PrimaryKey: &constraint{ + Name: "cached_image_pkey", + Columns: []string{"arcgis_id", "x", "y", "z"}, + Comment: "", + }, ForeignKeys: tileCachedImageForeignKeys{ TileCachedImageCachedImageArcgisIDFkey: foreignKey{ constraint: constraint{ @@ -98,13 +111,6 @@ var TileCachedImages = Table[ ForeignColumns: []string{"arcgis_id"}, }, }, - Uniques: tileCachedImageUniques{ - CachedImageArcgisIDXYZKey: constraint{ - Name: "cached_image_arcgis_id_x_y_z_key", - Columns: []string{"arcgis_id", "x", "y", "z"}, - Comment: "", - }, - }, Comment: "", } @@ -114,21 +120,22 @@ type tileCachedImageColumns struct { X column Y column Z column + IsEmpty column } func (c tileCachedImageColumns) AsSlice() []column { return []column{ - c.ArcgisID, c.X, c.Y, c.Z, + c.ArcgisID, c.X, c.Y, c.Z, c.IsEmpty, } } type tileCachedImageIndexes struct { - CachedImageArcgisIDXYZKey index + CachedImagePkey index } func (i tileCachedImageIndexes) AsSlice() []index { return []index{ - i.CachedImageArcgisIDXYZKey, + i.CachedImagePkey, } } @@ -142,14 +149,10 @@ func (f tileCachedImageForeignKeys) AsSlice() []foreignKey { } } -type tileCachedImageUniques struct { - CachedImageArcgisIDXYZKey constraint -} +type tileCachedImageUniques struct{} func (u tileCachedImageUniques) AsSlice() []constraint { - return []constraint{ - u.CachedImageArcgisIDXYZKey, - } + return []constraint{} } type tileCachedImageChecks struct{} diff --git a/db/migrations/00100_tile_empty.sql b/db/migrations/00100_tile_empty.sql new file mode 100644 index 00000000..9c669005 --- /dev/null +++ b/db/migrations/00100_tile_empty.sql @@ -0,0 +1,19 @@ +-- +goose Up +DROP TABLE tile.cached_image; +CREATE TABLE tile.cached_image ( + arcgis_id TEXT NOT NULL REFERENCES arcgis.service_map(arcgis_id), + x INTEGER NOT NULL, + y INTEGER NOT NULL, + z INTEGER NOT NULL, + is_empty BOOLEAN NOT NULL, + PRIMARY KEY (arcgis_id, x, y, z) +); +-- +goose Down +DROP TABLE tile.cached_image; +CREATE TABLE tile.cached_image ( + arcgis_id TEXT NOT NULL REFERENCES arcgis.service_map(arcgis_id), + x INTEGER NOT NULL, + y INTEGER NOT NULL, + z INTEGER NOT NULL, + UNIQUE(arcgis_id, x, y, z) +); diff --git a/db/models/arcgis.service_map.bob.go b/db/models/arcgis.service_map.bob.go index 116aa7b8..5a43bb03 100644 --- a/db/models/arcgis.service_map.bob.go +++ b/db/models/arcgis.service_map.bob.go @@ -627,6 +627,74 @@ func (arcgisServiceMap0 *ArcgisServiceMap) AttachArcgisMapServiceOrganizations(c return nil } +func insertArcgisServiceMapArcgisCachedImages0(ctx context.Context, exec bob.Executor, tileCachedImages1 []*TileCachedImageSetter, arcgisServiceMap0 *ArcgisServiceMap) (TileCachedImageSlice, error) { + for i := range tileCachedImages1 { + tileCachedImages1[i].ArcgisID = omit.From(arcgisServiceMap0.ArcgisID) + } + + ret, err := TileCachedImages.Insert(bob.ToMods(tileCachedImages1...)).All(ctx, exec) + if err != nil { + return ret, fmt.Errorf("insertArcgisServiceMapArcgisCachedImages0: %w", err) + } + + return ret, nil +} + +func attachArcgisServiceMapArcgisCachedImages0(ctx context.Context, exec bob.Executor, count int, tileCachedImages1 TileCachedImageSlice, arcgisServiceMap0 *ArcgisServiceMap) (TileCachedImageSlice, error) { + setter := &TileCachedImageSetter{ + ArcgisID: omit.From(arcgisServiceMap0.ArcgisID), + } + + err := tileCachedImages1.UpdateAll(ctx, exec, *setter) + if err != nil { + return nil, fmt.Errorf("attachArcgisServiceMapArcgisCachedImages0: %w", err) + } + + return tileCachedImages1, nil +} + +func (arcgisServiceMap0 *ArcgisServiceMap) InsertArcgisCachedImages(ctx context.Context, exec bob.Executor, related ...*TileCachedImageSetter) error { + if len(related) == 0 { + return nil + } + + var err error + + tileCachedImages1, err := insertArcgisServiceMapArcgisCachedImages0(ctx, exec, related, arcgisServiceMap0) + if err != nil { + return err + } + + arcgisServiceMap0.R.ArcgisCachedImages = append(arcgisServiceMap0.R.ArcgisCachedImages, tileCachedImages1...) + + for _, rel := range tileCachedImages1 { + rel.R.ArcgisServiceMap = arcgisServiceMap0 + } + return nil +} + +func (arcgisServiceMap0 *ArcgisServiceMap) AttachArcgisCachedImages(ctx context.Context, exec bob.Executor, related ...*TileCachedImage) error { + if len(related) == 0 { + return nil + } + + var err error + tileCachedImages1 := TileCachedImageSlice(related) + + _, err = attachArcgisServiceMapArcgisCachedImages0(ctx, exec, len(related), tileCachedImages1, arcgisServiceMap0) + if err != nil { + return err + } + + arcgisServiceMap0.R.ArcgisCachedImages = append(arcgisServiceMap0.R.ArcgisCachedImages, tileCachedImages1...) + + for _, rel := range related { + rel.R.ArcgisServiceMap = arcgisServiceMap0 + } + + return nil +} + type arcgisServiceMapWhere[Q psql.Filterable] struct { AccountID psql.WhereMod[Q, string] ArcgisID psql.WhereMod[Q, string] diff --git a/db/models/tile.cached_image.bob.go b/db/models/tile.cached_image.bob.go index 7e5b8009..12e7af8c 100644 --- a/db/models/tile.cached_image.bob.go +++ b/db/models/tile.cached_image.bob.go @@ -11,6 +11,7 @@ import ( "github.com/Gleipnir-Technology/bob" "github.com/Gleipnir-Technology/bob/dialect/psql" "github.com/Gleipnir-Technology/bob/dialect/psql/dialect" + "github.com/Gleipnir-Technology/bob/dialect/psql/dm" "github.com/Gleipnir-Technology/bob/dialect/psql/sm" "github.com/Gleipnir-Technology/bob/dialect/psql/um" "github.com/Gleipnir-Technology/bob/expr" @@ -21,10 +22,11 @@ import ( // TileCachedImage is an object representing the database table. type TileCachedImage struct { - ArcgisID string `db:"arcgis_id" ` - X int32 `db:"x" ` - Y int32 `db:"y" ` - Z int32 `db:"z" ` + ArcgisID string `db:"arcgis_id,pk" ` + X int32 `db:"x,pk" ` + Y int32 `db:"y,pk" ` + Z int32 `db:"z,pk" ` + IsEmpty bool `db:"is_empty" ` R tileCachedImageR `db:"-" ` } @@ -33,10 +35,10 @@ type TileCachedImage struct { // This should almost always be used instead of []*TileCachedImage. type TileCachedImageSlice []*TileCachedImage -// TileCachedImages contains methods to work with the cached_image view -var TileCachedImages = psql.NewViewx[*TileCachedImage, TileCachedImageSlice]("tile", "cached_image", buildTileCachedImageColumns("tile.cached_image")) +// TileCachedImages contains methods to work with the cached_image table +var TileCachedImages = psql.NewTablex[*TileCachedImage, TileCachedImageSlice, *TileCachedImageSetter]("tile", "cached_image", buildTileCachedImageColumns("tile.cached_image")) -// TileCachedImagesQuery is a query on the cached_image view +// TileCachedImagesQuery is a query on the cached_image table type TileCachedImagesQuery = *psql.ViewQuery[*TileCachedImage, TileCachedImageSlice] // tileCachedImageR is where relationships are stored. @@ -47,13 +49,14 @@ type tileCachedImageR struct { func buildTileCachedImageColumns(alias string) tileCachedImageColumns { return tileCachedImageColumns{ ColumnsExpr: expr.NewColumnsExpr( - "arcgis_id", "x", "y", "z", + "arcgis_id", "x", "y", "z", "is_empty", ).WithParent("tile.cached_image"), tableAlias: alias, ArcgisID: psql.Quote(alias, "arcgis_id"), X: psql.Quote(alias, "x"), Y: psql.Quote(alias, "y"), Z: psql.Quote(alias, "z"), + IsEmpty: psql.Quote(alias, "is_empty"), } } @@ -64,6 +67,7 @@ type tileCachedImageColumns struct { X psql.Expression Y psql.Expression Z psql.Expression + IsEmpty psql.Expression } func (c tileCachedImageColumns) Alias() string { @@ -78,14 +82,15 @@ func (tileCachedImageColumns) AliasedAs(alias string) tileCachedImageColumns { // All values are optional, and do not have to be set // Generated columns are not included type TileCachedImageSetter struct { - ArcgisID omit.Val[string] `db:"arcgis_id" ` - X omit.Val[int32] `db:"x" ` - Y omit.Val[int32] `db:"y" ` - Z omit.Val[int32] `db:"z" ` + ArcgisID omit.Val[string] `db:"arcgis_id,pk" ` + X omit.Val[int32] `db:"x,pk" ` + Y omit.Val[int32] `db:"y,pk" ` + Z omit.Val[int32] `db:"z,pk" ` + IsEmpty omit.Val[bool] `db:"is_empty" ` } func (s TileCachedImageSetter) SetColumns() []string { - vals := make([]string, 0, 4) + vals := make([]string, 0, 5) if s.ArcgisID.IsValue() { vals = append(vals, "arcgis_id") } @@ -98,6 +103,9 @@ func (s TileCachedImageSetter) SetColumns() []string { if s.Z.IsValue() { vals = append(vals, "z") } + if s.IsEmpty.IsValue() { + vals = append(vals, "is_empty") + } return vals } @@ -114,11 +122,18 @@ func (s TileCachedImageSetter) Overwrite(t *TileCachedImage) { if s.Z.IsValue() { t.Z = s.Z.MustGet() } + if s.IsEmpty.IsValue() { + t.IsEmpty = s.IsEmpty.MustGet() + } } func (s *TileCachedImageSetter) Apply(q *dialect.InsertQuery) { + q.AppendHooks(func(ctx context.Context, exec bob.Executor) (context.Context, error) { + return TileCachedImages.BeforeInsertHooks.RunHooks(ctx, exec, s) + }) + q.AppendValues(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { - vals := make([]bob.Expression, 4) + vals := make([]bob.Expression, 5) if s.ArcgisID.IsValue() { vals[0] = psql.Arg(s.ArcgisID.MustGet()) } else { @@ -143,6 +158,12 @@ func (s *TileCachedImageSetter) Apply(q *dialect.InsertQuery) { vals[3] = psql.Raw("DEFAULT") } + if s.IsEmpty.IsValue() { + vals[4] = psql.Arg(s.IsEmpty.MustGet()) + } else { + vals[4] = psql.Raw("DEFAULT") + } + return bob.ExpressSlice(ctx, w, d, start, vals, "", ", ", "") })) } @@ -152,7 +173,7 @@ func (s TileCachedImageSetter) UpdateMod() bob.Mod[*dialect.UpdateQuery] { } func (s TileCachedImageSetter) Expressions(prefix ...string) []bob.Expression { - exprs := make([]bob.Expression, 0, 4) + exprs := make([]bob.Expression, 0, 5) if s.ArcgisID.IsValue() { exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ @@ -182,9 +203,47 @@ func (s TileCachedImageSetter) Expressions(prefix ...string) []bob.Expression { }}) } + if s.IsEmpty.IsValue() { + exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{ + psql.Quote(append(prefix, "is_empty")...), + psql.Arg(s.IsEmpty), + }}) + } + return exprs } +// FindTileCachedImage retrieves a single record by primary key +// If cols is empty Find will return all columns. +func FindTileCachedImage(ctx context.Context, exec bob.Executor, ArcgisIDPK string, XPK int32, YPK int32, ZPK int32, cols ...string) (*TileCachedImage, error) { + if len(cols) == 0 { + return TileCachedImages.Query( + sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(ArcgisIDPK))), + sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(XPK))), + sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(YPK))), + sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(ZPK))), + ).One(ctx, exec) + } + + return TileCachedImages.Query( + sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(ArcgisIDPK))), + sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(XPK))), + sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(YPK))), + sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(ZPK))), + sm.Columns(TileCachedImages.Columns.Only(cols...)), + ).One(ctx, exec) +} + +// TileCachedImageExists checks the presence of a single record by primary key +func TileCachedImageExists(ctx context.Context, exec bob.Executor, ArcgisIDPK string, XPK int32, YPK int32, ZPK int32) (bool, error) { + return TileCachedImages.Query( + sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(ArcgisIDPK))), + sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(XPK))), + sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(YPK))), + sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(ZPK))), + ).Exists(ctx, exec) +} + // AfterQueryHook is called after TileCachedImage is retrieved from the database func (o *TileCachedImage) AfterQueryHook(ctx context.Context, exec bob.Executor, queryType bob.QueryType) error { var err error @@ -192,11 +251,69 @@ func (o *TileCachedImage) AfterQueryHook(ctx context.Context, exec bob.Executor, switch queryType { case bob.QueryTypeSelect: ctx, err = TileCachedImages.AfterSelectHooks.RunHooks(ctx, exec, TileCachedImageSlice{o}) + case bob.QueryTypeInsert: + ctx, err = TileCachedImages.AfterInsertHooks.RunHooks(ctx, exec, TileCachedImageSlice{o}) + case bob.QueryTypeUpdate: + ctx, err = TileCachedImages.AfterUpdateHooks.RunHooks(ctx, exec, TileCachedImageSlice{o}) + case bob.QueryTypeDelete: + ctx, err = TileCachedImages.AfterDeleteHooks.RunHooks(ctx, exec, TileCachedImageSlice{o}) } return err } +// primaryKeyVals returns the primary key values of the TileCachedImage +func (o *TileCachedImage) primaryKeyVals() bob.Expression { + return psql.ArgGroup( + o.ArcgisID, + o.X, + o.Y, + o.Z, + ) +} + +func (o *TileCachedImage) pkEQ() dialect.Expression { + return psql.Group(psql.Quote("tile.cached_image", "arcgis_id"), psql.Quote("tile.cached_image", "x"), psql.Quote("tile.cached_image", "y"), psql.Quote("tile.cached_image", "z")).EQ(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { + return o.primaryKeyVals().WriteSQL(ctx, w, d, start) + })) +} + +// Update uses an executor to update the TileCachedImage +func (o *TileCachedImage) Update(ctx context.Context, exec bob.Executor, s *TileCachedImageSetter) error { + v, err := TileCachedImages.Update(s.UpdateMod(), um.Where(o.pkEQ())).One(ctx, exec) + if err != nil { + return err + } + + o.R = v.R + *o = *v + + return nil +} + +// Delete deletes a single TileCachedImage record with an executor +func (o *TileCachedImage) Delete(ctx context.Context, exec bob.Executor) error { + _, err := TileCachedImages.Delete(dm.Where(o.pkEQ())).Exec(ctx, exec) + return err +} + +// Reload refreshes the TileCachedImage using the executor +func (o *TileCachedImage) Reload(ctx context.Context, exec bob.Executor) error { + o2, err := TileCachedImages.Query( + sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(o.ArcgisID))), + sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(o.X))), + sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(o.Y))), + sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(o.Z))), + ).One(ctx, exec) + if err != nil { + return err + } + o2.R = o.R + *o = *o2 + + return nil +} + // AfterQueryHook is called after TileCachedImageSlice is retrieved from the database func (o TileCachedImageSlice) AfterQueryHook(ctx context.Context, exec bob.Executor, queryType bob.QueryType) error { var err error @@ -204,11 +321,147 @@ func (o TileCachedImageSlice) AfterQueryHook(ctx context.Context, exec bob.Execu switch queryType { case bob.QueryTypeSelect: ctx, err = TileCachedImages.AfterSelectHooks.RunHooks(ctx, exec, o) + case bob.QueryTypeInsert: + ctx, err = TileCachedImages.AfterInsertHooks.RunHooks(ctx, exec, o) + case bob.QueryTypeUpdate: + ctx, err = TileCachedImages.AfterUpdateHooks.RunHooks(ctx, exec, o) + case bob.QueryTypeDelete: + ctx, err = TileCachedImages.AfterDeleteHooks.RunHooks(ctx, exec, o) } return err } +func (o TileCachedImageSlice) pkIN() dialect.Expression { + if len(o) == 0 { + return psql.Raw("NULL") + } + + return psql.Group(psql.Quote("tile.cached_image", "arcgis_id"), psql.Quote("tile.cached_image", "x"), psql.Quote("tile.cached_image", "y"), psql.Quote("tile.cached_image", "z")).In(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) { + pkPairs := make([]bob.Expression, len(o)) + for i, row := range o { + pkPairs[i] = row.primaryKeyVals() + } + return bob.ExpressSlice(ctx, w, d, start, pkPairs, "", ", ", "") + })) +} + +// copyMatchingRows finds models in the given slice that have the same primary key +// then it first copies the existing relationships from the old model to the new model +// and then replaces the old model in the slice with the new model +func (o TileCachedImageSlice) copyMatchingRows(from ...*TileCachedImage) { + for i, old := range o { + for _, new := range from { + if new.ArcgisID != old.ArcgisID { + continue + } + if new.X != old.X { + continue + } + if new.Y != old.Y { + continue + } + if new.Z != old.Z { + continue + } + new.R = old.R + o[i] = new + break + } + } +} + +// UpdateMod modifies an update query with "WHERE primary_key IN (o...)" +func (o TileCachedImageSlice) UpdateMod() bob.Mod[*dialect.UpdateQuery] { + return bob.ModFunc[*dialect.UpdateQuery](func(q *dialect.UpdateQuery) { + q.AppendHooks(func(ctx context.Context, exec bob.Executor) (context.Context, error) { + return TileCachedImages.BeforeUpdateHooks.RunHooks(ctx, exec, o) + }) + + q.AppendLoader(bob.LoaderFunc(func(ctx context.Context, exec bob.Executor, retrieved any) error { + var err error + switch retrieved := retrieved.(type) { + case *TileCachedImage: + o.copyMatchingRows(retrieved) + case []*TileCachedImage: + o.copyMatchingRows(retrieved...) + case TileCachedImageSlice: + o.copyMatchingRows(retrieved...) + default: + // If the retrieved value is not a TileCachedImage or a slice of TileCachedImage + // then run the AfterUpdateHooks on the slice + _, err = TileCachedImages.AfterUpdateHooks.RunHooks(ctx, exec, o) + } + + return err + })) + + q.AppendWhere(o.pkIN()) + }) +} + +// DeleteMod modifies an delete query with "WHERE primary_key IN (o...)" +func (o TileCachedImageSlice) DeleteMod() bob.Mod[*dialect.DeleteQuery] { + return bob.ModFunc[*dialect.DeleteQuery](func(q *dialect.DeleteQuery) { + q.AppendHooks(func(ctx context.Context, exec bob.Executor) (context.Context, error) { + return TileCachedImages.BeforeDeleteHooks.RunHooks(ctx, exec, o) + }) + + q.AppendLoader(bob.LoaderFunc(func(ctx context.Context, exec bob.Executor, retrieved any) error { + var err error + switch retrieved := retrieved.(type) { + case *TileCachedImage: + o.copyMatchingRows(retrieved) + case []*TileCachedImage: + o.copyMatchingRows(retrieved...) + case TileCachedImageSlice: + o.copyMatchingRows(retrieved...) + default: + // If the retrieved value is not a TileCachedImage or a slice of TileCachedImage + // then run the AfterDeleteHooks on the slice + _, err = TileCachedImages.AfterDeleteHooks.RunHooks(ctx, exec, o) + } + + return err + })) + + q.AppendWhere(o.pkIN()) + }) +} + +func (o TileCachedImageSlice) UpdateAll(ctx context.Context, exec bob.Executor, vals TileCachedImageSetter) error { + if len(o) == 0 { + return nil + } + + _, err := TileCachedImages.Update(vals.UpdateMod(), o.UpdateMod()).All(ctx, exec) + return err +} + +func (o TileCachedImageSlice) DeleteAll(ctx context.Context, exec bob.Executor) error { + if len(o) == 0 { + return nil + } + + _, err := TileCachedImages.Delete(o.DeleteMod()).Exec(ctx, exec) + return err +} + +func (o TileCachedImageSlice) ReloadAll(ctx context.Context, exec bob.Executor) error { + if len(o) == 0 { + return nil + } + + o2, err := TileCachedImages.Query(sm.Where(o.pkIN())).All(ctx, exec) + if err != nil { + return err + } + + o.copyMatchingRows(o2...) + + return nil +} + // ArcgisServiceMap starts a query for related objects on arcgis.service_map func (o *TileCachedImage) ArcgisServiceMap(mods ...bob.Mod[*dialect.SelectQuery]) ArcgisServiceMapsQuery { return ArcgisServiceMaps.Query(append(mods, @@ -233,11 +486,60 @@ func (os TileCachedImageSlice) ArcgisServiceMap(mods ...bob.Mod[*dialect.SelectQ )...) } +func attachTileCachedImageArcgisServiceMap0(ctx context.Context, exec bob.Executor, count int, tileCachedImage0 *TileCachedImage, arcgisServiceMap1 *ArcgisServiceMap) (*TileCachedImage, error) { + setter := &TileCachedImageSetter{ + ArcgisID: omit.From(arcgisServiceMap1.ArcgisID), + } + + err := tileCachedImage0.Update(ctx, exec, setter) + if err != nil { + return nil, fmt.Errorf("attachTileCachedImageArcgisServiceMap0: %w", err) + } + + return tileCachedImage0, nil +} + +func (tileCachedImage0 *TileCachedImage) InsertArcgisServiceMap(ctx context.Context, exec bob.Executor, related *ArcgisServiceMapSetter) error { + var err error + + arcgisServiceMap1, err := ArcgisServiceMaps.Insert(related).One(ctx, exec) + if err != nil { + return fmt.Errorf("inserting related objects: %w", err) + } + + _, err = attachTileCachedImageArcgisServiceMap0(ctx, exec, 1, tileCachedImage0, arcgisServiceMap1) + if err != nil { + return err + } + + tileCachedImage0.R.ArcgisServiceMap = arcgisServiceMap1 + + arcgisServiceMap1.R.ArcgisCachedImages = append(arcgisServiceMap1.R.ArcgisCachedImages, tileCachedImage0) + + return nil +} + +func (tileCachedImage0 *TileCachedImage) AttachArcgisServiceMap(ctx context.Context, exec bob.Executor, arcgisServiceMap1 *ArcgisServiceMap) error { + var err error + + _, err = attachTileCachedImageArcgisServiceMap0(ctx, exec, 1, tileCachedImage0, arcgisServiceMap1) + if err != nil { + return err + } + + tileCachedImage0.R.ArcgisServiceMap = arcgisServiceMap1 + + arcgisServiceMap1.R.ArcgisCachedImages = append(arcgisServiceMap1.R.ArcgisCachedImages, tileCachedImage0) + + return nil +} + type tileCachedImageWhere[Q psql.Filterable] struct { ArcgisID psql.WhereMod[Q, string] X psql.WhereMod[Q, int32] Y psql.WhereMod[Q, int32] Z psql.WhereMod[Q, int32] + IsEmpty psql.WhereMod[Q, bool] } func (tileCachedImageWhere[Q]) AliasedAs(alias string) tileCachedImageWhere[Q] { @@ -250,6 +552,7 @@ func buildTileCachedImageWhere[Q psql.Filterable](cols tileCachedImageColumns) t X: psql.Where[Q, int32](cols.X), Y: psql.Where[Q, int32](cols.Y), Z: psql.Where[Q, int32](cols.Z), + IsEmpty: psql.Where[Q, bool](cols.IsEmpty), } } diff --git a/html/template/sync/review/pool.html b/html/template/sync/review/pool.html index 29ee841c..b2526dde 100644 --- a/html/template/sync/review/pool.html +++ b/html/template/sync/review/pool.html @@ -205,8 +205,8 @@ try { const payload = { - taskId: this.selectedTask.id, - action: action, // 'reviewed' or 'discarded' + task_id: this.selectedTask.id, + status: action, // 'reviewed' or 'discarded' updates: {}, }; @@ -219,7 +219,7 @@ }); } - const response = await fetch("/api/review-task", { + const response = await fetch(`/api/review/`, { method: "POST", headers: { "Content-Type": "application/json", @@ -559,7 +559,7 @@ :disabled="!selectedTask || submitting" > - Mark Reviewed + Complete Review diff --git a/platform/imagetile/imagetile.go b/platform/imagetile/imagetile.go index d522cdda..e6c60fb4 100644 --- a/platform/imagetile/imagetile.go +++ b/platform/imagetile/imagetile.go @@ -3,64 +3,89 @@ package imagetile import ( "context" "embed" - "errors" "fmt" "github.com/Gleipnir-Technology/arcgis-go" "github.com/Gleipnir-Technology/arcgis-go/fieldseeker" "github.com/Gleipnir-Technology/nidus-sync/background" "github.com/Gleipnir-Technology/nidus-sync/db/models" - "github.com/rs/zerolog/log" + //"github.com/rs/zerolog/log" ) //go:embed empty-tile.png var emptyTileFS embed.FS -var ErrNoTile = errors.New("used placeholder tile") - var clientByOrgID = make(map[int32]*fieldseeker.FieldSeeker, 0) +var tileRasterPlaceholder *TileRaster -func ImageAtPoint(ctx context.Context, org *models.Organization, level uint, lat, lng float64) ([]byte, error) { +type TileRaster struct { + Content []byte + IsPlaceholder bool +} + +func ImageAtPoint(ctx context.Context, org *models.Organization, level uint, lat, lng float64) (*TileRaster, error) { fssync, err := getFieldseeker(ctx, org) if err != nil { - return []byte{}, fmt.Errorf("create fssync: %w", err) + return nil, fmt.Errorf("create fssync: %w", err) } map_service, err := aerialImageService(ctx, fssync.Arcgis) if err != nil { - return []byte{}, fmt.Errorf("no map service: %w", err) + return nil, fmt.Errorf("no map service: %w", err) } - return map_service.TileGPS(ctx, level, lat, lng) + data, e := map_service.TileGPS(ctx, level, lat, lng) + if e != nil { + return nil, fmt.Errorf("tilegps: %w", e) + } + if len(data) == 0 { + return TileRasterPlaceholder(), nil + } + return &TileRaster{ + Content: data, + IsPlaceholder: false, + }, nil } -func ImageAtTile(ctx context.Context, org *models.Organization, level, y, x uint) ([]byte, error) { +func ImageAtTile(ctx context.Context, org *models.Organization, level, y, x uint) (*TileRaster, error) { oauth, err := background.GetOAuthForOrg(ctx, org) if err != nil { - return []byte{}, fmt.Errorf("get oauth for org: %w", err) + return nil, fmt.Errorf("get oauth for org: %w", err) } fssync, err := background.NewFieldSeeker( ctx, oauth, ) if err != nil { - return []byte{}, fmt.Errorf("create fssync: %w", err) + return nil, fmt.Errorf("create fssync: %w", err) } map_service, err := aerialImageService(ctx, fssync.Arcgis) if err != nil { - return []byte{}, fmt.Errorf("no map service: %w", err) + return nil, fmt.Errorf("no map service: %w", err) } data, e := map_service.Tile(ctx, level, y, x) if e != nil { - log.Error().Err(e).Msg("error getting tile") - return []byte{}, fmt.Errorf("tile: %w", e) + return nil, fmt.Errorf("tile: %w", e) } // No data at this location, so supply the empty tile placeholder if len(data) == 0 { - empty, err := emptyTileFS.ReadFile("empty-tile.png") - if err != nil { - return []byte{}, fmt.Errorf("read empty tile: %w", err) - } - return empty, ErrNoTile + return TileRasterPlaceholder(), nil } - return data, nil + return &TileRaster{ + Content: data, + IsPlaceholder: false, + }, nil +} +func TileRasterPlaceholder() *TileRaster { + if tileRasterPlaceholder != nil { + return tileRasterPlaceholder + } + empty, err := emptyTileFS.ReadFile("empty-tile.png") + if err != nil { + panic(fmt.Sprintf("Failed to read empty-tile.png: %v", err)) + } + tileRasterPlaceholder = &TileRaster{ + Content: empty, + IsPlaceholder: true, + } + return tileRasterPlaceholder } func aerialImageService(ctx context.Context, gis *arcgis.ArcGIS) (*arcgis.MapService, error) { diff --git a/sync/tile.go b/sync/tile.go index fbe50472..0779ee49 100644 --- a/sync/tile.go +++ b/sync/tile.go @@ -46,8 +46,8 @@ func getTileGPS(w http.ResponseWriter, r *http.Request, org *models.Organization return } w.Header().Set("Content-Type", "image/png") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img))) - _, err = io.Copy(w, bytes.NewBuffer(img)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img.Content))) + _, err = io.Copy(w, bytes.NewBuffer(img.Content)) if err != nil { respondError(w, "copy bytes", err, http.StatusInternalServerError) return