Gleipnir/nidus-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Save tiles to the database to make empty tile load faster

Browse source
This commit is contained in:
Eli Ribble 2026-03-11 17:01:47 +00:00
parent a1e6f930cb
commit 3ccc05d4c5
No known key found for this signature in database
10 changed files with 559 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

6
api/compliance.go
View file

@ -98,10 +98,10 @@ func writeImage(ctx context.Context, w http.ResponseWriter, org *models.Organiza
if err != nil {
return fmt.Errorf("image at point: %w", err)
}
log.Info().Int("size", len(img)).Msg("image")
log.Info().Int("size", len(img.Content)).Msg("image")
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img)))
_, err = io.Copy(w, bytes.NewBuffer(img))
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img.Content)))
_, err = io.Copy(w, bytes.NewBuffer(img.Content))
if err != nil {
return fmt.Errorf("copy bytes: %w", err)
}

109
api/tile.go
View file

@ -3,7 +3,6 @@ package api
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
@ -11,7 +10,11 @@ import (
"path/filepath"
"strconv"
"github.com/aarondl/opt/omit"
//"github.com/Gleipnir-Technology/bob"
//"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/nidus-sync/config"
"github.com/Gleipnir-Technology/nidus-sync/db"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
"github.com/Gleipnir-Technology/nidus-sync/platform/imagetile"
"github.com/go-chi/chi/v5"
@ -50,49 +53,89 @@ func handleTile(ctx context.Context, w http.ResponseWriter, org *models.Organiza
return fmt.Errorf("no map service ID set")
}
map_service_id := org.ArcgisMapServiceID.MustGet()
tile_path := fmt.Sprintf("%s/tile-cache/%s/%d/%d/%d.raw", config.FilesDirectory, map_service_id, z, y, x)
file, err := os.Open(tile_path)
tile_path := tilePath(map_service_id, z, y, x)
tile_row, err := models.TileCachedImages.Query(
models.SelectWhere.TileCachedImages.ArcgisID.EQ(map_service_id),
models.SelectWhere.TileCachedImages.X.EQ(int32(x)),
models.SelectWhere.TileCachedImages.Y.EQ(int32(y)),
models.SelectWhere.TileCachedImages.Z.EQ(int32(z)),
).One(ctx, db.PGInstance.BobDB)
if err == nil {
var tile *imagetile.TileRaster
if tile_row.IsEmpty {
tile = imagetile.TileRasterPlaceholder()
} else {
tile, err = loadTileFromDisk(tile_path)
if err != nil {
return fmt.Errorf("load tile from disk: %w", err)
}
}
log.Debug().Uint("z", z).Uint("y", y).Uint("x", x).Bool("is empty", tile_row.IsEmpty).Msg("tile from cache")
return writeTile(w, tile)
}
if err.Error() != "sql: no rows in result set" {
return fmt.Errorf("query db: %w", err)
}
image, err := imagetile.ImageAtTile(ctx, org, uint(z), uint(y), uint(x))
if err != nil {
return fmt.Errorf("image at tile: %w", err)
}
if !image.IsPlaceholder {
err = saveTileToDisk(image, tile_path)
if err != nil {
return fmt.Errorf("save tile: %w", err)
}
}
_, err = models.TileCachedImages.Insert(&models.TileCachedImageSetter{
ArcgisID: omit.From(map_service_id),
X: omit.From(int32(x)),
Y: omit.From(int32(y)),
Z: omit.From(int32(z)),
IsEmpty: omit.From(image.IsPlaceholder),
}).One(ctx, db.PGInstance.BobDB)
if err != nil {
return fmt.Errorf("save to db: %w", err)
}
log.Debug().Uint("z", z).Uint("y", y).Uint("x", x).Bool("placeholder", image.IsPlaceholder).Msg("caching tile")
return writeTile(w, image)
}
func loadTileFromDisk(tile_path string) (*imagetile.TileRaster, error) {
file, err := os.Open(tile_path)
if err != nil {
return nil, fmt.Errorf("open: %w", err)
}
defer file.Close()
img, err := io.ReadAll(file)
if err != nil {
return fmt.Errorf("readall from %s: %w", tile_path, err)
}
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img)))
_, err = io.Copy(w, bytes.NewBuffer(img))
if err != nil {
return fmt.Errorf("copy bytes from %s: %w", tile_path)
}
return nil
}
content, err := imagetile.ImageAtTile(ctx, org, uint(z), uint(y), uint(x))
if err != nil {
if errors.Is(err, imagetile.ErrNoTile) {
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(content)))
_, err = io.Copy(w, bytes.NewBuffer(content))
if err != nil {
return fmt.Errorf("write image file: %w", err)
}
return nil
}
return fmt.Errorf("image at tile: %w", err)
return nil, fmt.Errorf("readall from %s: %w", tile_path, err)
}
return &imagetile.TileRaster{
Content: img,
IsPlaceholder: false,
}, nil
}
func saveTileToDisk(image *imagetile.TileRaster, tile_path string) error {
parent := filepath.Dir(tile_path)
err = os.MkdirAll(parent, 0750)
err := os.MkdirAll(parent, 0750)
if err != nil {
return fmt.Errorf("mkdirall: %w", err)
}
err = os.WriteFile(tile_path, content, 0644)
if err != nil {
return fmt.Errorf("write image file: %w", err)
}
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(content)))
_, err = io.Copy(w, bytes.NewBuffer(content))
err = os.WriteFile(tile_path, image.Content, 0644)
if err != nil {
return fmt.Errorf("write image file: %w", err)
}
return nil
}
func tilePath(map_service_id string, z, y, x uint) string {
return fmt.Sprintf("%s/tile-cache/%s/%d/%d/%d.raw", config.FilesDirectory, map_service_id, z, y, x)
}
func writeTile(w http.ResponseWriter, image *imagetile.TileRaster) error {
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(image.Content)))
_, err := io.Copy(w, bytes.NewBuffer(image.Content))
if err != nil {
return fmt.Errorf("io.copy: %w", err)
}
return nil
}

6
db/dberrors/tile.cached_image.bob.go
View file

@ -4,14 +4,14 @@
package dberrors
var TileCachedImageErrors = &tileCachedImageErrors{
ErrUniqueCachedImageArcgisIdXYZKey: &UniqueConstraintError{
ErrUniqueCachedImagePkey: &UniqueConstraintError{
schema: "tile",
table: "cached_image",
columns: []string{"arcgis_id", "x", "y", "z"},
s: "cached_image_arcgis_id_x_y_z_key",
s: "cached_image_pkey",
},
}
type tileCachedImageErrors struct {
ErrUniqueCachedImageArcgisIdXYZKey *UniqueConstraintError
ErrUniqueCachedImagePkey *UniqueConstraintError
}

41
db/dbinfo/tile.cached_image.bob.go
View file

@ -51,11 +51,20 @@ var TileCachedImages = Table[
Generated: false,
AutoIncr: false,
},
IsEmpty: column{
Name: "is_empty",
DBType: "boolean",
Default: "",
Comment: "",
Nullable: false,
Generated: false,
AutoIncr: false,
},
},
Indexes: tileCachedImageIndexes{
CachedImageArcgisIDXYZKey: index{
CachedImagePkey: index{
Type: "btree",
Name: "cached_image_arcgis_id_x_y_z_key",
Name: "cached_image_pkey",
Columns: []indexColumn{
{
Name: "arcgis_id",
@ -86,7 +95,11 @@ var TileCachedImages = Table[
Include: []string{},
},
},
PrimaryKey: &constraint{
Name: "cached_image_pkey",
Columns: []string{"arcgis_id", "x", "y", "z"},
Comment: "",
},
ForeignKeys: tileCachedImageForeignKeys{
TileCachedImageCachedImageArcgisIDFkey: foreignKey{
constraint: constraint{
@ -98,13 +111,6 @@ var TileCachedImages = Table[
ForeignColumns: []string{"arcgis_id"},
},
},
Uniques: tileCachedImageUniques{
CachedImageArcgisIDXYZKey: constraint{
Name: "cached_image_arcgis_id_x_y_z_key",
Columns: []string{"arcgis_id", "x", "y", "z"},
Comment: "",
},
},
Comment: "",
}
@ -114,21 +120,22 @@ type tileCachedImageColumns struct {
X column
Y column
Z column
IsEmpty column
}
func (c tileCachedImageColumns) AsSlice() []column {
return []column{
c.ArcgisID, c.X, c.Y, c.Z,
c.ArcgisID, c.X, c.Y, c.Z, c.IsEmpty,
}
}
type tileCachedImageIndexes struct {
CachedImageArcgisIDXYZKey index
CachedImagePkey index
}
func (i tileCachedImageIndexes) AsSlice() []index {
return []index{
i.CachedImageArcgisIDXYZKey,
i.CachedImagePkey,
}
}
@ -142,14 +149,10 @@ func (f tileCachedImageForeignKeys) AsSlice() []foreignKey {
}
}
type tileCachedImageUniques struct {
CachedImageArcgisIDXYZKey constraint
}
type tileCachedImageUniques struct{}
func (u tileCachedImageUniques) AsSlice() []constraint {
return []constraint{
u.CachedImageArcgisIDXYZKey,
}
return []constraint{}
}
type tileCachedImageChecks struct{}

19
db/migrations/00100_tile_empty.sql Normal file
View file

@ -0,0 +1,19 @@
-- +goose Up
DROP TABLE tile.cached_image;
CREATE TABLE tile.cached_image (
arcgis_id TEXT NOT NULL REFERENCES arcgis.service_map(arcgis_id),
x INTEGER NOT NULL,
y INTEGER NOT NULL,
z INTEGER NOT NULL,
is_empty BOOLEAN NOT NULL,
PRIMARY KEY (arcgis_id, x, y, z)
);
-- +goose Down
DROP TABLE tile.cached_image;
CREATE TABLE tile.cached_image (
arcgis_id TEXT NOT NULL REFERENCES arcgis.service_map(arcgis_id),
x INTEGER NOT NULL,
y INTEGER NOT NULL,
z INTEGER NOT NULL,
UNIQUE(arcgis_id, x, y, z)
);

68
db/models/arcgis.service_map.bob.go
View file

@ -627,6 +627,74 @@ func (arcgisServiceMap0 *ArcgisServiceMap) AttachArcgisMapServiceOrganizations(c
return nil
}
func insertArcgisServiceMapArcgisCachedImages0(ctx context.Context, exec bob.Executor, tileCachedImages1 []*TileCachedImageSetter, arcgisServiceMap0 *ArcgisServiceMap) (TileCachedImageSlice, error) {
for i := range tileCachedImages1 {
tileCachedImages1[i].ArcgisID = omit.From(arcgisServiceMap0.ArcgisID)
}
ret, err := TileCachedImages.Insert(bob.ToMods(tileCachedImages1...)).All(ctx, exec)
if err != nil {
return ret, fmt.Errorf("insertArcgisServiceMapArcgisCachedImages0: %w", err)
}
return ret, nil
}
func attachArcgisServiceMapArcgisCachedImages0(ctx context.Context, exec bob.Executor, count int, tileCachedImages1 TileCachedImageSlice, arcgisServiceMap0 *ArcgisServiceMap) (TileCachedImageSlice, error) {
setter := &TileCachedImageSetter{
ArcgisID: omit.From(arcgisServiceMap0.ArcgisID),
}
err := tileCachedImages1.UpdateAll(ctx, exec, *setter)
if err != nil {
return nil, fmt.Errorf("attachArcgisServiceMapArcgisCachedImages0: %w", err)
}
return tileCachedImages1, nil
}
func (arcgisServiceMap0 *ArcgisServiceMap) InsertArcgisCachedImages(ctx context.Context, exec bob.Executor, related ...*TileCachedImageSetter) error {
if len(related) == 0 {
return nil
}
var err error
tileCachedImages1, err := insertArcgisServiceMapArcgisCachedImages0(ctx, exec, related, arcgisServiceMap0)
if err != nil {
return err
}
arcgisServiceMap0.R.ArcgisCachedImages = append(arcgisServiceMap0.R.ArcgisCachedImages, tileCachedImages1...)
for _, rel := range tileCachedImages1 {
rel.R.ArcgisServiceMap = arcgisServiceMap0
}
return nil
}
func (arcgisServiceMap0 *ArcgisServiceMap) AttachArcgisCachedImages(ctx context.Context, exec bob.Executor, related ...*TileCachedImage) error {
if len(related) == 0 {
return nil
}
var err error
tileCachedImages1 := TileCachedImageSlice(related)
_, err = attachArcgisServiceMapArcgisCachedImages0(ctx, exec, len(related), tileCachedImages1, arcgisServiceMap0)
if err != nil {
return err
}
arcgisServiceMap0.R.ArcgisCachedImages = append(arcgisServiceMap0.R.ArcgisCachedImages, tileCachedImages1...)
for _, rel := range related {
rel.R.ArcgisServiceMap = arcgisServiceMap0
}
return nil
}
type arcgisServiceMapWhere[Q psql.Filterable] struct {
AccountID psql.WhereMod[Q, string]
ArcgisID psql.WhereMod[Q, string]

333
db/models/tile.cached_image.bob.go
View file

@ -11,6 +11,7 @@ import (
"github.com/Gleipnir-Technology/bob"
"github.com/Gleipnir-Technology/bob/dialect/psql"
"github.com/Gleipnir-Technology/bob/dialect/psql/dialect"
"github.com/Gleipnir-Technology/bob/dialect/psql/dm"
"github.com/Gleipnir-Technology/bob/dialect/psql/sm"
"github.com/Gleipnir-Technology/bob/dialect/psql/um"
"github.com/Gleipnir-Technology/bob/expr"
@ -21,10 +22,11 @@ import (
// TileCachedImage is an object representing the database table.
type TileCachedImage struct {
ArcgisID string `db:"arcgis_id" `
X int32 `db:"x" `
Y int32 `db:"y" `
Z int32 `db:"z" `
ArcgisID string `db:"arcgis_id,pk" `
X int32 `db:"x,pk" `
Y int32 `db:"y,pk" `
Z int32 `db:"z,pk" `
IsEmpty bool `db:"is_empty" `
R tileCachedImageR `db:"-" `
}
@ -33,10 +35,10 @@ type TileCachedImage struct {
// This should almost always be used instead of []*TileCachedImage.
type TileCachedImageSlice []*TileCachedImage
// TileCachedImages contains methods to work with the cached_image view
var TileCachedImages = psql.NewViewx[*TileCachedImage, TileCachedImageSlice]("tile", "cached_image", buildTileCachedImageColumns("tile.cached_image"))
// TileCachedImages contains methods to work with the cached_image table
var TileCachedImages = psql.NewTablex[*TileCachedImage, TileCachedImageSlice, *TileCachedImageSetter]("tile", "cached_image", buildTileCachedImageColumns("tile.cached_image"))
// TileCachedImagesQuery is a query on the cached_image view
// TileCachedImagesQuery is a query on the cached_image table
type TileCachedImagesQuery = *psql.ViewQuery[*TileCachedImage, TileCachedImageSlice]
// tileCachedImageR is where relationships are stored.
@ -47,13 +49,14 @@ type tileCachedImageR struct {
func buildTileCachedImageColumns(alias string) tileCachedImageColumns {
return tileCachedImageColumns{
ColumnsExpr: expr.NewColumnsExpr(
"arcgis_id", "x", "y", "z",
"arcgis_id", "x", "y", "z", "is_empty",
).WithParent("tile.cached_image"),
tableAlias: alias,
ArcgisID: psql.Quote(alias, "arcgis_id"),
X: psql.Quote(alias, "x"),
Y: psql.Quote(alias, "y"),
Z: psql.Quote(alias, "z"),
IsEmpty: psql.Quote(alias, "is_empty"),
}
}
@ -64,6 +67,7 @@ type tileCachedImageColumns struct {
X psql.Expression
Y psql.Expression
Z psql.Expression
IsEmpty psql.Expression
}
func (c tileCachedImageColumns) Alias() string {
@ -78,14 +82,15 @@ func (tileCachedImageColumns) AliasedAs(alias string) tileCachedImageColumns {
// All values are optional, and do not have to be set
// Generated columns are not included
type TileCachedImageSetter struct {
ArcgisID omit.Val[string] `db:"arcgis_id" `
X omit.Val[int32] `db:"x" `
Y omit.Val[int32] `db:"y" `
Z omit.Val[int32] `db:"z" `
ArcgisID omit.Val[string] `db:"arcgis_id,pk" `
X omit.Val[int32] `db:"x,pk" `
Y omit.Val[int32] `db:"y,pk" `
Z omit.Val[int32] `db:"z,pk" `
IsEmpty omit.Val[bool] `db:"is_empty" `
}
func (s TileCachedImageSetter) SetColumns() []string {
vals := make([]string, 0, 4)
vals := make([]string, 0, 5)
if s.ArcgisID.IsValue() {
vals = append(vals, "arcgis_id")
}
@ -98,6 +103,9 @@ func (s TileCachedImageSetter) SetColumns() []string {
if s.Z.IsValue() {
vals = append(vals, "z")
}
if s.IsEmpty.IsValue() {
vals = append(vals, "is_empty")
}
return vals
}
@ -114,11 +122,18 @@ func (s TileCachedImageSetter) Overwrite(t *TileCachedImage) {
if s.Z.IsValue() {
t.Z = s.Z.MustGet()
}
if s.IsEmpty.IsValue() {
t.IsEmpty = s.IsEmpty.MustGet()
}
}
func (s *TileCachedImageSetter) Apply(q *dialect.InsertQuery) {
q.AppendHooks(func(ctx context.Context, exec bob.Executor) (context.Context, error) {
return TileCachedImages.BeforeInsertHooks.RunHooks(ctx, exec, s)
})
q.AppendValues(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) {
vals := make([]bob.Expression, 4)
vals := make([]bob.Expression, 5)
if s.ArcgisID.IsValue() {
vals[0] = psql.Arg(s.ArcgisID.MustGet())
} else {
@ -143,6 +158,12 @@ func (s *TileCachedImageSetter) Apply(q *dialect.InsertQuery) {
vals[3] = psql.Raw("DEFAULT")
}
if s.IsEmpty.IsValue() {
vals[4] = psql.Arg(s.IsEmpty.MustGet())
} else {
vals[4] = psql.Raw("DEFAULT")
}
return bob.ExpressSlice(ctx, w, d, start, vals, "", ", ", "")
}))
}
@ -152,7 +173,7 @@ func (s TileCachedImageSetter) UpdateMod() bob.Mod[*dialect.UpdateQuery] {
}
func (s TileCachedImageSetter) Expressions(prefix ...string) []bob.Expression {
exprs := make([]bob.Expression, 0, 4)
exprs := make([]bob.Expression, 0, 5)
if s.ArcgisID.IsValue() {
exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{
@ -182,9 +203,47 @@ func (s TileCachedImageSetter) Expressions(prefix ...string) []bob.Expression {
}})
}
if s.IsEmpty.IsValue() {
exprs = append(exprs, expr.Join{Sep: " = ", Exprs: []bob.Expression{
psql.Quote(append(prefix, "is_empty")...),
psql.Arg(s.IsEmpty),
}})
}
return exprs
}
// FindTileCachedImage retrieves a single record by primary key
// If cols is empty Find will return all columns.
func FindTileCachedImage(ctx context.Context, exec bob.Executor, ArcgisIDPK string, XPK int32, YPK int32, ZPK int32, cols ...string) (*TileCachedImage, error) {
if len(cols) == 0 {
return TileCachedImages.Query(
sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(ArcgisIDPK))),
sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(XPK))),
sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(YPK))),
sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(ZPK))),
).One(ctx, exec)
}
return TileCachedImages.Query(
sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(ArcgisIDPK))),
sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(XPK))),
sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(YPK))),
sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(ZPK))),
sm.Columns(TileCachedImages.Columns.Only(cols...)),
).One(ctx, exec)
}
// TileCachedImageExists checks the presence of a single record by primary key
func TileCachedImageExists(ctx context.Context, exec bob.Executor, ArcgisIDPK string, XPK int32, YPK int32, ZPK int32) (bool, error) {
return TileCachedImages.Query(
sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(ArcgisIDPK))),
sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(XPK))),
sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(YPK))),
sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(ZPK))),
).Exists(ctx, exec)
}
// AfterQueryHook is called after TileCachedImage is retrieved from the database
func (o *TileCachedImage) AfterQueryHook(ctx context.Context, exec bob.Executor, queryType bob.QueryType) error {
var err error
@ -192,11 +251,69 @@ func (o *TileCachedImage) AfterQueryHook(ctx context.Context, exec bob.Executor,
switch queryType {
case bob.QueryTypeSelect:
ctx, err = TileCachedImages.AfterSelectHooks.RunHooks(ctx, exec, TileCachedImageSlice{o})
case bob.QueryTypeInsert:
ctx, err = TileCachedImages.AfterInsertHooks.RunHooks(ctx, exec, TileCachedImageSlice{o})
case bob.QueryTypeUpdate:
ctx, err = TileCachedImages.AfterUpdateHooks.RunHooks(ctx, exec, TileCachedImageSlice{o})
case bob.QueryTypeDelete:
ctx, err = TileCachedImages.AfterDeleteHooks.RunHooks(ctx, exec, TileCachedImageSlice{o})
}
return err
}
// primaryKeyVals returns the primary key values of the TileCachedImage
func (o *TileCachedImage) primaryKeyVals() bob.Expression {
return psql.ArgGroup(
o.ArcgisID,
o.X,
o.Y,
o.Z,
)
}
func (o *TileCachedImage) pkEQ() dialect.Expression {
return psql.Group(psql.Quote("tile.cached_image", "arcgis_id"), psql.Quote("tile.cached_image", "x"), psql.Quote("tile.cached_image", "y"), psql.Quote("tile.cached_image", "z")).EQ(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) {
return o.primaryKeyVals().WriteSQL(ctx, w, d, start)
}))
}
// Update uses an executor to update the TileCachedImage
func (o *TileCachedImage) Update(ctx context.Context, exec bob.Executor, s *TileCachedImageSetter) error {
v, err := TileCachedImages.Update(s.UpdateMod(), um.Where(o.pkEQ())).One(ctx, exec)
if err != nil {
return err
}
o.R = v.R
*o = *v
return nil
}
// Delete deletes a single TileCachedImage record with an executor
func (o *TileCachedImage) Delete(ctx context.Context, exec bob.Executor) error {
_, err := TileCachedImages.Delete(dm.Where(o.pkEQ())).Exec(ctx, exec)
return err
}
// Reload refreshes the TileCachedImage using the executor
func (o *TileCachedImage) Reload(ctx context.Context, exec bob.Executor) error {
o2, err := TileCachedImages.Query(
sm.Where(TileCachedImages.Columns.ArcgisID.EQ(psql.Arg(o.ArcgisID))),
sm.Where(TileCachedImages.Columns.X.EQ(psql.Arg(o.X))),
sm.Where(TileCachedImages.Columns.Y.EQ(psql.Arg(o.Y))),
sm.Where(TileCachedImages.Columns.Z.EQ(psql.Arg(o.Z))),
).One(ctx, exec)
if err != nil {
return err
}
o2.R = o.R
*o = *o2
return nil
}
// AfterQueryHook is called after TileCachedImageSlice is retrieved from the database
func (o TileCachedImageSlice) AfterQueryHook(ctx context.Context, exec bob.Executor, queryType bob.QueryType) error {
var err error
@ -204,11 +321,147 @@ func (o TileCachedImageSlice) AfterQueryHook(ctx context.Context, exec bob.Execu
switch queryType {
case bob.QueryTypeSelect:
ctx, err = TileCachedImages.AfterSelectHooks.RunHooks(ctx, exec, o)
case bob.QueryTypeInsert:
ctx, err = TileCachedImages.AfterInsertHooks.RunHooks(ctx, exec, o)
case bob.QueryTypeUpdate:
ctx, err = TileCachedImages.AfterUpdateHooks.RunHooks(ctx, exec, o)
case bob.QueryTypeDelete:
ctx, err = TileCachedImages.AfterDeleteHooks.RunHooks(ctx, exec, o)
}
return err
}
func (o TileCachedImageSlice) pkIN() dialect.Expression {
if len(o) == 0 {
return psql.Raw("NULL")
}
return psql.Group(psql.Quote("tile.cached_image", "arcgis_id"), psql.Quote("tile.cached_image", "x"), psql.Quote("tile.cached_image", "y"), psql.Quote("tile.cached_image", "z")).In(bob.ExpressionFunc(func(ctx context.Context, w io.StringWriter, d bob.Dialect, start int) ([]any, error) {
pkPairs := make([]bob.Expression, len(o))
for i, row := range o {
pkPairs[i] = row.primaryKeyVals()
}
return bob.ExpressSlice(ctx, w, d, start, pkPairs, "", ", ", "")
}))
}
// copyMatchingRows finds models in the given slice that have the same primary key
// then it first copies the existing relationships from the old model to the new model
// and then replaces the old model in the slice with the new model
func (o TileCachedImageSlice) copyMatchingRows(from ...*TileCachedImage) {
for i, old := range o {
for _, new := range from {
if new.ArcgisID != old.ArcgisID {
continue
}
if new.X != old.X {
continue
}
if new.Y != old.Y {
continue
}
if new.Z != old.Z {
continue
}
new.R = old.R
o[i] = new
break
}
}
}
// UpdateMod modifies an update query with "WHERE primary_key IN (o...)"
func (o TileCachedImageSlice) UpdateMod() bob.Mod[*dialect.UpdateQuery] {
return bob.ModFunc[*dialect.UpdateQuery](func(q *dialect.UpdateQuery) {
q.AppendHooks(func(ctx context.Context, exec bob.Executor) (context.Context, error) {
return TileCachedImages.BeforeUpdateHooks.RunHooks(ctx, exec, o)
})
q.AppendLoader(bob.LoaderFunc(func(ctx context.Context, exec bob.Executor, retrieved any) error {
var err error
switch retrieved := retrieved.(type) {
case *TileCachedImage:
o.copyMatchingRows(retrieved)
case []*TileCachedImage:
o.copyMatchingRows(retrieved...)
case TileCachedImageSlice:
o.copyMatchingRows(retrieved...)
default:
// If the retrieved value is not a TileCachedImage or a slice of TileCachedImage
// then run the AfterUpdateHooks on the slice
_, err = TileCachedImages.AfterUpdateHooks.RunHooks(ctx, exec, o)
}
return err
}))
q.AppendWhere(o.pkIN())
})
}
// DeleteMod modifies an delete query with "WHERE primary_key IN (o...)"
func (o TileCachedImageSlice) DeleteMod() bob.Mod[*dialect.DeleteQuery] {
return bob.ModFunc[*dialect.DeleteQuery](func(q *dialect.DeleteQuery) {
q.AppendHooks(func(ctx context.Context, exec bob.Executor) (context.Context, error) {
return TileCachedImages.BeforeDeleteHooks.RunHooks(ctx, exec, o)
})
q.AppendLoader(bob.LoaderFunc(func(ctx context.Context, exec bob.Executor, retrieved any) error {
var err error
switch retrieved := retrieved.(type) {
case *TileCachedImage:
o.copyMatchingRows(retrieved)
case []*TileCachedImage:
o.copyMatchingRows(retrieved...)
case TileCachedImageSlice:
o.copyMatchingRows(retrieved...)
default:
// If the retrieved value is not a TileCachedImage or a slice of TileCachedImage
// then run the AfterDeleteHooks on the slice
_, err = TileCachedImages.AfterDeleteHooks.RunHooks(ctx, exec, o)
}
return err
}))
q.AppendWhere(o.pkIN())
})
}
func (o TileCachedImageSlice) UpdateAll(ctx context.Context, exec bob.Executor, vals TileCachedImageSetter) error {
if len(o) == 0 {
return nil
}
_, err := TileCachedImages.Update(vals.UpdateMod(), o.UpdateMod()).All(ctx, exec)
return err
}
func (o TileCachedImageSlice) DeleteAll(ctx context.Context, exec bob.Executor) error {
if len(o) == 0 {
return nil
}
_, err := TileCachedImages.Delete(o.DeleteMod()).Exec(ctx, exec)
return err
}
func (o TileCachedImageSlice) ReloadAll(ctx context.Context, exec bob.Executor) error {
if len(o) == 0 {
return nil
}
o2, err := TileCachedImages.Query(sm.Where(o.pkIN())).All(ctx, exec)
if err != nil {
return err
}
o.copyMatchingRows(o2...)
return nil
}
// ArcgisServiceMap starts a query for related objects on arcgis.service_map
func (o *TileCachedImage) ArcgisServiceMap(mods ...bob.Mod[*dialect.SelectQuery]) ArcgisServiceMapsQuery {
return ArcgisServiceMaps.Query(append(mods,
@ -233,11 +486,60 @@ func (os TileCachedImageSlice) ArcgisServiceMap(mods ...bob.Mod[*dialect.SelectQ
)...)
}
func attachTileCachedImageArcgisServiceMap0(ctx context.Context, exec bob.Executor, count int, tileCachedImage0 *TileCachedImage, arcgisServiceMap1 *ArcgisServiceMap) (*TileCachedImage, error) {
setter := &TileCachedImageSetter{
ArcgisID: omit.From(arcgisServiceMap1.ArcgisID),
}
err := tileCachedImage0.Update(ctx, exec, setter)
if err != nil {
return nil, fmt.Errorf("attachTileCachedImageArcgisServiceMap0: %w", err)
}
return tileCachedImage0, nil
}
func (tileCachedImage0 *TileCachedImage) InsertArcgisServiceMap(ctx context.Context, exec bob.Executor, related *ArcgisServiceMapSetter) error {
var err error
arcgisServiceMap1, err := ArcgisServiceMaps.Insert(related).One(ctx, exec)
if err != nil {
return fmt.Errorf("inserting related objects: %w", err)
}
_, err = attachTileCachedImageArcgisServiceMap0(ctx, exec, 1, tileCachedImage0, arcgisServiceMap1)
if err != nil {
return err
}
tileCachedImage0.R.ArcgisServiceMap = arcgisServiceMap1
arcgisServiceMap1.R.ArcgisCachedImages = append(arcgisServiceMap1.R.ArcgisCachedImages, tileCachedImage0)
return nil
}
func (tileCachedImage0 *TileCachedImage) AttachArcgisServiceMap(ctx context.Context, exec bob.Executor, arcgisServiceMap1 *ArcgisServiceMap) error {
var err error
_, err = attachTileCachedImageArcgisServiceMap0(ctx, exec, 1, tileCachedImage0, arcgisServiceMap1)
if err != nil {
return err
}
tileCachedImage0.R.ArcgisServiceMap = arcgisServiceMap1
arcgisServiceMap1.R.ArcgisCachedImages = append(arcgisServiceMap1.R.ArcgisCachedImages, tileCachedImage0)
return nil
}
type tileCachedImageWhere[Q psql.Filterable] struct {
ArcgisID psql.WhereMod[Q, string]
X psql.WhereMod[Q, int32]
Y psql.WhereMod[Q, int32]
Z psql.WhereMod[Q, int32]
IsEmpty psql.WhereMod[Q, bool]
}
func (tileCachedImageWhere[Q]) AliasedAs(alias string) tileCachedImageWhere[Q] {
@ -250,6 +552,7 @@ func buildTileCachedImageWhere[Q psql.Filterable](cols tileCachedImageColumns) t
X: psql.Where[Q, int32](cols.X),
Y: psql.Where[Q, int32](cols.Y),
Z: psql.Where[Q, int32](cols.Z),
IsEmpty: psql.Where[Q, bool](cols.IsEmpty),
}
}

8
html/template/sync/review/pool.html
View file

@ -205,8 +205,8 @@
try {
const payload = {
taskId: this.selectedTask.id,
action: action, // 'reviewed' or 'discarded'
task_id: this.selectedTask.id,
status: action, // 'reviewed' or 'discarded'
updates: {},
};
@ -219,7 +219,7 @@
});
}
const response = await fetch("/api/review-task", {
const response = await fetch(`/api/review/`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@ -559,7 +559,7 @@
:disabled="!selectedTask || submitting"
>
<span x-show="!submitting">
<i class="bi bi-check-circle"></i> Mark Reviewed
<i class="bi bi-check-circle"></i> Complete Review
</span>
<span x-show="submitting">
<span class="spinner-border spinner-border-sm" role="status"></span>

59
platform/imagetile/imagetile.go
View file

@ -3,64 +3,89 @@ package imagetile
import (
"context"
"embed"
"errors"
"fmt"
"github.com/Gleipnir-Technology/arcgis-go"
"github.com/Gleipnir-Technology/arcgis-go/fieldseeker"
"github.com/Gleipnir-Technology/nidus-sync/background"
"github.com/Gleipnir-Technology/nidus-sync/db/models"
"github.com/rs/zerolog/log"
//"github.com/rs/zerolog/log"
)
//go:embed empty-tile.png
var emptyTileFS embed.FS
var ErrNoTile = errors.New("used placeholder tile")
var clientByOrgID = make(map[int32]*fieldseeker.FieldSeeker, 0)
var tileRasterPlaceholder *TileRaster
func ImageAtPoint(ctx context.Context, org *models.Organization, level uint, lat, lng float64) ([]byte, error) {
type TileRaster struct {
Content []byte
IsPlaceholder bool
}
func ImageAtPoint(ctx context.Context, org *models.Organization, level uint, lat, lng float64) (*TileRaster, error) {
fssync, err := getFieldseeker(ctx, org)
if err != nil {
return []byte{}, fmt.Errorf("create fssync: %w", err)
return nil, fmt.Errorf("create fssync: %w", err)
}
map_service, err := aerialImageService(ctx, fssync.Arcgis)
if err != nil {
return []byte{}, fmt.Errorf("no map service: %w", err)
return nil, fmt.Errorf("no map service: %w", err)
}
return map_service.TileGPS(ctx, level, lat, lng)
data, e := map_service.TileGPS(ctx, level, lat, lng)
if e != nil {
return nil, fmt.Errorf("tilegps: %w", e)
}
if len(data) == 0 {
return TileRasterPlaceholder(), nil
}
return &TileRaster{
Content: data,
IsPlaceholder: false,
}, nil
}
func ImageAtTile(ctx context.Context, org *models.Organization, level, y, x uint) ([]byte, error) {
func ImageAtTile(ctx context.Context, org *models.Organization, level, y, x uint) (*TileRaster, error) {
oauth, err := background.GetOAuthForOrg(ctx, org)
if err != nil {
return []byte{}, fmt.Errorf("get oauth for org: %w", err)
return nil, fmt.Errorf("get oauth for org: %w", err)
}
fssync, err := background.NewFieldSeeker(
ctx,
oauth,
)
if err != nil {
return []byte{}, fmt.Errorf("create fssync: %w", err)
return nil, fmt.Errorf("create fssync: %w", err)
}
map_service, err := aerialImageService(ctx, fssync.Arcgis)
if err != nil {
return []byte{}, fmt.Errorf("no map service: %w", err)
return nil, fmt.Errorf("no map service: %w", err)
}
data, e := map_service.Tile(ctx, level, y, x)
if e != nil {
log.Error().Err(e).Msg("error getting tile")
return []byte{}, fmt.Errorf("tile: %w", e)
return nil, fmt.Errorf("tile: %w", e)
}
// No data at this location, so supply the empty tile placeholder
if len(data) == 0 {
return TileRasterPlaceholder(), nil
}
return &TileRaster{
Content: data,
IsPlaceholder: false,
}, nil
}
func TileRasterPlaceholder() *TileRaster {
if tileRasterPlaceholder != nil {
return tileRasterPlaceholder
}
empty, err := emptyTileFS.ReadFile("empty-tile.png")
if err != nil {
return []byte{}, fmt.Errorf("read empty tile: %w", err)
panic(fmt.Sprintf("Failed to read empty-tile.png: %v", err))
}
return empty, ErrNoTile
tileRasterPlaceholder = &TileRaster{
Content: empty,
IsPlaceholder: true,
}
return data, nil
return tileRasterPlaceholder
}
func aerialImageService(ctx context.Context, gis *arcgis.ArcGIS) (*arcgis.MapService, error) {

4
sync/tile.go
View file

@ -46,8 +46,8 @@ func getTileGPS(w http.ResponseWriter, r *http.Request, org *models.Organization
return
}
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img)))
_, err = io.Copy(w, bytes.NewBuffer(img))
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(img.Content)))
_, err = io.Copy(w, bytes.NewBuffer(img.Content))
if err != nil {
respondError(w, "copy bytes", err, http.StatusInternalServerError)
return