diff --git a/background/arcgis.go b/background/arcgis.go index c96a4e95..e103ed56 100644 --- a/background/arcgis.go +++ b/background/arcgis.go @@ -22,6 +22,7 @@ import ( "github.com/Gleipnir-Technology/arcgis-go" "github.com/Gleipnir-Technology/arcgis-go/fieldseeker" + "github.com/Gleipnir-Technology/arcgis-go/response" "github.com/Gleipnir-Technology/bob/dialect/psql" "github.com/Gleipnir-Technology/bob/dialect/psql/dm" "github.com/Gleipnir-Technology/bob/dialect/psql/sm" @@ -36,6 +37,7 @@ import ( "github.com/aarondl/opt/omitnull" "github.com/alitto/pond/v2" "github.com/jackc/pgx/v5" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) @@ -139,7 +141,8 @@ func IsSyncOngoing(org_id int32) bool { } // This is a goroutine that is in charge of getting Fieldseeker data and keeping it fresh. -func refreshFieldseekerData(ctx context.Context, newOauthCh <-chan struct{}) { +func refreshFieldseekerData(background_ctx context.Context, newOauthCh <-chan struct{}) { + ctx := log.With().Str("component", "arcgis").Logger().Level(zerolog.InfoLevel).WithContext(background_ctx) syncStatusByOrg = make(map[int32]bool, 0) for { workerCtx, cancel := context.WithCancel(context.Background()) @@ -208,11 +211,8 @@ type SyncStats struct { } func downloadFieldseekerSchema(ctx context.Context, fieldseekerClient *fieldseeker.FieldSeeker, arcgis_id string) { - layers, err := fieldseekerClient.FeatureServerLayers(ctx) - if err != nil { - log.Error().Err(err).Msg("Failed to get fieldseeker layers") - return - } + layers := fieldseekerClient.Layers() + log.Debug().Int("len", len(layers)).Msg("Downloading fieldseeker schema") for _, layer := range layers { err := os.MkdirAll(filepath.Join(config.FieldseekerSchemaDirectory, arcgis_id), os.ModePerm) if err != nil { @@ -276,10 +276,9 @@ func generateCodeVerifier() string { // Find out what we can about this user func updateArcgisUserData(ctx context.Context, user *models.User, oauth *models.OauthToken) { - client, err := arcgis.NewArcGIS( + client, err := arcgis.NewArcGISAuth( ctx, - nil, - arcgis.AuthenticatorOAuth{ + &arcgis.AuthenticatorOAuth{ AccessToken: oauth.AccessToken, AccessTokenExpires: oauth.AccessTokenExpires, RefreshToken: oauth.RefreshToken, @@ -343,37 +342,25 @@ func updateArcgisUserData(ctx context.Context, user *models.User, oauth *models. return } - search, err := client.Search(ctx, "FieldseekerGIS") + fssync, err := fieldseeker.NewFieldSeekerFromAG(ctx, *client) if err != nil { - log.Error().Err(err).Msg("Failed to get search FieldseekerGIS data") - return + log.Error().Err(err).Msg("Failed to create fieldseeker") } - var fieldseeker_client *fieldseeker.FieldSeeker - for _, result := range search.Results { - log.Info().Str("name", result.Name).Msg("Got result") - if result.Name == "FieldSeekerGIS" { - log.Info().Str("url", result.URL).Msg("Found Fieldseeker") - setter := models.OrganizationSetter{ - FieldseekerURL: omitnull.From(result.URL), - } - err = org.Update(ctx, db.PGInstance.BobDB, &setter) - if err != nil { - log.Error().Err(err).Msg("Failed to create new organization") - return - } - fieldseeker_client, err = NewFieldSeeker(ctx, oauth) - if err != nil { - log.Error().Err(err).Msg("Failed to create new fieldseeker") - return - } - } + log.Info().Str("url", fssync.ServiceFeature.URL.String()).Msg("Found Fieldseeker") + setter := models.OrganizationSetter{ + FieldseekerURL: omitnull.From(fssync.ServiceFeature.URL.String()), + } + err = org.Update(ctx, db.PGInstance.BobDB, &setter) + if err != nil { + log.Error().Err(err).Msg("Failed to create new organization") + return } arcgis_id, ok := org.ArcgisID.Get() if !ok { log.Error().Int("org.id", int(org.ID)).Msg("Cannot get webhooks - ArcGIS ID is null") } - maybeCreateWebhook(ctx, fieldseeker_client) - downloadFieldseekerSchema(ctx, fieldseeker_client, arcgis_id) + maybeCreateWebhook(ctx, fssync) + downloadFieldseekerSchema(ctx, fssync, arcgis_id) notification.ClearOauth(ctx, user) newOAuthTokenChannel <- struct{}{} } @@ -394,9 +381,8 @@ func NewFieldSeeker(ctx context.Context, oauth *models.OauthToken) (*fieldseeker return nil, errors.New("Didn't get enough path parts") } context := pathParts[0] - ar, err := arcgis.NewArcGIS( + ar, err := arcgis.NewArcGISAuth( ctx, - &host, arcgis.AuthenticatorOAuth{ AccessToken: oauth.AccessToken, AccessTokenExpires: oauth.AccessTokenExpires, @@ -408,23 +394,18 @@ func NewFieldSeeker(ctx context.Context, oauth *models.OauthToken) (*fieldseeker return nil, fmt.Errorf("Failed to create ArcGIS client: %w", err) } log.Info().Str("context", context).Str("host", host).Msg("Using base fieldseeker URL") - fssync, err := fieldseeker.NewFieldSeeker(ar, row.FieldseekerURL.MustGet()) - if err != nil { - return nil, fmt.Errorf("Failed to create Fieldseeker client: %w", err) - } - // check that the authentication is valid - _, err = fssync.FeatureServerLayers(ctx) + fssync, err := fieldseeker.NewFieldSeekerFromURL(ctx, *ar, row.FieldseekerURL.MustGet()) if err != nil { if errors.Is(err, arcgis.ErrorInvalidAuthToken) { return nil, InvalidatedTokenError{} } else if errors.Is(err, arcgis.ErrorInvalidRefreshToken) { return nil, InvalidatedTokenError{} } - return nil, fmt.Errorf("Unrecognized error checking oauth validity: %w", err) + return nil, fmt.Errorf("Failed to create Fieldseeker client: %w", err) } return fssync, nil } -func updatePortalData(ctx context.Context, client *arcgis.ArcGIS, user_id int32) (*arcgis.PortalsResponse, error) { +func updatePortalData(ctx context.Context, client *arcgis.ArcGIS, user_id int32) (*response.Portal, error) { p, err := client.PortalsSelf(ctx) if err != nil { return nil, fmt.Errorf("Failed to get ArcGIS user data: %w", err) @@ -555,10 +536,7 @@ func exportFieldseekerData(ctx context.Context, fssync *fieldseeker.FieldSeeker, pool := pond.NewResultPool[SyncStats](20) group := pool.NewGroup() var ss SyncStats - layers, err := fssync.FeatureServerLayers(ctx) - if err != nil { - return fmt.Errorf("Failed to get layers: %w", err) - } + layers := fssync.Layers() for _, l := range layers { ss, err = exportFieldseekerLayer(ctx, group, org, fssync, l) if err != nil { @@ -1192,7 +1170,7 @@ func updateRowFromFeatureFS(ctx context.Context, transaction pgx.Tx, table strin return nil } -func exportFieldseekerLayer(ctx context.Context, group pond.ResultTaskGroup[SyncStats], org *models.Organization, fssync *fieldseeker.FieldSeeker, layer arcgis.LayerFeature) (SyncStats, error) { +func exportFieldseekerLayer(ctx context.Context, group pond.ResultTaskGroup[SyncStats], org *models.Organization, fssync *fieldseeker.FieldSeeker, layer response.Layer) (SyncStats, error) { var stats SyncStats count, err := fssync.QueryCount(ctx, layer.ID) if err != nil { diff --git a/go.mod b/go.mod index 5f893642..75f07ef0 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( golang.org/x/text v0.33.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + resty.dev/v3 v3.0.0-beta.6 // indirect ) // replace github.com/stephenafamo/bob v0.42.0 => ../bob diff --git a/go.sum b/go.sum index f2359a7f..025dd554 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Gleipnir-Technology/arcgis-go v0.0.6 h1:h21+ijW5NNAgRoBn2ktJwfmbgFX6f/CdlUojB4xQCWo= -github.com/Gleipnir-Technology/arcgis-go v0.0.6/go.mod h1:Stx2sn5Lvuyhy4SaTQpbLNCAfenboDINi/UU5gQvz4k= github.com/Gleipnir-Technology/bob v0.42.5 h1:fm4vH48E7scLwMFSJ4fX3+q2wSo+6Iphh+yVIrMgatE= github.com/Gleipnir-Technology/bob v0.42.5/go.mod h1:cjUNiSRIMBsk94NQQrpYoraCe0WxIc04C8A+PcJ5z8Q= github.com/Gleipnir-Technology/go-geojson2h3/v2 v2.0.0 h1:6OMVxoiX9r7dEkIyYYKtSu7I2UDq64dww4JxJTo3p78= @@ -379,3 +377,5 @@ modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +resty.dev/v3 v3.0.0-beta.6 h1:ghRdNpoE8/wBCv+kTKIOauW1aCrSIeTq7GxtfYgtevU= +resty.dev/v3 v3.0.0-beta.6/go.mod h1:NTOerrC/4T7/FE6tXIZGIysXXBdgNqwMZuKtxpea9NM= diff --git a/main.go b/main.go index cf70739a..eaa5ff55 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,6 @@ import ( "syscall" "time" - "github.com/Gleipnir-Technology/arcgis-go" "github.com/Gleipnir-Technology/nidus-sync/auth" "github.com/Gleipnir-Technology/nidus-sync/background" "github.com/Gleipnir-Technology/nidus-sync/config" @@ -144,9 +143,6 @@ func main() { log.Error().Err(err).Msg("Failed to start openAI client") os.Exit(8) } - custom_logger := log.With().Logger().Level(zerolog.InfoLevel) - background_ctx := arcgis.WithLogger(ctx, custom_logger) - background.Start(background_ctx) server := &http.Server{ Addr: config.Bind, Handler: r,