diff --git a/api/openapi.json b/api/openapi.json index ddc619b62..bd0bd267a 100644 --- a/api/openapi.json +++ b/api/openapi.json @@ -10638,6 +10638,7 @@ } }, { + "description": "Deprecated: served from a cache, so results may lag behind real-time. Use `last_account_id` for live, real-time pagination instead.", "in": "query", "name": "offset", "schema": { @@ -10647,6 +10648,16 @@ "minimum": 0, "type": "integer" } + }, + { + "description": "Cursor for pagination, always resolved live. Pass the `metadata.address` of the last jetton master from the previous page to get the next page. Preferred over `offset`.", + "in": "query", + "name": "last_account_id", + "schema": { + "example": "0:97264395BD65A255A429B11326C84128B7D70FFED7949ABAE3036D506BA38621", + "format": "address", + "type": "string" + } } ], "responses": { diff --git a/api/openapi.yml b/api/openapi.yml index 55ca4bc24..2781aa6d2 100644 --- a/api/openapi.yml +++ b/api/openapi.yml @@ -1587,12 +1587,24 @@ paths: minimum: 1 - name: offset in: query + description: >- + Deprecated: served from a cache, so results may lag behind real-time. Use `last_account_id` for + live, real-time pagination instead. schema: type: integer format: int32 default: 0 example: 10 minimum: 0 + - name: last_account_id + in: query + description: >- + Cursor for pagination, always resolved live. Pass the `metadata.address` of the last jetton master + from the previous page to get the next page. Preferred over `offset`. + schema: + type: string + format: address + example: 0:97264395BD65A255A429B11326C84128B7D70FFED7949ABAE3036D506BA38621 responses: '200': description: a list of jettons diff --git a/go.mod b/go.mod index 5290a2418..c15f670bb 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/sourcegraph/conc v0.3.0 github.com/stretchr/testify v1.11.1 github.com/tonkeeper/scam_backoffice_rules v0.0.12 - github.com/tonkeeper/tongo v1.22.47 + github.com/tonkeeper/tongo v1.22.48 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 diff --git a/go.sum b/go.sum index 75334a9e1..8277b44d7 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/tonkeeper/scam_backoffice_rules v0.0.12 h1:vCxE77SEddtPTWCaTH1E3tZ9Ss github.com/tonkeeper/scam_backoffice_rules v0.0.12/go.mod h1:SqZXYO9vbID8ku+xnnaKXeNGmehxigODGrk5V1KqbRA= github.com/tonkeeper/tongo v1.22.47 h1:srweJ0PlupSw7AW+/zNCjHzA8uZuQA4kALayplqM2R0= github.com/tonkeeper/tongo v1.22.47/go.mod h1:nHmdEXPfT0/EvkEaBzPiY599/0OYjQSW4dWR7aX+OII= +github.com/tonkeeper/tongo v1.22.48 h1:yqjBXKOIfax7hb3e2NCEOyoPq0pU+eC+aLNpiqnKhJA= +github.com/tonkeeper/tongo v1.22.48/go.mod h1:nHmdEXPfT0/EvkEaBzPiY599/0OYjQSW4dWR7aX+OII= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= diff --git a/pkg/api/event_converters.go b/pkg/api/event_converters.go index 370d55fb9..b49d8ecb9 100644 --- a/pkg/api/event_converters.go +++ b/pkg/api/event_converters.go @@ -103,6 +103,7 @@ func (h *Handler) convertRisk(ctx context.Context, risk wallet.Risk, walletAddre var curPrice float64 var todayRates map[string]float64 var err error + currencyKnown := currency != nil if currency != nil { todayRates, _, _, _, err = h.getRates() if err != nil { @@ -111,7 +112,8 @@ func (h *Handler) convertRisk(ctx context.Context, risk wallet.Risk, walletAddre var prs bool curPrice, prs = todayRates[strings.ToUpper(*currency)] if !prs { - return oas.Risk{}, fmt.Errorf("can't calculate risk: unknown currency") + h.logger.Warn("can't calculate risk: unknown currency", zap.String("currency", *currency)) + currencyKnown = false } } if risk.TransferAllRemainingBalance { @@ -155,7 +157,7 @@ func (h *Handler) convertRisk(ctx context.Context, risk wallet.Risk, walletAddre } oasRisk.Jettons = append(oasRisk.Jettons, jettonQuantity) } - if currency != nil { + if currencyKnown { oasRisk.TotalEquivalent = oas.NewOptFloat32(float32(total / curPrice)) } if len(risk.Nfts) > 0 { diff --git a/pkg/api/handler.go b/pkg/api/handler.go index 879b34321..1c290445e 100644 --- a/pkg/api/handler.go +++ b/pkg/api/handler.go @@ -79,6 +79,10 @@ type Handler struct { mu sync.Mutex dns *dns.DNS // todo: update when blockchain config changes configPool *sync.Pool + + // jettonMasters serves legacy offset-based /v2/jettons requests. May be nil, in which case + // offset-based requests fail rather than being silently served as page one. + jettonMasters jettonMastersOffsetSource } func (h *Handler) NewError(ctx context.Context, err error) *oas.ErrorStatusCode { @@ -108,6 +112,8 @@ type Options struct { parallelTraceProcessing bool archiveLiteServers []config.LiteServer publicAPIURL string + + jettonMastersOffsetSource jettonMastersOffsetSource } type Option func(o *Options) @@ -216,6 +222,14 @@ func WithPublicAPIURL(publicAPIURL string) Option { } } +// WithJettonMastersOffsetSource registers the source used to serve legacy offset-based +// /v2/jettons requests. When not set, offset-based requests fail with an error. +func WithJettonMastersOffsetSource(source jettonMastersOffsetSource) Option { + return func(o *Options) { + o.jettonMastersOffsetSource = source + } +} + func NewHandler(logger *zap.Logger, opts ...Option) (*Handler, error) { options := &Options{} for _, o := range opts { @@ -319,6 +333,7 @@ func NewHandler(logger *zap.Logger, opts ...Option) (*Handler, error) { configPool: configPool, rewards: rwd, stats: rewards.NewStats(liteapi.WithLiteServers(options.archiveLiteServers)), + jettonMasters: options.jettonMastersOffsetSource, }, nil } diff --git a/pkg/api/interfaces.go b/pkg/api/interfaces.go index 22281003c..86831e266 100644 --- a/pkg/api/interfaces.go +++ b/pkg/api/interfaces.go @@ -101,7 +101,7 @@ type storage interface { GetWalletAddressesByPubkeys(ctx context.Context, pubKeys []ed25519.PublicKey) (map[string]map[ton.AccountID]abi.ContractInterface, error) GetSubscriptionsV2(ctx context.Context, address tongo.AccountID) ([]core.SubscriptionV2, error) GetSubscriptionsV1(ctx context.Context, address tongo.AccountID) ([]core.SubscriptionV1, error) - GetJettonMasters(ctx context.Context, limit, offset int) ([]core.JettonMaster, error) + GetJettonMasters(ctx context.Context, limit int, lastAccountID *tongo.AccountID) ([]core.JettonMaster, error) GetJettonMastersByAddresses(ctx context.Context, addresses []ton.AccountID) ([]core.JettonMaster, error) GetLastConfig(ctx context.Context) (ton.BlockchainConfig, error) @@ -153,6 +153,11 @@ type liteStorageRaw interface { GetOutMsgQueueSizes(ctx context.Context) (liteclient.LiteServerOutMsgQueueSizesC, error) } +// jettonMastersOffsetSource serves legacy offset-based /v2/jettons pagination without ever running an OFFSET query against storage +type jettonMastersOffsetSource interface { + Slice(limit, offset int) []core.JettonMaster +} + // chainState provides current blockchain state which change very rarely or slow like staking APY income type chainState interface { GetAPY() float64 diff --git a/pkg/api/jetton_handlers.go b/pkg/api/jetton_handlers.go index 6963f8843..a2a705f30 100644 --- a/pkg/api/jetton_handlers.go +++ b/pkg/api/jetton_handlers.go @@ -198,11 +198,27 @@ func (h *Handler) GetJettons(ctx context.Context, params oas.GetJettonsParams) ( limit = 1000 } } - offset := 0 - if params.Offset.IsSet() { - offset = max(int(params.Offset.Value), 0) + var jettons []core.JettonMaster + var err error + switch { + case params.LastAccountID.IsSet(): + var account ton.Address + account, err = tongo.ParseAddress(params.LastAccountID.Value) + if err != nil { + return nil, toError(http.StatusBadRequest, fmt.Errorf("last_account_id: %w", err)) + } + jettons, err = h.storage.GetJettonMasters(ctx, limit, &account.ID) + case params.Offset.IsSet() && params.Offset.Value > 0: + if h.jettonMasters != nil { + offset := max(int(params.Offset.Value), 0) + jettons = h.jettonMasters.Slice(limit, offset) + } else { + h.logger.Error("received an offset-based /v2/jettons request, but no jettonMastersOffsetSource is configured") + return nil, toError(http.StatusInternalServerError, fmt.Errorf("offset-based pagination is currently not available")) + } + default: + jettons, err = h.storage.GetJettonMasters(ctx, limit, nil) } - jettons, err := h.storage.GetJettonMasters(ctx, limit, offset) if err != nil { return nil, toError(http.StatusInternalServerError, err) } diff --git a/pkg/litestorage/jetton.go b/pkg/litestorage/jetton.go index aa5184a27..b2d118267 100644 --- a/pkg/litestorage/jetton.go +++ b/pkg/litestorage/jetton.go @@ -183,7 +183,7 @@ func (s *LiteStorage) JettonMastersForWallets(ctx context.Context, wallets []ton return masters, nil } -func (s *LiteStorage) GetJettonMasters(ctx context.Context, limit, offset int) ([]core.JettonMaster, error) { +func (s *LiteStorage) GetJettonMasters(ctx context.Context, limit int, lastAccountID *tongo.AccountID) ([]core.JettonMaster, error) { // TODO: implement return []core.JettonMaster{}, nil } diff --git a/pkg/oas/oas_client_gen.go b/pkg/oas/oas_client_gen.go index 0be223871..1d17b76c0 100644 --- a/pkg/oas/oas_client_gen.go +++ b/pkg/oas/oas_client_gen.go @@ -8016,6 +8016,23 @@ func (c *Client) sendGetJettons(ctx context.Context, params GetJettonsParams) (r return res, errors.Wrap(err, "encode query") } } + { + // Encode "last_account_id" parameter. + cfg := uri.QueryParameterEncodingConfig{ + Name: "last_account_id", + Style: uri.QueryStyleForm, + Explode: true, + } + + if err := q.EncodeParam(cfg, func(e uri.Encoder) error { + if val, ok := params.LastAccountID.Get(); ok { + return e.EncodeValue(conv.StringToString(val)) + } + return nil + }); err != nil { + return res, errors.Wrap(err, "encode query") + } + } u.RawQuery = q.Values().Encode() stage = "EncodeRequest" diff --git a/pkg/oas/oas_handlers_gen.go b/pkg/oas/oas_handlers_gen.go index fb33920ac..666492a0d 100644 --- a/pkg/oas/oas_handlers_gen.go +++ b/pkg/oas/oas_handlers_gen.go @@ -10189,6 +10189,10 @@ func (s *Server) handleGetJettonsRequest(args [0]string, argsEscaped bool, w htt Name: "offset", In: "query", }: params.Offset, + { + Name: "last_account_id", + In: "query", + }: params.LastAccountID, }, Raw: r, } diff --git a/pkg/oas/oas_parameters_gen.go b/pkg/oas/oas_parameters_gen.go index 024c92c38..833d0cc00 100644 --- a/pkg/oas/oas_parameters_gen.go +++ b/pkg/oas/oas_parameters_gen.go @@ -8316,8 +8316,13 @@ func decodeGetJettonTransferPayloadParams(args [2]string, argsEscaped bool, r *h // GetJettonsParams is parameters of getJettons operation. type GetJettonsParams struct { - Limit OptInt32 `json:",omitempty,omitzero"` + Limit OptInt32 `json:",omitempty,omitzero"` + // Deprecated: served from a cache, so results may lag behind real-time. Use `last_account_id` for + // live, real-time pagination instead. Offset OptInt32 `json:",omitempty,omitzero"` + // Cursor for pagination, always resolved live. Pass the `metadata.address` of the last jetton master + // from the previous page to get the next page. Preferred over `offset`. + LastAccountID OptString `json:",omitempty,omitzero"` } func unpackGetJettonsParams(packed middleware.Parameters) (params GetJettonsParams) { @@ -8339,6 +8344,15 @@ func unpackGetJettonsParams(packed middleware.Parameters) (params GetJettonsPara params.Offset = v.(OptInt32) } } + { + key := middleware.ParameterKey{ + Name: "last_account_id", + In: "query", + } + if v, ok := packed[key]; ok { + params.LastAccountID = v.(OptString) + } + } return params } @@ -8486,6 +8500,47 @@ func decodeGetJettonsParams(args [0]string, argsEscaped bool, r *http.Request) ( Err: err, } } + // Decode query: last_account_id. + if err := func() error { + cfg := uri.QueryParameterDecodingConfig{ + Name: "last_account_id", + Style: uri.QueryStyleForm, + Explode: true, + } + + if err := q.HasParam(cfg); err == nil { + if err := q.DecodeParam(cfg, func(d uri.Decoder) error { + var paramsDotLastAccountIDVal string + if err := func() error { + val, err := d.DecodeValue() + if err != nil { + return err + } + + c, err := conv.ToString(val) + if err != nil { + return err + } + + paramsDotLastAccountIDVal = c + return nil + }(); err != nil { + return err + } + params.LastAccountID.SetTo(paramsDotLastAccountIDVal) + return nil + }); err != nil { + return err + } + } + return nil + }(); err != nil { + return params, &ogenerrors.DecodeParamError{ + Name: "last_account_id", + In: "query", + Err: err, + } + } return params, nil }