feat: add SAE last executed and last settled height metrics

RELEASES.md

@@ -2,6 +2,10 @@
 
 ## Pending (v1.14.3)
 
+### API
+
+- Added `avalanche_<vm>_sae_last_executed_height` and `avalanche_<vm>_sae_last_settled_height` gauges, exposing SAE async-execution and settlement heights
+
 ### Metrics
 
 - Renamed Coreth and Subnet-EVM state-sync p2p metrics (`{vmName}` is `evm` for Coreth/C-Chain and `subnetevm` for Subnet-EVM chains):

vms/saevm/sae/BUILD.bazel

@@ -23,6 +23,7 @@ go_library(
     importpath = "github.com/ava-labs/avalanchego/vms/saevm/sae",
     visibility = ["//visibility:public"],
     deps = [
+        "//api/metrics",
         "//database",
         "//ids",
         "//network/p2p",
@@ -109,6 +110,7 @@ go_test(
         "//vms/saevm/saedb",
         "//vms/saevm/saetest",
         "//vms/saevm/saetest/escrow",
+        "//vms/saevm/saexec",
         "//vms/saevm/txgossip/txgossiptest",
         "//vms/saevm/types",
         "@com_github_arr4n_shed//testerr",
@@ -139,6 +141,8 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_prometheus_client_model//go",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",
@@ -194,6 +198,7 @@ go_test(
         "//vms/saevm/saedb",
         "//vms/saevm/saetest",
         "//vms/saevm/saetest/escrow",
+        "//vms/saevm/saexec",
         "//vms/saevm/txgossip/txgossiptest",
         "//vms/saevm/types",
         "//vms/saevm/worstcase",
@@ -224,6 +229,8 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_prometheus_client_model//go",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",

vms/saevm/sae/consensus.go

@@ -75,6 +75,7 @@ func (vm *VM) AcceptBlock(ctx context.Context, b *blocks.Block) error {
 		if err := s.MarkSettled(&vm.last.settled); err != nil {
 			return err
 		}
+		vm.exec.MarkSettled(s.Height())
 	}
 
 	// I(s ∈ S) above, before I(b ∈ A) before X(b ∈ A)

vms/saevm/sae/vm.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ava-labs/libevm/params"
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/ava-labs/avalanchego/api/metrics"
 	"github.com/ava-labs/avalanchego/network/p2p"
 	"github.com/ava-labs/avalanchego/network/p2p/gossip"
 	"github.com/ava-labs/avalanchego/snow"
@@ -52,10 +53,10 @@ type VM struct {
 	Peers          *p2p.Peers
 	ValidatorPeers *p2p.Validators
 
-	hooks   hook.Points
-	config  Config
-	snowCtx *snow.Context
-	metrics *prometheus.Registry
+	hooks          hook.Points
+	config         Config
+	snowCtx        *snow.Context
+	metricRegistry *prometheus.Registry
 
 	db  ethdb.Database
 	xdb saetypes.ExecutionResults
@@ -123,23 +124,23 @@ func NewVM[T hook.Transaction](
 	if cfg.Now == nil {
 		cfg.Now = time.Now
 	}
+	reg, err := metrics.MakeAndRegister(snowCtx.Metrics, "sae")
+	if err != nil {
+		return nil, err
+	}
 	vm := &VM{
-		hooks:   hooks,
-		config:  cfg,
-		snowCtx: snowCtx,
-		metrics: prometheus.NewRegistry(),
-		db:      db,
+		hooks:          hooks,
+		config:         cfg,
+		snowCtx:        snowCtx,
+		metricRegistry: reg,
+		db:             db,
 	}
 	defer func() {
 		if retErr != nil {
 			retErr = errors.Join(retErr, vm.close())
 		}
 	}()
 
-	if err := snowCtx.Metrics.Register("sae", vm.metrics); err != nil {
-		return nil, err
-	}
-
 	xdb, err := hooks.ExecutionResultsDB(
 		filepath.Join(snowCtx.ChainDataDir, "sae_execution_results"),
 	)
@@ -180,6 +181,7 @@ func NewVM[T hook.Transaction](
 			cfg.DBConfig,
 			hooks,
 			snowCtx.Log,
+			vm.metricRegistry,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("saexec.New(...): %v", err)
@@ -201,6 +203,8 @@ func NewVM[T hook.Transaction](
 		vm.last.settled.Store(lastSettled)
 		vm.last.accepted.Store(head)
 		vm.preference.Store(head)
+		// [saexec.New] already records the initial executed height.
+		vm.exec.MarkSettled(lastSettled.Height())
 	}
 
 	{ // ==========  Mempool  ==========
@@ -214,11 +218,11 @@ func NewVM[T hook.Transaction](
 		}
 		vm.toClose = append(vm.toClose, txPool)
 
-		metrics, err := bloom.NewMetrics("mempool", vm.metrics)
+		bloomMetrics, err := bloom.NewMetrics("mempool", vm.metricRegistry)
 		if err != nil {
 			return nil, err
 		}
-		conf := gossip.BloomSetConfig{Metrics: metrics}
+		conf := gossip.BloomSetConfig{Metrics: bloomMetrics}
 		pool, err := txgossip.NewSet(txPool, conf)
 		if err != nil {
 			return nil, err
@@ -239,7 +243,7 @@ func NewVM[T hook.Transaction](
 	}
 
 	{ // ==========  P2P Gossip  ==========
-		network, peers, validatorPeers, err := newNetwork(snowCtx, sender, vm.metrics)
+		network, peers, validatorPeers, err := newNetwork(snowCtx, sender, vm.metricRegistry)
 		if err != nil {
 			return nil, fmt.Errorf("newNetwork(...): %v", err)
 		}
@@ -253,7 +257,7 @@ func NewVM[T hook.Transaction](
 			txgossip.Marshaller{},
 			gossip.SystemConfig{
 				Log:           snowCtx.Log,
-				Registry:      vm.metrics,
+				Registry:      vm.metricRegistry,
 				Namespace:     "gossip",
 				RequestPeriod: pullGossipPeriod,
 			},

vms/saevm/sae/vm_test.go

@@ -10,6 +10,7 @@ import (
 	"math/big"
 	"net/http/httptest"
 	"os"
+	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -32,6 +33,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/holiman/uint256"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -54,12 +56,14 @@ import (
 	"github.com/ava-labs/avalanchego/vms/saevm/hook"
 	"github.com/ava-labs/avalanchego/vms/saevm/hook/hookstest"
 	"github.com/ava-labs/avalanchego/vms/saevm/saetest"
+	"github.com/ava-labs/avalanchego/vms/saevm/saexec"
 	"github.com/ava-labs/avalanchego/vms/saevm/txgossip/txgossiptest"
 
 	snowcommon "github.com/ava-labs/avalanchego/snow/engine/common"
 	saeparams "github.com/ava-labs/avalanchego/vms/saevm/params"
 	saetypes "github.com/ava-labs/avalanchego/vms/saevm/types"
 	libevmhookstest "github.com/ava-labs/libevm/libevm/hookstest"
+	dto "github.com/prometheus/client_model/go"
 )
 
 func TestMain(m *testing.M) {
@@ -915,6 +919,34 @@ func TestGossip(t *testing.T) {
 	requireNotReceiveTx(t, nonValidators[1:], tx.Hash())
 }
 
+func TestSettlementMetric(t *testing.T) {
+	opt, vmTime := withVMTime(t, time.Unix(saeparams.TauSeconds, 0))
+	ctx, sut := newSUT(t, 1, opt)
+
+	executed := sut.runConsensusLoop(t)
+	require.NoErrorf(t, executed.WaitUntilExecuted(ctx), "%T.WaitUntilExecuted()", executed)
+
+	vmTime.advanceToSettle(ctx, t, executed)
+	settledBy := sut.runConsensusLoop(t)
+	require.NoErrorf(t, settledBy.WaitUntilExecuted(ctx), "%T.WaitUntilExecuted()", settledBy)
+	require.Equal(t, float64(executed.Height()), gaugeValue(t, sut.rawVM.metricRegistry, saexec.LastSettledHeightName), "last settled height")
+}
+
+// gaugeValue returns the current value of a single-series gauge from `g` by
+// name, failing the test if it is missing or has more than one series.
+func gaugeValue(t *testing.T, g prometheus.Gatherer, name string) float64 {
+	t.Helper()
+	mfs, err := g.Gather()
+	require.NoError(t, err, "Gather()")
+	i := slices.IndexFunc(mfs, func(mf *dto.MetricFamily) bool {
+		return mf.GetName() == name
+	})
+	require.GreaterOrEqualf(t, i, 0, "metric %q not found", name)
+	series := mfs[i].GetMetric()
+	require.Lenf(t, series, 1, "metric %q series count", name)
+	return series[0].GetGauge().GetValue()
+}
+
 func TestBlockSources(t *testing.T) {
 	opt, vmTime := withVMTime(t, time.Unix(saeparams.TauSeconds, 0))
 	ctx, sut := newSUT(t, 1, opt)

vms/saevm/saexec/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     srcs = [
         "context.go",
         "execution.go",
+        "metrics.go",
         "receipts.go",
         "saexec.go",
         "subscription.go",
@@ -32,6 +33,7 @@ go_library(
         "@com_github_ava_labs_libevm//libevm/eventual",
         "@com_github_ava_labs_libevm//params",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus",
         "@org_uber_go_zap//:zap",
     ],
 )
@@ -71,6 +73,8 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_prometheus_client_golang//prometheus/testutil",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",

vms/saevm/saexec/execution.go

@@ -292,16 +292,18 @@ func (e *Executor) afterExecution(b *blocks.Block, r *ExecutionResults) error {
 	// post-execution state to no longer be consensus-critical.
 	e.Tracker.Track(root)
 
-	// The strict ordering of the next 3 calls guarantees invariants that MUST
-	// NOT be broken:
+	// The strict ordering of the following calls guarantees invariants that
+	// MUST NOT be broken:
 	//
 	// 1. [blocks.Block.MarkExecuted] guarantees disk then in-memory changes.
 	// 2. Internal indicator of last executed MUST follow in-memory change.
-	// 3. External indicator of last executed MUST follow internal indicator.
+	// 3. Metrics indicator of last executed MUST follow internal indicator.
+	// 4. External indicator of last executed MUST follow internal indicator.
 	if err := b.MarkExecuted(e.db, e.xdb, r.FinishBy.Gas.Clone(), r.FinishBy.Wall, r.BaseFee.ToBig(), r.Receipts, root, &e.lastExecuted /* (2) */); err != nil {
 		return err
 	}
-	e.sendPostExecutionEvents(b.EthBlock(), r.Receipts) // (3)
+	e.metrics.markExecuted(b.Height())                  // (3)
+	e.sendPostExecutionEvents(b.EthBlock(), r.Receipts) // (4)
 	return nil
 }
 

vms/saevm/saexec/metrics.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2019, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package saexec
+
+import (
+	"errors"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Exported metric names for the SAE lifecycle bucket. Cross-package callers
+// (tests, alert/dashboard wiring) should reference these constants rather
+// than the raw strings.
+const (
+	LastExecutedHeightName = "last_executed_height"
+	LastSettledHeightName  = "last_settled_height"
+)
+
+// metrics holds the SAE block-lifecycle gauges. Both live here — the lowest
+// package that updates a member of the bucket — so one site owns
+// registration. Execution events update last_executed_height in-package;
+// settlement updates last_settled_height through [Executor.MarkSettled].
+type metrics struct {
+	lastExecutedHeight prometheus.Gauge
+	lastSettledHeight  prometheus.Gauge
+}
+
+func newMetrics(reg prometheus.Registerer) (*metrics, error) {
+	m := &metrics{
+		lastExecutedHeight: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: LastExecutedHeightName,
+			Help: "Height of the latest block that completed async execution.",
+		}),
+		lastSettledHeight: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: LastSettledHeightName,
+			Help: "Height of the latest block that has settled.",
+		}),
+	}
+	return m, errors.Join(
+		reg.Register(m.lastExecutedHeight),
+		reg.Register(m.lastSettledHeight),
+	)
+}
+
+func (m *metrics) markExecuted(height uint64) {
+	m.lastExecutedHeight.Set(float64(height))
+}
+
+func (m *metrics) markSettled(height uint64) {
+	m.lastSettledHeight.Set(float64(height))
+}

vms/saevm/saexec/saexec.go

@@ -8,6 +8,7 @@
 package saexec
 
 import (
+	"fmt"
 	"io"
 	"sync/atomic"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/ava-labs/libevm/event"
 	"github.com/ava-labs/libevm/libevm/eventual"
 	"github.com/ava-labs/libevm/params"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/ava-labs/avalanchego/cache/lru"
 	"github.com/ava-labs/avalanchego/utils/logging"
@@ -49,6 +51,7 @@ type Executor struct {
 	chainConfig  *params.ChainConfig
 	db           ethdb.Database
 	xdb          saetypes.ExecutionResults
+	metrics      *metrics
 }
 
 // New constructs and starts a new [Executor]. Call [Executor.Close] to release
@@ -66,12 +69,18 @@ func New(
 	saedbConfig saedb.Config,
 	hooks hook.Points,
 	log logging.Logger,
+	reg prometheus.Registerer,
 ) (*Executor, error) {
 	t, err := saedb.NewTracker(db, saedbConfig, lastExecuted.PostExecutionStateRoot(), log)
 	if err != nil {
 		return nil, err
 	}
 
+	m, err := newMetrics(reg)
+	if err != nil {
+		return nil, fmt.Errorf("registering saexec metrics: %w", err)
+	}
+
 	e := &Executor{
 		Tracker: t,
 		quit:    make(chan struct{}), // closed by [Executor.Close]
@@ -90,14 +99,22 @@ func New(
 		chainConfig: chainConfig,
 		db:          db,
 		xdb:         xdb,
+		metrics:     m,
 		receipts:    newSyncMap[common.Hash, eventual.Value[*Receipt]](),
 	}
 	e.lastExecuted.Store(lastExecuted)
+	e.metrics.markExecuted(lastExecuted.Height())
 
 	go e.processQueue()
 	return e, nil
 }
 
+// MarkSettled records height as the latest settled height. See [metrics]
+// for why the lifecycle gauge is colocated in this package.
+func (e *Executor) MarkSettled(height uint64) {
+	e.metrics.markSettled(height)
+}
+
 var _ io.Closer = (*Executor)(nil)
 
 // Close shuts down the [Executor], waits for the currently executing block