feat: add SAE last executed and last settled height metrics

RELEASES.md

@@ -2,6 +2,10 @@
 
 ## Pending (v1.14.3)
 
+### API
+
+- Added `avalanche_<vm>_sae_last_executed_height` and `avalanche_<vm>_sae_last_settled_height` gauges, exposing SAE async-execution and settlement heights
+
 ### Metrics
 
 - Renamed Coreth and Subnet-EVM state-sync p2p metrics (`{vmName}` is `evm` for Coreth/C-Chain and `subnetevm` for Subnet-EVM chains):

vms/saevm/metrics/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//.bazel:defs.bzl", "go_test")
+
+go_library(
+    name = "metrics",
+    srcs = ["metrics.go"],
+    importpath = "github.com/ava-labs/avalanchego/vms/saevm/metrics",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_prometheus_client_golang//prometheus"],
+)
+
+go_test(
+    name = "metrics_test",
+    srcs = ["metrics_test.go"],
+    embed = [":metrics"],
+    deps = [
+        "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_prometheus_client_golang//prometheus/testutil",
+        "@com_github_stretchr_testify//require",
+    ],
+)

vms/saevm/metrics/metrics.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2019, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// metrics defines SAE Prometheus collectors. These lifecycle
+// metrics are useful in their own right for configuring dashboards
+// and alerts and can also be compared with Snowman metrics to derive
+// additional metrics like settlement lag.
+package metrics
+
+import (
+	"errors"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds SAE Prometheus collectors and provides update methods.
+type Metrics struct {
+	LastExecutedHeight prometheus.Gauge
+	LastSettledHeight  prometheus.Gauge
+}
+
+// New constructs and registers SAE metrics.
+func New(reg prometheus.Registerer) (*Metrics, error) {
+	m := &Metrics{
+		LastExecutedHeight: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "last_executed_height",
+			Help: "Height of the latest block that completed async execution.",
+		}),
+		LastSettledHeight: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "last_settled_height",
+			Help: "Height of the latest block that has settled.",
+		}),
+	}
+
+	return m, errors.Join(
+		reg.Register(m.LastExecutedHeight),
+		reg.Register(m.LastSettledHeight),
+	)
+}
+
+// MarkBlockExecuted updates metrics for a block that completed async execution.
+func (m *Metrics) MarkBlockExecuted(height uint64) {
+	m.LastExecutedHeight.Set(float64(height))
+}
+
+// MarkBlockSettled updates metrics for a block that has settled.
+func (m *Metrics) MarkBlockSettled(height uint64) {
+	m.LastSettledHeight.Set(float64(height))
+}

vms/saevm/metrics/metrics_test.go

@@ -0,0 +1,23 @@
+// Copyright (C) 2019, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetrics(t *testing.T) {
+	metrics, err := New(prometheus.NewRegistry())
+	require.NoError(t, err, "New()")
+
+	metrics.MarkBlockExecuted(6)
+	metrics.MarkBlockSettled(7)
+
+	require.Equal(t, float64(6), testutil.ToFloat64(metrics.LastExecutedHeight), "last executed height")
+	require.Equal(t, float64(7), testutil.ToFloat64(metrics.LastSettledHeight), "last settled height")
+}

vms/saevm/sae/BUILD.bazel

@@ -23,6 +23,7 @@ go_library(
     importpath = "github.com/ava-labs/avalanchego/vms/saevm/sae",
     visibility = ["//visibility:public"],
     deps = [
+        "//api/metrics",
         "//database",
         "//ids",
         "//network/p2p",
@@ -40,6 +41,7 @@ go_library(
         "//vms/saevm/adaptor",
         "//vms/saevm/blocks",
         "//vms/saevm/hook",
+        "//vms/saevm/metrics",
         "//vms/saevm/params",
         "//vms/saevm/proxytime",
         "//vms/saevm/sae/rpc",
@@ -137,6 +139,7 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus/testutil",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",
@@ -221,6 +224,7 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus/testutil",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",

vms/saevm/sae/consensus.go

@@ -75,6 +75,7 @@ func (vm *VM) AcceptBlock(ctx context.Context, b *blocks.Block) error {
 		if err := s.MarkSettled(&vm.last.settled); err != nil {
 			return err
 		}
+		vm.metrics.MarkBlockSettled(s.Height())
 	}
 
 	// I(s ∈ S) above, before I(b ∈ A) before X(b ∈ A)

vms/saevm/sae/vm.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ava-labs/libevm/params"
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/ava-labs/avalanchego/api/metrics"
 	"github.com/ava-labs/avalanchego/network/p2p"
 	"github.com/ava-labs/avalanchego/network/p2p/gossip"
 	"github.com/ava-labs/avalanchego/snow"
@@ -41,6 +42,7 @@ import (
 	"github.com/ava-labs/avalanchego/vms/saevm/txgossip"
 
 	snowcommon "github.com/ava-labs/avalanchego/snow/engine/common"
+	saemetrics "github.com/ava-labs/avalanchego/vms/saevm/metrics"
 	saetypes "github.com/ava-labs/avalanchego/vms/saevm/types"
 )
 
@@ -52,10 +54,11 @@ type VM struct {
 	Peers          *p2p.Peers
 	ValidatorPeers *p2p.Validators
 
-	hooks   hook.Points
-	config  Config
-	snowCtx *snow.Context
-	metrics *prometheus.Registry
+	hooks          hook.Points
+	config         Config
+	snowCtx        *snow.Context
+	metricRegistry *prometheus.Registry
+	metrics        *saemetrics.Metrics
 
 	db  ethdb.Database
 	xdb saetypes.ExecutionResults
@@ -123,21 +126,26 @@ func NewVM[T hook.Transaction](
 	if cfg.Now == nil {
 		cfg.Now = time.Now
 	}
+	reg, err := metrics.MakeAndRegister(snowCtx.Metrics, "sae")
+	if err != nil {
+		return nil, err
+	}
 	vm := &VM{
-		hooks:   hooks,
-		config:  cfg,
-		snowCtx: snowCtx,
-		metrics: prometheus.NewRegistry(),
-		db:      db,
+		hooks:          hooks,
+		config:         cfg,
+		snowCtx:        snowCtx,
+		metricRegistry: reg,
+		db:             db,
 	}
 	defer func() {
 		if retErr != nil {
 			retErr = errors.Join(retErr, vm.close())
 		}
 	}()
 
-	if err := snowCtx.Metrics.Register("sae", vm.metrics); err != nil {
-		return nil, err
+	vm.metrics, err = saemetrics.New(vm.metricRegistry)
+	if err != nil {
+		return nil, fmt.Errorf("new metrics: %w", err)
 	}
 
 	xdb, err := hooks.ExecutionResultsDB(
@@ -180,6 +188,7 @@ func NewVM[T hook.Transaction](
 			cfg.DBConfig,
 			hooks,
 			snowCtx.Log,
+			vm.metrics,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("saexec.New(...): %v", err)
@@ -201,6 +210,8 @@ func NewVM[T hook.Transaction](
 		vm.last.settled.Store(lastSettled)
 		vm.last.accepted.Store(head)
 		vm.preference.Store(head)
+		vm.metrics.MarkBlockExecuted(head.Height())
+		vm.metrics.MarkBlockSettled(lastSettled.Height())
 	}
 
 	{ // ==========  Mempool  ==========
@@ -214,11 +225,11 @@ func NewVM[T hook.Transaction](
 		}
 		vm.toClose = append(vm.toClose, txPool)
 
-		metrics, err := bloom.NewMetrics("mempool", vm.metrics)
+		bloomMetrics, err := bloom.NewMetrics("mempool", vm.metricRegistry)
 		if err != nil {
 			return nil, err
 		}
-		conf := gossip.BloomSetConfig{Metrics: metrics}
+		conf := gossip.BloomSetConfig{Metrics: bloomMetrics}
 		pool, err := txgossip.NewSet(txPool, conf)
 		if err != nil {
 			return nil, err
@@ -239,7 +250,7 @@ func NewVM[T hook.Transaction](
 	}
 
 	{ // ==========  P2P Gossip  ==========
-		network, peers, validatorPeers, err := newNetwork(snowCtx, sender, vm.metrics)
+		network, peers, validatorPeers, err := newNetwork(snowCtx, sender, vm.metricRegistry)
 		if err != nil {
 			return nil, fmt.Errorf("newNetwork(...): %v", err)
 		}
@@ -253,7 +264,7 @@ func NewVM[T hook.Transaction](
 			txgossip.Marshaller{},
 			gossip.SystemConfig{
 				Log:           snowCtx.Log,
-				Registry:      vm.metrics,
+				Registry:      vm.metricRegistry,
 				Namespace:     "gossip",
 				RequestPeriod: pullGossipPeriod,
 			},

vms/saevm/sae/vm_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/holiman/uint256"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -915,6 +916,20 @@ func TestGossip(t *testing.T) {
 	requireNotReceiveTx(t, nonValidators[1:], tx.Hash())
 }
 
+func TestSettlementMetric(t *testing.T) {
+	opt, vmTime := withVMTime(t, time.Unix(saeparams.TauSeconds, 0))
+	ctx, sut := newSUT(t, 1, opt)
+	metrics := sut.rawVM.metrics
+
+	executed := sut.runConsensusLoop(t)
+	require.NoErrorf(t, executed.WaitUntilExecuted(ctx), "%T.WaitUntilExecuted()", executed)
+
+	vmTime.advanceToSettle(ctx, t, executed)
+	settledBy := sut.runConsensusLoop(t)
+	require.NoErrorf(t, settledBy.WaitUntilExecuted(ctx), "%T.WaitUntilExecuted()", settledBy)
+	require.Equal(t, float64(executed.Height()), testutil.ToFloat64(metrics.LastSettledHeight), "last settled height")
+}
+
 func TestBlockSources(t *testing.T) {
 	opt, vmTime := withVMTime(t, time.Unix(saeparams.TauSeconds, 0))
 	ctx, sut := newSUT(t, 1, opt)

vms/saevm/saexec/BUILD.bazel

@@ -19,6 +19,7 @@ go_library(
         "//vms/saevm/blocks",
         "//vms/saevm/gastime",
         "//vms/saevm/hook",
+        "//vms/saevm/metrics",
         "//vms/saevm/saedb",
         "//vms/saevm/types",
         "@com_github_ava_labs_libevm//common",
@@ -49,6 +50,7 @@ go_test(
         "//vms/saevm/cmputils",
         "//vms/saevm/gastime",
         "//vms/saevm/hook/hookstest",
+        "//vms/saevm/metrics",
         "//vms/saevm/proxytime",
         "//vms/saevm/saedb",
         "//vms/saevm/saetest",
@@ -71,6 +73,8 @@ go_test(
         "@com_github_google_go_cmp//cmp",
         "@com_github_google_go_cmp//cmp/cmpopts",
         "@com_github_holiman_uint256//:uint256",
+        "@com_github_prometheus_client_golang//prometheus",
+        "@com_github_prometheus_client_golang//prometheus/testutil",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",

vms/saevm/saexec/execution.go

@@ -292,16 +292,18 @@ func (e *Executor) afterExecution(b *blocks.Block, r *ExecutionResults) error {
 	// post-execution state to no longer be consensus-critical.
 	e.Tracker.Track(root)
 
-	// The strict ordering of the next 3 calls guarantees invariants that MUST
-	// NOT be broken:
+	// The strict ordering of the following calls guarantees invariants that
+	// MUST NOT be broken:
 	//
 	// 1. [blocks.Block.MarkExecuted] guarantees disk then in-memory changes.
 	// 2. Internal indicator of last executed MUST follow in-memory change.
-	// 3. External indicator of last executed MUST follow internal indicator.
+	// 3. Metrics indicator of last executed MUST follow internal indicator.
+	// 4. External indicator of last executed MUST follow internal indicator.
 	if err := b.MarkExecuted(e.db, e.xdb, r.FinishBy.Gas.Clone(), r.FinishBy.Wall, r.BaseFee.ToBig(), r.Receipts, root, &e.lastExecuted /* (2) */); err != nil {
 		return err
 	}
-	e.sendPostExecutionEvents(b.EthBlock(), r.Receipts) // (3)
+	e.metrics.MarkBlockExecuted(b.Height())             // (3)
+	e.sendPostExecutionEvents(b.EthBlock(), r.Receipts) // (4)
 	return nil
 }
 

vms/saevm/saexec/saexec.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ava-labs/avalanchego/vms/saevm/hook"
 	"github.com/ava-labs/avalanchego/vms/saevm/saedb"
 
+	saemetrics "github.com/ava-labs/avalanchego/vms/saevm/metrics"
 	saetypes "github.com/ava-labs/avalanchego/vms/saevm/types"
 )
 
@@ -49,6 +50,7 @@ type Executor struct {
 	chainConfig  *params.ChainConfig
 	db           ethdb.Database
 	xdb          saetypes.ExecutionResults
+	metrics      *saemetrics.Metrics
 }
 
 // New constructs and starts a new [Executor]. Call [Executor.Close] to release
@@ -66,6 +68,7 @@ func New(
 	saedbConfig saedb.Config,
 	hooks hook.Points,
 	log logging.Logger,
+	metrics *saemetrics.Metrics,
 ) (*Executor, error) {
 	t, err := saedb.NewTracker(db, saedbConfig, lastExecuted.PostExecutionStateRoot(), log)
 	if err != nil {
@@ -90,6 +93,7 @@ func New(
 		chainConfig: chainConfig,
 		db:          db,
 		xdb:         xdb,
+		metrics:     metrics,
 		receipts:    newSyncMap[common.Hash, eventual.Value[*Receipt]](),
 	}
 	e.lastExecuted.Store(lastExecuted)