feat(tracing): enrich OTel resource, compress OTLP, annotate spans

cmd/bee/cmd/cmd.go

@@ -40,10 +40,10 @@ const (
 	optionWelcomeMessage                   = "welcome-message"
 	optionCORSAllowedOrigins               = "cors-allowed-origins"
 	optionNameTracingEnabled               = "tracing-enable"
-	optionNameTracingOTLPEndpoint          = "tracing-otlp-endpoint"
-	optionNameTracingOTLPInsecure          = "tracing-otlp-insecure"
-	optionNameTracingOTLPCAFile            = "tracing-otlp-ca-file"
-	optionNameTracingOTLPProtocol          = "tracing-otlp-protocol"
+	optionNameTracingEndpoint              = "tracing-endpoint"
+	optionNameTracingInsecure              = "tracing-insecure"
+	optionNameTracingCAFile                = "tracing-ca-file"
+	optionNameTracingProtocol              = "tracing-protocol"
 	optionNameTracingSamplingRatio         = "tracing-sampling-ratio"
 	optionNameTracingServiceName           = "tracing-service-name"
 	optionNameVerbosity                    = "verbosity"
@@ -108,10 +108,10 @@ const (
 
 	// tracing
 	configKeyTracingEnabled       = "tracing.enable"
-	configKeyTracingOTLPEndpoint  = "tracing.otlp-endpoint"
-	configKeyTracingOTLPInsecure  = "tracing.otlp-insecure"
-	configKeyTracingOTLPCAFile    = "tracing.otlp-ca-file"
-	configKeyTracingOTLPProtocol  = "tracing.otlp-protocol"
+	configKeyTracingEndpoint      = "tracing.endpoint"
+	configKeyTracingInsecure      = "tracing.insecure"
+	configKeyTracingCAFile        = "tracing.ca-file"
+	configKeyTracingProtocol      = "tracing.protocol"
 	configKeyTracingSamplingRatio = "tracing.sampling-ratio"
 	configKeyTracingServiceName   = "tracing.service-name"
 )
@@ -126,28 +126,30 @@ var blockchainRpcConfigPairs = []struct{ flat, dotted string }{
 
 var tracingConfigPairs = []struct{ flat, dotted string }{
 	{optionNameTracingEnabled, configKeyTracingEnabled},
-	{optionNameTracingOTLPEndpoint, configKeyTracingOTLPEndpoint},
-	{optionNameTracingOTLPInsecure, configKeyTracingOTLPInsecure},
-	{optionNameTracingOTLPCAFile, configKeyTracingOTLPCAFile},
-	{optionNameTracingOTLPProtocol, configKeyTracingOTLPProtocol},
+	{optionNameTracingEndpoint, configKeyTracingEndpoint},
+	{optionNameTracingInsecure, configKeyTracingInsecure},
+	{optionNameTracingCAFile, configKeyTracingCAFile},
+	{optionNameTracingProtocol, configKeyTracingProtocol},
 	{optionNameTracingSamplingRatio, configKeyTracingSamplingRatio},
 	{optionNameTracingServiceName, configKeyTracingServiceName},
 }
 
 // Deprecated tracing options, removed in the OpenTelemetry migration. They are
 // kept only as hidden no-op flags so that existing configs do not break node
 // startup; their values are ignored (see deprecatedTracingKeys).
+//
+// Note: tracing-endpoint is intentionally NOT here — it is reused as the active
+// OTLP endpoint flag (see optionNameTracingEndpoint). Its meaning changed from a
+// Jaeger agent address to an OTLP collector endpoint.
 const (
-	optionNameTracingEndpoint = "tracing-endpoint"
-	optionNameTracingHost     = "tracing-host"
-	optionNameTracingPort     = "tracing-port"
+	optionNameTracingHost = "tracing-host"
+	optionNameTracingPort = "tracing-port"
 )
 
 // deprecatedTracingKeys are the pre-OpenTelemetry tracing options. They are
 // registered as hidden no-op flags so stale configs still start the node, but
-// their values are ignored; operators should migrate to the tracing-otlp-* keys.
+// their values are ignored; operators should migrate to the tracing-* keys.
 var deprecatedTracingKeys = []string{
-	optionNameTracingEndpoint,
 	optionNameTracingHost,
 	optionNameTracingPort,
 }
@@ -326,17 +328,17 @@ func (c *command) setAllFlags(cmd *cobra.Command) {
 	cmd.Flags().Uint64(optionNameNetworkID, chaincfg.Mainnet.NetworkID, "ID of the Swarm network")
 	cmd.Flags().StringSlice(optionCORSAllowedOrigins, []string{}, "origins with CORS headers enabled")
 	cmd.Flags().Bool(optionNameTracingEnabled, false, "enable tracing")
-	cmd.Flags().String(optionNameTracingOTLPEndpoint, "127.0.0.1:4318", "OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc")
-	cmd.Flags().Bool(optionNameTracingOTLPInsecure, false, "disable TLS for the OTLP exporter (useful for a local collector); when false, set --tracing-otlp-ca-file to verify the collector certificate against a private CA")
-	cmd.Flags().String(optionNameTracingOTLPCAFile, "", "path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when --tracing-otlp-insecure=true")
-	cmd.Flags().String(optionNameTracingOTLPProtocol, "http", "OTLP exporter transport: http or grpc")
+	cmd.Flags().String(optionNameTracingEndpoint, "127.0.0.1:4318", "OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc")
+	cmd.Flags().Bool(optionNameTracingInsecure, false, "disable TLS for the OTLP exporter (useful for a local collector); when false, set --tracing-ca-file to verify the collector certificate against a private CA")
+	cmd.Flags().String(optionNameTracingCAFile, "", "path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when --tracing-insecure=true")
+	cmd.Flags().String(optionNameTracingProtocol, "http", "OTLP exporter transport: http or grpc")
 	cmd.Flags().Float64(optionNameTracingSamplingRatio, 1.0, "head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything")
 	cmd.Flags().String(optionNameTracingServiceName, "bee", "service name identifier for tracing")
 	// Deprecated, no-op tracing flags kept for backward compatibility so that
 	// existing configs do not break node startup. Their values are ignored.
 	for _, name := range deprecatedTracingKeys {
-		cmd.Flags().String(name, "", "deprecated and ignored; use the tracing-otlp-* options")
-		_ = cmd.Flags().MarkDeprecated(name, "no longer used after the OpenTelemetry migration; use the tracing-otlp-* options")
+		cmd.Flags().String(name, "", "deprecated and ignored; use the tracing-* options")
+		_ = cmd.Flags().MarkDeprecated(name, "no longer used after the OpenTelemetry migration; use the tracing-* options")
 	}
 	cmd.Flags().String(optionNameVerbosity, "info", "log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace")
 	cmd.Flags().String(optionWelcomeMessage, "", "send a welcome message string during handshakes")
@@ -423,8 +425,8 @@ func (c *command) bindBlockchainRpcConfig(cmd *cobra.Command) {
 	c.bindNestedConfig(cmd, blockchainRpcConfigPairs)
 }
 
-// bindTracingConfig supports both flat (tracing-otlp-endpoint) and nested
-// (tracing.otlp-endpoint) YAML forms, with nested taking precedence.
+// bindTracingConfig supports both flat (tracing-endpoint) and nested
+// (tracing.endpoint) YAML forms, with nested taking precedence.
 func (c *command) bindTracingConfig(cmd *cobra.Command) {
 	c.bindNestedConfig(cmd, tracingConfigPairs)
 }
@@ -436,7 +438,7 @@ func (c *command) bindTracingConfig(cmd *cobra.Command) {
 func (c *command) warnDeprecatedTracingKeys() {
 	for _, key := range deprecatedTracingKeys {
 		if c.config.InConfig(key) {
-			c.logger.Warning("deprecated tracing config key is ignored; use the tracing-otlp-* options", "key", key)
+			c.logger.Warning("deprecated tracing config key is ignored; use the tracing-* options", "key", key)
 		}
 	}
 }
@@ -445,9 +447,9 @@ func (c *command) warnDeprecatedTracingKeys() {
 // CA bundle, in which case the OTLP exporter falls back to the system root CAs.
 func (c *command) warnTracingTLSWithoutCA() {
 	if c.config.GetBool(configKeyTracingEnabled) &&
-		!c.config.GetBool(configKeyTracingOTLPInsecure) &&
-		c.config.GetString(configKeyTracingOTLPCAFile) == "" {
-		c.logger.Warning("tracing: TLS is enabled but no CA bundle is configured; the OTLP exporter will rely on the system root CAs. Provide --tracing-otlp-ca-file, set --tracing-otlp-insecure=true for a plaintext local collector, or disable tracing.")
+		!c.config.GetBool(configKeyTracingInsecure) &&
+		c.config.GetString(configKeyTracingCAFile) == "" {
+		c.logger.Warning("tracing: TLS is enabled but no CA bundle is configured; the OTLP exporter will rely on the system root CAs. Provide --tracing-ca-file, set --tracing-insecure=true for a plaintext local collector, or disable tracing.")
 	}
 }
 

cmd/bee/cmd/start.go

@@ -343,10 +343,10 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo
 		SwapInitialDeposit:            c.config.GetString(optionNameSwapInitialDeposit),
 		TargetNeighborhood:            c.config.GetString(optionNameTargetNeighborhood),
 		TracingEnabled:                c.config.GetBool(configKeyTracingEnabled),
-		TracingEndpoint:               c.config.GetString(configKeyTracingOTLPEndpoint),
-		TracingInsecure:               c.config.GetBool(configKeyTracingOTLPInsecure),
-		TracingCAFile:                 c.config.GetString(configKeyTracingOTLPCAFile),
-		TracingProtocol:               c.config.GetString(configKeyTracingOTLPProtocol),
+		TracingEndpoint:               c.config.GetString(configKeyTracingEndpoint),
+		TracingInsecure:               c.config.GetBool(configKeyTracingInsecure),
+		TracingCAFile:                 c.config.GetString(configKeyTracingCAFile),
+		TracingProtocol:               c.config.GetString(configKeyTracingProtocol),
 		TracingSamplingRatio:          c.config.GetFloat64(configKeyTracingSamplingRatio),
 		TracingServiceName:            c.config.GetString(configKeyTracingServiceName),
 		TrxDebugMode:                  c.config.GetBool(optionNameTransactionDebugMode),

packaging/bee.yaml

@@ -111,14 +111,14 @@ password-file: "/var/lib/bee/password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

packaging/homebrew-amd64/bee.yaml

@@ -111,14 +111,14 @@ password-file: "/usr/local/var/lib/swarm-bee/password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

packaging/homebrew-arm64/bee.yaml

@@ -111,14 +111,14 @@ password-file: "/opt/homebrew/var/lib/swarm-bee/password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

packaging/scoop/bee.yaml

@@ -111,14 +111,14 @@ password-file: "./password"
 # tracing:
 #   ## enable tracing
 #   enable: false
-#   ## OTLP endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
-#   otlp-endpoint: 127.0.0.1:4318
+#   ## OTLP collector endpoint to send tracing data (host:port); default port is 4318 for http, 4317 for grpc
+#   endpoint: 127.0.0.1:4318
 #   ## disable TLS for the OTLP exporter (useful for a local collector)
-#   otlp-insecure: false
-#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when otlp-insecure is true
-#   otlp-ca-file: ""
+#   insecure: false
+#   ## path to a PEM-encoded CA bundle used to verify the OTLP collector certificate; ignored when insecure is true
+#   ca-file: ""
 #   ## OTLP exporter transport: http or grpc
-#   otlp-protocol: http
+#   protocol: http
 #   ## head-based sampling ratio in [0,1]; 0 samples nothing, 1 samples everything
 #   sampling-ratio: 1.0
 #   ## service name identifier for tracing

pkg/api/api.go

@@ -772,6 +772,9 @@ func (p *putterSessionWrapper) Put(ctx context.Context, chunk swarm.Chunk) error
 	if err != nil {
 		return err
 	}
+	// Attach the batch id as baggage so it follows the chunk across hops (e.g.
+	// direct upload -> pushsync). Best effort: a baggage error must not fail the put.
+	ctx, _ = tracing.WithBaggageMember(ctx, "batch_id", hex.EncodeToString(stamp.BatchID()))
 	return p.PutterSession.Put(ctx, chunk.WithStamp(stamp))
 }
 

pkg/node/node.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	bee "github.com/ethersphere/bee/v2"
 	"github.com/ethersphere/bee/v2/pkg/accesscontrol"
 	"github.com/ethersphere/bee/v2/pkg/accounting"
 	"github.com/ethersphere/bee/v2/pkg/addressbook"
@@ -219,6 +220,19 @@ const (
 	maxAllowedDoubling            = 1
 )
 
+// tracingEnvironment maps a network id to the deployment.environment trace
+// attribute. Unknown ids are reported as "private".
+func tracingEnvironment(networkID uint64) string {
+	switch networkID {
+	case config.Mainnet.NetworkID:
+		return "mainnet"
+	case config.Testnet.NetworkID:
+		return "testnet"
+	default:
+		return "private"
+	}
+}
+
 func NewBee(
 	ctx context.Context,
 	addr string,
@@ -237,19 +251,6 @@ func NewBee(
 
 	nodeMetrics := newMetrics()
 
-	tracer, tracerCloser, err := tracing.NewTracer(&tracing.Options{
-		Enabled:       o.TracingEnabled,
-		Endpoint:      o.TracingEndpoint,
-		Insecure:      o.TracingInsecure,
-		CAFile:        o.TracingCAFile,
-		Protocol:      o.TracingProtocol,
-		SamplingRatio: o.TracingSamplingRatio,
-		ServiceName:   o.TracingServiceName,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("tracer: %w", err)
-	}
-
 	if err := validatePublicAddress(o.NATAddr); err != nil {
 		return nil, fmt.Errorf("invalid NAT address %s: %w", o.NATAddr, err)
 	}
@@ -283,7 +284,6 @@ func NewBee(
 		logger:         logger,
 		ctxCancel:      ctxCancel,
 		errorLogWriter: sink,
-		tracerCloser:   tracerCloser,
 		syncingStopped: syncutil.NewSignaler(),
 	}
 
@@ -394,6 +394,24 @@ func NewBee(
 		return nil, fmt.Errorf("check overlay address: %w", err)
 	}
 
+	tracer, tracerCloser, err := tracing.NewTracer(&tracing.Options{
+		Enabled:        o.TracingEnabled,
+		Endpoint:       o.TracingEndpoint,
+		Insecure:       o.TracingInsecure,
+		CAFile:         o.TracingCAFile,
+		Protocol:       o.TracingProtocol,
+		SamplingRatio:  o.TracingSamplingRatio,
+		ServiceName:    o.TracingServiceName,
+		ServiceVersion: bee.Version,
+		Environment:    tracingEnvironment(networkID),
+		InstanceID:     swarmAddress.String(),
+		Logger:         logger,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("tracer: %w", err)
+	}
+	b.tracerCloser = tracerCloser
+
 	var (
 		chequebookService chequebook.Service = new(noOpChequebookService)
 		chequeStore       chequebook.ChequeStore
@@ -795,7 +813,7 @@ func NewBee(
 	eventListener = listener.New(b.syncingStopped, logger, chainBackend, postageStampContractAddress, postageStampContractABI, o.BlockTime, postageSyncingStallingTimeout, postageSyncingBackoffTimeout)
 	b.listenerCloser = eventListener
 
-	batchSvc, err = batchservice.New(stateStore, batchStore, logger, eventListener, overlayEthAddress.Bytes(), post, sha3.New256, o.Resync)
+	batchSvc, err = batchservice.New(stateStore, batchStore, logger, eventListener, overlayEthAddress.Bytes(), post, sha3.New256, o.Resync, tracer)
 	if err != nil {
 		return nil, fmt.Errorf("init batch service: %w", err)
 	}
@@ -824,7 +842,7 @@ func NewBee(
 
 	var swapService *swap.Service
 
-	kad, err := kademlia.New(swarmAddress, addressbook, hive, p2ps, detector, logger,
+	kad, err := kademlia.New(swarmAddress, addressbook, hive, p2ps, detector, logger, tracer,
 		kademlia.Options{Bootnodes: bootnodes, BootnodeMode: o.BootnodeMode, StaticNodes: o.StaticNodes, DataDir: o.DataDir})
 	if err != nil {
 		return nil, fmt.Errorf("unable to create kademlia: %w", err)
@@ -905,7 +923,7 @@ func NewBee(
 
 		snapshotEventListener := listener.New(b.syncingStopped, logger, chainBackend, postageStampContractAddress, postageStampContractABI, o.BlockTime, postageSyncingStallingTimeout, postageSyncingBackoffTimeout)
 
-		snapshotBatchSvc, err := batchservice.New(stateStore, batchStore, logger, snapshotEventListener, overlayEthAddress.Bytes(), post, sha3.New256, o.Resync)
+		snapshotBatchSvc, err := batchservice.New(stateStore, batchStore, logger, snapshotEventListener, overlayEthAddress.Bytes(), post, sha3.New256, o.Resync, tracer)
 		if err != nil {
 			logger.Error(err, "failed to initialize batch service from snapshot, continuing outside snapshot block...")
 		} else {
@@ -1113,7 +1131,7 @@ func NewBee(
 		return nil, fmt.Errorf("status service: %w", err)
 	}
 
-	saludService := salud.New(nodeStatus, kad, localStore, logger, detector, api.FullMode.String(), salud.DefaultDurPercentile, salud.DefaultConnsPercentile)
+	saludService := salud.New(nodeStatus, kad, localStore, logger, detector, api.FullMode.String(), salud.DefaultDurPercentile, salud.DefaultConnsPercentile, tracer)
 	b.saludCloser = saludService
 
 	rC, unsub := saludService.SubscribeNetworkStorageRadius()

pkg/p2p/p2p.go

@@ -226,6 +226,10 @@ const (
 	// from the legacy OpenTracing/Jaeger payload, so the distinct key prevents
 	// mixed-version peers from decoding each other's incompatible payloads.
 	HeaderNameTracingSpanContext = "tracing-span-context-v2"
+	// HeaderNameTracingBaggage carries W3C baggage (a string of key=value pairs)
+	// alongside the span context. It is additive: peers that do not understand it
+	// simply ignore the header, so it never breaks the stream.
+	HeaderNameTracingBaggage = "tracing-baggage"
 )
 
 // NewSwarmStreamName constructs a libp2p compatible stream name out of

pkg/pingpong/pingpong.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/pingpong/pb"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
 	"github.com/ethersphere/bee/v2/pkg/tracing"
+	"go.opentelemetry.io/otel/attribute"
 )
 
 // loggerName is the tree path name of the logger for this package.
@@ -65,6 +66,7 @@ func (s *Service) Protocol() p2p.ProtocolSpec {
 
 func (s *Service) Ping(ctx context.Context, address swarm.Address, msgs ...string) (rtt time.Duration, err error) {
 	span, _, ctx := s.tracer.StartSpanFromContext(ctx, "pingpong-p2p-ping", s.logger)
+	span.SetAttributes(attribute.String("peer_address", address.String()))
 	defer span.End()
 
 	start := time.Now()
@@ -104,6 +106,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er
 	defer stream.FullClose()
 
 	span, _, ctx := s.tracer.StartSpanFromContext(ctx, "pingpong-p2p-handler", s.logger)
+	span.SetAttributes(attribute.String("peer_address", p.Address.String()))
 	defer span.End()
 
 	var ping pb.Ping