diff --git a/.gitignore b/.gitignore index 60e6ee752..3205883ce 100644 --- a/.gitignore +++ b/.gitignore @@ -20,13 +20,21 @@ Session.vim .netrwhist *~ -*.xo +# IDE and scratch files .vscode/*.log .idea/ +*.xo z-scratch/ -# binary files +# Binary files cmd/benchmark/benchmark +examples/storage/storage + +# Gorums generated backup files cmd/protoc-gen-gorums/gengorums/template_static.go.bak +# Ignore the generated code for the tutorial example +examples/gorumsexample/storage.pb.go +examples/gorumsexample/storage_gorums.pb.go + go.work.sum diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 36c57f41c..f39cbb6a5 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,11 +1,9 @@ { "recommendations": [ - "xaver.clang-format", "streetsidesoftware.code-spell-checker", "golang.go", "davidanson.vscode-markdownlint", - "zxh404.vscode-proto3", - "github.vscode-pull-request-github", - "redhat.vscode-yaml" + "drblury.protobuf-vsc", + "github.vscode-pull-request-github" ] } diff --git a/.vscode/gorums.txt b/.vscode/gorums.txt index 09d412eab..b76530943 100644 --- a/.vscode/gorums.txt +++ b/.vscode/gorums.txt @@ -1,5 +1,7 @@ Ansible benchgraph +benchtest +benchtime bootstrapgorums calltype calltypes @@ -19,6 +21,7 @@ errgroup extendee Frausing's fset +GCFLAGS gengorums genproto GOBIN @@ -51,11 +54,14 @@ memprofile mktemp multicast naddr +netrwhist obj's +objc oneway Paxos Pedersen pkgs +pluginpb proto protobuf protoc @@ -78,10 +84,13 @@ RPC's RPCs semver shlex +skipcq srvs stressdev stressgen +stresstest struct +subtests syncrq testprotos testrace diff --git a/.vscode/settings.json b/.vscode/settings.json index 045c987d2..e263ed484 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -30,6 +30,7 @@ }, "files.exclude": { "**/.git": true, + "**/.jj": true, "**/.DS_Store": true, }, "files.insertFinalNewline": true, @@ -53,19 +54,24 @@ "usePlaceholders": true, "staticcheck": false }, - "protoc": { - "compile_on_save": false, - "options": [ - "--proto_path=${workspaceRoot}", - "--proto_path=${workspaceRoot}/cmd/protoc-gen-gorums/dev", - "--proto_path=${workspaceRoot}/third_party" - ] - }, - "clang-format.executable": "/usr/bin/clang-format", - "clang-format.style": "{ IndentWidth: 4, BasedOnStyle: google, AlignConsecutiveAssignments: true, ColumnLimit: 120 }", - "[proto3]": { - "editor.defaultFormatter": "zxh404.vscode-proto3" - }, + "protobuf.protoc.path": "protoc", + "protobuf.formatOnSave": true, + "protobuf.formatter.enabled": true, + "protobuf.formatter.alignFields": true, + "protobuf.protoc.options": [ + "--proto_path=${workspaceFolder}", + "--proto_path=${workspaceFolder}/cmd/protoc-gen-gorums/dev", + "--proto_path=${workspaceFolder}/third_party", + "--go_opt=paths=source_relative", + "--go-grpc_opt=paths=source_relative", + "--gorums_opt=paths=source_relative", + "--go_opt=default_api_level=API_OPAQUE" + ], + "protobuf.buf.path": "buf", + "protobuf.externalLinter.bufPath": "buf", + "protobuf.externalLinter.enabled": true, + "protobuf.externalLinter.runOnSave": true, + "protobuf.externalLinter.linter": "buf", "markdownlint.config": { "MD024": false, "default": true, diff --git a/AGENTS.md b/AGENTS.md index 5ff5924bc..16ee8efd5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,7 @@ # Agent Instructions for Gorums Project -You are working on the Gorums project - a framework for building fault-tolerant distributed systems using quorum-based abstractions. This document provides critical context and rules for AI coding assistants. +Gorums is a framework for building fault-tolerant distributed systems using quorum-based abstractions. +This document provides context and rules for AI coding assistants. ## Project Overview @@ -34,7 +35,7 @@ gorums/ └── *.go # Core library files ``` -## Critical Development Rules +## Development Rules ### Code Generation Workflow @@ -43,21 +44,16 @@ gorums/ These files are generated from templates. Instead: 1. **For Template Changes:** - - Edit the corresponding template in `cmd/protoc-gen-gorums/gengorums/template_*.go` - - Run `make` to regenerate code - - The `zorums_*` files will be automatically updated + - Edit template in `cmd/protoc-gen-gorums/gengorums/template_*.go` + - Run `make dev` to regenerate `zorums_*_gorums.pb.go` files 2. **For Static Code Changes:** - Edit files in `cmd/protoc-gen-gorums/dev/` that are NOT prefixed with `zorums_*` - - Run `make` to bundle changes into `template_static.go` - - Changes propagate to generated code automatically + - Run `make dev` to bundle changes into `template_static.go` 3. **After Any Template or Static Code Changes:** - - ```bash - make # Normal rebuild - make -B # Force rebuild (e.g., after protoc update) - ``` + - Run `make dev` to regenerate `zorums_*_gorums.pb.go` files + - Or run `make genproto` to regenerate all _gorums.pb.go files ### Testing Requirements @@ -65,6 +61,9 @@ These files are generated from templates. Instead: - Tests verify the correctness and stability of generated code - ALL test failures must be addressed before considering work complete - Never delete failing tests - fix the underlying issue +- If suitable, tests should be table-driven +- If suitable, tests should be organized as subtests +- Test names should be capitalized, like TestFileNameFeatureName, e.g., TestQuorumCallFeatureName, for some feature in `quorumcall_test.go` ### Code Style and Conventions @@ -105,6 +104,12 @@ These files are generated from templates. Instead: ### Common Commands ```bash +# Generate `zorums_*_gorums.pb.go` files in `cmd/protoc-gen-gorums/dev/` +make dev + +# Generate _gorums.pb.go files across the project +make genproto + # Build everything make @@ -114,6 +119,9 @@ make -B # Run tests make test +# Ensure tests are actually run (not skipped by cache) +go test ./... -count=1 + # Install protoc-gen-gorums plugin make installgorums @@ -152,7 +160,7 @@ Gorums provides custom protobuf options defined in `gorums.proto`: - Method-level options for quorum call types - Configuration options for RPC behavior -- See `doc/method-options.md` for details +- See `doc/user-guide.md` for details ## Documentation @@ -161,8 +169,9 @@ Before making significant changes, consult: - `doc/user-guide.md` - Understanding the API and usage patterns - `doc/dev-guide.md` - Development workflow and architecture - `doc/design-doc-layering.md` - System architecture and layering -- `doc/method-options.md` - Protocol buffer options reference + - `README.md` - Project overview and getting started +- When editing markdown files, use one sentences per line, so that diffs are easier to read. ## Common Pitfalls to Avoid diff --git a/Makefile b/Makefile index b08ed4df5..12bf73b96 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,11 @@ static_file := $(gen_path)/template_static.go static_files := $(shell find $(dev_path) -name "*.go" -not -name "zorums*" -not -name "*_test.go") proto_path := $(dev_path):third_party:. -plugin_deps := gorums.pb.go internal/correctable/opts.pb.go $(static_file) +plugin_deps := gorums.pb.go $(static_file) runtime_deps := ordering/ordering.pb.go ordering/ordering_grpc.pb.go benchmark_deps := benchmark/benchmark.pb.go benchmark/benchmark_gorums.pb.go -.PHONY: all dev tools bootstrapgorums installgorums benchmark test compiletests genproto +.PHONY: all dev tools bootstrapgorums installgorums benchmark test compiletests genproto benchtest bench all: dev benchmark compiletests @@ -56,12 +56,35 @@ endif compiletests: installgorums @$(MAKE) --no-print-directory -C ./internal/tests all -test: compiletests +test: compiletests benchtest @go test ./... testrace: compiletests go test -race -cpu=1,2,4 ./... +# Run benchmarks with validation (short runs to verify they don't fail). +# Uses -benchtime=100x for limited iterations to avoid port exhaustion on macOS. +# Suppresses output on success; shows details on failure. +benchtest: compiletests + @if ! go test -run=^$$ -bench=. -benchtime=100x -count=1 ./... > /tmp/benchtest.out 2>&1; then \ + echo "Benchmark validation failed:"; \ + cat /tmp/benchtest.out; \ + exit 1; \ + fi + @echo "Benchmark validation passed" + +# Run benchmarks with proper measurement (longer runs for performance analysis). +# Use -count=10 or more for statistically significant results. +# This only runs benchmarks in the main module and in internal/tests/oneway +# (when adding benchmarks elsewhere, update this target accordingly). +bench: compiletests + go test -run=^$$ -bench=. -benchtime=1s -count=10 . ./internal/tests/oneway + +# Run stress tests that use longer durations for thorough testing. +# These tests are excluded from normal test runs via the 'stress' build tag. +stresstest: compiletests + go test -tags=stress ./... + # Warning: will probably run for 10 minutes; the timeout does not work stressdev: tools go test -c $(dev_path) diff --git a/async.go b/async.go index 0d16c982f..50d99591e 100644 --- a/async.go +++ b/async.go @@ -1,31 +1,25 @@ package gorums -import ( - "context" - - "github.com/relab/gorums/ordering" - "google.golang.org/protobuf/proto" -) - -// Async encapsulates the state of an asynchronous quorum call, -// and has methods for checking the status of the call or waiting for it to complete. +// Async is a generic future type for asynchronous quorum calls. +// It encapsulates the state of an asynchronous call and provides methods +// for checking the status or waiting for completion. // -// This struct should only be used by generated code. -type Async struct { - reply proto.Message +// Type parameter Resp is the response type from nodes. +type Async[Resp any] struct { + reply Resp err error c chan struct{} } // Get returns the reply and any error associated with the called method. // The method blocks until a reply or error is available. -func (f *Async) Get() (proto.Message, error) { +func (f *Async[Resp]) Get() (Resp, error) { <-f.c return f.reply, f.err } // Done reports if a reply and/or error is available for the called method. -func (f *Async) Done() bool { +func (f *Async[Resp]) Done() bool { select { case <-f.c: return true @@ -34,74 +28,39 @@ func (f *Async) Done() bool { } } -type asyncCallState struct { - md *ordering.Metadata - data QuorumCallData - replyChan <-chan response - expectedReplies int +// AsyncMajority returns an Async future that resolves when a majority quorum is reached. +// Messages are sent immediately (synchronously) to preserve ordering when multiple +// async calls are created in sequence. +func (r *Responses[Resp]) AsyncMajority() *Async[Resp] { + quorumSize := r.size/2 + 1 + return r.AsyncThreshold(quorumSize) } -// AsyncCall starts an asynchronous quorum call, returning an Async object that can be used to retrieve the results. -// -// This function should only be used by generated code. -func (c RawConfiguration) AsyncCall(ctx context.Context, d QuorumCallData) *Async { - expectedReplies := len(c) - md := ordering.NewGorumsMetadata(ctx, c.getMsgID(), d.Method) - replyChan := make(chan response, expectedReplies) - - for _, n := range c { - msg := d.Message - if d.PerNodeArgFn != nil { - msg = d.PerNodeArgFn(d.Message, n.id) - if !msg.ProtoReflect().IsValid() { - expectedReplies-- - continue // don't send if no msg - } - } - n.channel.enqueue(request{ctx: ctx, msg: NewRequestMessage(md, msg)}, replyChan) - } - - fut := &Async{c: make(chan struct{}, 1)} - - go c.handleAsyncCall(ctx, fut, asyncCallState{ - md: md, - data: d, - replyChan: replyChan, - expectedReplies: expectedReplies, - }) +// AsyncFirst returns an Async future that resolves when the first response is received. +// Messages are sent immediately (synchronously) to preserve ordering. +func (r *Responses[Resp]) AsyncFirst() *Async[Resp] { + return r.AsyncThreshold(1) +} - return fut +// AsyncAll returns an Async future that resolves when all nodes have responded. +// Messages are sent immediately (synchronously) to preserve ordering. +func (r *Responses[Resp]) AsyncAll() *Async[Resp] { + return r.AsyncThreshold(r.size) } -func (RawConfiguration) handleAsyncCall(ctx context.Context, fut *Async, state asyncCallState) { - defer close(fut.c) +// AsyncThreshold returns an Async future that resolves when the threshold is reached. +// Messages are sent immediately (synchronously) to preserve ordering when multiple +// async calls are created in sequence. +func (r *Responses[Resp]) AsyncThreshold(threshold int) *Async[Resp] { + // Send messages synchronously before spawning the goroutine to preserve ordering + r.sendNow() - var ( - resp proto.Message - errs []nodeError - quorum bool - replies = make(map[uint32]proto.Message) - ) + fut := &Async[Resp]{c: make(chan struct{}, 1)} - for { - select { - case r := <-state.replyChan: - if r.err != nil { - errs = append(errs, nodeError{nodeID: r.nid, cause: r.err}) - break - } - replies[r.nid] = r.msg - if resp, quorum = state.data.QuorumFunction(state.data.Message, replies); quorum { - fut.reply, fut.err = resp, nil - return - } - case <-ctx.Done(): - fut.reply, fut.err = resp, QuorumCallError{cause: ctx.Err(), errors: errs, replies: len(replies)} - return - } - if len(errs)+len(replies) == state.expectedReplies { - fut.reply, fut.err = resp, QuorumCallError{cause: Incomplete, errors: errs, replies: len(replies)} - return - } - } + go func() { + defer close(fut.c) + fut.reply, fut.err = r.Threshold(threshold) + }() + + return fut } diff --git a/async_test.go b/async_test.go new file mode 100644 index 000000000..41d757752 --- /dev/null +++ b/async_test.go @@ -0,0 +1,122 @@ +package gorums_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/relab/gorums" + "github.com/relab/gorums/internal/testutils/mock" + pb "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestAsync(t *testing.T) { + // a type alias short hand for the responses type + type respType = *gorums.Responses[*pb.StringValue] + tests := []struct { + name string + call func(respType) *gorums.Async[*pb.StringValue] + numNodes int + wantValue string + wantErr bool + }{ + { + name: "Majority", + call: respType.AsyncMajority, + numNodes: 3, + wantValue: "echo: test", + }, + { + name: "First", + call: respType.AsyncFirst, + numNodes: 3, + wantValue: "echo: test", + }, + { + name: "All", + call: respType.AsyncAll, + numNodes: 3, + wantValue: "echo: test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := gorums.TestConfiguration(t, tt.numNodes, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + ) + + future := tt.call(responses) + + reply, err := future.Get() + if !checkQuorumCall(t, ctx.Err(), err) { + return + } + + if reply.GetValue() != tt.wantValue { + t.Errorf("Expected %q, got %q", tt.wantValue, reply.GetValue()) + } + }) + } +} + +func TestAsync_Error(t *testing.T) { + // Use a configuration with no servers to force an error (or timeout) + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + ) + + future := responses.AsyncMajority() + _, err := future.Get() + if err == nil { + t.Error("Expected error, got nil") + } +} + +func BenchmarkAsyncQuorumCall(b *testing.B) { + for _, numNodes := range []int{3, 5, 7, 9} { + config := gorums.TestConfiguration(b, numNodes, gorums.EchoServerFn) + cfgCtx := config.Context(b.Context()) + + b.Run(fmt.Sprintf("AsyncMajority/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + future := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ).AsyncMajority() + _, err := future.Get() + if err != nil { + b.Fatalf("AsyncMajority error: %v", err) + } + } + }) + + // Compare with blocking Majority + b.Run(fmt.Sprintf("BlockingMajority/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _, err := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ).Majority() + if err != nil { + b.Fatalf("Majority error: %v", err) + } + } + }) + } +} diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go index bdc3b96b1..667cd358f 100644 --- a/benchmark/benchmark.go +++ b/benchmark/benchmark.go @@ -2,12 +2,15 @@ package benchmark import ( context "context" + "maps" "regexp" "runtime" + "slices" "sort" "sync/atomic" "time" + "github.com/relab/gorums" "golang.org/x/sync/errgroup" ) @@ -32,14 +35,15 @@ type Bench struct { type ( benchFunc func(Options) (*Result, error) - qcFunc func(context.Context, *Echo) (*Echo, error) - asyncQCFunc func(context.Context, *Echo) *AsyncEcho + qcFunc func(*gorums.ConfigContext, *Echo, int, ...gorums.CallOption) (*Echo, error) + asyncQCFunc func(*gorums.ConfigContext, *Echo, int, ...gorums.CallOption) AsyncEcho serverFunc func(context.Context, *TimedMsg) ) -func runQCBenchmark(opts Options, cfg *Configuration, f qcFunc) (*Result, error) { +func runQCBenchmark(opts Options, config Configuration, f qcFunc) (*Result, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + cfgCtx := config.Context(ctx) msg := Echo_builder{Payload: make([]byte, opts.Payload)}.Build() s := &Stats{} var g errgroup.Group @@ -48,7 +52,7 @@ func runQCBenchmark(opts Options, cfg *Configuration, f qcFunc) (*Result, error) g.Go(func() error { warmupEnd := time.Now().Add(opts.Warmup) for !time.Now().After(warmupEnd) { - _, err := f(ctx, msg) + _, err := f(cfgCtx, msg, opts.QuorumSize) if err != nil { return err } @@ -56,14 +60,12 @@ func runQCBenchmark(opts Options, cfg *Configuration, f qcFunc) (*Result, error) return nil }) } - - err := g.Wait() - if err != nil { + if err := g.Wait(); err != nil { return nil, err } if opts.Remote { - _, err := cfg.StartBenchmark(ctx, &StartRequest{}) + _, err := StartBenchmark(cfgCtx, &StartRequest{}).All() if err != nil { return nil, err } @@ -75,7 +77,7 @@ func runQCBenchmark(opts Options, cfg *Configuration, f qcFunc) (*Result, error) endTime := time.Now().Add(opts.Duration) for !time.Now().After(endTime) { start := time.Now() - _, err := f(ctx, msg) + _, err := f(cfgCtx, msg, opts.QuorumSize) if err != nil { return err } @@ -84,28 +86,24 @@ func runQCBenchmark(opts Options, cfg *Configuration, f qcFunc) (*Result, error) return nil }) } - - err = g.Wait() - s.End() - if err != nil { + if err := g.Wait(); err != nil { return nil, err } + s.End() result := s.GetResult() if opts.Remote { - memStats, err := cfg.StopBenchmark(ctx, &StopRequest{}) - if err != nil { - return nil, err - } - result.SetServerStats(memStats.GetMemoryStats()) + replies := StopBenchmark(cfgCtx, &StopRequest{}).CollectAll() + result.SetServerStats(slices.Collect(maps.Values(replies))) } return result, nil } -func runAsyncQCBenchmark(opts Options, cfg *Configuration, f asyncQCFunc) (*Result, error) { +func runAsyncQCBenchmark(opts Options, config Configuration, f asyncQCFunc) (*Result, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + cfgCtx := config.Context(ctx) msg := Echo_builder{Payload: make([]byte, opts.Payload)}.Build() s := &Stats{} var g errgroup.Group @@ -116,7 +114,7 @@ func runAsyncQCBenchmark(opts Options, cfg *Configuration, f asyncQCFunc) (*Resu var warmupFunc func() error warmupFunc = func() error { for ; !time.Now().After(warmupEnd) && atomic.LoadUint64(&async) < uint64(opts.MaxAsync); atomic.AddUint64(&async, 1) { - fut := f(ctx, msg) + fut := f(cfgCtx, msg, opts.QuorumSize) g.Go(func() error { _, err := fut.Get() if err != nil { @@ -133,13 +131,12 @@ func runAsyncQCBenchmark(opts Options, cfg *Configuration, f asyncQCFunc) (*Resu for range opts.Concurrent { g.Go(warmupFunc) } - err := g.Wait() - if err != nil { + if err := g.Wait(); err != nil { return nil, err } if opts.Remote { - _, err := cfg.StartBenchmark(ctx, &StartRequest{}) + _, err := StartBenchmark(cfgCtx, &StartRequest{}).All() if err != nil { return nil, err } @@ -150,7 +147,7 @@ func runAsyncQCBenchmark(opts Options, cfg *Configuration, f asyncQCFunc) (*Resu benchmarkFunc = func() error { for ; !time.Now().After(endTime) && atomic.LoadUint64(&async) < uint64(opts.MaxAsync); atomic.AddUint64(&async, 1) { start := time.Now() - fut := f(ctx, msg) + fut := f(cfgCtx, msg, opts.QuorumSize) g.Go(func() error { _, err := fut.Get() if err != nil { @@ -169,29 +166,25 @@ func runAsyncQCBenchmark(opts Options, cfg *Configuration, f asyncQCFunc) (*Resu for range opts.Concurrent { g.Go(benchmarkFunc) } - err = g.Wait() - s.End() - if err != nil { + if err := g.Wait(); err != nil { return nil, err } + s.End() result := s.GetResult() if opts.Remote { - memStats, err := cfg.StopBenchmark(ctx, &StopRequest{}) - if err != nil { - return nil, err - } - result.SetServerStats(memStats.GetMemoryStats()) + replies := StopBenchmark(cfgCtx, &StopRequest{}).CollectAll() + result.SetServerStats(slices.Collect(maps.Values(replies))) } return result, nil } -func runServerBenchmark(opts Options, cfg *Configuration, f serverFunc) (*Result, error) { +func runServerBenchmark(opts Options, config Configuration, f serverFunc) (*Result, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + cfgCtx := config.Context(ctx) payload := make([]byte, opts.Payload) - var g errgroup.Group var start runtime.MemStats var end runtime.MemStats @@ -204,30 +197,23 @@ func runServerBenchmark(opts Options, cfg *Configuration, f serverFunc) (*Result warmupEnd := time.Now().Add(opts.Warmup) for range opts.Concurrent { - go benchmarkFunc(warmupEnd) - } - err := g.Wait() - if err != nil { - return nil, err + benchmarkFunc(warmupEnd) } - _, err = cfg.StartServerBenchmark(ctx, &StartRequest{}) + _, err := StartServerBenchmark(cfgCtx, &StartRequest{}).All() if err != nil { return nil, err } - runtime.ReadMemStats(&start) + endTime := time.Now().Add(opts.Duration) for range opts.Concurrent { benchmarkFunc(endTime) } - err = g.Wait() - if err != nil { - return nil, err - } runtime.ReadMemStats(&end) - resp, err := cfg.StopServerBenchmark(ctx, &StopRequest{}) + replies := StopServerBenchmark(cfgCtx, &StopRequest{}).CollectAll() + resp, err := StopServerBenchmarkQF(replies) if err != nil { return nil, err } @@ -241,28 +227,43 @@ func runServerBenchmark(opts Options, cfg *Configuration, f serverFunc) (*Result } // GetBenchmarks returns a list of Benchmarks that can be performed on the configuration -func GetBenchmarks(cfg *Configuration) []Bench { +func GetBenchmarks(config Configuration) []Bench { m := []Bench{ { Name: "QuorumCall", Description: "NodeStream based quorum call implementation with FIFO ordering", - runBench: func(opts Options) (*Result, error) { return runQCBenchmark(opts, cfg, cfg.QuorumCall) }, + runBench: func(opts Options) (*Result, error) { + return runQCBenchmark(opts, config, func(ctx *gorums.ConfigContext, in *Echo, quorumSize int, callOpts ...gorums.CallOption) (*Echo, error) { + return QuorumCall(ctx, in, callOpts...).Threshold(quorumSize) + }) + }, }, { Name: "AsyncQuorumCall", Description: "NodeStream based async quorum call implementation with FIFO ordering", - runBench: func(opts Options) (*Result, error) { return runAsyncQCBenchmark(opts, cfg, cfg.AsyncQuorumCall) }, + runBench: func(opts Options) (*Result, error) { + return runAsyncQCBenchmark(opts, config, func(ctx *gorums.ConfigContext, in *Echo, quorumSize int, callOpts ...gorums.CallOption) AsyncEcho { + return QuorumCall(ctx, in, callOpts...).AsyncThreshold(quorumSize) + }) + }, }, { Name: "SlowServer", Description: "Quorum Call with a 10s processing time on the server", - runBench: func(opts Options) (*Result, error) { return runQCBenchmark(opts, cfg, cfg.SlowServer) }, + runBench: func(opts Options) (*Result, error) { + return runQCBenchmark(opts, config, func(ctx *gorums.ConfigContext, in *Echo, quorumSize int, callOpts ...gorums.CallOption) (*Echo, error) { + return SlowServer(ctx, in, callOpts...).Threshold(quorumSize) + }) + }, }, { Name: "Multicast", Description: "NodeStream based multicast implementation (servers measure latency and throughput)", runBench: func(opts Options) (*Result, error) { - return runServerBenchmark(opts, cfg, func(ctx context.Context, msg *TimedMsg) { cfg.Multicast(ctx, msg) }) + return runServerBenchmark(opts, config, func(ctx context.Context, msg *TimedMsg) { + cfgCtx := config.Context(ctx) + Multicast(cfgCtx, msg, gorums.IgnoreErrors()) + }) }, }, } @@ -270,8 +271,8 @@ func GetBenchmarks(cfg *Configuration) []Bench { } // RunBenchmarks runs all the benchmarks that match the given regex with the given options -func RunBenchmarks(benchRegex *regexp.Regexp, options Options, cfg *Configuration) ([]*Result, error) { - benchmarks := GetBenchmarks(cfg) +func RunBenchmarks(benchRegex *regexp.Regexp, options Options, config Configuration) ([]*Result, error) { + benchmarks := GetBenchmarks(config) var results []*Result for _, b := range benchmarks { if benchRegex.MatchString(b.Name) { diff --git a/benchmark/benchmark.pb.go b/benchmark/benchmark.pb.go index b67b6a796..251310811 100644 --- a/benchmark/benchmark.pb.go +++ b/benchmark/benchmark.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: benchmark/benchmark.proto package benchmark @@ -22,6 +22,7 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Echo is a simple message used for echo benchmarks. type Echo struct { state protoimpl.MessageState `protogen:"opaque.v1"` xxx_hidden_Payload []byte `protobuf:"bytes,1,opt,name=payload"` @@ -82,9 +83,10 @@ func (b0 Echo_builder) Build() *Echo { return m0 } +// TimedMsg is a message with a send time and a payload used for multicast benchmarks. type TimedMsg struct { state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_SendTime int64 `protobuf:"varint,1,opt,name=SendTime"` + xxx_hidden_SendTime int64 `protobuf:"varint,1,opt,name=send_time,json=sendTime"` xxx_hidden_Payload []byte `protobuf:"bytes,2,opt,name=payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -156,6 +158,7 @@ func (b0 TimedMsg_builder) Build() *TimedMsg { return m0 } +// StartRequest is an empty message for starting a benchmarking campaign. type StartRequest struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields @@ -199,6 +202,7 @@ func (b0 StartRequest_builder) Build() *StartRequest { return m0 } +// StartResponse is an empty message to acknowledge the start of a benchmarking campaign. type StartResponse struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields @@ -242,6 +246,7 @@ func (b0 StartResponse_builder) Build() *StartResponse { return m0 } +// StopRequest is an empty message for stopping a benchmarking campaign. type StopRequest struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields @@ -285,17 +290,18 @@ func (b0 StopRequest_builder) Build() *StopRequest { return m0 } +// Result contains the results of a server-side benchmarking campaign. type Result struct { state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Name string `protobuf:"bytes,1,opt,name=Name"` - xxx_hidden_TotalOps uint64 `protobuf:"varint,2,opt,name=TotalOps"` - xxx_hidden_TotalTime int64 `protobuf:"varint,3,opt,name=TotalTime"` - xxx_hidden_Throughput float64 `protobuf:"fixed64,4,opt,name=Throughput"` - xxx_hidden_LatencyAvg float64 `protobuf:"fixed64,5,opt,name=LatencyAvg"` - xxx_hidden_LatencyVar float64 `protobuf:"fixed64,6,opt,name=LatencyVar"` - xxx_hidden_AllocsPerOp uint64 `protobuf:"varint,7,opt,name=AllocsPerOp"` - xxx_hidden_MemPerOp uint64 `protobuf:"varint,8,opt,name=MemPerOp"` - xxx_hidden_ServerStats *[]*MemoryStat `protobuf:"bytes,9,rep,name=ServerStats"` + xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name"` + xxx_hidden_TotalOps uint64 `protobuf:"varint,2,opt,name=total_ops,json=totalOps"` + xxx_hidden_TotalTime int64 `protobuf:"varint,3,opt,name=total_time,json=totalTime"` + xxx_hidden_Throughput float64 `protobuf:"fixed64,4,opt,name=throughput"` + xxx_hidden_LatencyAvg float64 `protobuf:"fixed64,5,opt,name=latency_avg,json=latencyAvg"` + xxx_hidden_LatencyVar float64 `protobuf:"fixed64,6,opt,name=latency_var,json=latencyVar"` + xxx_hidden_AllocsPerOp uint64 `protobuf:"varint,7,opt,name=allocs_per_op,json=allocsPerOp"` + xxx_hidden_MemPerOp uint64 `protobuf:"varint,8,opt,name=mem_per_op,json=memPerOp"` + xxx_hidden_ServerStats *[]*MemoryStat `protobuf:"bytes,9,rep,name=server_stats,json=serverStats"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -456,10 +462,11 @@ func (b0 Result_builder) Build() *Result { return m0 } +// MemoryStat contains memory statistics for a single server. type MemoryStat struct { state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Allocs uint64 `protobuf:"varint,1,opt,name=Allocs"` - xxx_hidden_Memory uint64 `protobuf:"varint,2,opt,name=Memory"` + xxx_hidden_Allocs uint64 `protobuf:"varint,1,opt,name=allocs"` + xxx_hidden_Memory uint64 `protobuf:"varint,2,opt,name=memory"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -527,148 +534,82 @@ func (b0 MemoryStat_builder) Build() *MemoryStat { return m0 } -type MemoryStatList struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_MemoryStats *[]*MemoryStat `protobuf:"bytes,1,rep,name=MemoryStats"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *MemoryStatList) Reset() { - *x = MemoryStatList{} - mi := &file_benchmark_benchmark_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *MemoryStatList) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MemoryStatList) ProtoMessage() {} - -func (x *MemoryStatList) ProtoReflect() protoreflect.Message { - mi := &file_benchmark_benchmark_proto_msgTypes[7] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -func (x *MemoryStatList) GetMemoryStats() []*MemoryStat { - if x != nil { - if x.xxx_hidden_MemoryStats != nil { - return *x.xxx_hidden_MemoryStats - } - } - return nil -} - -func (x *MemoryStatList) SetMemoryStats(v []*MemoryStat) { - x.xxx_hidden_MemoryStats = &v -} - -type MemoryStatList_builder struct { - _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - - MemoryStats []*MemoryStat -} - -func (b0 MemoryStatList_builder) Build() *MemoryStatList { - m0 := &MemoryStatList{} - b, x := &b0, m0 - _, _ = b, x - x.xxx_hidden_MemoryStats = &b.MemoryStats - return m0 -} - var File_benchmark_benchmark_proto protoreflect.FileDescriptor const file_benchmark_benchmark_proto_rawDesc = "" + "\n" + "\x19benchmark/benchmark.proto\x12\tbenchmark\x1a\x1bgoogle/protobuf/empty.proto\x1a\fgorums.proto\" \n" + "\x04Echo\x12\x18\n" + - "\apayload\x18\x01 \x01(\fR\apayload\"@\n" + - "\bTimedMsg\x12\x1a\n" + - "\bSendTime\x18\x01 \x01(\x03R\bSendTime\x12\x18\n" + + "\apayload\x18\x01 \x01(\fR\apayload\"A\n" + + "\bTimedMsg\x12\x1b\n" + + "\tsend_time\x18\x01 \x01(\x03R\bsendTime\x12\x18\n" + "\apayload\x18\x02 \x01(\fR\apayload\"\x0e\n" + "\fStartRequest\"\x0f\n" + "\rStartResponse\"\r\n" + - "\vStopRequest\"\xad\x02\n" + + "\vStopRequest\"\xb6\x02\n" + "\x06Result\x12\x12\n" + - "\x04Name\x18\x01 \x01(\tR\x04Name\x12\x1a\n" + - "\bTotalOps\x18\x02 \x01(\x04R\bTotalOps\x12\x1c\n" + - "\tTotalTime\x18\x03 \x01(\x03R\tTotalTime\x12\x1e\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1b\n" + + "\ttotal_ops\x18\x02 \x01(\x04R\btotalOps\x12\x1d\n" + "\n" + - "Throughput\x18\x04 \x01(\x01R\n" + - "Throughput\x12\x1e\n" + + "total_time\x18\x03 \x01(\x03R\ttotalTime\x12\x1e\n" + "\n" + - "LatencyAvg\x18\x05 \x01(\x01R\n" + - "LatencyAvg\x12\x1e\n" + + "throughput\x18\x04 \x01(\x01R\n" + + "throughput\x12\x1f\n" + + "\vlatency_avg\x18\x05 \x01(\x01R\n" + + "latencyAvg\x12\x1f\n" + + "\vlatency_var\x18\x06 \x01(\x01R\n" + + "latencyVar\x12\"\n" + + "\rallocs_per_op\x18\a \x01(\x04R\vallocsPerOp\x12\x1c\n" + "\n" + - "LatencyVar\x18\x06 \x01(\x01R\n" + - "LatencyVar\x12 \n" + - "\vAllocsPerOp\x18\a \x01(\x04R\vAllocsPerOp\x12\x1a\n" + - "\bMemPerOp\x18\b \x01(\x04R\bMemPerOp\x127\n" + - "\vServerStats\x18\t \x03(\v2\x15.benchmark.MemoryStatR\vServerStats\"<\n" + + "mem_per_op\x18\b \x01(\x04R\bmemPerOp\x128\n" + + "\fserver_stats\x18\t \x03(\v2\x15.benchmark.MemoryStatR\vserverStats\"<\n" + "\n" + "MemoryStat\x12\x16\n" + - "\x06Allocs\x18\x01 \x01(\x04R\x06Allocs\x12\x16\n" + - "\x06Memory\x18\x02 \x01(\x04R\x06Memory\"I\n" + - "\x0eMemoryStatList\x127\n" + - "\vMemoryStats\x18\x01 \x03(\v2\x15.benchmark.MemoryStatR\vMemoryStats2\xb2\x04\n" + + "\x06allocs\x18\x01 \x01(\x04R\x06allocs\x12\x16\n" + + "\x06memory\x18\x02 \x01(\x04R\x06memory2\xe1\x03\n" + "\tBenchmark\x12O\n" + "\x14StartServerBenchmark\x12\x17.benchmark.StartRequest\x1a\x18.benchmark.StartResponse\"\x04\xa0\xb5\x18\x01\x12F\n" + "\x13StopServerBenchmark\x12\x16.benchmark.StopRequest\x1a\x11.benchmark.Result\"\x04\xa0\xb5\x18\x01\x12I\n" + - "\x0eStartBenchmark\x12\x17.benchmark.StartRequest\x1a\x18.benchmark.StartResponse\"\x04\xa0\xb5\x18\x01\x12V\n" + - "\rStopBenchmark\x12\x16.benchmark.StopRequest\x1a\x15.benchmark.MemoryStat\"\x16\xa0\xb5\x18\x01\xf2\xb6\x18\x0eMemoryStatList\x124\n" + + "\x0eStartBenchmark\x12\x17.benchmark.StartRequest\x1a\x18.benchmark.StartResponse\"\x04\xa0\xb5\x18\x01\x12D\n" + + "\rStopBenchmark\x12\x16.benchmark.StopRequest\x1a\x15.benchmark.MemoryStat\"\x04\xa0\xb5\x18\x01\x124\n" + "\n" + - "QuorumCall\x12\x0f.benchmark.Echo\x1a\x0f.benchmark.Echo\"\x04\xa0\xb5\x18\x01\x12=\n" + - "\x0fAsyncQuorumCall\x12\x0f.benchmark.Echo\x1a\x0f.benchmark.Echo\"\b\xa0\xb5\x18\x01е\x18\x01\x124\n" + + "QuorumCall\x12\x0f.benchmark.Echo\x1a\x0f.benchmark.Echo\"\x04\xa0\xb5\x18\x01\x124\n" + "\n" + "SlowServer\x12\x0f.benchmark.Echo\x1a\x0f.benchmark.Echo\"\x04\xa0\xb5\x18\x01\x12>\n" + - "\tMulticast\x12\x13.benchmark.TimedMsg\x1a\x16.google.protobuf.Empty\"\x04\x98\xb5\x18\x01B(Z!github.com/relab/gorums/benchmark\x92\x03\x02\b\x02b\beditionsp\xe8\a" + "\tMulticast\x12\x13.benchmark.TimedMsg\x1a\x16.google.protobuf.Empty\"\x04\x98\xb5\x18\x01B(Z!github.com/relab/gorums/benchmark\x92\x03\x02\b\x02b\beditionsp\xe9\a" -var file_benchmark_benchmark_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_benchmark_benchmark_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_benchmark_benchmark_proto_goTypes = []any{ - (*Echo)(nil), // 0: benchmark.Echo - (*TimedMsg)(nil), // 1: benchmark.TimedMsg - (*StartRequest)(nil), // 2: benchmark.StartRequest - (*StartResponse)(nil), // 3: benchmark.StartResponse - (*StopRequest)(nil), // 4: benchmark.StopRequest - (*Result)(nil), // 5: benchmark.Result - (*MemoryStat)(nil), // 6: benchmark.MemoryStat - (*MemoryStatList)(nil), // 7: benchmark.MemoryStatList - (*emptypb.Empty)(nil), // 8: google.protobuf.Empty + (*Echo)(nil), // 0: benchmark.Echo + (*TimedMsg)(nil), // 1: benchmark.TimedMsg + (*StartRequest)(nil), // 2: benchmark.StartRequest + (*StartResponse)(nil), // 3: benchmark.StartResponse + (*StopRequest)(nil), // 4: benchmark.StopRequest + (*Result)(nil), // 5: benchmark.Result + (*MemoryStat)(nil), // 6: benchmark.MemoryStat + (*emptypb.Empty)(nil), // 7: google.protobuf.Empty } var file_benchmark_benchmark_proto_depIdxs = []int32{ - 6, // 0: benchmark.Result.ServerStats:type_name -> benchmark.MemoryStat - 6, // 1: benchmark.MemoryStatList.MemoryStats:type_name -> benchmark.MemoryStat - 2, // 2: benchmark.Benchmark.StartServerBenchmark:input_type -> benchmark.StartRequest - 4, // 3: benchmark.Benchmark.StopServerBenchmark:input_type -> benchmark.StopRequest - 2, // 4: benchmark.Benchmark.StartBenchmark:input_type -> benchmark.StartRequest - 4, // 5: benchmark.Benchmark.StopBenchmark:input_type -> benchmark.StopRequest - 0, // 6: benchmark.Benchmark.QuorumCall:input_type -> benchmark.Echo - 0, // 7: benchmark.Benchmark.AsyncQuorumCall:input_type -> benchmark.Echo - 0, // 8: benchmark.Benchmark.SlowServer:input_type -> benchmark.Echo - 1, // 9: benchmark.Benchmark.Multicast:input_type -> benchmark.TimedMsg - 3, // 10: benchmark.Benchmark.StartServerBenchmark:output_type -> benchmark.StartResponse - 5, // 11: benchmark.Benchmark.StopServerBenchmark:output_type -> benchmark.Result - 3, // 12: benchmark.Benchmark.StartBenchmark:output_type -> benchmark.StartResponse - 6, // 13: benchmark.Benchmark.StopBenchmark:output_type -> benchmark.MemoryStat - 0, // 14: benchmark.Benchmark.QuorumCall:output_type -> benchmark.Echo - 0, // 15: benchmark.Benchmark.AsyncQuorumCall:output_type -> benchmark.Echo - 0, // 16: benchmark.Benchmark.SlowServer:output_type -> benchmark.Echo - 8, // 17: benchmark.Benchmark.Multicast:output_type -> google.protobuf.Empty - 10, // [10:18] is the sub-list for method output_type - 2, // [2:10] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 6, // 0: benchmark.Result.server_stats:type_name -> benchmark.MemoryStat + 2, // 1: benchmark.Benchmark.StartServerBenchmark:input_type -> benchmark.StartRequest + 4, // 2: benchmark.Benchmark.StopServerBenchmark:input_type -> benchmark.StopRequest + 2, // 3: benchmark.Benchmark.StartBenchmark:input_type -> benchmark.StartRequest + 4, // 4: benchmark.Benchmark.StopBenchmark:input_type -> benchmark.StopRequest + 0, // 5: benchmark.Benchmark.QuorumCall:input_type -> benchmark.Echo + 0, // 6: benchmark.Benchmark.SlowServer:input_type -> benchmark.Echo + 1, // 7: benchmark.Benchmark.Multicast:input_type -> benchmark.TimedMsg + 3, // 8: benchmark.Benchmark.StartServerBenchmark:output_type -> benchmark.StartResponse + 5, // 9: benchmark.Benchmark.StopServerBenchmark:output_type -> benchmark.Result + 3, // 10: benchmark.Benchmark.StartBenchmark:output_type -> benchmark.StartResponse + 6, // 11: benchmark.Benchmark.StopBenchmark:output_type -> benchmark.MemoryStat + 0, // 12: benchmark.Benchmark.QuorumCall:output_type -> benchmark.Echo + 0, // 13: benchmark.Benchmark.SlowServer:output_type -> benchmark.Echo + 7, // 14: benchmark.Benchmark.Multicast:output_type -> google.protobuf.Empty + 8, // [8:15] is the sub-list for method output_type + 1, // [1:8] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_benchmark_benchmark_proto_init() } @@ -682,7 +623,7 @@ func file_benchmark_benchmark_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_benchmark_benchmark_proto_rawDesc), len(file_benchmark_benchmark_proto_rawDesc)), NumEnums: 0, - NumMessages: 8, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/benchmark/benchmark.proto b/benchmark/benchmark.proto index d64e5132a..fcf76a430 100644 --- a/benchmark/benchmark.proto +++ b/benchmark/benchmark.proto @@ -1,84 +1,88 @@ -edition = "2023"; +edition = "2024"; package benchmark; - option go_package = "github.com/relab/gorums/benchmark"; - option features.field_presence = IMPLICIT; import "google/protobuf/empty.proto"; import "gorums.proto"; +// Echo is a simple message used for echo benchmarks. message Echo { - bytes payload = 1; + bytes payload = 1; } +// TimedMsg is a message with a send time and a payload used for multicast benchmarks. message TimedMsg { - int64 SendTime = 1; - bytes payload = 2; + int64 send_time = 1; + bytes payload = 2; } +// StartRequest is an empty message for starting a benchmarking campaign. message StartRequest {} +// StartResponse is an empty message to acknowledge the start of a benchmarking campaign. message StartResponse {} +// StopRequest is an empty message for stopping a benchmarking campaign. message StopRequest {} +// Result contains the results of a server-side benchmarking campaign. message Result { - string Name = 1; - uint64 TotalOps = 2; - int64 TotalTime = 3; - double Throughput = 4; - double LatencyAvg = 5; - double LatencyVar = 6; - uint64 AllocsPerOp = 7; - uint64 MemPerOp = 8; - repeated MemoryStat ServerStats = 9; + string name = 1; + uint64 total_ops = 2; + int64 total_time = 3; + double throughput = 4; + double latency_avg = 5; + double latency_var = 6; + uint64 allocs_per_op = 7; + uint64 mem_per_op = 8; + repeated MemoryStat server_stats = 9; } +// MemoryStat contains memory statistics for a single server. message MemoryStat { - uint64 Allocs = 1; - uint64 Memory = 2; -} - -message MemoryStatList { - repeated MemoryStat MemoryStats = 1; + uint64 allocs = 1; + uint64 memory = 2; } +// Benchmark is a service for running various benchmarks. service Benchmark { - rpc StartServerBenchmark(StartRequest) returns (StartResponse) { - option (gorums.quorumcall) = true; - } - - rpc StopServerBenchmark(StopRequest) returns (Result) { - option (gorums.quorumcall) = true; - } - - rpc StartBenchmark(StartRequest) returns (StartResponse) { - option (gorums.quorumcall) = true; - } - - rpc StopBenchmark(StopRequest) returns (MemoryStat) { - option (gorums.quorumcall) = true; - option (gorums.custom_return_type) = "MemoryStatList"; - } - - // benchmarks - rpc QuorumCall(Echo) returns (Echo) { - option (gorums.quorumcall) = true; - } - - rpc AsyncQuorumCall(Echo) returns (Echo) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - } - - rpc SlowServer(Echo) returns (Echo) { - option (gorums.quorumcall) = true; - } - - rpc Multicast(TimedMsg) returns (google.protobuf.Empty) { - option (gorums.multicast) = true; - } + // StartServerBenchmark starts a server-side benchmark campaign. + rpc StartServerBenchmark(StartRequest) returns (StartResponse) { + option (gorums.quorumcall) = true; + } + + // StopServerBenchmark stops a server-side benchmark campaign. + rpc StopServerBenchmark(StopRequest) returns (Result) { + option (gorums.quorumcall) = true; + } + + // StartBenchmark starts a client-side benchmark campaign. + rpc StartBenchmark(StartRequest) returns (StartResponse) { + option (gorums.quorumcall) = true; + } + + // StopBenchmark stops a client-side benchmark campaign. + rpc StopBenchmark(StopRequest) returns (MemoryStat) { + option (gorums.quorumcall) = true; + } + + // That actual benchmark RPCs: + + // QuorumCall performs an echo quorum call on all servers. + rpc QuorumCall(Echo) returns (Echo) { + option (gorums.quorumcall) = true; + } + + // SlowServer performs an echo quorum call on slow servers. + rpc SlowServer(Echo) returns (Echo) { + option (gorums.quorumcall) = true; + } + + // Multicast performs a multicast call to all servers. + rpc Multicast(TimedMsg) returns (google.protobuf.Empty) { + option (gorums.multicast) = true; + } } diff --git a/benchmark/benchmark_gorums.pb.go b/benchmark/benchmark_gorums.pb.go index 9a3a81001..44da7918e 100644 --- a/benchmark/benchmark_gorums.pb.go +++ b/benchmark/benchmark_gorums.pb.go @@ -1,17 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: benchmark/benchmark.proto package benchmark import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" - proto "google.golang.org/protobuf/proto" emptypb "google.golang.org/protobuf/types/known/emptypb" ) @@ -22,366 +18,135 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// AsyncQuorumCall asynchronously invokes a quorum call on configuration c -// and returns a AsyncEcho, which can be used to inspect the quorum call -// reply and error when available. -func (c *Configuration) AsyncQuorumCall(ctx context.Context, in *Echo) *AsyncEcho { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.AsyncQuorumCall", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Echo, len(replies)) - for k, v := range replies { - r[k] = v.(*Echo) - } - return c.qspec.AsyncQuorumCallQF(req.(*Echo), r) - } +// AsyncEcho is a future for async quorum calls returning *Echo. +type AsyncEcho = *gorums.Async[*Echo] - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncEcho{fut} -} +// AsyncMemoryStat is a future for async quorum calls returning *MemoryStat. +type AsyncMemoryStat = *gorums.Async[*MemoryStat] -// BenchmarkClient is the client interface for the Benchmark service. -type BenchmarkClient interface { - StartServerBenchmark(ctx context.Context, in *StartRequest) (resp *StartResponse, err error) - StopServerBenchmark(ctx context.Context, in *StopRequest) (resp *Result, err error) - StartBenchmark(ctx context.Context, in *StartRequest) (resp *StartResponse, err error) - StopBenchmark(ctx context.Context, in *StopRequest) (resp *MemoryStatList, err error) - QuorumCall(ctx context.Context, in *Echo) (resp *Echo, err error) - AsyncQuorumCall(ctx context.Context, in *Echo) *AsyncEcho - SlowServer(ctx context.Context, in *Echo) (resp *Echo, err error) - Multicast(ctx context.Context, in *TimedMsg, opts ...gorums.CallOption) -} +// AsyncResult is a future for async quorum calls returning *Result. +type AsyncResult = *gorums.Async[*Result] -// enforce interface compliance -var _ BenchmarkClient = (*Configuration)(nil) +// AsyncStartResponse is a future for async quorum calls returning *StartResponse. +type AsyncStartResponse = *gorums.Async[*StartResponse] -// Reference imports to suppress errors if they are not otherwise used. -var _ emptypb.Empty - -// Multicast is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) Multicast(ctx context.Context, in *TimedMsg, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.Multicast", - } +// CorrectableEcho is a correctable object for quorum calls returning *Echo. +type CorrectableEcho = *gorums.Correctable[*Echo] - c.RawConfiguration.Multicast(ctx, cd, opts...) -} +// CorrectableMemoryStat is a correctable object for quorum calls returning *MemoryStat. +type CorrectableMemoryStat = *gorums.Correctable[*MemoryStat] -// QuorumSpec is the interface of quorum functions for Benchmark. -type QuorumSpec interface { - gorums.ConfigOption +// CorrectableResult is a correctable object for quorum calls returning *Result. +type CorrectableResult = *gorums.Correctable[*Result] - // StartServerBenchmarkQF is the quorum function for the StartServerBenchmark - // quorum call method. The in parameter is the request object - // supplied to the StartServerBenchmark method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *StartRequest'. - StartServerBenchmarkQF(in *StartRequest, replies map[uint32]*StartResponse) (*StartResponse, bool) +// CorrectableStartResponse is a correctable object for quorum calls returning *StartResponse. +type CorrectableStartResponse = *gorums.Correctable[*StartResponse] - // StopServerBenchmarkQF is the quorum function for the StopServerBenchmark - // quorum call method. The in parameter is the request object - // supplied to the StopServerBenchmark method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *StopRequest'. - StopServerBenchmarkQF(in *StopRequest, replies map[uint32]*Result) (*Result, bool) - - // StartBenchmarkQF is the quorum function for the StartBenchmark - // quorum call method. The in parameter is the request object - // supplied to the StartBenchmark method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *StartRequest'. - StartBenchmarkQF(in *StartRequest, replies map[uint32]*StartResponse) (*StartResponse, bool) - - // StopBenchmarkQF is the quorum function for the StopBenchmark - // quorum call method. The in parameter is the request object - // supplied to the StopBenchmark method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *StopRequest'. - StopBenchmarkQF(in *StopRequest, replies map[uint32]*MemoryStat) (*MemoryStatList, bool) - - // QuorumCallQF is the quorum function for the QuorumCall - // quorum call method. The in parameter is the request object - // supplied to the QuorumCall method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Echo'. - QuorumCallQF(in *Echo, replies map[uint32]*Echo) (*Echo, bool) - - // AsyncQuorumCallQF is the quorum function for the AsyncQuorumCall - // asynchronous quorum call method. The in parameter is the request object - // supplied to the AsyncQuorumCall method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Echo'. - AsyncQuorumCallQF(in *Echo, replies map[uint32]*Echo) (*Echo, bool) +// Reference imports to suppress errors if they are not otherwise used. +var _ emptypb.Empty - // SlowServerQF is the quorum function for the SlowServer - // quorum call method. The in parameter is the request object - // supplied to the SlowServer method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Echo'. - SlowServerQF(in *Echo, replies map[uint32]*Echo) (*Echo, bool) +// StartServerBenchmark starts a server-side benchmark campaign. +func StartServerBenchmark(ctx *gorums.ConfigContext, in *StartRequest, opts ...gorums.CallOption) *gorums.Responses[*StartResponse] { + return gorums.QuorumCall[*StartRequest, *StartResponse]( + ctx, in, "benchmark.Benchmark.StartServerBenchmark", + opts..., + ) } -// StartServerBenchmark is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) StartServerBenchmark(ctx context.Context, in *StartRequest) (resp *StartResponse, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.StartServerBenchmark", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*StartResponse, len(replies)) - for k, v := range replies { - r[k] = v.(*StartResponse) - } - return c.qspec.StartServerBenchmarkQF(req.(*StartRequest), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*StartResponse), err +// StopServerBenchmark stops a server-side benchmark campaign. +func StopServerBenchmark(ctx *gorums.ConfigContext, in *StopRequest, opts ...gorums.CallOption) *gorums.Responses[*Result] { + return gorums.QuorumCall[*StopRequest, *Result]( + ctx, in, "benchmark.Benchmark.StopServerBenchmark", + opts..., + ) } -// StopServerBenchmark is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) StopServerBenchmark(ctx context.Context, in *StopRequest) (resp *Result, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.StopServerBenchmark", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Result, len(replies)) - for k, v := range replies { - r[k] = v.(*Result) - } - return c.qspec.StopServerBenchmarkQF(req.(*StopRequest), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Result), err +// StartBenchmark starts a client-side benchmark campaign. +func StartBenchmark(ctx *gorums.ConfigContext, in *StartRequest, opts ...gorums.CallOption) *gorums.Responses[*StartResponse] { + return gorums.QuorumCall[*StartRequest, *StartResponse]( + ctx, in, "benchmark.Benchmark.StartBenchmark", + opts..., + ) } -// StartBenchmark is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) StartBenchmark(ctx context.Context, in *StartRequest) (resp *StartResponse, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.StartBenchmark", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*StartResponse, len(replies)) - for k, v := range replies { - r[k] = v.(*StartResponse) - } - return c.qspec.StartBenchmarkQF(req.(*StartRequest), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*StartResponse), err +// StopBenchmark stops a client-side benchmark campaign. +func StopBenchmark(ctx *gorums.ConfigContext, in *StopRequest, opts ...gorums.CallOption) *gorums.Responses[*MemoryStat] { + return gorums.QuorumCall[*StopRequest, *MemoryStat]( + ctx, in, "benchmark.Benchmark.StopBenchmark", + opts..., + ) } -// StopBenchmark is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) StopBenchmark(ctx context.Context, in *StopRequest) (resp *MemoryStatList, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.StopBenchmark", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*MemoryStat, len(replies)) - for k, v := range replies { - r[k] = v.(*MemoryStat) - } - return c.qspec.StopBenchmarkQF(req.(*StopRequest), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*MemoryStatList), err +// QuorumCall performs an echo quorum call on all servers. +func QuorumCall(ctx *gorums.ConfigContext, in *Echo, opts ...gorums.CallOption) *gorums.Responses[*Echo] { + return gorums.QuorumCall[*Echo, *Echo]( + ctx, in, "benchmark.Benchmark.QuorumCall", + opts..., + ) } -// benchmarks -func (c *Configuration) QuorumCall(ctx context.Context, in *Echo) (resp *Echo, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.QuorumCall", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Echo, len(replies)) - for k, v := range replies { - r[k] = v.(*Echo) - } - return c.qspec.QuorumCallQF(req.(*Echo), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Echo), err +// SlowServer performs an echo quorum call on slow servers. +func SlowServer(ctx *gorums.ConfigContext, in *Echo, opts ...gorums.CallOption) *gorums.Responses[*Echo] { + return gorums.QuorumCall[*Echo, *Echo]( + ctx, in, "benchmark.Benchmark.SlowServer", + opts..., + ) } -// SlowServer is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) SlowServer(ctx context.Context, in *Echo) (resp *Echo, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "benchmark.Benchmark.SlowServer", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Echo, len(replies)) - for k, v := range replies { - r[k] = v.(*Echo) - } - return c.qspec.SlowServerQF(req.(*Echo), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Echo), err +// Multicast performs a multicast call to all servers. +func Multicast(ctx *gorums.ConfigContext, in *TimedMsg, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "benchmark.Benchmark.Multicast", opts...) } // Benchmark is the server-side API for the Benchmark Service @@ -391,7 +156,6 @@ type BenchmarkServer interface { StartBenchmark(ctx gorums.ServerCtx, request *StartRequest) (response *StartResponse, err error) StopBenchmark(ctx gorums.ServerCtx, request *StopRequest) (response *MemoryStat, err error) QuorumCall(ctx gorums.ServerCtx, request *Echo) (response *Echo, err error) - AsyncQuorumCall(ctx gorums.ServerCtx, request *Echo) (response *Echo, err error) SlowServer(ctx gorums.ServerCtx, request *Echo) (response *Echo, err error) Multicast(ctx gorums.ServerCtx, request *TimedMsg) } @@ -422,11 +186,6 @@ func RegisterBenchmarkServer(srv *gorums.Server, impl BenchmarkServer) { resp, err := impl.QuorumCall(ctx, req) return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) - srv.RegisterHandler("benchmark.Benchmark.AsyncQuorumCall", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Echo](in) - resp, err := impl.AsyncQuorumCall(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) srv.RegisterHandler("benchmark.Benchmark.SlowServer", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*Echo](in) resp, err := impl.SlowServer(ctx, req) @@ -438,42 +197,3 @@ func RegisterBenchmarkServer(srv *gorums.Server, impl BenchmarkServer) { return nil, nil }) } - -type internalEcho struct { - nid uint32 - reply *Echo - err error -} - -type internalMemoryStat struct { - nid uint32 - reply *MemoryStat - err error -} - -type internalResult struct { - nid uint32 - reply *Result - err error -} - -type internalStartResponse struct { - nid uint32 - reply *StartResponse - err error -} - -// AsyncEcho is a async object for processing replies. -type AsyncEcho struct { - *gorums.Async -} - -// Get returns the reply and any error associated with the called method. -// The method blocks until a reply or error is available. -func (f *AsyncEcho) Get() (*Echo, error) { - resp, err := f.Async.Get() - if err != nil { - return nil, err - } - return resp.(*Echo), err -} diff --git a/benchmark/qspec.go b/benchmark/qspec.go index 8957978d1..76de3b509 100644 --- a/benchmark/qspec.go +++ b/benchmark/qspec.go @@ -1,43 +1,14 @@ package benchmark -// QSpec is the quorum specification object for the benchmark -type QSpec struct { - CfgSize int - QSize int -} - -func (qspec *QSpec) qf(replies map[uint32]*Echo) (*Echo, bool) { - if len(replies) < qspec.QSize { - return nil, false - } - var resp *Echo - for _, r := range replies { - resp = r - break - } - return resp, true -} - -// StartServerBenchmarkQF is the quorum function for the StartServerBenchmark quorumcall. -// It requires a response from all nodes. -func (qspec *QSpec) StartServerBenchmarkQF(_ *StartRequest, replies map[uint32]*StartResponse) (*StartResponse, bool) { - if len(replies) < qspec.CfgSize { - return nil, false - } - // return any response - var resp *StartResponse - for _, r := range replies { - resp = r - break - } - return resp, true -} +import ( + "github.com/relab/gorums" +) -// StopServerBenchmarkQF is the quorum function for the StopServerBenchmark quorumcall. -// It requires a response from all nodes. -func (qspec *QSpec) StopServerBenchmarkQF(_ *StopRequest, replies map[uint32]*Result) (*Result, bool) { - if len(replies) < qspec.CfgSize { - return nil, false +// StopServerBenchmarkQF aggregates StopServerBenchmark responses from all nodes. +// It combines results, calculating averages and pooled variance. +func StopServerBenchmarkQF(replies map[uint32]*Result) (*Result, error) { + if len(replies) == 0 { + return nil, gorums.ErrIncomplete } // combine results, calculating averages and pooled variance resp := &Result{} @@ -62,48 +33,5 @@ func (qspec *QSpec) StopServerBenchmarkQF(_ *StopRequest, replies map[uint32]*Re resp.SetTotalOps(resp.GetTotalOps() / uint64(len(replies))) resp.SetTotalTime(resp.GetTotalTime() / int64(len(replies))) resp.SetThroughput(resp.GetThroughput() / float64(len(replies))) - return resp, true -} - -// StartBenchmarkQF is the quorum function for the StartBenchmark quorumcall. -// It requires a response from all nodes. -func (qspec *QSpec) StartBenchmarkQF(_ *StartRequest, replies map[uint32]*StartResponse) (*StartResponse, bool) { - if len(replies) < qspec.CfgSize { - return nil, false - } - // return any response - var resp *StartResponse - for _, r := range replies { - resp = r - break - } - return resp, true -} - -// StopBenchmarkQF is the quorum function for the StopBenchmark quorumcall. -// It requires a response from all nodes. -func (qspec *QSpec) StopBenchmarkQF(_ *StopRequest, replies map[uint32]*MemoryStat) (*MemoryStatList, bool) { - if len(replies) < qspec.CfgSize { - return nil, false - } - replyList := make([]*MemoryStat, 0, len(replies)) - for _, v := range replies { - replyList = append(replyList, v) - } - return MemoryStatList_builder{MemoryStats: replyList}.Build(), true -} - -// QuorumCallQF is the quorum function for the QuorumCall quorumcall -func (qspec *QSpec) QuorumCallQF(_ *Echo, replies map[uint32]*Echo) (*Echo, bool) { - return qspec.qf(replies) -} - -// AsyncQuorumCallQF is the quorum function for the AsyncQuorumCall quorumcall -func (qspec *QSpec) AsyncQuorumCallQF(_ *Echo, replies map[uint32]*Echo) (*Echo, bool) { - return qspec.qf(replies) -} - -// SlowServerQF is the quorum function for the SlowServer quorumcall -func (qspec *QSpec) SlowServerQF(_ *Echo, replies map[uint32]*Echo) (*Echo, bool) { - return qspec.qf(replies) + return resp, nil } diff --git a/callopts.go b/callopts.go index 77e34040e..5c7efc411 100644 --- a/callopts.go +++ b/callopts.go @@ -1,30 +1,33 @@ package gorums -import "google.golang.org/protobuf/runtime/protoimpl" +import ( + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/runtime/protoimpl" +) type callOptions struct { callType *protoimpl.ExtensionInfo - waitSendDone bool + ignoreErrors bool + interceptors []any // Type-erased interceptors, restored by QuorumCall } // mustWaitSendDone returns true if the caller of a one-way call type must wait -// for send completion. This is the default behavior unless the WithNoSendWaiting +// for send completion. This is the default behavior unless the IgnoreErrors // call option is set. This always returns false for two-way call types, since // they should always wait for actual server responses. func (o callOptions) mustWaitSendDone() bool { - if o.callType == E_Rpc || o.callType == E_Quorumcall || o.callType == E_Correctable { - return false - } - return o.callType != nil && o.waitSendDone + // must wait for send completion if we are not ignoring errors + // and the call type is Unicast or Multicast + return !o.ignoreErrors && (o.callType == E_Unicast || o.callType == E_Multicast) } // CallOption is a function that sets a value in the given callOptions struct type CallOption func(*callOptions) -func getCallOptions(callType *protoimpl.ExtensionInfo, opts []CallOption) callOptions { +func getCallOptions(callType *protoimpl.ExtensionInfo, opts ...CallOption) callOptions { o := callOptions{ callType: callType, - waitSendDone: true, // default: wait for send completion + ignoreErrors: false, // default: return error and wait for send completion } for _, opt := range opts { opt(&o) @@ -32,11 +35,29 @@ func getCallOptions(callType *protoimpl.ExtensionInfo, opts []CallOption) callOp return o } -// WithNoSendWaiting is a CallOption that makes Unicast or Multicast methods -// return immediately instead of blocking until the message has been sent. -// By default, Unicast and Multicast methods wait for send completion. -func WithNoSendWaiting() CallOption { +// IgnoreErrors ignores send errors from Unicast or Multicast methods and +// returns immediately instead of blocking until the message has been sent. +// By default, Unicast and Multicast methods return an error if the message +// could not be sent or the context was canceled. +func IgnoreErrors() CallOption { + return func(o *callOptions) { + o.ignoreErrors = true + } +} + +// Interceptors returns a CallOption that adds quorum call interceptors. +// Interceptors are executed in the order provided, modifying the Responses object +// before the user calls a terminal method. +// +// Example: +// +// resp, err := ReadQC(ctx, req, +// gorums.Interceptors(loggingInterceptor, filterInterceptor), +// ).Majority() +func Interceptors[Req, Resp proto.Message](interceptors ...QuorumInterceptor[Req, Resp]) CallOption { return func(o *callOptions) { - o.waitSendDone = false + for _, interceptor := range interceptors { + o.interceptors = append(o.interceptors, interceptor) + } } } diff --git a/callopts_test.go b/callopts_test.go index 7f21a7737..35418268a 100644 --- a/callopts_test.go +++ b/callopts_test.go @@ -1,6 +1,9 @@ package gorums -import "testing" +import ( + "fmt" + "testing" +) func TestCallOptionsMustWaitSendDone(t *testing.T) { tests := []struct { @@ -9,17 +12,15 @@ func TestCallOptionsMustWaitSendDone(t *testing.T) { wantWaitSendDone bool }{ // One-way call types - {name: "Unicast/WithSendWaiting", callOpts: getCallOptions(E_Unicast, nil), wantWaitSendDone: true}, - {name: "Unicast/WithNoSendWaiting", callOpts: getCallOptions(E_Unicast, []CallOption{WithNoSendWaiting()}), wantWaitSendDone: false}, - {name: "Multicast/WithSendWaiting", callOpts: getCallOptions(E_Multicast, nil), wantWaitSendDone: true}, - {name: "Multicast/WithNoSendWaiting", callOpts: getCallOptions(E_Multicast, []CallOption{WithNoSendWaiting()}), wantWaitSendDone: false}, + {name: "Unicast/Default", callOpts: getCallOptions(E_Unicast), wantWaitSendDone: true}, + {name: "Unicast/IgnoreErrors", callOpts: getCallOptions(E_Unicast, IgnoreErrors()), wantWaitSendDone: false}, + {name: "Multicast/Default", callOpts: getCallOptions(E_Multicast), wantWaitSendDone: true}, + {name: "Multicast/IgnoreErrors", callOpts: getCallOptions(E_Multicast, IgnoreErrors()), wantWaitSendDone: false}, // Two-way call types (never wait for send completion, regardless of option) - {name: "Rpc/WithSendWaiting", callOpts: getCallOptions(E_Rpc, nil), wantWaitSendDone: false}, - {name: "Rpc/WithNoSendWaiting", callOpts: getCallOptions(E_Rpc, []CallOption{WithNoSendWaiting()}), wantWaitSendDone: false}, - {name: "Quorumcall/WithSendWaiting", callOpts: getCallOptions(E_Quorumcall, nil), wantWaitSendDone: false}, - {name: "Quorumcall/WithNoSendWaiting", callOpts: getCallOptions(E_Quorumcall, []CallOption{WithNoSendWaiting()}), wantWaitSendDone: false}, - {name: "Correctable/WithSendWaiting", callOpts: getCallOptions(E_Correctable, nil), wantWaitSendDone: false}, - {name: "Correctable/WithNoSendWaiting", callOpts: getCallOptions(E_Correctable, []CallOption{WithNoSendWaiting()}), wantWaitSendDone: false}, + {name: "Rpc/Default", callOpts: getCallOptions(E_Rpc), wantWaitSendDone: false}, + {name: "Rpc/IgnoreErrors", callOpts: getCallOptions(E_Rpc, IgnoreErrors()), wantWaitSendDone: false}, + {name: "Quorumcall/Default", callOpts: getCallOptions(E_Quorumcall), wantWaitSendDone: false}, + {name: "Quorumcall/IgnoreErrors", callOpts: getCallOptions(E_Quorumcall, IgnoreErrors()), wantWaitSendDone: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -30,3 +31,25 @@ func TestCallOptionsMustWaitSendDone(t *testing.T) { }) } } + +func BenchmarkGetCallOptions(b *testing.B) { + interceptor := func(_ *ClientCtx[msg, msg], next ResponseSeq[msg]) ResponseSeq[msg] { return next } + tests := []struct { + numOpts int + }{ + {0}, {1}, {2}, {3}, {4}, {5}, + } + + for _, tc := range tests { + opts := make([]CallOption, tc.numOpts) + for i := range tc.numOpts { + opts[i] = Interceptors(interceptor) + } + b.Run(fmt.Sprintf("options=%d", tc.numOpts), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = getCallOptions(E_Quorumcall, opts...) + } + }) + } +} diff --git a/channel.go b/channel.go index 6c6b54a3b..6e032bcf9 100644 --- a/channel.go +++ b/channel.go @@ -2,7 +2,6 @@ package gorums import ( "context" - "fmt" "sync" "time" @@ -14,48 +13,75 @@ import ( "google.golang.org/protobuf/proto" ) -var streamDownErr = status.Error(codes.Unavailable, "stream is down") - -type request struct { - ctx context.Context - msg *Message - opts callOptions - streaming bool +// NodeResponse wraps a response value from node ID, and an error if any. +type NodeResponse[T any] struct { + NodeID uint32 + Value T + Err error } -type response struct { - nid uint32 - msg proto.Message - err error +// newNodeResponse converts a NodeResponse[msg] to a NodeResponse[Resp]. +// This is necessary because the channel layer's response router returns a +// NodeResponse[msg], while the calltype expects a NodeResponse[Resp]. +func newNodeResponse[Resp msg](r NodeResponse[msg]) NodeResponse[Resp] { + res := NodeResponse[Resp]{ + NodeID: r.NodeID, + Err: r.Err, + } + if r.Err == nil { + if val, ok := r.Value.(Resp); ok { + res.Value = val + } else { + res.Err = ErrTypeMismatch + } + } + return res } -type responseRouter struct { - c chan<- response - streaming bool +var ( + nodeClosedErr = status.Error(codes.Unavailable, "node closed") + streamDownErr = status.Error(codes.Unavailable, "stream is down") +) + +type request struct { + ctx context.Context + msg *Message + waitSendDone bool + streaming bool + responseChan chan<- NodeResponse[proto.Message] + sendTime time.Time } type channel struct { sendQ chan request - node *RawNode + id uint32 + + // Connection lifecycle management: node close() cancels the + // connection context to stop all goroutines and the NodeStream + conn *grpc.ClientConn + connCtx context.Context + connCancel context.CancelFunc // Error and latency tracking mu sync.Mutex lastError error latency time.Duration - // Stream management for FIFO ordering - // gorumsStream is a bidirectional stream for FIFO message delivery - gorumsStream grpc.ClientStream - streamMut sync.RWMutex + // Stream lifecycle management for FIFO ordered message delivery + // gorumsStream is a bidirectional stream for + // sending and receiving ordering.Metadata messages. + gorumsStream ordering.Gorums_NodeStreamClient + streamMut sync.Mutex streamCtx context.Context - cancelStream context.CancelFunc + streamCancel context.CancelFunc + streamReady chan struct{} // signals receiver when stream becomes available - // Response routing - responseRouters map[uint64]responseRouter + // Response routing; the map holds pending requests waiting for responses. + // The request contains the responseChan on which to send the response + // to the caller. + responseRouters map[uint64]request responseMut sync.Mutex - - // Lifecycle management: node close() cancels this context - parentCtx context.Context + closeOnceFunc func() error } // newChannel creates a new channel for the given node and starts the sender @@ -65,34 +91,73 @@ type channel struct { // have not yet been established. This is to prevent deadlock when invoking // a call type. The sender blocks on the sendQ and the receiver waits for // the stream to become available. -func newChannel(n *RawNode) *channel { +func newChannel(parentCtx context.Context, conn *grpc.ClientConn, id uint32, sendBufferSize uint) *channel { + ctx, connCancel := context.WithCancel(parentCtx) c := &channel{ - sendQ: make(chan request, n.mgr.opts.sendBuffer), - node: n, + sendQ: make(chan request, sendBufferSize), + id: id, + conn: conn, + connCtx: ctx, + connCancel: connCancel, latency: -1 * time.Second, - responseRouters: make(map[uint64]responseRouter), + responseRouters: make(map[uint64]request), + streamReady: make(chan struct{}, 1), } - // parentCtx controls the channel and is used to shut it down - c.parentCtx = n.newContext() + c.closeOnceFunc = sync.OnceValue(func() error { + // important to cancel first to stop goroutines + c.connCancel() + if c.conn != nil { + return c.conn.Close() + } + return nil + }) go c.sender() go c.receiver() return c } -// newNodeStream creates a new stream for this channel. -// The receiver goroutine will detect the new stream and start using it. -func (c *channel) newNodeStream() (err error) { +// close closes the channel and the underlying connection exactly once. +func (c *channel) close() error { + return c.closeOnceFunc() +} + +// ensureStream ensures there is an active NodeStream for the sender and +// receiver goroutines, and signals the receiver when the stream is ready. +// gRPC automatically handles TCP connection state when creating the stream. +// This method is safe for concurrent use. +func (c *channel) ensureStream() error { + if err := c.ensureConnectedNodeStream(); err != nil { + return err + } + // signal receiver that stream is ready (non-blocking) + select { + case c.streamReady <- struct{}{}: + default: + // channel already has a signal pending, no need to add another + } + return nil +} + +// ensureConnectedNodeStream ensures there is an active and connected +// NodeStream, or creates a new stream if one doesn't already exist. +// This method is safe for concurrent use. +func (c *channel) ensureConnectedNodeStream() (err error) { c.streamMut.Lock() - c.streamCtx, c.cancelStream = context.WithCancel(c.parentCtx) - c.gorumsStream, err = ordering.NewGorumsClient(c.node.conn).NodeStream(c.streamCtx) - c.streamMut.Unlock() + defer c.streamMut.Unlock() + // if we already have a ready connection and an active stream, do nothing + // (cannot reuse isConnected() here since it uses the streamMut lock) + if c.conn.GetState() == connectivity.Ready && c.gorumsStream != nil { + return nil + } + c.streamCtx, c.streamCancel = context.WithCancel(c.connCtx) + c.gorumsStream, err = ordering.NewGorumsClient(c.conn).NodeStream(c.streamCtx) return err } // getStream returns the current stream, or nil if no stream is available. func (c *channel) getStream() grpc.ClientStream { - c.streamMut.RLock() - defer c.streamMut.RUnlock() + c.streamMut.Lock() + defer c.streamMut.Unlock() return c.gorumsStream } @@ -100,71 +165,65 @@ func (c *channel) getStream() grpc.ClientStream { // This triggers reconnection on the next send attempt. func (c *channel) clearStream() { c.streamMut.Lock() - c.cancelStream() + c.streamCancel() c.gorumsStream = nil c.streamMut.Unlock() } -// ensureStream ensures there's an active NodeStream for the sender and receiver goroutines. -// gRPC automatically handles TCP connection state when creating the stream. -func (c *channel) ensureStream() error { - if c.isConnected() { - return nil - } - return c.newNodeStream() -} - // isConnected returns true if the gRPC connection is in Ready state and we have an active stream. +// This method is safe for concurrent use. func (c *channel) isConnected() bool { - return c.node.conn.GetState() == connectivity.Ready && c.getStream() != nil + return c.conn.GetState() == connectivity.Ready && c.getStream() != nil } -// enqueue adds the request to the send queue and sets up a response router if needed. +// enqueue adds the request to the send queue and sets up response routing if needed. // If the node is closed, it responds with an error instead. -func (c *channel) enqueue(req request, responseChan chan<- response) { - msgID := req.msg.GetMessageID() - if responseChan != nil { - // allocate before critical section - router := responseRouter{responseChan, req.streaming} +func (c *channel) enqueue(req request) { + if req.responseChan != nil { + req.sendTime = time.Now() + msgID := req.msg.GetMessageID() c.responseMut.Lock() - c.responseRouters[msgID] = router + c.responseRouters[msgID] = req c.responseMut.Unlock() } // either enqueue the request on the sendQ or respond // with error if the node is closed select { - case <-c.parentCtx.Done(): + case <-c.connCtx.Done(): // the node's close() method was called: respond with error instead of enqueueing - c.routeResponse(msgID, response{nid: c.node.ID(), err: fmt.Errorf("node closed")}) + c.routeResponse(req.msg.GetMessageID(), NodeResponse[proto.Message]{NodeID: c.id, Err: nodeClosedErr}) return case c.sendQ <- req: // enqueued successfully } } -// routeResponse routes the response to the appropriate response router based on msgID. -// If no router is found, the response is discarded. -func (c *channel) routeResponse(msgID uint64, resp response) { +// routeResponse routes the response to the appropriate response channel based on msgID. +// If no matching request is found, the response is discarded. +func (c *channel) routeResponse(msgID uint64, resp NodeResponse[proto.Message]) { c.responseMut.Lock() defer c.responseMut.Unlock() - if router, ok := c.responseRouters[msgID]; ok { - router.c <- resp + if req, ok := c.responseRouters[msgID]; ok { + if resp.Err == nil { + c.updateLatency(time.Since(req.sendTime)) + } + req.responseChan <- resp // delete the router if we are only expecting a single reply message - if !router.streaming { + if !req.streaming { delete(c.responseRouters, msgID) } } } -// cancelPendingMsgs cancels all pending messages by sending an error response to each router. -// This is typically called when the stream goes down to notify all waiting calls. -func (c *channel) cancelPendingMsgs() { +// cancelPendingMsgs cancels all pending messages by sending an error response to each +// associated request. This is called when the stream goes down to notify all waiting calls. +func (c *channel) cancelPendingMsgs(err error) { c.responseMut.Lock() defer c.responseMut.Unlock() - for msgID, router := range c.responseRouters { - router.c <- response{nid: c.node.ID(), err: streamDownErr} + for msgID, req := range c.responseRouters { + req.responseChan <- NodeResponse[proto.Message]{NodeID: c.id, Err: err} // delete the router if we are only expecting a single reply message - if !router.streaming { + if !req.streaming { delete(c.responseRouters, msgID) } } @@ -181,21 +240,24 @@ func (c *channel) deleteRouter(msgID uint64) { // sender goroutine takes requests from the sendQ and sends them on the stream. // If the stream is down, it tries to re-establish it. func (c *channel) sender() { + // eager connect; ignored if stream is down (will be retried on send) + _ = c.ensureStream() + var req request for { select { - case <-c.parentCtx.Done(): + case <-c.connCtx.Done(): // the node's close() method was called: exit sender goroutine return case req = <-c.sendQ: // take next request from sendQ } if err := c.ensureStream(); err != nil { - c.routeResponse(req.msg.GetMessageID(), response{nid: c.node.ID(), err: err}) + c.routeResponse(req.msg.GetMessageID(), NodeResponse[proto.Message]{NodeID: c.id, Err: err}) continue } if err := c.sendMsg(req); err != nil { - c.routeResponse(req.msg.GetMessageID(), response{nid: c.node.ID(), err: err}) + c.routeResponse(req.msg.GetMessageID(), NodeResponse[proto.Message]{NodeID: c.id, Err: err}) } } } @@ -207,24 +269,29 @@ func (c *channel) receiver() { for { stream := c.getStream() if stream == nil { - // stream not yet available, wait and retry - time.Sleep(10 * time.Millisecond) - continue + // Stream not yet available; wait for signal or shutdown + select { + case <-c.streamReady: + // Stream is now available, continue to get it + continue + case <-c.connCtx.Done(): + // the node's close() method was called: exit receiver goroutine + return + } } resp := newMessage(responseType) if err := stream.RecvMsg(resp); err != nil { - c.logf("RecvMsg error: %v", err) c.setLastErr(err) - c.cancelPendingMsgs() + c.cancelPendingMsgs(err) c.clearStream() } else { err := resp.GetStatus().Err() - c.routeResponse(resp.GetMessageID(), response{nid: c.node.ID(), msg: resp.GetProtoMessage(), err: err}) + c.routeResponse(resp.GetMessageID(), NodeResponse[proto.Message]{NodeID: c.id, Value: resp.GetProtoMessage(), Err: err}) } select { - case <-c.parentCtx.Done(): + case <-c.connCtx.Done(): // the node's close() method was called: exit receiver goroutine return default: @@ -241,16 +308,16 @@ func (c *channel) sendMsg(req request) (err error) { // - If send fails (err != nil): sender() goroutine will deliver the error // - Provides synchronous error handling with immediate feedback // - // 2. WithNoSendWaiting option (mustWaitSendDone=false): Return immediately after enqueuing + // 2. With the IgnoreErrors option (mustWaitSendDone=false): Return immediately after enqueuing // - Caller returns as soon as message is queued to sendQ // - Any errors are delivered asynchronously by sender() goroutine // - Provides fire-and-forget semantics // // Note: Two-way call types (RPCCall, QuorumCall) do not use this mechanism, they always - // wait for actual server responses, so mustWaitSendDone() returns false for them. - if req.opts.mustWaitSendDone() && err == nil { + // wait for actual server responses, so waitSendDone is false for them. + if req.waitSendDone && err == nil { // Send succeeded: unblock the caller and clean up the responseRouter - c.routeResponse(req.msg.GetMessageID(), response{}) + c.routeResponse(req.msg.GetMessageID(), NodeResponse[proto.Message]{}) } }() @@ -282,14 +349,12 @@ func (c *channel) sendMsg(req request) (err error) { // false alarm default: // trigger reconnect - c.logf("request context cancelled: %v", req.ctx.Err()) c.clearStream() } } }() if err = stream.SendMsg(req.msg); err != nil { - c.logf("SendMsg error: %v", err) c.setLastErr(err) c.clearStream() } @@ -298,13 +363,6 @@ func (c *channel) sendMsg(req request) (err error) { return err } -func (c *channel) logf(format string, args ...any) { - if c.node.mgr.logger == nil { - return - } - c.node.mgr.logger.Printf("Node %d: %s", c.node.ID(), fmt.Sprintf(format, args...)) -} - func (c *channel) setLastErr(err error) { c.mu.Lock() defer c.mu.Unlock() @@ -318,9 +376,22 @@ func (c *channel) lastErr() error { return c.lastError } -// channelLatency returns the latency between the client and this channel. +// channelLatency returns the latency between the client and the server associated with this channel. func (c *channel) channelLatency() time.Duration { c.mu.Lock() defer c.mu.Unlock() return c.latency } + +// updateLatency updates the latency between the client and the server associated with this channel. +// It uses a simple moving average to calculate the latency. +func (c *channel) updateLatency(rtt time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + if c.latency < 0 { + c.latency = rtt + } else { + // Use simple moving average (alpha=0.2) + c.latency = time.Duration(0.8*float64(c.latency) + 0.2*float64(rtt)) + } +} diff --git a/channel_test.go b/channel_test.go index 6ecf2a62f..f4c24309a 100644 --- a/channel_test.go +++ b/channel_test.go @@ -16,31 +16,20 @@ import ( pb "google.golang.org/protobuf/types/known/wrapperspb" ) -const defaultTestTimeout = 3 * time.Second - -var ( - waitSendDone = getCallOptions(E_Multicast, nil) - noSendWaiting = getCallOptions(E_Multicast, []CallOption{WithNoSendWaiting()}) +const ( + defaultTestTimeout = 3 * time.Second + streamConnectDelay = 10 * time.Millisecond ) -// newNodeWithServer creates a node connected to a live server that will delay -// responding by the given duration. If delay is 0, the server responds immediately. -func newNodeWithServer(t testing.TB, delay time.Duration) *RawNode { - t.Helper() - node, teardown := newNodeWithStoppableServer(t, delay) - t.Cleanup(teardown) - return node -} - type mockSrv struct{} func (mockSrv) Test(_ ServerCtx, req proto.Message) (proto.Message, error) { return pb.String(mock.GetVal(req) + "-mocked-"), nil } -func newNodeWithStoppableServer(t testing.TB, delay time.Duration) (*RawNode, func()) { - t.Helper() - addrs, teardown := TestSetup(t, 1, func(_ int) ServerIface { +// delayServerFn returns a server function that delays responses by the given duration. +func delayServerFn(delay time.Duration) func(_ int) ServerIface { + return func(_ int) ServerIface { mockSrv := &mockSrv{} srv := NewServer() srv.RegisterHandler(mock.TestMethod, func(ctx ServerCtx, in *Message) (*Message, error) { @@ -50,35 +39,34 @@ func newNodeWithStoppableServer(t testing.TB, delay time.Duration) (*RawNode, fu return NewResponseMessage(in.GetMetadata(), resp), err }) return srv - }) - - return NewNode(t, addrs[0]), teardown + } } -func sendRequest(t testing.TB, node *RawNode, req request, msgID uint64) response { +func sendRequest(t testing.TB, node *Node, req request, msgID uint64) NodeResponse[proto.Message] { t.Helper() if req.ctx == nil { req.ctx = t.Context() } req.msg = NewRequestMessage(ordering.NewGorumsMetadata(req.ctx, msgID, mock.TestMethod), nil) - replyChan := make(chan response, 1) - node.channel.enqueue(req, replyChan) + replyChan := make(chan NodeResponse[proto.Message], 1) + req.responseChan = replyChan + node.channel.enqueue(req) select { case resp := <-replyChan: return resp case <-time.After(defaultTestTimeout): t.Fatalf("timeout waiting for response to message %d", msgID) - return response{} + return NodeResponse[proto.Message]{} } } type msgResponse struct { msgID uint64 - resp response + resp NodeResponse[proto.Message] } -func send(t testing.TB, results chan<- msgResponse, node *RawNode, goroutineID, msgsToSend int, req request) { +func sendReq(t testing.TB, results chan<- msgResponse, node *Node, goroutineID, msgsToSend int, req request) { for j := range msgsToSend { msgID := uint64(goroutineID*1000 + j) resp := sendRequest(t, node, req, msgID) @@ -86,32 +74,52 @@ func send(t testing.TB, results chan<- msgResponse, node *RawNode, goroutineID, } } +// testNodeWithoutServer creates a node for the given server address and +// adds it to a new manager. This is useful for testing node and channel +// behavior without an active server. The manager is automatically closed +// when the test finishes. +func testNodeWithoutServer(t testing.TB, opts ...ManagerOption) *Node { + t.Helper() + mgrOpts := append([]ManagerOption{InsecureDialOptions(t)}, opts...) + mgr := NewManager(mgrOpts...) + t.Cleanup(Closer(t, mgr)) + // Use a high port number that's unlikely to have anything listening. + // We use a fixed ID for simplicity. + node, err := mgr.newNode("127.0.0.1:59999", 1) + if err != nil { + t.Fatal(err) + } + return node +} + // Helper functions for accessing channel internals -func routerExists(node *RawNode, msgID uint64) bool { +func routerExists(node *Node, msgID uint64) bool { node.channel.responseMut.Lock() defer node.channel.responseMut.Unlock() _, exists := node.channel.responseRouters[msgID] return exists } -func getStream(node *RawNode) grpc.ClientStream { +func getStream(node *Node) grpc.ClientStream { return node.channel.getStream() } func TestChannelCreation(t *testing.T) { - node := NewNode(t, "127.0.0.1:5000") + node := testNodeWithoutServer(t) // send message when server is down - resp := sendRequest(t, node, request{opts: waitSendDone}, 1) - if resp.err == nil { + resp := sendRequest(t, node, request{waitSendDone: true}, 1) + if resp.Err == nil { t.Error("response err: got , want error") } } // TestChannelShutdown verifies proper cleanup when channel is closed. func TestChannelShutdown(t *testing.T) { - node := newNodeWithServer(t, 0) + node := TestNode(t, delayServerFn(0)) + // Wait for stream to be established + time.Sleep(streamConnectDelay) if !node.channel.isConnected() { t.Error("node should be connected") } @@ -122,8 +130,8 @@ func TestChannelShutdown(t *testing.T) { for i := range numMessages { wg.Go(func() { resp := sendRequest(t, node, request{}, uint64(i)) - if resp.err != nil { - t.Errorf("unexpected error for message %d, got error: %v", i, resp.err) + if resp.Err != nil { + t.Errorf("unexpected error for message %d, got error: %v", i, resp.Err) } }) } @@ -136,10 +144,10 @@ func TestChannelShutdown(t *testing.T) { // try to send a message after node closure resp := sendRequest(t, node, request{}, 999) - if resp.err == nil { + if resp.Err == nil { t.Error("expected error when sending to closed channel") - } else if resp.err.Error() != "node closed" { - t.Errorf("expected 'node closed' error, got: %v", resp.err) + } else if !strings.Contains(resp.Err.Error(), "node closed") { + t.Errorf("expected 'node closed' error, got: %v", resp.Err) } if node.channel.isConnected() { @@ -147,24 +155,73 @@ func TestChannelShutdown(t *testing.T) { } } +// TestChannelSendBufferSize verifies that setting the send buffer size option +// works as expected (at least that it doesn't panic and channels are functional). +func TestChannelSendBufferSize(t *testing.T) { + bufferSizes := []uint{0, 1, 10, 100, 1024} + + for _, size := range bufferSizes { + t.Run(string(rune(size)), func(t *testing.T) { + node := TestNode(t, EchoServerFn, WithSendBufferSize(size)) + + ctx := TestContext(t, time.Second) + _, err := RPCCall(node.Context(ctx), pb.String("test"), mock.TestMethod) + if err != nil { + t.Errorf("RPCCall failed with buffer size %d: %v", size, err) + } + }) + } +} + +// TestChannelLatency verifies that the channel latency is updated after requests. +func TestChannelLatency(t *testing.T) { + const minDelay = 20 * time.Millisecond + node := TestNode(t, delayServerFn(minDelay)) + + // Initial latency should be -1 + if latency := node.Latency(); latency != -1*time.Second { + t.Errorf("Initial latency = %v, expected -1s", latency) + } + + // Send a few requests to update latency; we need a few samples for the + // exponential weighted moving average to stabilize to the real RTT. + for i := range 10 { + ctx := TestContext(t, time.Second) + _, err := RPCCall(node.Context(ctx), pb.String("ping"), mock.TestMethod) + if err != nil { + t.Fatalf("RPCCall %d failed: %v", i, err) + } + } + + // Latency should be positive and roughly > 20ms + latency := node.Latency() + if latency <= 0 { + t.Errorf("Latency = %v, expected > 0", latency) + } + if latency < minDelay { + t.Errorf("Latency = %v, expected >= %v (server delay)", latency, minDelay) + } + t.Logf("Measured latency: %v", latency) +} + // TestChannelSendCompletionWaiting verifies the behavior of send completion waiting. func TestChannelSendCompletionWaiting(t *testing.T) { - node := newNodeWithServer(t, 0) + node := TestNode(t, delayServerFn(0)) tests := []struct { - name string - opts callOptions + name string + waitSendDone bool }{ - {name: "WaitForSend", opts: waitSendDone}, - {name: "NoSendWaiting", opts: noSendWaiting}, + {name: "WaitForSend", waitSendDone: true}, + {name: "NoSendWaiting", waitSendDone: false}, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { start := time.Now() - resp := sendRequest(t, node, request{opts: tt.opts}, uint64(i)) + resp := sendRequest(t, node, request{waitSendDone: tt.waitSendDone}, uint64(i)) elapsed := time.Since(start) - if resp.err != nil { - t.Errorf("unexpected error: %v", resp.err) + if resp.Err != nil { + t.Errorf("unexpected error: %v", resp.Err) } t.Logf("response received in %v", elapsed) }) @@ -175,20 +232,20 @@ func TestChannelSendCompletionWaiting(t *testing.T) { func TestChannelErrors(t *testing.T) { tests := []struct { name string - setup func(t *testing.T) *RawNode + setup func(t *testing.T) *Node wantErr string }{ { name: "EnqueueWithoutServer", - setup: func(t *testing.T) *RawNode { - return NewNode(t, "127.0.0.1:5002") + setup: func(t *testing.T) *Node { + return testNodeWithoutServer(t) }, wantErr: "connect: connection refused", }, { name: "EnqueueToClosedChannel", - setup: func(t *testing.T) *RawNode { - node := NewNode(t, "127.0.0.1:5000") + setup: func(t *testing.T) *Node { + node := testNodeWithoutServer(t) err := node.close() if err != nil { t.Errorf("failed to close node: %v", err) @@ -199,8 +256,8 @@ func TestChannelErrors(t *testing.T) { }, { name: "EnqueueToServerWithClosedNode", - setup: func(t *testing.T) *RawNode { - node := newNodeWithServer(t, 0) + setup: func(t *testing.T) *Node { + node := TestNode(t, delayServerFn(0)) err := node.close() if err != nil { t.Errorf("failed to close node: %v", err) @@ -211,11 +268,12 @@ func TestChannelErrors(t *testing.T) { }, { name: "ServerFailureDuringCommunication", - setup: func(t *testing.T) *RawNode { - node, stopServer := newNodeWithStoppableServer(t, 0) - resp := sendRequest(t, node, request{opts: waitSendDone}, 1) - if resp.err != nil { - t.Errorf("first message should succeed, got error: %v", resp.err) + setup: func(t *testing.T) *Node { + var stopServer func(...int) + node := TestNode(t, delayServerFn(0), WithStopFunc(t, &stopServer)) + resp := sendRequest(t, node, request{waitSendDone: true}, 1) + if resp.Err != nil { + t.Errorf("first message should succeed, got error: %v", resp.Err) } stopServer() return node @@ -229,11 +287,11 @@ func TestChannelErrors(t *testing.T) { time.Sleep(100 * time.Millisecond) // Send message and verify error - resp := sendRequest(t, node, request{opts: waitSendDone}, uint64(i)) - if resp.err == nil { + resp := sendRequest(t, node, request{waitSendDone: true}, uint64(i)) + if resp.Err == nil { t.Errorf("expected error '%s' but got nil", tt.wantErr) - } else if !strings.Contains(resp.err.Error(), tt.wantErr) { - t.Errorf("expected error '%s', got: %v", tt.wantErr, resp.err) + } else if !strings.Contains(resp.Err.Error(), tt.wantErr) { + t.Errorf("expected error '%s', got: %v", tt.wantErr, resp.Err) } }) } @@ -242,10 +300,14 @@ func TestChannelErrors(t *testing.T) { // TestChannelEnsureStream verifies that ensureStream correctly manages stream lifecycle. func TestChannelEnsureStream(t *testing.T) { // Helper to prepare a fresh node with no stream - newNodeWithoutStream := func(t *testing.T) *RawNode { - node := newNodeWithServer(t, 0) - node.cancel() // ensure sender and receiver goroutines are stopped - node.channel = newChannel(node) + newNodeWithoutStream := func(t *testing.T) *Node { + node := TestNode(t, delayServerFn(0)) + // ensure sender and receiver goroutines are stopped + node.channel.connCancel() + // Extract grpc.ClientConn from existing channel + conn := node.channel.conn + // Create new channel with test context without metadata (real implementation captures metadata) + node.channel = newChannel(t.Context(), conn, node.id, 10) return node } @@ -267,14 +329,14 @@ func TestChannelEnsureStream(t *testing.T) { tests := []struct { name string - setup func(t *testing.T) *RawNode - action func(node *RawNode) (first, second grpc.ClientStream) + setup func(t *testing.T) *Node + action func(node *Node) (first, second grpc.ClientStream) wantSame bool }{ { name: "UnconnectedNodeHasNoStream", - setup: func(t *testing.T) *RawNode { return NewNode(t, "") }, - action: func(node *RawNode) (grpc.ClientStream, grpc.ClientStream) { + setup: func(t *testing.T) *Node { return testNodeWithoutServer(t) }, + action: func(node *Node) (grpc.ClientStream, grpc.ClientStream) { if err := node.channel.ensureStream(); err == nil { t.Error("ensureStream succeeded unexpectedly") } @@ -287,7 +349,7 @@ func TestChannelEnsureStream(t *testing.T) { { name: "CreatesStreamWhenConnected", setup: newNodeWithoutStream, - action: func(node *RawNode) (grpc.ClientStream, grpc.ClientStream) { + action: func(node *Node) (grpc.ClientStream, grpc.ClientStream) { if err := node.channel.ensureStream(); err != nil { t.Errorf("ensureStream failed: %v", err) } @@ -297,7 +359,7 @@ func TestChannelEnsureStream(t *testing.T) { { name: "RepeatedCallsReturnSameStream", setup: newNodeWithoutStream, - action: func(node *RawNode) (grpc.ClientStream, grpc.ClientStream) { + action: func(node *Node) (grpc.ClientStream, grpc.ClientStream) { if err := node.channel.ensureStream(); err != nil { t.Errorf("first ensureStream failed: %v", err) } @@ -312,7 +374,7 @@ func TestChannelEnsureStream(t *testing.T) { { name: "StreamDisconnectionCreatesNewStream", setup: newNodeWithoutStream, - action: func(node *RawNode) (grpc.ClientStream, grpc.ClientStream) { + action: func(node *Node) (grpc.ClientStream, grpc.ClientStream) { if err := node.channel.ensureStream(); err != nil { t.Errorf("initial ensureStream failed: %v", err) } @@ -330,9 +392,6 @@ func TestChannelEnsureStream(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { node := tt.setup(t) - if stream := getStream(node); stream != nil { - t.Fatal("stream should be nil initially") - } first, second := tt.action(node) cmpStream(t, first, second, tt.wantSame) }) @@ -343,38 +402,40 @@ func TestChannelEnsureStream(t *testing.T) { func TestChannelConnectionState(t *testing.T) { tests := []struct { name string - node *RawNode - setup func(node *RawNode) + setup func(t *testing.T) *Node wantConnected bool }{ { name: "WithoutServer", - node: NewNode(t, "127.0.0.1:5003"), + setup: func(t *testing.T) *Node { return testNodeWithoutServer(t) }, wantConnected: false, }, { name: "WithLiveServer", - node: newNodeWithServer(t, 0), + setup: func(t *testing.T) *Node { return TestNode(t, delayServerFn(0)) }, wantConnected: true, }, { name: "RequiresBothReadyAndStream", - node: newNodeWithServer(t, 0), - setup: func(node *RawNode) { + setup: func(t *testing.T) *Node { + node := TestNode(t, delayServerFn(0)) + // Wait for stream to be established + time.Sleep(streamConnectDelay) if !node.channel.isConnected() { t.Fatal("node should be connected before clearing stream") } node.channel.clearStream() + return node }, wantConnected: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if tt.setup != nil { - tt.setup(tt.node) - } - connected := tt.node.channel.isConnected() + node := tt.setup(t) + // Wait for the stream to be established, as it is now done asynchronously in sender + time.Sleep(streamConnectDelay) + connected := node.channel.isConnected() if connected != tt.wantConnected { t.Errorf("isConnected() = %v, want %v", connected, tt.wantConnected) } @@ -384,7 +445,7 @@ func TestChannelConnectionState(t *testing.T) { // TestChannelConcurrentSends tests sending multiple messages concurrently from multiple goroutines. func TestChannelConcurrentSends(t *testing.T) { - node := newNodeWithServer(t, 0) + node := TestNode(t, delayServerFn(0)) const numMessages = 1000 const numGoroutines = 10 @@ -393,16 +454,16 @@ func TestChannelConcurrentSends(t *testing.T) { results := make(chan msgResponse, numMessages) for goID := range numGoroutines { go func() { - send(t, results, node, goID, msgsPerGoroutine, request{opts: waitSendDone}) - send(t, results, node, goID, msgsPerGoroutine, request{opts: noSendWaiting}) + sendReq(t, results, node, goID, msgsPerGoroutine, request{waitSendDone: true}) + sendReq(t, results, node, goID, msgsPerGoroutine, request{waitSendDone: false}) }() } var errs []error for range numMessages { res := <-results - if res.resp.err != nil { - errs = append(errs, res.resp.err) + if res.resp.Err != nil { + errs = append(errs, res.resp.Err) } } @@ -437,35 +498,35 @@ func TestChannelContext(t *testing.T) { name string serverDelay time.Duration contextSetup func(context.Context) (context.Context, context.CancelFunc) - callOpts callOptions + waitSendDone bool wantErr error }{ { name: "CancelBeforeSend/WaitSending", serverDelay: 0, contextSetup: cancelledContext, - callOpts: waitSendDone, + waitSendDone: true, wantErr: context.Canceled, }, { name: "CancelBeforeSend/NoSendWaiting", serverDelay: 0, contextSetup: cancelledContext, - callOpts: noSendWaiting, + waitSendDone: false, wantErr: context.Canceled, }, { name: "CancelDuringSend/WaitSending", serverDelay: 3 * time.Second, contextSetup: expireBeforeSend, - callOpts: waitSendDone, + waitSendDone: true, wantErr: context.DeadlineExceeded, }, { name: "CancelDuringSend/NoSendWaiting", serverDelay: 3 * time.Second, contextSetup: expireBeforeSend, - callOpts: noSendWaiting, + waitSendDone: false, wantErr: context.DeadlineExceeded, }, } @@ -475,10 +536,10 @@ func TestChannelContext(t *testing.T) { ctx, cancel := tt.contextSetup(t.Context()) t.Cleanup(cancel) - node := newNodeWithServer(t, tt.serverDelay) - resp := sendRequest(t, node, request{ctx: ctx, opts: tt.callOpts}, uint64(i)) - if !errors.Is(resp.err, tt.wantErr) { - t.Errorf("expected %v, got: %v", tt.wantErr, resp.err) + node := TestNode(t, delayServerFn(tt.serverDelay)) + resp := sendRequest(t, node, request{ctx: ctx, waitSendDone: tt.waitSendDone}, uint64(i)) + if !errors.Is(resp.Err, tt.wantErr) { + t.Errorf("expected %v, got: %v", tt.wantErr, resp.Err) } }) } @@ -498,13 +559,15 @@ func TestChannelContext(t *testing.T) { // 3. Sending multiple messages concurrently to trigger the deadlock condition // 4. Verifying all goroutines can successfully enqueue without hanging func TestChannelDeadlock(t *testing.T) { - node := newNodeWithServer(t, 0) + node := TestNode(t, delayServerFn(0)) + // Wait for the stream to be established + time.Sleep(streamConnectDelay) if !node.channel.isConnected() { t.Error("node should be connected") } // Send a message to activate the stream - sendRequest(t, node, request{opts: waitSendDone}, 1) + sendRequest(t, node, request{waitSendDone: true}, 1) // Break the stream, forcing a reconnection on next send node.channel.clearStream() @@ -515,9 +578,7 @@ func TestChannelDeadlock(t *testing.T) { doneChan := make(chan bool, 10) for id := range 10 { go func() { - ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) - defer cancel() - + ctx := TestContext(t, 3*time.Second) md := ordering.NewGorumsMetadata(ctx, uint64(100+id), mock.TestMethod) req := request{ctx: ctx, msg: NewRequestMessage(md, nil)} @@ -556,30 +617,32 @@ func TestChannelDeadlock(t *testing.T) { // TestChannelRouterLifecycle tests router creation, persistence, and cleanup behavior. func TestChannelRouterLifecycle(t *testing.T) { - node := newNodeWithServer(t, 0) + node := TestNode(t, delayServerFn(0)) + // Wait for the stream to be established + time.Sleep(streamConnectDelay) if !node.channel.isConnected() { t.Error("node should be connected") } tests := []struct { - name string - opts callOptions - streaming bool - afterSend func(t *testing.T, node *RawNode, msgID uint64) - wantRouter bool + name string + waitSendDone bool + streaming bool + afterSend func(t *testing.T, node *Node, msgID uint64) + wantRouter bool }{ - {name: "WaitSendDone/NonStreamingAutoCleanup", opts: waitSendDone, streaming: false, wantRouter: false}, - {name: "WaitSendDone/StreamingKeepsRouterAlive", opts: waitSendDone, streaming: true, wantRouter: true}, - {name: "NoSendWaiting/NonStreamingAutoCleanup", opts: noSendWaiting, streaming: false, wantRouter: false}, - {name: "NoSendWaiting/StreamingKeepsRouterAlive", opts: noSendWaiting, streaming: true, wantRouter: true}, + {name: "WaitSendDone/NonStreamingAutoCleanup", waitSendDone: true, streaming: false, wantRouter: false}, + {name: "WaitSendDone/StreamingKeepsRouterAlive", waitSendDone: true, streaming: true, wantRouter: true}, + {name: "NoSendWaiting/NonStreamingAutoCleanup", waitSendDone: false, streaming: false, wantRouter: false}, + {name: "NoSendWaiting/StreamingKeepsRouterAlive", waitSendDone: false, streaming: true, wantRouter: true}, } for i, tt := range tests { name := fmt.Sprintf("msgID=%d/%s/streaming=%t", i, tt.name, tt.streaming) t.Run(name, func(t *testing.T) { msgID := uint64(i) - resp := sendRequest(t, node, request{opts: tt.opts, streaming: tt.streaming}, msgID) - if resp.err != nil { - t.Errorf("unexpected error: %v", resp.err) + resp := sendRequest(t, node, request{waitSendDone: tt.waitSendDone, streaming: tt.streaming}, msgID) + if resp.Err != nil { + t.Errorf("unexpected error: %v", resp.Err) } if exists := routerExists(node, msgID); exists != tt.wantRouter { t.Errorf("router exists = %v, want %v", exists, tt.wantRouter) @@ -592,21 +655,21 @@ func TestChannelRouterLifecycle(t *testing.T) { // TestChannelResponseRouting sends multiple messages and verifies that // responses are correctly routed to their callers. func TestChannelResponseRouting(t *testing.T) { - node := newNodeWithServer(t, 0) + node := TestNode(t, delayServerFn(0)) const numMessages = 20 results := make(chan msgResponse, numMessages) for i := range numMessages { - go send(t, results, node, i, 1, request{}) + go sendReq(t, results, node, i, 1, request{}) } // Collect and verify results received := make(map[uint64]bool) for range numMessages { result := <-results - if result.resp.err != nil { - t.Errorf("message %d got error: %v", result.msgID, result.resp.err) + if result.resp.Err != nil { + t.Errorf("message %d got error: %v", result.msgID, result.resp.Err) } if received[result.msgID] { t.Errorf("message %d received twice", result.msgID) @@ -619,3 +682,200 @@ func TestChannelResponseRouting(t *testing.T) { t.Errorf("got %d unique responses, want %d", len(received), numMessages) } } + +// TestChannelStreamReadySignaling verifies that the receiver goroutine is properly notified +// when a stream becomes available. This tests the stream-ready signaling mechanism +// that replaces the old time.Sleep polling approach. +func TestChannelStreamReadySignaling(t *testing.T) { + node := TestNode(t, delayServerFn(0)) + + // The first request triggers stream creation. We measure how quickly + // the receiver starts processing after the stream is ready. + start := time.Now() + resp := sendRequest(t, node, request{}, 1) + firstLatency := time.Since(start) + + if resp.Err != nil { + t.Fatalf("unexpected error on first request: %v", resp.Err) + } + + // Second request should be faster since stream is already established + start = time.Now() + resp = sendRequest(t, node, request{}, 2) + secondLatency := time.Since(start) + + if resp.Err != nil { + t.Fatalf("unexpected error on second request: %v", resp.Err) + } + + t.Logf("first request latency: %v", firstLatency) + t.Logf("second request latency: %v", secondLatency) + + // The first request should complete in reasonable time (not waiting for polling timeout). + // With the old 10ms polling, the first request could take 10-20ms just waiting for the stream. + // With proper signaling, it should be much faster (sub-millisecond for the signal itself). + const maxAcceptableLatency = 100 * time.Millisecond // generous bound for CI environments + if firstLatency > maxAcceptableLatency { + t.Errorf("first request took %v, expected < %v (possible stream-ready polling delay)", firstLatency, maxAcceptableLatency) + } +} + +// TestChannelStreamReadyAfterReconnect verifies that the receiver is properly notified +// when a stream is re-established after being cleared (simulating reconnection). +func TestChannelStreamReadyAfterReconnect(t *testing.T) { + node := TestNode(t, delayServerFn(0)) + + // First request to establish the stream + resp := sendRequest(t, node, request{}, 1) + if resp.Err != nil { + t.Fatalf("unexpected error on first request: %v", resp.Err) + } + + // Clear the stream to simulate disconnection. + // Note: In production, clearStream is called by the receiver when it detects an error. + // The next request will trigger stream re-creation via ensureStream(). + node.channel.clearStream() + + // The sender's ensureStream() will recreate the stream. + // We may need to retry a few times since there's a race between + // clearStream and the sender's stream check. + var reconnectLatency time.Duration + var lastErr error + start := time.Now() + for i := range 5 { + resp = sendRequest(t, node, request{}, uint64(i+2)) + if resp.Err == nil { + reconnectLatency = time.Since(start) + break + } + lastErr = resp.Err + // Give the sender a chance to reconnect + time.Sleep(5 * time.Millisecond) + } + if resp.Err != nil { + t.Fatalf("failed to reconnect after 5 attempts: %v", lastErr) + } + + t.Logf("reconnect latency: %v", reconnectLatency) + + // Even after reconnection, the latency should be reasonable + const maxAcceptableLatency = 200 * time.Millisecond + if reconnectLatency > maxAcceptableLatency { + t.Errorf("reconnect took %v, expected < %v", reconnectLatency, maxAcceptableLatency) + } +} + +// BenchmarkChannelStreamReadyFirstRequest measures the latency of the first request, +// which includes stream creation and the stream-ready signaling. +// +// This benchmark creates a new server and node per iteration to measure true +// "cold start" latency. Due to TCP port exhaustion on macOS (ephemeral ports +// enter TIME_WAIT state and take time to recycle), this benchmark should be +// run with limited iterations (e.g., -benchtime=100x). +// +// Note: This benchmark includes server setup overhead, so absolute numbers +// should be interpreted with caution. The goal is to detect regressions. +func BenchmarkChannelStreamReadyFirstRequest(b *testing.B) { + if b.N > 500 { + b.Skip("Skipping to avoid port exhaustion; use -benchtime=100x") + } + + for b.Loop() { + var stopServer func(...int) + node := TestNode(b, delayServerFn(0), WithStopFunc(b, &stopServer)) + + // Use a fresh context for the benchmark request + ctx := TestContext(b, defaultTestTimeout) + req := request{ctx: ctx} + req.msg = NewRequestMessage(ordering.NewGorumsMetadata(ctx, 1, mock.TestMethod), nil) + replyChan := make(chan NodeResponse[proto.Message], 1) + req.responseChan = replyChan + node.channel.enqueue(req) + + select { + case resp := <-replyChan: + if resp.Err != nil { + b.Logf("request error (may occur during rapid cycles): %v", resp.Err) + } + case <-ctx.Done(): + b.Logf("timeout (may occur during rapid cycles)") + } + + // Close the node before stopping the server to ensure clean shutdown + _ = node.close() + stopServer() + } +} + +// BenchmarkChannelStreamReadySubsequentRequest measures the latency of requests +// after the stream is already established (steady-state performance). +func BenchmarkChannelStreamReadySubsequentRequest(b *testing.B) { + node := TestNode(b, delayServerFn(0)) + + // Warm up: establish the stream + resp := sendRequest(b, node, request{}, 0) + if resp.Err != nil { + b.Fatalf("warmup error: %v", resp.Err) + } + + b.ResetTimer() + for i := range b.N { + resp := sendRequest(b, node, request{}, uint64(i+1)) + if resp.Err != nil { + b.Fatalf("unexpected error: %v", resp.Err) + } + } +} + +// BenchmarkChannelStreamReadyReconnect measures the latency of reconnecting +// after the stream has been cleared. +// Note: This benchmark has inherent variability due to the race between +// clearStream and the sender's ensureStream call. +func BenchmarkChannelStreamReadyReconnect(b *testing.B) { + node := TestNode(b, delayServerFn(0)) + + // Establish initial stream with a fresh context + ctx := context.Background() + req := request{ctx: ctx} + req.msg = NewRequestMessage(ordering.NewGorumsMetadata(ctx, 0, mock.TestMethod), nil) + replyChan := make(chan NodeResponse[proto.Message], 1) + req.responseChan = replyChan + node.channel.enqueue(req) + + select { + case resp := <-replyChan: + if resp.Err != nil { + b.Fatalf("initial request error: %v", resp.Err) + } + case <-time.After(defaultTestTimeout): + b.Fatal("timeout on initial request") + } + + b.ResetTimer() + for i := range b.N { + node.channel.clearStream() + + // Wait a tiny bit for the receiver to notice the stream is gone + // and be ready for the signal. This simulates real-world behavior + // where the receiver detects the error before reconnection. + time.Sleep(100 * time.Microsecond) + + // Now send a request which will trigger ensureStream -> newNodeStream -> signal + ctx := context.Background() + req := request{ctx: ctx} + req.msg = NewRequestMessage(ordering.NewGorumsMetadata(ctx, uint64(i+1), mock.TestMethod), nil) + replyChan := make(chan NodeResponse[proto.Message], 1) + req.responseChan = replyChan + node.channel.enqueue(req) + + select { + case resp := <-replyChan: + if resp.Err != nil { + // Stream down error is expected sometimes due to race; just continue + b.Logf("request %d error: %v", i, resp.Err) + } + case <-time.After(500 * time.Millisecond): + b.Fatalf("timeout on request %d", i) + } + } +} diff --git a/client_interceptor.go b/client_interceptor.go new file mode 100644 index 000000000..968265992 --- /dev/null +++ b/client_interceptor.go @@ -0,0 +1,308 @@ +package gorums + +import ( + "context" + "slices" + "sync" + + "github.com/relab/gorums/ordering" + "google.golang.org/protobuf/proto" +) + +// QuorumInterceptor intercepts and processes quorum calls, allowing modification of +// requests, responses, and aggregation logic. Interceptors can be chained together. +// +// Type parameters: +// - Req: The request message type sent to nodes +// - Resp: The response message type from individual nodes +// +// The interceptor receives the ClientCtx for metadata access, the current response +// iterator (next), and returns a new response iterator. This pattern allows +// interceptors to wrap the response stream with custom logic. +// +// Custom interceptors can be created like this: +// +// func LoggingInterceptor[Req, Resp proto.Message]( +// ctx *gorums.ClientCtx[Req, Resp], +// next gorums.ResponseSeq[Resp], +// ) gorums.ResponseSeq[Resp] { +// return func(yield func(gorums.NodeResponse[Resp]) bool) { +// for resp := range next { +// log.Printf("Response from node %d", resp.NodeID) +// if !yield(resp) { return } +// } +// } +// } +type QuorumInterceptor[Req, Resp msg] func(ctx *ClientCtx[Req, Resp], next ResponseSeq[Resp]) ResponseSeq[Resp] + +// ClientCtx provides context and access to the quorum call state for interceptors. +// It exposes the request, configuration, metadata about the call, and the response iterator. +type ClientCtx[Req, Resp msg] struct { + context.Context + config Configuration + request Req + method string + md *ordering.Metadata + replyChan chan NodeResponse[msg] + + // reqTransforms holds request transformation functions registered by interceptors. + reqTransforms []func(Req, *Node) Req + + // responseSeq is the iterator that yields node responses. + // Interceptors can wrap this iterator to modify responses. + responseSeq ResponseSeq[Resp] + + // expectedReplies is the number of responses expected from nodes. + // It is set when messages are sent and may be lower than config size + // if some nodes are skipped by request transformations. + expectedReplies int + + // streaming indicates whether this is a streaming call (for correctable streams). + streaming bool + + // waitSendDone indicates whether the caller waits for send completion (for multicast). + waitSendDone bool + + // sendOnce ensures messages are sent exactly once, on the first + // call to Responses(). This deferred sending allows interceptors + // to register request transformations before dispatch. + sendOnce sync.Once +} + +// clientCtxBuilder provides an interface for constructing ClientCtx instances. +type clientCtxBuilder[Req, Resp msg] struct { + c *ClientCtx[Req, Resp] + // chanMultiplier is the buffer multiplier for the reply channel. + // Default is 1; streaming calls use a larger multiplier. + chanMultiplier int +} + +// newClientCtxBuilder creates a new builder for constructing a ClientCtx. +// The required parameters are provided upfront; optional settings use builder methods. +// The metadata and reply channel are created at Build() time. +func newClientCtxBuilder[Req, Resp msg]( + ctx *ConfigContext, + req Req, + method string, +) *clientCtxBuilder[Req, Resp] { + return &clientCtxBuilder[Req, Resp]{ + c: &ClientCtx[Req, Resp]{ + Context: ctx, + config: ctx.Configuration(), + request: req, + method: method, + expectedReplies: ctx.Configuration().Size(), + }, + chanMultiplier: 1, + } +} + +// WithStreaming configures the clientCtx for streaming responses. +// When enabled, the response iterator continues until context cancellation +// rather than stopping after expectedReplies responses. +// It also increases the reply channel buffer size (10x) to handle streaming volume. +func (b *clientCtxBuilder[Req, Resp]) WithStreaming() *clientCtxBuilder[Req, Resp] { + b.c.streaming = true + b.chanMultiplier = 10 + return b +} + +// WithWaitSendDone configures the clientCtx to wait for send completion. +// Used by multicast calls to ensure messages are sent before returning. +func (b *clientCtxBuilder[Req, Resp]) WithWaitSendDone(waitSendDone bool) *clientCtxBuilder[Req, Resp] { + b.c.waitSendDone = waitSendDone + return b +} + +// Build finalizes the ClientCtx configuration and returns the constructed instance. +// It creates the metadata and reply channel, and sets up the appropriate response iterator. +func (b *clientCtxBuilder[Req, Resp]) Build() *ClientCtx[Req, Resp] { + // Create metadata and reply channel at build time + b.c.md = ordering.NewGorumsMetadata(b.c.Context, b.c.config.nextMsgID(), b.c.method) + b.c.replyChan = make(chan NodeResponse[msg], b.c.config.Size()*b.chanMultiplier) + + if b.c.streaming { + b.c.responseSeq = b.c.streamingResponseSeq() + } else { + b.c.responseSeq = b.c.defaultResponseSeq() + } + return b.c +} + +// ------------------------------------------------------------------------- +// ClientCtx Methods +// ------------------------------------------------------------------------- + +// Request returns the original request message for this quorum call. +func (c *ClientCtx[Req, Resp]) Request() Req { + return c.request +} + +// Config returns the configuration (set of nodes) for this quorum call. +func (c *ClientCtx[Req, Resp]) Config() Configuration { + return c.config +} + +// Method returns the name of the RPC method being called. +func (c *ClientCtx[Req, Resp]) Method() string { + return c.method +} + +// Nodes returns the slice of nodes in this configuration. +func (c *ClientCtx[Req, Resp]) Nodes() []*Node { + return c.config.Nodes() +} + +// Node returns the node with the given ID. +func (c *ClientCtx[Req, Resp]) Node(id uint32) *Node { + nodes := c.config.Nodes() + index := slices.IndexFunc(nodes, func(n *Node) bool { + return n.ID() == id + }) + if index != -1 { + return nodes[index] + } + return nil +} + +// Size returns the number of nodes in this configuration. +func (c *ClientCtx[Req, Resp]) Size() int { + return c.config.Size() +} + +// applyTransforms returns the transformed request as a proto.Message, or nil if the result is +// invalid or the node should be skipped. It applies the registered transformation functions to +// the given request for the specified node. Transformation functions are applied in the order +// they were registered. +func (c *ClientCtx[Req, Resp]) applyTransforms(req Req, node *Node) proto.Message { + result := req + for _, transform := range c.reqTransforms { + result = transform(result, node) + } + if protoMsg, ok := any(result).(proto.Message); ok { + if protoMsg.ProtoReflect().IsValid() { + return protoMsg + } + } + return nil +} + +// applyInterceptors chains the given interceptors, wrapping the response sequence. +// Each interceptor receives the current response sequence and returns a new one. +// Interceptors are applied in order, with each wrapping the previous result. +func (c *ClientCtx[Req, Resp]) applyInterceptors(interceptors []any) { + responseSeq := c.responseSeq + for _, ic := range interceptors { + interceptor := ic.(QuorumInterceptor[Req, Resp]) + responseSeq = interceptor(c, responseSeq) + } + c.responseSeq = responseSeq +} + +// send dispatches requests to all nodes, applying any registered transformations. +// It updates expectedReplies based on how many nodes actually receive requests +// (nodes may be skipped if a transformation returns nil). +func (c *ClientCtx[Req, Resp]) send() { + var expected int + for _, n := range c.config { + msg := c.applyTransforms(c.request, n) + if msg == nil { + continue // Skip node if transformation returns nil + } + expected++ + n.channel.enqueue(request{ + ctx: c.Context, + msg: NewRequestMessage(c.md, msg), + streaming: c.streaming, + waitSendDone: c.waitSendDone, + responseChan: c.replyChan, + }) + } + c.expectedReplies = expected +} + +// defaultResponseSeq returns an iterator that yields at most c.expectedReplies responses +// from nodes until the context is canceled or all expected responses are received. +func (c *ClientCtx[Req, Resp]) defaultResponseSeq() ResponseSeq[Resp] { + return func(yield func(NodeResponse[Resp]) bool) { + // Trigger sending on first iteration + c.sendOnce.Do(c.send) + for range c.expectedReplies { + select { + case r := <-c.replyChan: + res := newNodeResponse[Resp](r) + if !yield(res) { + return // Consumer stopped iteration + } + case <-c.Done(): + return // Context canceled + } + } + } +} + +// streamingResponseSeq returns an iterator that yields responses as they arrive +// from nodes until the context is canceled or breaking from the range loop. +func (c *ClientCtx[Req, Resp]) streamingResponseSeq() ResponseSeq[Resp] { + return func(yield func(NodeResponse[Resp]) bool) { + // Trigger sending on first iteration + c.sendOnce.Do(c.send) + for { + select { + case r := <-c.replyChan: + res := newNodeResponse[Resp](r) + if !yield(res) { + return // Consumer stopped iteration + } + case <-c.Done(): + return // Context canceled + } + } + } +} + +// ------------------------------------------------------------------------- +// Interceptors (Middleware) +// ------------------------------------------------------------------------- + +// MapRequest returns an interceptor that applies per-node request transformations. +// Multiple interceptors can be chained together, with transforms applied in order. +// +// The fn receives the original request and a node, and returns the transformed +// request to send to that node. If the function returns an invalid message or nil, +// the request to that node is skipped. +func MapRequest[Req, Resp msg](fn func(Req, *Node) Req) QuorumInterceptor[Req, Resp] { + return func(ctx *ClientCtx[Req, Resp], next ResponseSeq[Resp]) ResponseSeq[Resp] { + if fn != nil { + ctx.reqTransforms = append(ctx.reqTransforms, fn) + } + return next + } +} + +// MapResponse returns an interceptor that applies per-node response transformations. +// +// The fn receives the response from a node and the node itself, and returns the +// transformed response. +func MapResponse[Req, Resp msg](fn func(Resp, *Node) Resp) QuorumInterceptor[Req, Resp] { + return func(ctx *ClientCtx[Req, Resp], next ResponseSeq[Resp]) ResponseSeq[Resp] { + if fn == nil { + return next + } + // Wrap the response iterator with the transformation logic. + return func(yield func(NodeResponse[Resp]) bool) { + for resp := range next { + // We only apply the transformation if there is no error. + // Errors are passed through as-is. + if resp.Err == nil { + if node := ctx.Node(resp.NodeID); node != nil { + resp.Value = fn(resp.Value, node) + } + } + if !yield(resp) { + return + } + } + } + } +} diff --git a/client_interceptor_test.go b/client_interceptor_test.go new file mode 100644 index 000000000..9755da869 --- /dev/null +++ b/client_interceptor_test.go @@ -0,0 +1,181 @@ +package gorums_test + +import ( + "testing" + "time" + + "github.com/relab/gorums" + "github.com/relab/gorums/internal/testutils/mock" + "google.golang.org/protobuf/proto" + pb "google.golang.org/protobuf/types/known/wrapperspb" +) + +// LoggingInterceptor is a custom interceptor that logs each response. +func LoggingInterceptor[Req, Resp proto.Message]( + ctx *gorums.ClientCtx[Req, Resp], + next gorums.ResponseSeq[Resp], +) gorums.ResponseSeq[Resp] { + _ = ctx.Method() // Access method name (could be used for logging) + return func(yield func(gorums.NodeResponse[Resp]) bool) { + for resp := range next { + // In a real interceptor, you would log here + if !yield(resp) { + return + } + } + } +} + +// FilterInterceptor returns an interceptor that filters responses based on a predicate. +func FilterInterceptor[Req, Resp proto.Message]( + keep func(resp gorums.NodeResponse[Resp]) bool, +) gorums.QuorumInterceptor[Req, Resp] { + return func(ctx *gorums.ClientCtx[Req, Resp], next gorums.ResponseSeq[Resp]) gorums.ResponseSeq[Resp] { + _ = ctx.Method() // Access method name (could be used for filtering) + return func(yield func(gorums.NodeResponse[Resp]) bool) { + for resp := range next { + if keep(resp) { + if !yield(resp) { + return + } + } + } + } + } +} + +// CountingInterceptor counts the number of responses passing through. +func CountingInterceptor[Req, Resp proto.Message]( + counter *int, +) gorums.QuorumInterceptor[Req, Resp] { + return func(_ *gorums.ClientCtx[Req, Resp], next gorums.ResponseSeq[Resp]) gorums.ResponseSeq[Resp] { + return func(yield func(gorums.NodeResponse[Resp]) bool) { + for resp := range next { + *counter++ + if !yield(resp) { + return + } + } + } + } +} + +// TestCustomLoggingInterceptor verifies that custom interceptors can be +// created and used from an external package. +func TestCustomLoggingInterceptor(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + // Use the custom logging interceptor from this external package + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + gorums.Interceptors(LoggingInterceptor[*pb.StringValue, *pb.StringValue]), + ) + + result, err := responses.Majority() + if err != nil { + t.Fatalf("QuorumCall failed: %v", err) + } + if result.GetValue() != "echo: test" { + t.Errorf("Expected 'echo: test', got %q", result.GetValue()) + } +} + +// TestCustomFilterInterceptor verifies that filter interceptors work correctly. +func TestCustomFilterInterceptor(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + // Use a filter interceptor that only keeps responses from node 1 + // (In practice, this would filter based on response content) + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + gorums.Interceptors(FilterInterceptor[*pb.StringValue]( + func(resp gorums.NodeResponse[*pb.StringValue]) bool { + return resp.Err == nil // Only keep successful responses + }, + )), + ) + + result, err := responses.First() + if err != nil { + t.Fatalf("QuorumCall failed: %v", err) + } + if result.GetValue() != "echo: test" { + t.Errorf("Expected 'echo: test', got %q", result.GetValue()) + } +} + +// TestInterceptorChaining verifies that multiple custom interceptors can be chained. +func TestInterceptorChaining(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + var count int + + // Chain multiple custom interceptors + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + gorums.Interceptors( + LoggingInterceptor[*pb.StringValue, *pb.StringValue], + CountingInterceptor[*pb.StringValue, *pb.StringValue](&count), + ), + ) + + result, err := responses.Majority() + if err != nil { + t.Fatalf("QuorumCall failed: %v", err) + } + if result.GetValue() != "echo: test" { + t.Errorf("Expected 'echo: test', got %q", result.GetValue()) + } + + // Majority requires 2 responses out of 3, so count should be at least 2 + if count < 2 { + t.Errorf("Expected at least 2 responses counted, got %d", count) + } +} + +// TestCustomInterceptorWithMapRequest verifies custom interceptors work with built-in ones. +func TestCustomInterceptorWithMapRequest(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + var count int + + // Mix custom interceptor with built-in MapRequest + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + gorums.Interceptors( + // Custom: count responses + CountingInterceptor[*pb.StringValue, *pb.StringValue](&count), + // Built-in: transform request (identity transform for this test) + gorums.MapRequest[*pb.StringValue, *pb.StringValue]( + func(req *pb.StringValue, node *gorums.Node) *pb.StringValue { + return req + }, + ), + ), + ) + + result, err := responses.All() + if err != nil { + t.Fatalf("QuorumCall failed: %v", err) + } + if result.GetValue() != "echo: test" { + t.Errorf("Expected 'echo: test', got %q", result.GetValue()) + } + + // All requires 3 responses + if count != 3 { + t.Errorf("Expected 3 responses counted, got %d", count) + } +} diff --git a/cmd/benchmark/main.go b/cmd/benchmark/main.go index a7b64744e..6b8d4dc7c 100644 --- a/cmd/benchmark/main.go +++ b/cmd/benchmark/main.go @@ -190,21 +190,16 @@ func main() { } mgrOpts := []gorums.ManagerOption{ - gorums.WithGrpcDialOptions( + gorums.WithDialOptions( grpc.WithTransportCredentials(insecure.NewCredentials()), ), gorums.WithSendBufferSize(*sendBuffer), } - mgr := benchmark.NewManager(mgrOpts...) + mgr := gorums.NewManager(mgrOpts...) defer mgr.Close() - qspec := &benchmark.QSpec{ - QSize: options.QuorumSize, - CfgSize: options.NumNodes, - } - - cfg, err := mgr.NewConfiguration(qspec, gorums.WithNodeList(remotes[:options.NumNodes])) + cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(remotes[:options.NumNodes])) checkf("Failed to create configuration: %v", err) results, err := benchmark.RunBenchmarks(benchReg, options, cfg) diff --git a/cmd/protoc-gen-gorums/dev/aliases.go b/cmd/protoc-gen-gorums/dev/aliases.go new file mode 100644 index 000000000..94608a0d5 --- /dev/null +++ b/cmd/protoc-gen-gorums/dev/aliases.go @@ -0,0 +1,54 @@ +package dev + +import gorums "github.com/relab/gorums" + +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) + +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) + +// NewManager returns a new Manager for managing connection to nodes added +// to the manager. This function accepts manager options used to configure +// various aspects of the manager. +func NewManager(opts ...gorums.ManagerOption) *Manager { + return gorums.NewManager(opts...) +} + +// NewConfiguration returns a configuration based on the provided list of nodes. +// Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. +// A new configuration can also be created from an existing configuration, +// using the And, WithNewNodes, Except, and WithoutNodes methods. +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) +} + +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) +} diff --git a/cmd/protoc-gen-gorums/dev/config.go b/cmd/protoc-gen-gorums/dev/config.go deleted file mode 100644 index 3ec3d3df2..000000000 --- a/cmd/protoc-gen-gorums/dev/config.go +++ /dev/null @@ -1,58 +0,0 @@ -package dev - -import ( - "fmt" - - "github.com/relab/gorums" -) - -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} diff --git a/cmd/protoc-gen-gorums/dev/generated_code_test.go b/cmd/protoc-gen-gorums/dev/generated_code_test.go new file mode 100644 index 000000000..d0d52e5b3 --- /dev/null +++ b/cmd/protoc-gen-gorums/dev/generated_code_test.go @@ -0,0 +1,110 @@ +// Package dev_test contains integration tests for the generated Gorums code. +// These tests validate that the protoc-gen-gorums code generator produces +// correct and functional code. They exercise the generated QuorumCall method +// and its terminal methods (Majority, All, Threshold) along with custom +// aggregation patterns using CollectAll. +// +// NOTE: These tests are intentionally separate from the core library tests +// in the repository root. While they test similar functionality, they serve +// a different purpose: verifying the code generation pipeline end-to-end. +package dev_test + +import ( + "testing" + "time" + + "github.com/relab/gorums" + "github.com/relab/gorums/cmd/protoc-gen-gorums/dev" +) + +const quorumCallMethod = "dev.ZorumsService.QuorumCall" + +func quorumCallServer(_ int) gorums.ServerIface { + srv := gorums.NewServer() + srv.RegisterHandler(quorumCallMethod, func(_ gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + req := gorums.AsProto[*dev.Request](in) + resp := &dev.Response{} + resp.SetResult(int64(len(req.GetValue()))) + return gorums.NewResponseMessage(in.GetMetadata(), resp), nil + }) + return srv +} + +func TestQuorumCallWithMajority(t *testing.T) { + config := gorums.TestConfiguration(t, 3, quorumCallServer) + ctx := config.Context(gorums.TestContext(t, 2*time.Second)) + + req := &dev.Request{} + req.SetValue("test") + + // Call QuorumCall and wait for a majority to respond + resp, err := dev.QuorumCall(ctx, req).Majority() + if err != nil { + t.Fatalf("QuorumCall.Majority() failed: %v", err) + } + + // The server returns len(req.Value) which is 4 for "test" + if resp.GetResult() != 4 { + t.Errorf("Expected result 4, got %d", resp.GetResult()) + } +} + +func TestQuorumCallWithAll(t *testing.T) { + config := gorums.TestConfiguration(t, 3, quorumCallServer) + ctx := config.Context(gorums.TestContext(t, 2*time.Second)) + + req := &dev.Request{} + req.SetValue("test") + + // Call QuorumCall and wait for all responses + resp, err := dev.QuorumCall(ctx, req).All() + if err != nil { + t.Fatalf("QuorumCall.All() failed: %v", err) + } + + // The server returns len(req.Value) which is 4 for "test" + if resp.GetResult() != 4 { + t.Errorf("Expected result 4, got %d", resp.GetResult()) + } +} + +func TestQuorumCallWithThreshold(t *testing.T) { + config := gorums.TestConfiguration(t, 3, quorumCallServer) + ctx := config.Context(gorums.TestContext(t, 2*time.Second)) + + req := &dev.Request{} + req.SetValue("hello") + + // Use Threshold to wait for at least 2 responses + resp, err := dev.QuorumCall(ctx, req).Threshold(2) + if err != nil { + t.Fatalf("QuorumCall.Threshold() failed: %v", err) + } + + // The server returns len(req.Value) which is 5 for "hello" + if resp.GetResult() != 5 { + t.Errorf("Expected result 5, got %d", resp.GetResult()) + } +} + +func TestQuorumCallWithCustomAggregation(t *testing.T) { + config := gorums.TestConfiguration(t, 3, quorumCallServer) + ctx := config.Context(gorums.TestContext(t, 2*time.Second)) + + req := &dev.Request{} + req.SetValue("hello") + + // Use CollectAll for custom aggregation (sum all results) + responses := dev.QuorumCall(ctx, req) + results := responses.CollectAll() + + var total int64 + for _, resp := range results { + total += resp.GetResult() + } + + // Each server returns 5 (len("hello")), so 3 servers = 15 + if total != 15 { + t.Errorf("Expected total result 15, got %d", total) + } +} diff --git a/cmd/protoc-gen-gorums/dev/mgr.go b/cmd/protoc-gen-gorums/dev/mgr.go deleted file mode 100644 index fc603ac45..000000000 --- a/cmd/protoc-gen-gorums/dev/mgr.go +++ /dev/null @@ -1,79 +0,0 @@ -package dev - -import ( - "fmt" - - "github.com/relab/gorums" - "google.golang.org/grpc/encoding" -) - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} - -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} - -// NewManager returns a new Manager for managing connection to nodes added -// to the manager. This function accepts manager options used to configure -// various aspects of the manager. -func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } -} - -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. -// Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. -// A new configuration can also be created from an existing configuration, -// using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} diff --git a/cmd/protoc-gen-gorums/dev/node.go b/cmd/protoc-gen-gorums/dev/node.go deleted file mode 100644 index e5acc464c..000000000 --- a/cmd/protoc-gen-gorums/dev/node.go +++ /dev/null @@ -1,9 +0,0 @@ -package dev - -import "github.com/relab/gorums" - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} diff --git a/cmd/protoc-gen-gorums/dev/zorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums.pb.go index f199fad03..042594bf7 100644 --- a/cmd/protoc-gen-gorums/dev/zorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: zorums.proto package dev @@ -204,51 +204,23 @@ const file_zorums_proto_rawDesc = "" + "\x06Result\x18\x01 \x01(\x03R\x06Result\"\"\n" + "\n" + "MyResponse\x12\x14\n" + - "\x05Value\x18\x01 \x01(\tR\x05Value2\x86\x11\n" + + "\x05Value\x18\x01 \x01(\tR\x05Value2\x81\x06\n" + "\rZorumsService\x12)\n" + "\bGRPCCall\x12\f.dev.Request\x1a\r.dev.Response\"\x00\x12/\n" + "\n" + - "QuorumCall\x12\f.dev.Request\x1a\r.dev.Response\"\x04\xa0\xb5\x18\x01\x12=\n" + - "\x14QuorumCallPerNodeArg\x12\f.dev.Request\x1a\r.dev.Response\"\b\xa0\xb5\x18\x01\xa0\xb6\x18\x01\x12M\n" + - "\x1aQuorumCallCustomReturnType\x12\f.dev.Request\x1a\r.dev.Response\"\x12\xa0\xb5\x18\x01\xf2\xb6\x18\n" + - "MyResponse\x12F\n" + - "\x0fQuorumCallCombo\x12\f.dev.Request\x1a\r.dev.Response\"\x16\xa0\xb5\x18\x01\xa0\xb6\x18\x01\xf2\xb6\x18\n" + - "MyResponse\x12>\n" + + "QuorumCall\x12\f.dev.Request\x1a\r.dev.Response\"\x04\xa0\xb5\x18\x01\x12>\n" + "\x0fQuorumCallEmpty\x12\x16.google.protobuf.Empty\x1a\r.dev.Response\"\x04\xa0\xb5\x18\x01\x12>\n" + "\x10QuorumCallEmpty2\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\x04\xa0\xb5\x18\x01\x12.\n" + - "\tMulticast\x12\f.dev.Request\x1a\r.dev.Response\"\x04\x98\xb5\x18\x01\x12<\n" + - "\x13MulticastPerNodeArg\x12\f.dev.Request\x1a\r.dev.Response\"\b\x98\xb5\x18\x01\xa0\xb6\x18\x01\x12/\n" + + "\tMulticast\x12\f.dev.Request\x1a\r.dev.Response\"\x04\x98\xb5\x18\x01\x12/\n" + "\n" + "Multicast2\x12\f.dev.Request\x1a\r.dev.Response\"\x04\x98\xb5\x18\x01\x128\n" + "\n" + "Multicast3\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\x04\x98\xb5\x18\x01\x12B\n" + "\n" + - "Multicast4\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x04\x98\xb5\x18\x01\x128\n" + - "\x0fQuorumCallAsync\x12\f.dev.Request\x1a\r.dev.Response\"\b\xa0\xb5\x18\x01е\x18\x01\x12F\n" + - "\x19QuorumCallAsyncPerNodeArg\x12\f.dev.Request\x1a\r.dev.Response\"\f\xa0\xb5\x18\x01е\x18\x01\xa0\xb6\x18\x01\x12V\n" + - "\x1fQuorumCallAsyncCustomReturnType\x12\f.dev.Request\x1a\r.dev.Response\"\x16\xa0\xb5\x18\x01е\x18\x01\xf2\xb6\x18\n" + - "MyResponse\x12O\n" + - "\x14QuorumCallAsyncCombo\x12\f.dev.Request\x1a\r.dev.Response\"\x1a\xa0\xb5\x18\x01е\x18\x01\xa0\xb6\x18\x01\xf2\xb6\x18\n" + - "MyResponse\x129\n" + - "\x10QuorumCallAsync2\x12\f.dev.Request\x1a\r.dev.Response\"\b\xa0\xb5\x18\x01е\x18\x01\x12F\n" + - "\x14QuorumCallAsyncEmpty\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\b\xa0\xb5\x18\x01е\x18\x01\x12H\n" + - "\x15QuorumCallAsyncEmpty2\x12\x16.google.protobuf.Empty\x1a\r.dev.Response\"\b\xa0\xb5\x18\x01е\x18\x01\x120\n" + - "\vCorrectable\x12\f.dev.Request\x1a\r.dev.Response\"\x04\xa8\xb5\x18\x01\x12>\n" + - "\x15CorrectablePerNodeArg\x12\f.dev.Request\x1a\r.dev.Response\"\b\xa8\xb5\x18\x01\xa0\xb6\x18\x01\x12N\n" + - "\x1bCorrectableCustomReturnType\x12\f.dev.Request\x1a\r.dev.Response\"\x12\xa8\xb5\x18\x01\xf2\xb6\x18\n" + - "MyResponse\x12G\n" + - "\x10CorrectableCombo\x12\f.dev.Request\x1a\r.dev.Response\"\x16\xa8\xb5\x18\x01\xa0\xb6\x18\x01\xf2\xb6\x18\n" + - "MyResponse\x12>\n" + - "\x10CorrectableEmpty\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\x04\xa8\xb5\x18\x01\x12@\n" + - "\x11CorrectableEmpty2\x12\x16.google.protobuf.Empty\x1a\r.dev.Response\"\x04\xa8\xb5\x18\x01\x128\n" + - "\x11CorrectableStream\x12\f.dev.Request\x1a\r.dev.Response\"\x04\xa8\xb5\x18\x010\x01\x12F\n" + - "\x1bCorrectableStreamPerNodeArg\x12\f.dev.Request\x1a\r.dev.Response\"\b\xa8\xb5\x18\x01\xa0\xb6\x18\x010\x01\x12V\n" + - "!CorrectableStreamCustomReturnType\x12\f.dev.Request\x1a\r.dev.Response\"\x12\xa8\xb5\x18\x01\xf2\xb6\x18\n" + - "MyResponse0\x01\x12O\n" + - "\x16CorrectableStreamCombo\x12\f.dev.Request\x1a\r.dev.Response\"\x16\xa8\xb5\x18\x01\xa0\xb6\x18\x01\xf2\xb6\x18\n" + - "MyResponse0\x01\x12F\n" + - "\x16CorrectableStreamEmpty\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\x04\xa8\xb5\x18\x010\x01\x12H\n" + - "\x17CorrectableStreamEmpty2\x12\x16.google.protobuf.Empty\x1a\r.dev.Response\"\x04\xa8\xb5\x18\x010\x01\x12,\n" + + "Multicast4\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x04\x98\xb5\x18\x01\x127\n" + + "\x10QuorumCallStream\x12\f.dev.Request\x1a\r.dev.Response\"\x04\xa0\xb5\x18\x010\x01\x12I\n" + + "\x19QuorumCallStreamWithEmpty\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\x04\xa0\xb5\x18\x010\x01\x12K\n" + + "\x1aQuorumCallStreamWithEmpty2\x12\x16.google.protobuf.Empty\x1a\r.dev.Response\"\x04\xa0\xb5\x18\x010\x01\x12,\n" + "\aUnicast\x12\f.dev.Request\x1a\r.dev.Response\"\x04\x90\xb5\x18\x01\x126\n" + "\bUnicast2\x12\f.dev.Request\x1a\x16.google.protobuf.Empty\"\x04\x90\xb5\x18\x01B Z\x19cmd/protoc-gen-gorums/dev\x92\x03\x02\b\x02b\beditionsp\xe8\a" @@ -262,72 +234,32 @@ var file_zorums_proto_goTypes = []any{ var file_zorums_proto_depIdxs = []int32{ 0, // 0: dev.ZorumsService.GRPCCall:input_type -> dev.Request 0, // 1: dev.ZorumsService.QuorumCall:input_type -> dev.Request - 0, // 2: dev.ZorumsService.QuorumCallPerNodeArg:input_type -> dev.Request - 0, // 3: dev.ZorumsService.QuorumCallCustomReturnType:input_type -> dev.Request - 0, // 4: dev.ZorumsService.QuorumCallCombo:input_type -> dev.Request - 3, // 5: dev.ZorumsService.QuorumCallEmpty:input_type -> google.protobuf.Empty - 0, // 6: dev.ZorumsService.QuorumCallEmpty2:input_type -> dev.Request - 0, // 7: dev.ZorumsService.Multicast:input_type -> dev.Request - 0, // 8: dev.ZorumsService.MulticastPerNodeArg:input_type -> dev.Request - 0, // 9: dev.ZorumsService.Multicast2:input_type -> dev.Request - 0, // 10: dev.ZorumsService.Multicast3:input_type -> dev.Request - 3, // 11: dev.ZorumsService.Multicast4:input_type -> google.protobuf.Empty - 0, // 12: dev.ZorumsService.QuorumCallAsync:input_type -> dev.Request - 0, // 13: dev.ZorumsService.QuorumCallAsyncPerNodeArg:input_type -> dev.Request - 0, // 14: dev.ZorumsService.QuorumCallAsyncCustomReturnType:input_type -> dev.Request - 0, // 15: dev.ZorumsService.QuorumCallAsyncCombo:input_type -> dev.Request - 0, // 16: dev.ZorumsService.QuorumCallAsync2:input_type -> dev.Request - 0, // 17: dev.ZorumsService.QuorumCallAsyncEmpty:input_type -> dev.Request - 3, // 18: dev.ZorumsService.QuorumCallAsyncEmpty2:input_type -> google.protobuf.Empty - 0, // 19: dev.ZorumsService.Correctable:input_type -> dev.Request - 0, // 20: dev.ZorumsService.CorrectablePerNodeArg:input_type -> dev.Request - 0, // 21: dev.ZorumsService.CorrectableCustomReturnType:input_type -> dev.Request - 0, // 22: dev.ZorumsService.CorrectableCombo:input_type -> dev.Request - 0, // 23: dev.ZorumsService.CorrectableEmpty:input_type -> dev.Request - 3, // 24: dev.ZorumsService.CorrectableEmpty2:input_type -> google.protobuf.Empty - 0, // 25: dev.ZorumsService.CorrectableStream:input_type -> dev.Request - 0, // 26: dev.ZorumsService.CorrectableStreamPerNodeArg:input_type -> dev.Request - 0, // 27: dev.ZorumsService.CorrectableStreamCustomReturnType:input_type -> dev.Request - 0, // 28: dev.ZorumsService.CorrectableStreamCombo:input_type -> dev.Request - 0, // 29: dev.ZorumsService.CorrectableStreamEmpty:input_type -> dev.Request - 3, // 30: dev.ZorumsService.CorrectableStreamEmpty2:input_type -> google.protobuf.Empty - 0, // 31: dev.ZorumsService.Unicast:input_type -> dev.Request - 0, // 32: dev.ZorumsService.Unicast2:input_type -> dev.Request - 1, // 33: dev.ZorumsService.GRPCCall:output_type -> dev.Response - 1, // 34: dev.ZorumsService.QuorumCall:output_type -> dev.Response - 1, // 35: dev.ZorumsService.QuorumCallPerNodeArg:output_type -> dev.Response - 1, // 36: dev.ZorumsService.QuorumCallCustomReturnType:output_type -> dev.Response - 1, // 37: dev.ZorumsService.QuorumCallCombo:output_type -> dev.Response - 1, // 38: dev.ZorumsService.QuorumCallEmpty:output_type -> dev.Response - 3, // 39: dev.ZorumsService.QuorumCallEmpty2:output_type -> google.protobuf.Empty - 1, // 40: dev.ZorumsService.Multicast:output_type -> dev.Response - 1, // 41: dev.ZorumsService.MulticastPerNodeArg:output_type -> dev.Response - 1, // 42: dev.ZorumsService.Multicast2:output_type -> dev.Response - 3, // 43: dev.ZorumsService.Multicast3:output_type -> google.protobuf.Empty - 3, // 44: dev.ZorumsService.Multicast4:output_type -> google.protobuf.Empty - 1, // 45: dev.ZorumsService.QuorumCallAsync:output_type -> dev.Response - 1, // 46: dev.ZorumsService.QuorumCallAsyncPerNodeArg:output_type -> dev.Response - 1, // 47: dev.ZorumsService.QuorumCallAsyncCustomReturnType:output_type -> dev.Response - 1, // 48: dev.ZorumsService.QuorumCallAsyncCombo:output_type -> dev.Response - 1, // 49: dev.ZorumsService.QuorumCallAsync2:output_type -> dev.Response - 3, // 50: dev.ZorumsService.QuorumCallAsyncEmpty:output_type -> google.protobuf.Empty - 1, // 51: dev.ZorumsService.QuorumCallAsyncEmpty2:output_type -> dev.Response - 1, // 52: dev.ZorumsService.Correctable:output_type -> dev.Response - 1, // 53: dev.ZorumsService.CorrectablePerNodeArg:output_type -> dev.Response - 1, // 54: dev.ZorumsService.CorrectableCustomReturnType:output_type -> dev.Response - 1, // 55: dev.ZorumsService.CorrectableCombo:output_type -> dev.Response - 3, // 56: dev.ZorumsService.CorrectableEmpty:output_type -> google.protobuf.Empty - 1, // 57: dev.ZorumsService.CorrectableEmpty2:output_type -> dev.Response - 1, // 58: dev.ZorumsService.CorrectableStream:output_type -> dev.Response - 1, // 59: dev.ZorumsService.CorrectableStreamPerNodeArg:output_type -> dev.Response - 1, // 60: dev.ZorumsService.CorrectableStreamCustomReturnType:output_type -> dev.Response - 1, // 61: dev.ZorumsService.CorrectableStreamCombo:output_type -> dev.Response - 3, // 62: dev.ZorumsService.CorrectableStreamEmpty:output_type -> google.protobuf.Empty - 1, // 63: dev.ZorumsService.CorrectableStreamEmpty2:output_type -> dev.Response - 1, // 64: dev.ZorumsService.Unicast:output_type -> dev.Response - 3, // 65: dev.ZorumsService.Unicast2:output_type -> google.protobuf.Empty - 33, // [33:66] is the sub-list for method output_type - 0, // [0:33] is the sub-list for method input_type + 3, // 2: dev.ZorumsService.QuorumCallEmpty:input_type -> google.protobuf.Empty + 0, // 3: dev.ZorumsService.QuorumCallEmpty2:input_type -> dev.Request + 0, // 4: dev.ZorumsService.Multicast:input_type -> dev.Request + 0, // 5: dev.ZorumsService.Multicast2:input_type -> dev.Request + 0, // 6: dev.ZorumsService.Multicast3:input_type -> dev.Request + 3, // 7: dev.ZorumsService.Multicast4:input_type -> google.protobuf.Empty + 0, // 8: dev.ZorumsService.QuorumCallStream:input_type -> dev.Request + 0, // 9: dev.ZorumsService.QuorumCallStreamWithEmpty:input_type -> dev.Request + 3, // 10: dev.ZorumsService.QuorumCallStreamWithEmpty2:input_type -> google.protobuf.Empty + 0, // 11: dev.ZorumsService.Unicast:input_type -> dev.Request + 0, // 12: dev.ZorumsService.Unicast2:input_type -> dev.Request + 1, // 13: dev.ZorumsService.GRPCCall:output_type -> dev.Response + 1, // 14: dev.ZorumsService.QuorumCall:output_type -> dev.Response + 1, // 15: dev.ZorumsService.QuorumCallEmpty:output_type -> dev.Response + 3, // 16: dev.ZorumsService.QuorumCallEmpty2:output_type -> google.protobuf.Empty + 1, // 17: dev.ZorumsService.Multicast:output_type -> dev.Response + 1, // 18: dev.ZorumsService.Multicast2:output_type -> dev.Response + 3, // 19: dev.ZorumsService.Multicast3:output_type -> google.protobuf.Empty + 3, // 20: dev.ZorumsService.Multicast4:output_type -> google.protobuf.Empty + 1, // 21: dev.ZorumsService.QuorumCallStream:output_type -> dev.Response + 3, // 22: dev.ZorumsService.QuorumCallStreamWithEmpty:output_type -> google.protobuf.Empty + 1, // 23: dev.ZorumsService.QuorumCallStreamWithEmpty2:output_type -> dev.Response + 1, // 24: dev.ZorumsService.Unicast:output_type -> dev.Response + 3, // 25: dev.ZorumsService.Unicast2:output_type -> google.protobuf.Empty + 13, // [13:26] is the sub-list for method output_type + 0, // [0:13] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/cmd/protoc-gen-gorums/dev/zorums.proto b/cmd/protoc-gen-gorums/dev/zorums.proto index b0019447f..9d85a5cac 100644 --- a/cmd/protoc-gen-gorums/dev/zorums.proto +++ b/cmd/protoc-gen-gorums/dev/zorums.proto @@ -40,22 +40,6 @@ service ZorumsService { rpc QuorumCall(Request) returns (Response) { option (gorums.quorumcall) = true; } - // QuorumCall with per_node_arg option. - rpc QuorumCallPerNodeArg(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.per_node_arg) = true; - } - // QuorumCall with custom_return_type option. - rpc QuorumCallCustomReturnType(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // QuorumCallCombo with all supported options. - rpc QuorumCallCombo(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } // QuorumCallEmpty for testing imported message type. rpc QuorumCallEmpty(google.protobuf.Empty) returns (Response) { option (gorums.quorumcall) = true; @@ -67,20 +51,12 @@ service ZorumsService { // ------------------------------------------------------------------ // Multicast variants - // The multicast call type does not support the 'custom_return_type' - // option since multicast is one-way and so it doesn't make sense to - // operate on response types. // ------------------------------------------------------------------ // Multicast plain. Response type is not needed here. rpc Multicast(Request) returns (Response) { option (gorums.multicast) = true; } - // MulticastPerNodeArg with per_node_arg option. - rpc MulticastPerNodeArg(Request) returns (Response) { - option (gorums.multicast) = true; - option (gorums.per_node_arg) = true; - } // Multicast2 is testing whether multiple streams work. rpc Multicast2(Request) returns (Response) { option (gorums.multicast) = true; @@ -95,116 +71,21 @@ service ZorumsService { } // --------------------------------------------------------------- - // Async variants + // Streaming quorum call variants // --------------------------------------------------------------- - // QuorumCallAsync plain. - rpc QuorumCallAsync(Request) returns (Response) { + // QuorumCallStream plain. + rpc QuorumCallStream(Request) returns (stream Response) { option (gorums.quorumcall) = true; - option (gorums.async) = true; } - // QuorumCallAsyncPerNodeArg with per_node_arg option. - rpc QuorumCallAsyncPerNodeArg(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - option (gorums.per_node_arg) = true; - } - // QuorumCallAsyncCustomReturnType with custom_return_type option. - rpc QuorumCallAsyncCustomReturnType(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // QuorumCallAsyncCombo with all supported options. - rpc QuorumCallAsyncCombo(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // QuorumCallAsync2 plain; with same return type: Response. - rpc QuorumCallAsync2(Request) returns (Response) { + // QuorumCallStreamWithEmpty for testing imported message type. + rpc QuorumCallStreamWithEmpty(Request) returns (stream google.protobuf.Empty) { option (gorums.quorumcall) = true; - option (gorums.async) = true; } - // QuorumCallAsyncEmpty for testing imported message type. - rpc QuorumCallAsyncEmpty(Request) returns (google.protobuf.Empty) { + // QuorumCallStreamWithEmpty2 for testing imported message type; with same return + // type as QuorumCallStream: Response. + rpc QuorumCallStreamWithEmpty2(google.protobuf.Empty) returns (stream Response) { option (gorums.quorumcall) = true; - option (gorums.async) = true; - } - // QuorumCallAsyncEmpty2 for testing imported message type; with same return - // type as QuorumCallAsync: Response. - rpc QuorumCallAsyncEmpty2(google.protobuf.Empty) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - } - - // --------------------------------------------------------------- - // Correctable variants - // --------------------------------------------------------------- - - // Correctable plain. - rpc Correctable(Request) returns (Response) { - option (gorums.correctable) = true; - } - // CorrectablePerNodeArg with per_node_arg option. - rpc CorrectablePerNodeArg(Request) returns (Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - } - // CorrectableCustomReturnType with custom_return_type option. - rpc CorrectableCustomReturnType(Request) returns (Response) { - option (gorums.correctable) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableCombo with all supported options. - rpc CorrectableCombo(Request) returns (Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableEmpty for testing imported message type. - rpc CorrectableEmpty(Request) returns (google.protobuf.Empty) { - option (gorums.correctable) = true; - } - // CorrectableEmpty2 for testing imported message type; with same return - // type as Correctable: Response. - rpc CorrectableEmpty2(google.protobuf.Empty) returns (Response) { - option (gorums.correctable) = true; - } - - // --------------------------------------------------------------- - // CorrectableStream variants - // --------------------------------------------------------------- - - // CorrectableStream plain. - rpc CorrectableStream(Request) returns (stream Response) { - option (gorums.correctable) = true; - } - // CorrectablePerNodeArg with per_node_arg option. - rpc CorrectableStreamPerNodeArg(Request) returns (stream Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - } - // CorrectableCustomReturnType with custom_return_type option. - rpc CorrectableStreamCustomReturnType(Request) returns (stream Response) { - option (gorums.correctable) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableCombo with all supported options. - rpc CorrectableStreamCombo(Request) returns (stream Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableEmpty for testing imported message type. - rpc CorrectableStreamEmpty(Request) returns (stream google.protobuf.Empty) { - option (gorums.correctable) = true; - } - // CorrectableEmpty2 for testing imported message type; with same return - // type as Correctable: Response. - rpc CorrectableStreamEmpty2(google.protobuf.Empty) returns (stream Response) { - option (gorums.correctable) = true; } // --------------------------------------------------------------- diff --git a/cmd/protoc-gen-gorums/dev/zorums_async_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_async_gorums.pb.go deleted file mode 100644 index 68b78199d..000000000 --- a/cmd/protoc-gen-gorums/dev/zorums_async_gorums.pb.go +++ /dev/null @@ -1,154 +0,0 @@ -// Code generated by protoc-gen-gorums. DO NOT EDIT. -// versions: -// protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 -// source: zorums.proto - -package dev - -import ( - context "context" - gorums "github.com/relab/gorums" - proto "google.golang.org/protobuf/proto" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = gorums.EnforceVersion(10 - gorums.MinVersion) - // Verify that the gorums runtime is sufficiently up-to-date. - _ = gorums.EnforceVersion(gorums.MaxVersion - 10) -) - -// QuorumCallAsync plain. -func (c *Configuration) QuorumCallAsync(ctx context.Context, in *Request) *AsyncResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsync", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallAsyncQF(req.(*Request), r) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncResponse{fut} -} - -// QuorumCallAsyncPerNodeArg with per_node_arg option. -func (c *Configuration) QuorumCallAsyncPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *AsyncResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsyncPerNodeArg", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallAsyncPerNodeArgQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncResponse{fut} -} - -// QuorumCallAsyncCustomReturnType with custom_return_type option. -func (c *Configuration) QuorumCallAsyncCustomReturnType(ctx context.Context, in *Request) *AsyncMyResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsyncCustomReturnType", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallAsyncCustomReturnTypeQF(req.(*Request), r) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncMyResponse{fut} -} - -// QuorumCallAsyncCombo with all supported options. -func (c *Configuration) QuorumCallAsyncCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *AsyncMyResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsyncCombo", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallAsyncComboQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncMyResponse{fut} -} - -// QuorumCallAsync2 plain; with same return type: Response. -func (c *Configuration) QuorumCallAsync2(ctx context.Context, in *Request) *AsyncResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsync2", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallAsync2QF(req.(*Request), r) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncResponse{fut} -} - -// QuorumCallAsyncEmpty for testing imported message type. -func (c *Configuration) QuorumCallAsyncEmpty(ctx context.Context, in *Request) *AsyncEmpty { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsyncEmpty", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*emptypb.Empty, len(replies)) - for k, v := range replies { - r[k] = v.(*emptypb.Empty) - } - return c.qspec.QuorumCallAsyncEmptyQF(req.(*Request), r) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncEmpty{fut} -} - -// QuorumCallAsyncEmpty2 for testing imported message type; with same return -// type as QuorumCallAsync: Response. -func (c *Configuration) QuorumCallAsyncEmpty2(ctx context.Context, in *emptypb.Empty) *AsyncResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallAsyncEmpty2", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallAsyncEmpty2QF(req.(*emptypb.Empty), r) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncResponse{fut} -} diff --git a/cmd/protoc-gen-gorums/dev/zorums_client_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_client_gorums.pb.go deleted file mode 100644 index ff3c24621..000000000 --- a/cmd/protoc-gen-gorums/dev/zorums_client_gorums.pb.go +++ /dev/null @@ -1,67 +0,0 @@ -// Code generated by protoc-gen-gorums. DO NOT EDIT. -// versions: -// protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 -// source: zorums.proto - -package dev - -import ( - context "context" - gorums "github.com/relab/gorums" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = gorums.EnforceVersion(10 - gorums.MinVersion) - // Verify that the gorums runtime is sufficiently up-to-date. - _ = gorums.EnforceVersion(gorums.MaxVersion - 10) -) - -// ZorumsServiceClient is the client interface for the ZorumsService service. -type ZorumsServiceClient interface { - QuorumCall(ctx context.Context, in *Request) (resp *Response, err error) - QuorumCallPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) (resp *Response, err error) - QuorumCallCustomReturnType(ctx context.Context, in *Request) (resp *MyResponse, err error) - QuorumCallCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) (resp *MyResponse, err error) - QuorumCallEmpty(ctx context.Context, in *emptypb.Empty) (resp *Response, err error) - QuorumCallEmpty2(ctx context.Context, in *Request) (resp *emptypb.Empty, err error) - Multicast(ctx context.Context, in *Request, opts ...gorums.CallOption) - MulticastPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request, opts ...gorums.CallOption) - Multicast2(ctx context.Context, in *Request, opts ...gorums.CallOption) - Multicast3(ctx context.Context, in *Request, opts ...gorums.CallOption) - Multicast4(ctx context.Context, in *emptypb.Empty, opts ...gorums.CallOption) - QuorumCallAsync(ctx context.Context, in *Request) *AsyncResponse - QuorumCallAsyncPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *AsyncResponse - QuorumCallAsyncCustomReturnType(ctx context.Context, in *Request) *AsyncMyResponse - QuorumCallAsyncCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *AsyncMyResponse - QuorumCallAsync2(ctx context.Context, in *Request) *AsyncResponse - QuorumCallAsyncEmpty(ctx context.Context, in *Request) *AsyncEmpty - QuorumCallAsyncEmpty2(ctx context.Context, in *emptypb.Empty) *AsyncResponse - Correctable(ctx context.Context, in *Request) *CorrectableResponse - CorrectablePerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableResponse - CorrectableCustomReturnType(ctx context.Context, in *Request) *CorrectableMyResponse - CorrectableCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableMyResponse - CorrectableEmpty(ctx context.Context, in *Request) *CorrectableEmpty - CorrectableEmpty2(ctx context.Context, in *emptypb.Empty) *CorrectableResponse - CorrectableStream(ctx context.Context, in *Request) *CorrectableStreamResponse - CorrectableStreamPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableStreamResponse - CorrectableStreamCustomReturnType(ctx context.Context, in *Request) *CorrectableStreamMyResponse - CorrectableStreamCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableStreamMyResponse - CorrectableStreamEmpty(ctx context.Context, in *Request) *CorrectableStreamEmpty - CorrectableStreamEmpty2(ctx context.Context, in *emptypb.Empty) *CorrectableStreamResponse -} - -// enforce interface compliance -var _ ZorumsServiceClient = (*Configuration)(nil) - -// ZorumsServiceNodeClient is the single node client interface for the ZorumsService service. -type ZorumsServiceNodeClient interface { - GRPCCall(ctx context.Context, in *Request) (resp *Response, err error) - Unicast(ctx context.Context, in *Request, opts ...gorums.CallOption) - Unicast2(ctx context.Context, in *Request, opts ...gorums.CallOption) -} - -// enforce interface compliance -var _ ZorumsServiceNodeClient = (*Node)(nil) diff --git a/cmd/protoc-gen-gorums/dev/zorums_correctable_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_correctable_gorums.pb.go deleted file mode 100644 index 7856f6658..000000000 --- a/cmd/protoc-gen-gorums/dev/zorums_correctable_gorums.pb.go +++ /dev/null @@ -1,263 +0,0 @@ -// Code generated by protoc-gen-gorums. DO NOT EDIT. -// versions: -// protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 -// source: zorums.proto - -package dev - -import ( - context "context" - gorums "github.com/relab/gorums" - proto "google.golang.org/protobuf/proto" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = gorums.EnforceVersion(10 - gorums.MinVersion) - // Verify that the gorums runtime is sufficiently up-to-date. - _ = gorums.EnforceVersion(gorums.MaxVersion - 10) -) - -// Correctable plain. -func (c *Configuration) Correctable(ctx context.Context, in *Request) *CorrectableResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.Correctable", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableQF(req.(*Request), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableResponse{corr} -} - -// CorrectablePerNodeArg with per_node_arg option. -func (c *Configuration) CorrectablePerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectablePerNodeArg", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectablePerNodeArgQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableResponse{corr} -} - -// CorrectableCustomReturnType with custom_return_type option. -func (c *Configuration) CorrectableCustomReturnType(ctx context.Context, in *Request) *CorrectableMyResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableCustomReturnType", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableCustomReturnTypeQF(req.(*Request), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableMyResponse{corr} -} - -// CorrectableCombo with all supported options. -func (c *Configuration) CorrectableCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableMyResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableCombo", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableComboQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableMyResponse{corr} -} - -// CorrectableEmpty for testing imported message type. -func (c *Configuration) CorrectableEmpty(ctx context.Context, in *Request) *CorrectableEmpty { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableEmpty", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*emptypb.Empty, len(replies)) - for k, v := range replies { - r[k] = v.(*emptypb.Empty) - } - return c.qspec.CorrectableEmptyQF(req.(*Request), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableEmpty{corr} -} - -// CorrectableEmpty2 for testing imported message type; with same return -// type as Correctable: Response. -func (c *Configuration) CorrectableEmpty2(ctx context.Context, in *emptypb.Empty) *CorrectableResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableEmpty2", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableEmpty2QF(req.(*emptypb.Empty), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableResponse{corr} -} - -// CorrectableStream plain. -func (c *Configuration) CorrectableStream(ctx context.Context, in *Request) *CorrectableStreamResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableStream", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableStreamQF(req.(*Request), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamResponse{corr} -} - -// CorrectablePerNodeArg with per_node_arg option. -func (c *Configuration) CorrectableStreamPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableStreamResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableStreamPerNodeArg", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableStreamPerNodeArgQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamResponse{corr} -} - -// CorrectableCustomReturnType with custom_return_type option. -func (c *Configuration) CorrectableStreamCustomReturnType(ctx context.Context, in *Request) *CorrectableStreamMyResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableStreamCustomReturnType", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableStreamCustomReturnTypeQF(req.(*Request), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamMyResponse{corr} -} - -// CorrectableCombo with all supported options. -func (c *Configuration) CorrectableStreamCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) *CorrectableStreamMyResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableStreamCombo", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableStreamComboQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamMyResponse{corr} -} - -// CorrectableEmpty for testing imported message type. -func (c *Configuration) CorrectableStreamEmpty(ctx context.Context, in *Request) *CorrectableStreamEmpty { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableStreamEmpty", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*emptypb.Empty, len(replies)) - for k, v := range replies { - r[k] = v.(*emptypb.Empty) - } - return c.qspec.CorrectableStreamEmptyQF(req.(*Request), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamEmpty{corr} -} - -// CorrectableEmpty2 for testing imported message type; with same return -// type as Correctable: Response. -func (c *Configuration) CorrectableStreamEmpty2(ctx context.Context, in *emptypb.Empty) *CorrectableStreamResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "dev.ZorumsService.CorrectableStreamEmpty2", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.CorrectableStreamEmpty2QF(req.(*emptypb.Empty), r) - } - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamResponse{corr} -} diff --git a/cmd/protoc-gen-gorums/dev/zorums_multicast_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_multicast_gorums.pb.go index 4b4465405..fd9d55071 100644 --- a/cmd/protoc-gen-gorums/dev/zorums_multicast_gorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums_multicast_gorums.pb.go @@ -1,15 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: zorums.proto package dev import ( - context "context" gorums "github.com/relab/gorums" - proto "google.golang.org/protobuf/proto" emptypb "google.golang.org/protobuf/types/known/emptypb" ) @@ -20,62 +18,25 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// Multicast plain. Response type is not needed here. -func (c *Configuration) Multicast(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.Multicast", - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) -} - -// MulticastPerNodeArg with per_node_arg option. -func (c *Configuration) MulticastPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.MulticastPerNodeArg", - } - - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } +// Reference imports to suppress errors if they are not otherwise used. +var _ emptypb.Empty - c.RawConfiguration.Multicast(ctx, cd, opts...) +// Multicast plain. Response type is not needed here. +func Multicast(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "dev.ZorumsService.Multicast", opts...) } // Multicast2 is testing whether multiple streams work. -func (c *Configuration) Multicast2(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.Multicast2", - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) +func Multicast2(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "dev.ZorumsService.Multicast2", opts...) } -// Reference imports to suppress errors if they are not otherwise used. -var _ emptypb.Empty - // Multicast3 is testing imported message type. -func (c *Configuration) Multicast3(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.Multicast3", - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) +func Multicast3(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "dev.ZorumsService.Multicast3", opts...) } -// Reference imports to suppress errors if they are not otherwise used. -var _ emptypb.Empty - // Multicast4 is testing imported message type. -func (c *Configuration) Multicast4(ctx context.Context, in *emptypb.Empty, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.Multicast4", - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) +func Multicast4(ctx *gorums.ConfigContext, in *emptypb.Empty, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "dev.ZorumsService.Multicast4", opts...) } diff --git a/cmd/protoc-gen-gorums/dev/zorums_qspec_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_qspec_gorums.pb.go deleted file mode 100644 index 06acfaad7..000000000 --- a/cmd/protoc-gen-gorums/dev/zorums_qspec_gorums.pb.go +++ /dev/null @@ -1,199 +0,0 @@ -// Code generated by protoc-gen-gorums. DO NOT EDIT. -// versions: -// protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 -// source: zorums.proto - -package dev - -import ( - gorums "github.com/relab/gorums" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = gorums.EnforceVersion(10 - gorums.MinVersion) - // Verify that the gorums runtime is sufficiently up-to-date. - _ = gorums.EnforceVersion(gorums.MaxVersion - 10) -) - -// QuorumSpec is the interface of quorum functions for ZorumsService. -type QuorumSpec interface { - gorums.ConfigOption - - // QuorumCallQF is the quorum function for the QuorumCall - // quorum call method. The in parameter is the request object - // supplied to the QuorumCall method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallQF(in *Request, replies map[uint32]*Response) (*Response, bool) - - // QuorumCallPerNodeArgQF is the quorum function for the QuorumCallPerNodeArg - // quorum call method. The in parameter is the request object - // supplied to the QuorumCallPerNodeArg method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallPerNodeArgQF(in *Request, replies map[uint32]*Response) (*Response, bool) - - // QuorumCallCustomReturnTypeQF is the quorum function for the QuorumCallCustomReturnType - // quorum call method. The in parameter is the request object - // supplied to the QuorumCallCustomReturnType method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallCustomReturnTypeQF(in *Request, replies map[uint32]*Response) (*MyResponse, bool) - - // QuorumCallComboQF is the quorum function for the QuorumCallCombo - // quorum call method. The in parameter is the request object - // supplied to the QuorumCallCombo method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallComboQF(in *Request, replies map[uint32]*Response) (*MyResponse, bool) - - // QuorumCallEmptyQF is the quorum function for the QuorumCallEmpty - // quorum call method. The in parameter is the request object - // supplied to the QuorumCallEmpty method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *emptypb.Empty'. - QuorumCallEmptyQF(in *emptypb.Empty, replies map[uint32]*Response) (*Response, bool) - - // QuorumCallEmpty2QF is the quorum function for the QuorumCallEmpty2 - // quorum call method. The in parameter is the request object - // supplied to the QuorumCallEmpty2 method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallEmpty2QF(in *Request, replies map[uint32]*emptypb.Empty) (*emptypb.Empty, bool) - - // QuorumCallAsyncQF is the quorum function for the QuorumCallAsync - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsync method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallAsyncQF(in *Request, replies map[uint32]*Response) (*Response, bool) - - // QuorumCallAsyncPerNodeArgQF is the quorum function for the QuorumCallAsyncPerNodeArg - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsyncPerNodeArg method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallAsyncPerNodeArgQF(in *Request, replies map[uint32]*Response) (*Response, bool) - - // QuorumCallAsyncCustomReturnTypeQF is the quorum function for the QuorumCallAsyncCustomReturnType - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsyncCustomReturnType method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallAsyncCustomReturnTypeQF(in *Request, replies map[uint32]*Response) (*MyResponse, bool) - - // QuorumCallAsyncComboQF is the quorum function for the QuorumCallAsyncCombo - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsyncCombo method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallAsyncComboQF(in *Request, replies map[uint32]*Response) (*MyResponse, bool) - - // QuorumCallAsync2QF is the quorum function for the QuorumCallAsync2 - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsync2 method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallAsync2QF(in *Request, replies map[uint32]*Response) (*Response, bool) - - // QuorumCallAsyncEmptyQF is the quorum function for the QuorumCallAsyncEmpty - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsyncEmpty method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QuorumCallAsyncEmptyQF(in *Request, replies map[uint32]*emptypb.Empty) (*emptypb.Empty, bool) - - // QuorumCallAsyncEmpty2QF is the quorum function for the QuorumCallAsyncEmpty2 - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QuorumCallAsyncEmpty2 method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *emptypb.Empty'. - QuorumCallAsyncEmpty2QF(in *emptypb.Empty, replies map[uint32]*Response) (*Response, bool) - - // CorrectableQF is the quorum function for the Correctable - // correctable quorum call method. The in parameter is the request object - // supplied to the Correctable method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableQF(in *Request, replies map[uint32]*Response) (*Response, int, bool) - - // CorrectablePerNodeArgQF is the quorum function for the CorrectablePerNodeArg - // correctable quorum call method. The in parameter is the request object - // supplied to the CorrectablePerNodeArg method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectablePerNodeArgQF(in *Request, replies map[uint32]*Response) (*Response, int, bool) - - // CorrectableCustomReturnTypeQF is the quorum function for the CorrectableCustomReturnType - // correctable quorum call method. The in parameter is the request object - // supplied to the CorrectableCustomReturnType method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableCustomReturnTypeQF(in *Request, replies map[uint32]*Response) (*MyResponse, int, bool) - - // CorrectableComboQF is the quorum function for the CorrectableCombo - // correctable quorum call method. The in parameter is the request object - // supplied to the CorrectableCombo method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableComboQF(in *Request, replies map[uint32]*Response) (*MyResponse, int, bool) - - // CorrectableEmptyQF is the quorum function for the CorrectableEmpty - // correctable quorum call method. The in parameter is the request object - // supplied to the CorrectableEmpty method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableEmptyQF(in *Request, replies map[uint32]*emptypb.Empty) (*emptypb.Empty, int, bool) - - // CorrectableEmpty2QF is the quorum function for the CorrectableEmpty2 - // correctable quorum call method. The in parameter is the request object - // supplied to the CorrectableEmpty2 method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *emptypb.Empty'. - CorrectableEmpty2QF(in *emptypb.Empty, replies map[uint32]*Response) (*Response, int, bool) - - // CorrectableStreamQF is the quorum function for the CorrectableStream - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStream method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableStreamQF(in *Request, replies map[uint32]*Response) (*Response, int, bool) - - // CorrectableStreamPerNodeArgQF is the quorum function for the CorrectableStreamPerNodeArg - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStreamPerNodeArg method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableStreamPerNodeArgQF(in *Request, replies map[uint32]*Response) (*Response, int, bool) - - // CorrectableStreamCustomReturnTypeQF is the quorum function for the CorrectableStreamCustomReturnType - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStreamCustomReturnType method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableStreamCustomReturnTypeQF(in *Request, replies map[uint32]*Response) (*MyResponse, int, bool) - - // CorrectableStreamComboQF is the quorum function for the CorrectableStreamCombo - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStreamCombo method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableStreamComboQF(in *Request, replies map[uint32]*Response) (*MyResponse, int, bool) - - // CorrectableStreamEmptyQF is the quorum function for the CorrectableStreamEmpty - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStreamEmpty method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - CorrectableStreamEmptyQF(in *Request, replies map[uint32]*emptypb.Empty) (*emptypb.Empty, int, bool) - - // CorrectableStreamEmpty2QF is the quorum function for the CorrectableStreamEmpty2 - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStreamEmpty2 method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *emptypb.Empty'. - CorrectableStreamEmpty2QF(in *emptypb.Empty, replies map[uint32]*Response) (*Response, int, bool) -} diff --git a/cmd/protoc-gen-gorums/dev/zorums_quorumcall_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_quorumcall_gorums.pb.go index d92c4369f..501094797 100644 --- a/cmd/protoc-gen-gorums/dev/zorums_quorumcall_gorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums_quorumcall_gorums.pb.go @@ -1,15 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: zorums.proto package dev import ( - context "context" gorums "github.com/relab/gorums" - proto "google.golang.org/protobuf/proto" emptypb "google.golang.org/protobuf/types/known/emptypb" ) @@ -21,133 +19,50 @@ const ( ) // QuorumCall plain. -func (c *Configuration) QuorumCall(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCall", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallQF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err +func QuorumCall(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCall[*Request, *Response]( + ctx, in, "dev.ZorumsService.QuorumCall", + opts..., + ) } -// QuorumCall with per_node_arg option. -func (c *Configuration) QuorumCallPerNodeArg(ctx context.Context, in *Request, f func(*Request, uint32) *Request) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallPerNodeArg", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallPerNodeArgQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err +// QuorumCallEmpty for testing imported message type. +func QuorumCallEmpty(ctx *gorums.ConfigContext, in *emptypb.Empty, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCall[*emptypb.Empty, *Response]( + ctx, in, "dev.ZorumsService.QuorumCallEmpty", + opts..., + ) } -// QuorumCall with custom_return_type option. -func (c *Configuration) QuorumCallCustomReturnType(ctx context.Context, in *Request) (resp *MyResponse, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallCustomReturnType", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallCustomReturnTypeQF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*MyResponse), err +// QuorumCallEmpty2 for testing imported message type. +func QuorumCallEmpty2(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*emptypb.Empty] { + return gorums.QuorumCall[*Request, *emptypb.Empty]( + ctx, in, "dev.ZorumsService.QuorumCallEmpty2", + opts..., + ) } -// QuorumCallCombo with all supported options. -func (c *Configuration) QuorumCallCombo(ctx context.Context, in *Request, f func(*Request, uint32) *Request) (resp *MyResponse, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallCombo", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallComboQF(req.(*Request), r) - } - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*MyResponse), err +// QuorumCallStream plain. +func QuorumCallStream(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCallStream[*Request, *Response]( + ctx, in, "dev.ZorumsService.QuorumCallStream", + opts..., + ) } -// QuorumCallEmpty for testing imported message type. -func (c *Configuration) QuorumCallEmpty(ctx context.Context, in *emptypb.Empty) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallEmpty", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QuorumCallEmptyQF(req.(*emptypb.Empty), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err +// QuorumCallStreamWithEmpty for testing imported message type. +func QuorumCallStreamWithEmpty(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*emptypb.Empty] { + return gorums.QuorumCallStream[*Request, *emptypb.Empty]( + ctx, in, "dev.ZorumsService.QuorumCallStreamWithEmpty", + opts..., + ) } -// QuorumCallEmpty2 for testing imported message type. -func (c *Configuration) QuorumCallEmpty2(ctx context.Context, in *Request) (resp *emptypb.Empty, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "dev.ZorumsService.QuorumCallEmpty2", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*emptypb.Empty, len(replies)) - for k, v := range replies { - r[k] = v.(*emptypb.Empty) - } - return c.qspec.QuorumCallEmpty2QF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*emptypb.Empty), err +// QuorumCallStreamWithEmpty2 for testing imported message type; with same return +// type as QuorumCallStream: Response. +func QuorumCallStreamWithEmpty2(ctx *gorums.ConfigContext, in *emptypb.Empty, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCallStream[*emptypb.Empty, *Response]( + ctx, in, "dev.ZorumsService.QuorumCallStreamWithEmpty2", + opts..., + ) } diff --git a/cmd/protoc-gen-gorums/dev/zorums_rpc_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_rpc_gorums.pb.go index b5ec6efef..7ba99a06c 100644 --- a/cmd/protoc-gen-gorums/dev/zorums_rpc_gorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums_rpc_gorums.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: zorums.proto package dev import ( - context "context" gorums "github.com/relab/gorums" ) @@ -20,13 +19,8 @@ const ( // GRPCCall plain gRPC call; testing that Gorums can ignore these, but that // they are added to the _grpc.pb.go generated file. -func (n *Node) GRPCCall(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.CallData{ - Message: in, - Method: "dev.ZorumsService.GRPCCall", - } - - res, err := n.RawNode.RPCCall(ctx, cd) +func GRPCCall(ctx *gorums.NodeContext, in *Request) (resp *Response, err error) { + res, err := gorums.RPCCall(ctx, in, "dev.ZorumsService.GRPCCall") if err != nil { return nil, err } diff --git a/cmd/protoc-gen-gorums/dev/zorums_server_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_server_gorums.pb.go index 745d35a4c..1711da4c4 100644 --- a/cmd/protoc-gen-gorums/dev/zorums_server_gorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums_server_gorums.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: zorums.proto package dev @@ -23,35 +23,15 @@ const ( type ZorumsServiceServer interface { GRPCCall(ctx gorums.ServerCtx, request *Request) (response *Response, err error) QuorumCall(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallPerNodeArg(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallCustomReturnType(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallCombo(ctx gorums.ServerCtx, request *Request) (response *Response, err error) QuorumCallEmpty(ctx gorums.ServerCtx, request *emptypb.Empty) (response *Response, err error) QuorumCallEmpty2(ctx gorums.ServerCtx, request *Request) (response *emptypb.Empty, err error) Multicast(ctx gorums.ServerCtx, request *Request) - MulticastPerNodeArg(ctx gorums.ServerCtx, request *Request) Multicast2(ctx gorums.ServerCtx, request *Request) Multicast3(ctx gorums.ServerCtx, request *Request) Multicast4(ctx gorums.ServerCtx, request *emptypb.Empty) - QuorumCallAsync(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallAsyncPerNodeArg(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallAsyncCustomReturnType(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallAsyncCombo(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallAsync2(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QuorumCallAsyncEmpty(ctx gorums.ServerCtx, request *Request) (response *emptypb.Empty, err error) - QuorumCallAsyncEmpty2(ctx gorums.ServerCtx, request *emptypb.Empty) (response *Response, err error) - Correctable(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - CorrectablePerNodeArg(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - CorrectableCustomReturnType(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - CorrectableCombo(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - CorrectableEmpty(ctx gorums.ServerCtx, request *Request) (response *emptypb.Empty, err error) - CorrectableEmpty2(ctx gorums.ServerCtx, request *emptypb.Empty) (response *Response, err error) - CorrectableStream(ctx gorums.ServerCtx, request *Request, send func(response *Response) error) error - CorrectableStreamPerNodeArg(ctx gorums.ServerCtx, request *Request, send func(response *Response) error) error - CorrectableStreamCustomReturnType(ctx gorums.ServerCtx, request *Request, send func(response *Response) error) error - CorrectableStreamCombo(ctx gorums.ServerCtx, request *Request, send func(response *Response) error) error - CorrectableStreamEmpty(ctx gorums.ServerCtx, request *Request, send func(response *emptypb.Empty) error) error - CorrectableStreamEmpty2(ctx gorums.ServerCtx, request *emptypb.Empty, send func(response *Response) error) error + QuorumCallStream(ctx gorums.ServerCtx, request *Request, send func(response *Response) error) error + QuorumCallStreamWithEmpty(ctx gorums.ServerCtx, request *Request, send func(response *emptypb.Empty) error) error + QuorumCallStreamWithEmpty2(ctx gorums.ServerCtx, request *emptypb.Empty, send func(response *Response) error) error Unicast(ctx gorums.ServerCtx, request *Request) Unicast2(ctx gorums.ServerCtx, request *Request) } @@ -67,21 +47,6 @@ func RegisterZorumsServiceServer(srv *gorums.Server, impl ZorumsServiceServer) { resp, err := impl.QuorumCall(ctx, req) return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallPerNodeArg", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallPerNodeArg(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallCustomReturnType", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallCustomReturnType(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallCombo", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallCombo(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) srv.RegisterHandler("dev.ZorumsService.QuorumCallEmpty", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*emptypb.Empty](in) resp, err := impl.QuorumCallEmpty(ctx, req) @@ -97,11 +62,6 @@ func RegisterZorumsServiceServer(srv *gorums.Server, impl ZorumsServiceServer) { impl.Multicast(ctx, req) return nil, nil }) - srv.RegisterHandler("dev.ZorumsService.MulticastPerNodeArg", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - impl.MulticastPerNodeArg(ctx, req) - return nil, nil - }) srv.RegisterHandler("dev.ZorumsService.Multicast2", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*Request](in) impl.Multicast2(ctx, req) @@ -117,119 +77,27 @@ func RegisterZorumsServiceServer(srv *gorums.Server, impl ZorumsServiceServer) { impl.Multicast4(ctx, req) return nil, nil }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsync", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallAsync(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsyncPerNodeArg", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallAsyncPerNodeArg(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsyncCustomReturnType", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallAsyncCustomReturnType(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsyncCombo", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallAsyncCombo(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsync2", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallAsync2(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsyncEmpty", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QuorumCallAsyncEmpty(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.QuorumCallAsyncEmpty2", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*emptypb.Empty](in) - resp, err := impl.QuorumCallAsyncEmpty2(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.Correctable", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.Correctable(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectablePerNodeArg", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.CorrectablePerNodeArg(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableCustomReturnType", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.CorrectableCustomReturnType(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableCombo", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.CorrectableCombo(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableEmpty", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.CorrectableEmpty(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableEmpty2", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*emptypb.Empty](in) - resp, err := impl.CorrectableEmpty2(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableStream", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - err := impl.CorrectableStream(ctx, req, func(resp *Response) error { - // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg - md := proto.CloneOf(in.GetMetadata()) - return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) - }) - return nil, err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableStreamPerNodeArg", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - err := impl.CorrectableStreamPerNodeArg(ctx, req, func(resp *Response) error { - // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg - md := proto.CloneOf(in.GetMetadata()) - return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) - }) - return nil, err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableStreamCustomReturnType", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - err := impl.CorrectableStreamCustomReturnType(ctx, req, func(resp *Response) error { - // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg - md := proto.CloneOf(in.GetMetadata()) - return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) - }) - return nil, err - }) - srv.RegisterHandler("dev.ZorumsService.CorrectableStreamCombo", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + srv.RegisterHandler("dev.ZorumsService.QuorumCallStream", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*Request](in) - err := impl.CorrectableStreamCombo(ctx, req, func(resp *Response) error { + err := impl.QuorumCallStream(ctx, req, func(resp *Response) error { // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg md := proto.CloneOf(in.GetMetadata()) return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) }) return nil, err }) - srv.RegisterHandler("dev.ZorumsService.CorrectableStreamEmpty", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + srv.RegisterHandler("dev.ZorumsService.QuorumCallStreamWithEmpty", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*Request](in) - err := impl.CorrectableStreamEmpty(ctx, req, func(resp *emptypb.Empty) error { + err := impl.QuorumCallStreamWithEmpty(ctx, req, func(resp *emptypb.Empty) error { // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg md := proto.CloneOf(in.GetMetadata()) return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) }) return nil, err }) - srv.RegisterHandler("dev.ZorumsService.CorrectableStreamEmpty2", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + srv.RegisterHandler("dev.ZorumsService.QuorumCallStreamWithEmpty2", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*emptypb.Empty](in) - err := impl.CorrectableStreamEmpty2(ctx, req, func(resp *Response) error { + err := impl.QuorumCallStreamWithEmpty2(ctx, req, func(resp *Response) error { // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg md := proto.CloneOf(in.GetMetadata()) return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) diff --git a/cmd/protoc-gen-gorums/dev/zorums_types_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_types_gorums.pb.go index ab2f6e80b..14ff8c467 100644 --- a/cmd/protoc-gen-gorums/dev/zorums_types_gorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums_types_gorums.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: zorums.proto package dev @@ -18,167 +18,14 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -type internalEmpty struct { - nid uint32 - reply *emptypb.Empty - err error -} +// AsyncEmpty is a future for async quorum calls returning *emptypb.Empty. +type AsyncEmpty = *gorums.Async[*emptypb.Empty] -type internalResponse struct { - nid uint32 - reply *Response - err error -} +// AsyncResponse is a future for async quorum calls returning *Response. +type AsyncResponse = *gorums.Async[*Response] -// AsyncEmpty is a async object for processing replies. -type AsyncEmpty struct { - *gorums.Async -} +// CorrectableEmpty is a correctable object for quorum calls returning *emptypb.Empty. +type CorrectableEmpty = *gorums.Correctable[*emptypb.Empty] -// Get returns the reply and any error associated with the called method. -// The method blocks until a reply or error is available. -func (f *AsyncEmpty) Get() (*emptypb.Empty, error) { - resp, err := f.Async.Get() - if err != nil { - return nil, err - } - return resp.(*emptypb.Empty), err -} - -// AsyncMyResponse is a async object for processing replies. -type AsyncMyResponse struct { - *gorums.Async -} - -// Get returns the reply and any error associated with the called method. -// The method blocks until a reply or error is available. -func (f *AsyncMyResponse) Get() (*MyResponse, error) { - resp, err := f.Async.Get() - if err != nil { - return nil, err - } - return resp.(*MyResponse), err -} - -// AsyncResponse is a async object for processing replies. -type AsyncResponse struct { - *gorums.Async -} - -// Get returns the reply and any error associated with the called method. -// The method blocks until a reply or error is available. -func (f *AsyncResponse) Get() (*Response, error) { - resp, err := f.Async.Get() - if err != nil { - return nil, err - } - return resp.(*Response), err -} - -// CorrectableEmpty is a correctable object for processing replies. -type CorrectableEmpty struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableEmpty) Get() (*emptypb.Empty, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*emptypb.Empty), level, err -} - -// CorrectableMyResponse is a correctable object for processing replies. -type CorrectableMyResponse struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableMyResponse) Get() (*MyResponse, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*MyResponse), level, err -} - -// CorrectableResponse is a correctable object for processing replies. -type CorrectableResponse struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableResponse) Get() (*Response, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*Response), level, err -} - -// CorrectableStreamEmpty is a correctable object for processing replies. -type CorrectableStreamEmpty struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableStreamEmpty) Get() (*emptypb.Empty, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*emptypb.Empty), level, err -} - -// CorrectableStreamMyResponse is a correctable object for processing replies. -type CorrectableStreamMyResponse struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableStreamMyResponse) Get() (*MyResponse, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*MyResponse), level, err -} - -// CorrectableStreamResponse is a correctable object for processing replies. -type CorrectableStreamResponse struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableStreamResponse) Get() (*Response, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*Response), level, err -} +// CorrectableResponse is a correctable object for quorum calls returning *Response. +type CorrectableResponse = *gorums.Correctable[*Response] diff --git a/cmd/protoc-gen-gorums/dev/zorums_unicast_gorums.pb.go b/cmd/protoc-gen-gorums/dev/zorums_unicast_gorums.pb.go index cf3ab4f27..563769853 100644 --- a/cmd/protoc-gen-gorums/dev/zorums_unicast_gorums.pb.go +++ b/cmd/protoc-gen-gorums/dev/zorums_unicast_gorums.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: zorums.proto package dev import ( - context "context" gorums "github.com/relab/gorums" emptypb "google.golang.org/protobuf/types/known/emptypb" ) @@ -19,27 +18,17 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// Unicast is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (n *Node) Unicast(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.CallData{ - Message: in, - Method: "dev.ZorumsService.Unicast", - } - - n.RawNode.Unicast(ctx, cd, opts...) -} - // Reference imports to suppress errors if they are not otherwise used. var _ emptypb.Empty -// Unicast2 is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (n *Node) Unicast2(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.CallData{ - Message: in, - Method: "dev.ZorumsService.Unicast2", - } +// Unicast is a unicast call invoked on the node in ctx. +// No reply is returned to the client. +func Unicast(ctx *gorums.NodeContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Unicast(ctx, in, "dev.ZorumsService.Unicast", opts...) +} - n.RawNode.Unicast(ctx, cd, opts...) +// Unicast2 is a unicast call invoked on the node in ctx. +// No reply is returned to the client. +func Unicast2(ctx *gorums.NodeContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Unicast(ctx, in, "dev.ZorumsService.Unicast2", opts...) } diff --git a/cmd/protoc-gen-gorums/gengorums/gorums.go b/cmd/protoc-gen-gorums/gengorums/gorums.go index 2c80786e0..5425b95ff 100644 --- a/cmd/protoc-gen-gorums/gengorums/gorums.go +++ b/cmd/protoc-gen-gorums/gengorums/gorums.go @@ -4,11 +4,9 @@ package gengorums import ( "fmt" "log" - "sort" "strings" "github.com/relab/gorums" - "github.com/relab/gorums/internal/correctable" "github.com/relab/gorums/internal/version" "google.golang.org/protobuf/compiler/protogen" "google.golang.org/protobuf/proto" @@ -25,7 +23,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) { } filename := file.GeneratedFilenamePrefix + "_gorums.pb.go" g := gen.NewGeneratedFile(filename, file.GoImportPath) - genGeneratedHeader(gen, g, file) + genGeneratedHeader(gen, file, g) g.P("package ", file.GoPackageName) g.P() genVersionCheck(g) @@ -34,10 +32,10 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) { for path, ident := range pkgIdentMap { addImport(path, ident, g) } - generateFileContent(gen, file, g) + generateFileContent(file, g) } -func genGeneratedHeader(gen *protogen.Plugin, g *protogen.GeneratedFile, f *protogen.File) { +func genGeneratedHeader(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile) { g.P("// Code generated by protoc-gen-gorums. DO NOT EDIT.") if GenerateVersionMarkers { @@ -50,10 +48,10 @@ func genGeneratedHeader(gen *protogen.Plugin, g *protogen.GeneratedFile, f *prot g.P("// \tprotoc ", protocVersion) } - if f.Proto.GetOptions().GetDeprecated() { - g.P("// ", f.Desc.Path(), " is a deprecated file.") + if file.Proto.GetOptions().GetDeprecated() { + g.P("// ", file.Desc.Path(), " is a deprecated file.") } else { - g.P("// source: ", f.Desc.Path()) + g.P("// source: ", file.Desc.Path()) } g.P() } @@ -97,83 +95,104 @@ func gorumsGuard(file *protogen.File) bool { } // GenerateFileContent generates the Gorums service definitions, excluding the package statement. -func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile) { - // sort the gorums types so that output remains stable across rebuilds - sortedTypes := make([]string, 0, len(gorumsCallTypesInfo)) - for gorumsType := range gorumsCallTypesInfo { - sortedTypes = append(sortedTypes, gorumsType) +func generateFileContent(file *protogen.File, g *protogen.GeneratedFile) { + // 1. Data Types (Async/Correctable aliases) + genGorumsType(g, file.Services, "types") + + // 2. Reference Imports + genReferenceImports(g, file.Services) + + // 3. Client Methods (in proto order) + genGorumsMethods(g, file.Services, "") + + // 4. Server Interface and Registration + genGorumsType(g, file.Services, "server") +} + +// genReferenceImports generates a dummy reference to suppress unused import errors. +// These are only needed for oneway calls (unicast/multicast) because their +// output type is qualified by the commonVariables template but may be unused. +func genReferenceImports(g *protogen.GeneratedFile, services []*protogen.Service) { + seen := make(map[string]bool) + var idents []string + for _, service := range services { + for _, method := range service.Methods { + if !hasMethodOption(method, gorums.E_Multicast, gorums.E_Unicast) { + continue + } + qualified := g.QualifiedGoIdent(method.Output.GoIdent) + if !strings.Contains(qualified, ".") { + continue + } + pkg := qualified[:strings.LastIndex(qualified, ".")] + if seen[pkg] { + continue + } + seen[pkg] = true + idents = append(idents, qualified) + } } - sort.Strings(sortedTypes) - for _, gorumsType := range sortedTypes { - genGorumsType(g, file.Services, gorumsType) + if len(idents) > 0 { + g.P("// Reference imports to suppress errors if they are not otherwise used.") + for _, ident := range idents { + g.P("var _ ", ident) + } + g.P() } } -// servicesData hold the services to generate and a reference to the file in which -// the services should be generated. This is data to be used by template generator. -type servicesData struct { - GenFile *protogen.GeneratedFile - Services []*protogen.Service +// genGorumsType generates Gorums methods and corresponding data structures for the given call type. +func genGorumsType(g *protogen.GeneratedFile, services []*protogen.Service, callType string) { + if callTypeInfo := gorumsCallTypesInfo[callType]; callTypeInfo.extInfo == nil { + // servicesData hold the services to generate and a reference to the file in which + // the services should be generated. This is data to be used by template generator. + type servicesData struct { + GenFile *protogen.GeneratedFile + Services []*protogen.Service + } + g.P(mustExecute(parseTemplate(callType, callTypeInfo.template), servicesData{g, services})) + } } -// genGorumsType generates Gorums methods and corresponding data structures for gorumsType. -func genGorumsType(g *protogen.GeneratedFile, services []*protogen.Service, gorumsType string) { - data := servicesData{g, services} - if callTypeInfo := gorumsCallTypesInfo[gorumsType]; callTypeInfo.extInfo == nil { - g.P(mustExecute(parseTemplate(gorumsType, callTypeInfo.template), data)) - } else { - genGorumsMethods(gorumsType, data, callTypeInfo) +// genGorumsMethods generates Gorums methods for the given call type. +func genGorumsMethods(g *protogen.GeneratedFile, services []*protogen.Service, callType string) { + for _, service := range services { + for _, method := range service.Methods { + genMethod(g, method, callType) + } } - g.P() } -// genGorumsMethods generates Gorums methods for the given call type. -func genGorumsMethods(gorumsType string, data servicesData, callTypeInfo *callTypeInfo) { +// genMethod generates the Gorums method for the given method and call type. +func genMethod(g *protogen.GeneratedFile, method *protogen.Method, targetType string) { + typeName, info := callType(method) + if info == nil { + return + } + if targetType != "" && targetType != typeName { + return + } + if err := validateOptions(method); err != nil { + log.Fatal(err) + } type methodData struct { GenFile *protogen.GeneratedFile Method *protogen.Method } - g := data.GenFile - for _, service := range data.Services { - for _, method := range service.Methods { - err := validateOptions(method) - if err != nil { - log.Fatal(err) - } - callType := callTypeInfo.deriveCallType(method) - if callType.check(method) { - g.P(mustExecute(parseTemplate(gorumsType, callType.template), methodData{g, method})) - } - } - } + g.P(mustExecute(parseTemplate(typeName, info.template), methodData{g, method})) } // callType returns call type information for the given method. // If the given method has specified a Gorums method option that // correspond to a call type, this call type is returned. Further, // if the call type has a sub call type, then this is returned instead. -func callType(method *protogen.Method) *callTypeInfo { - for _, callTypeInfo := range callTypeOptions(method) { - callType := callTypeInfo.deriveCallType(method) - if callType.check(method) { - return callType - } - } - panic(fmt.Sprintf("unknown call type for method %s\n", method.GoName)) -} - -// callTypeOptions returns all Gorums call types for the given method. -func callTypeOptions(method *protogen.Method) []*callTypeInfo { - methExt := protoimpl.X.MessageOf(method.Desc.Options()).Interface() - var options []*callTypeInfo - for _, callType := range gorumsCallTypesInfo { - if callType.extInfo != nil { - if proto.HasExtension(methExt, callType.extInfo) { - options = append(options, callType) - } +func callType(method *protogen.Method) (string, *callTypeInfo) { + for name, callTypeInfo := range gorumsCallTypesInfo { + if callType := callTypeInfo.deriveCallType(method); callType.check(method) { + return name, callType } } - return options + return "", nil } // hasGorumsMethods returns true if one of the methods in the set of services @@ -181,11 +200,8 @@ func callTypeOptions(method *protogen.Method) []*callTypeInfo { func hasGorumsMethods(services []*protogen.Service) bool { for _, service := range services { for _, method := range service.Methods { - for _, callTypeInfo := range gorumsCallTypesInfo { - callType := callTypeInfo.deriveCallType(method) - if callType.check(method) { - return true - } + if _, callType := callType(method); callType != nil { + return true } } } @@ -200,7 +216,6 @@ type callTypeInfo struct { extInfo *protoimpl.ExtensionInfo docName string template string - outPrefix string chkFn func(m *protogen.Method) bool nestedCallType map[string]*callTypeInfo } @@ -216,11 +231,9 @@ func (c *callTypeInfo) check(m *protogen.Method) bool { // deriveCallType resolves the nested call type if any. func (c *callTypeInfo) deriveCallType(m *protogen.Method) *callTypeInfo { if c != nil { - if c.nestedCallType != nil { - for _, nestedCallType := range c.nestedCallType { - if nestedCallType.chkFn(m) { - return nestedCallType - } + for _, nestedCallType := range c.nestedCallType { + if nestedCallType.chkFn(m) { + return nestedCallType } } } @@ -239,10 +252,8 @@ func callTypeName(ext *protoimpl.ExtensionInfo) string { // The entries in this map is used to generate dev/zorums_{type}.pb.go // files for the different keys. var gorumsCallTypesInfo = map[string]*callTypeInfo{ - "qspec": {template: qspecInterface}, "types": {template: dataTypes}, "server": {template: server}, - "client": {template: client}, callTypeName(gorums.E_Rpc): { extInfo: gorums.E_Rpc, @@ -253,44 +264,25 @@ var gorumsCallTypesInfo = map[string]*callTypeInfo{ }, }, callTypeName(gorums.E_Quorumcall): { - extInfo: gorums.E_Quorumcall, - docName: "quorum", - template: quorumCall, + extInfo: gorums.E_Quorumcall, chkFn: func(m *protogen.Method) bool { - return hasMethodOption(m, gorums.E_Quorumcall) && !hasMethodOption(m, gorums.E_Async) - }, - }, - callTypeName(gorums.E_Async): { - extInfo: gorums.E_Async, - docName: "asynchronous quorum", - template: asyncCall, - outPrefix: "Async", - chkFn: func(m *protogen.Method) bool { - return hasAllMethodOption(m, gorums.E_Quorumcall, gorums.E_Async) - }, - }, - callTypeName(gorums.E_Correctable): { - extInfo: gorums.E_Correctable, - chkFn: func(m *protogen.Method) bool { - return hasMethodOption(m, gorums.E_Correctable) + return hasMethodOption(m, gorums.E_Quorumcall) }, nestedCallType: map[string]*callTypeInfo{ - callTypeName(correctable.E_Correctable): { - extInfo: correctable.E_Correctable, - docName: "correctable quorum", - template: correctableCall, - outPrefix: "Correctable", + "quorumcall": { + extInfo: gorums.E_Quorumcall, + docName: "quorum", + template: quorumCall, chkFn: func(m *protogen.Method) bool { - return hasMethodOption(m, gorums.E_Correctable) && !m.Desc.IsStreamingServer() + return hasMethodOption(m, gorums.E_Quorumcall) && !m.Desc.IsStreamingServer() }, }, - callTypeName(correctable.E_CorrectableStream): { - extInfo: correctable.E_CorrectableStream, - docName: "correctable stream quorum", - template: correctableCall, - outPrefix: "CorrectableStream", + "quorumcall_stream": { + extInfo: gorums.E_Quorumcall, + docName: "streaming quorum", + template: quorumCallStream, chkFn: func(m *protogen.Method) bool { - return hasMethodOption(m, gorums.E_Correctable) && m.Desc.IsStreamingServer() + return hasMethodOption(m, gorums.E_Quorumcall) && m.Desc.IsStreamingServer() }, }, }, @@ -317,38 +309,10 @@ var gorumsCallTypesInfo = map[string]*callTypeInfo{ // These are considered mutually incompatible. var gorumsCallTypes = []*protoimpl.ExtensionInfo{ gorums.E_Quorumcall, - gorums.E_Async, - gorums.E_Correctable, gorums.E_Multicast, gorums.E_Unicast, } -// callTypesWithInternal should list all available call types that -// has a quorum function and hence need an internal type that wraps -// the return type with additional information. -var callTypesWithInternal = []*protoimpl.ExtensionInfo{ - gorums.E_Quorumcall, - gorums.E_Async, - gorums.E_Correctable, -} - -// callTypesWithPromiseObject lists all call types that returns -// a promise (async or correctable) object. -var callTypesWithPromiseObject = []*protoimpl.ExtensionInfo{ - gorums.E_Async, - gorums.E_Correctable, -} - -// hasGorumsCallType returns true if the given method has specified -// one of the call types supported by Gorums. -func hasGorumsCallType(method *protogen.Method) bool { - return hasMethodOption(method, gorumsCallTypes...) -} - -func hasConfigurationCallType(method *protogen.Method) bool { - return hasGorumsCallType(method) && !hasMethodOption(method, gorums.E_Unicast) -} - // hasMethodOption returns true if the method has one of the given method options. func hasMethodOption(method *protogen.Method, methodOptions ...*protoimpl.ExtensionInfo) bool { ext := protoimpl.X.MessageOf(method.Desc.Options()).Interface() @@ -360,32 +324,15 @@ func hasMethodOption(method *protogen.Method, methodOptions ...*protoimpl.Extens return false } -// hasAllMethodOption returns true if the method has all of the given method options -func hasAllMethodOption(method *protogen.Method, methodOptions ...*protoimpl.ExtensionInfo) bool { - ext := protoimpl.X.MessageOf(method.Desc.Options()).Interface() - for _, callType := range methodOptions { - if !proto.HasExtension(ext, callType) { - return false - } - } - return true -} - // validateOptions returns an error if the extension options // for the provided method are incompatible. func validateOptions(method *protogen.Method) error { switch { - case hasMethodOption(method, gorums.E_Async) && !hasMethodOption(method, gorums.E_Quorumcall): - return optionErrorf("is required for async methods", method, gorums.E_Quorumcall) - case !hasMethodOption(method, gorums.E_Multicast) && method.Desc.IsStreamingClient(): return optionErrorf("is required for client-server stream methods", method, gorums.E_Multicast) - case !hasMethodOption(method, gorums.E_Correctable) && method.Desc.IsStreamingServer(): - return optionErrorf("is required for server-client stream methods", method, gorums.E_Correctable) - - case hasMethodOption(method, gorums.E_Correctable) && method.Desc.IsStreamingClient(): - return optionErrorf("is only valid for server-client stream methods", method, gorums.E_Correctable) + case !hasMethodOption(method, gorums.E_Quorumcall) && method.Desc.IsStreamingServer(): + return optionErrorf("is required for server-client stream methods", method, gorums.E_Quorumcall) } return nil } diff --git a/cmd/protoc-gen-gorums/gengorums/gorums_dev.go b/cmd/protoc-gen-gorums/gengorums/gorums_dev.go index d6e10f619..809acd018 100644 --- a/cmd/protoc-gen-gorums/gengorums/gorums_dev.go +++ b/cmd/protoc-gen-gorums/gengorums/gorums_dev.go @@ -1,6 +1,7 @@ package gengorums import ( + "github.com/relab/gorums" "google.golang.org/protobuf/compiler/protogen" ) @@ -19,9 +20,13 @@ func generateDevFile(gen *protogen.Plugin, file *protogen.File, gorumsType strin // generate dev file for given gorumsType filename := file.GeneratedFilenamePrefix + "_" + gorumsType + "_gorums.pb.go" g := gen.NewGeneratedFile(filename, file.GoImportPath) - genGeneratedHeader(gen, g, file) + genGeneratedHeader(gen, file, g) g.P("package ", file.GoPackageName) g.P() genVersionCheck(g) + if gorumsType == callTypeName(gorums.E_Multicast) || gorumsType == callTypeName(gorums.E_Unicast) { + genReferenceImports(g, file.Services) + } genGorumsType(g, file.Services, gorumsType) + genGorumsMethods(g, file.Services, gorumsType) } diff --git a/cmd/protoc-gen-gorums/gengorums/gorums_func_map.go b/cmd/protoc-gen-gorums/gengorums/gorums_func_map.go index 355f1e763..becfa5d8d 100644 --- a/cmd/protoc-gen-gorums/gengorums/gorums_func_map.go +++ b/cmd/protoc-gen-gorums/gengorums/gorums_func_map.go @@ -8,8 +8,6 @@ import ( "github.com/relab/gorums" "google.golang.org/protobuf/compiler/protogen" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/runtime/protoimpl" ) // importMap holds the mapping between short-hand import name @@ -59,42 +57,15 @@ var funcMap = template.FuncMap{ } return g.QualifiedGoIdent(pkg.Ident(ident)) }, - "hasPerNodeArg": func(method *protogen.Method) bool { - return hasMethodOption(method, gorums.E_PerNodeArg) - }, - "perNodeArg": func(method *protogen.Method, arg string) string { - if hasMethodOption(method, gorums.E_PerNodeArg) { - return arg - } - return "" - }, - "perNodeFnType": func(g *protogen.GeneratedFile, method *protogen.Method, arg string) string { - if hasMethodOption(method, gorums.E_PerNodeArg) { - inType := g.QualifiedGoIdent(method.Input.GoIdent) - return arg + " func(*" + inType + ", uint32) *" + inType - } - return "" - }, - "correctableStream": func(method *protogen.Method) bool { - return hasMethodOption(method, gorums.E_Correctable) && method.Desc.IsStreamingServer() - }, - "isCorrectable": func(method *protogen.Method) bool { - return hasMethodOption(method, gorums.E_Correctable) - }, - "withCorrectable": func(method *protogen.Method, arg string) string { - if hasMethodOption(method, gorums.E_Correctable) { - return arg - } - return "" - }, - "withPromise": func(method *protogen.Method, arg string) string { - if hasMethodOption(method, callTypesWithPromiseObject...) { - return arg - } - return "" + "isStreamingServer": func(method *protogen.Method) bool { + return method.Desc.IsStreamingServer() }, "docName": func(method *protogen.Method) string { - return callType(method).docName + _, info := callType(method) + if info != nil { + return info.docName + } + return "unknown" }, "fullName": func(method *protogen.Method) string { return fmt.Sprintf("/%s/%s", method.Parent.Desc.FullName(), method.Desc.Name()) @@ -108,25 +79,12 @@ var funcMap = template.FuncMap{ "isOneway": func(method *protogen.Method) bool { return hasMethodOption(method, gorums.E_Multicast, gorums.E_Unicast) }, - "isAsync": func(method *protogen.Method) bool { - return hasMethodOption(method, gorums.E_Async) - }, - "out": out, - "outType": outType, - "internalOut": internalOut, - "customOut": customOut, - "mapInternalOutType": mapInternalOutType, - "mapCorrectableOutType": mapCorrectableOutType, - "mapAsyncOutType": mapAsyncOutType, - "qspecMethods": qspecMethods, - "qspecServices": qspecServices, - "unexport": unexport, - "contains": strings.Contains, - "field": field, - "configurationsServices": configurationsServices, - "configurationMethods": configurationMethods, - "nodeServices": nodeServices, - "nodeMethods": nodeMethods, + "out": out, + "mapAsyncOutType": mapAsyncOutType, + "mapCorrectableOutType": mapCorrectableOutType, + "unexport": unexport, + "contains": strings.Contains, + "field": field, } type mapFunc func(*protogen.GeneratedFile, *protogen.Method, map[string]string) @@ -146,54 +104,26 @@ func out(g *protogen.GeneratedFile, method *protogen.Method) string { return g.QualifiedGoIdent(method.Output.GoIdent) } -func outType(method *protogen.Method, out string) string { - return fmt.Sprintf("%s%s", callType(method).outPrefix, field(out)) -} - -func internalOut(out string) string { - return fmt.Sprintf("internal%s", field(out)) -} - -// customOut returns the output type to be used for the given method. -// This may be the output type specified in the rpc line, -// or if a custom_return_type option is provided for the method, -// this provided custom type will be returned. -func customOut(g *protogen.GeneratedFile, method *protogen.Method) string { - ext := protoimpl.X.MessageOf(method.Desc.Options()).Interface() - customOutType := fmt.Sprintf("%v", proto.GetExtension(ext, gorums.E_CustomReturnType)) - outType := method.Output.GoIdent - if customOutType != "" { - outType.GoName = customOutType - } - return g.QualifiedGoIdent(outType) -} - -func mapInternalOutType(g *protogen.GeneratedFile, services []*protogen.Service) (s map[string]string) { - return mapType(g, services, func(g *protogen.GeneratedFile, method *protogen.Method, s map[string]string) { - if hasMethodOption(method, callTypesWithInternal...) { - out := out(g, method) - intOut := internalOut(out) - s[intOut] = out - } - }) -} - func mapAsyncOutType(g *protogen.GeneratedFile, services []*protogen.Service) (s map[string]string) { return mapType(g, services, func(g *protogen.GeneratedFile, method *protogen.Method, s map[string]string) { - if hasAllMethodOption(method, gorums.E_Quorumcall, gorums.E_Async) { - out := customOut(g, method) - futOut := outType(method, out) - s[futOut] = out + // Generate Async type aliases for quorumcall methods since + // users can call .AsyncMajority() on any quorum call result + if hasMethodOption(method, gorums.E_Quorumcall) { + o := out(g, method) + futOut := fmt.Sprintf("Async%s", field(o)) + s[futOut] = o } }) } func mapCorrectableOutType(g *protogen.GeneratedFile, services []*protogen.Service) (s map[string]string) { return mapType(g, services, func(g *protogen.GeneratedFile, method *protogen.Method, s map[string]string) { - if hasMethodOption(method, gorums.E_Correctable) { - out := customOut(g, method) - corrOut := outType(method, out) - s[corrOut] = out + // Generate Correctable type aliases for quorumcall methods since + // users can call .Correctable() on any quorum call result + if hasMethodOption(method, gorums.E_Quorumcall) { + o := out(g, method) + corrOut := fmt.Sprintf("Correctable%s", field(o)) + s[corrOut] = o } }) } diff --git a/cmd/protoc-gen-gorums/gengorums/gorums_test.go b/cmd/protoc-gen-gorums/gengorums/gorums_test.go deleted file mode 100644 index 1f7a27d84..000000000 --- a/cmd/protoc-gen-gorums/gengorums/gorums_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package gengorums - -import ( - "testing" -) - -//TODO(meling) Test various generator functions that are non-trivial -// methods to tests: -// qspecMethods -// qspecServices - -func TestQSpecMethods(t *testing.T) { - t.Skip("Not implemented yet") -} diff --git a/cmd/protoc-gen-gorums/gengorums/template_async.go b/cmd/protoc-gen-gorums/gengorums/template_async.go deleted file mode 100644 index fa6e842dd..000000000 --- a/cmd/protoc-gen-gorums/gengorums/template_async.go +++ /dev/null @@ -1,59 +0,0 @@ -package gengorums - -var asyncCallComment = ` -{{$comments := .Method.Comments.Leading}} -{{if ne $comments ""}} -{{$comments -}} -{{else}} -{{if hasPerNodeArg .Method}} -// {{$method}} asynchronously invokes a quorum call on each node in -// configuration c, with the argument returned by the provided function f -// and returns the result as a {{$asyncOut}}, which can be used to inspect -// the quorum call reply and error when available. -// The provide per node function f takes the provided {{$in}} argument -// and returns an {{$out}} object to be passed to the given nodeID. -// The per node function f should be thread-safe. -{{else}} -// {{$method}} asynchronously invokes a quorum call on configuration c -// and returns a {{$asyncOut}}, which can be used to inspect the quorum call -// reply and error when available. -{{end -}} -{{end -}} -` - -var asyncSignature = `func (c *Configuration) {{$method}}(` + - `ctx {{$context}}, in *{{$in}}` + - `{{perNodeFnType .GenFile .Method ", f"}}) ` + - `*{{$asyncOut}} {` - -var asyncVar = qcVar + ` -{{$asyncOut := outType .Method $customOut}} -` - -var asyncBody = ` cd := {{$callData}}{ - Message: in, - Method: "{{$fullName}}", - } - cd.QuorumFunction = func(req {{$protoMessage}}, replies map[uint32]{{$protoMessage}}) ({{$protoMessage}}, bool) { - r := make(map[uint32]*{{$out}}, len(replies)) - for k, v := range replies { - r[k] = v.(*{{$out}}) - } - return c.qspec.{{$method}}QF(req.(*{{$in}}), r) - } -{{- if hasPerNodeArg .Method}} - cd.PerNodeArgFn = func(req {{$protoMessage}}, nid uint32) {{$protoMessage}} { - return f(req.(*{{$in}}), nid) - } -{{- end}} - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &{{$asyncOut}}{fut} -} -` - -var asyncCall = commonVariables + - asyncVar + - asyncCallComment + - asyncSignature + - asyncBody diff --git a/cmd/protoc-gen-gorums/gengorums/template_client.go b/cmd/protoc-gen-gorums/gengorums/template_client.go deleted file mode 100644 index db1e65829..000000000 --- a/cmd/protoc-gen-gorums/gengorums/template_client.go +++ /dev/null @@ -1,101 +0,0 @@ -package gengorums - -import ( - "google.golang.org/protobuf/compiler/protogen" -) - -// gorums need to be imported in the zorums file -var clientVariables = ` -{{$context := use "context.Context" .GenFile}} -{{$_ := use "gorums.EnforceVersion" .GenFile}} -{{$callOpt := use "gorums.CallOption" .GenFile}} -` - -var clientConfigurationInterface = ` -{{- $genFile := .GenFile}} -{{- range configurationsServices .Services}} - {{- $service := .GoName}} - {{- $interfaceName := printf "%sClient" $service}} - // {{$interfaceName}} is the client interface for the {{$service}} service. - type {{$interfaceName}} interface { - {{- range configurationMethods .Methods}} - {{- $method := .GoName}} - {{- if isOneway .}} - {{$method}}(ctx {{$context}}, in *{{in $genFile .}} {{perNodeFnType $genFile . ", f"}}, opts ...{{$callOpt}}) - {{- else if or (isCorrectable .) (isAsync .)}} - {{- $customOut := outType . (customOut $genFile .)}} - {{$method}}(ctx {{$context}}, in *{{in $genFile .}} {{perNodeFnType $genFile . ", f"}}) *{{$customOut}} - {{- else}} - {{- $customOut := customOut $genFile .}} - {{$method}}(ctx {{$context}}, in *{{in $genFile .}} {{perNodeFnType $genFile . ", f"}}) (resp *{{$customOut}}, err error) - {{- end}} - {{- end}} - } - // enforce interface compliance - var _ {{$interfaceName}} = (*Configuration)(nil) -{{- end}} -` - -var clientNodeInterface = ` -{{- $genFile := .GenFile}} -{{- range nodeServices .Services}} - {{- $service := .GoName}} - {{- $interfaceName := printf "%sNodeClient" $service}} - // {{$interfaceName}} is the single node client interface for the {{$service}} service. - type {{$interfaceName}} interface { - {{- range nodeMethods .Methods}} - {{- $method := .GoName}} - {{- if isOneway .}} - {{$method}}(ctx {{$context}}, in *{{in $genFile .}} {{perNodeFnType $genFile . ", f"}}, opts ...{{$callOpt}}) - {{- else}} - {{- $customOut := customOut $genFile .}} - {{$method}}(ctx {{$context}}, in *{{in $genFile .}} {{perNodeFnType $genFile . ", f"}}) (resp *{{$customOut}}, err error) - {{- end}} - {{- end}} - } - // enforce interface compliance - var _ {{$interfaceName}} = (*Node)(nil) -{{- end}} -` - -var client = clientVariables + clientConfigurationInterface + clientNodeInterface - -// configurationsServices returns all services containing at least one multi node method. -func configurationsServices(services []*protogen.Service) (s []*protogen.Service) { - for _, service := range services { - if len(configurationMethods(service.Methods)) > 0 { - s = append(s, service) - } - } - return s -} - -// configurationMethods returns all multi node methods, such as multicast, quorumcall, correctable, and async methods. -func configurationMethods(methods []*protogen.Method) (s []*protogen.Method) { - for _, method := range methods { - if hasConfigurationCallType(method) { - s = append(s, method) - } - } - return s -} - -// nodeServices returns all services containing at least one single node method. -func nodeServices(services []*protogen.Service) (s []*protogen.Service) { - for _, service := range services { - if len(nodeMethods(service.Methods)) > 0 { - s = append(s, service) - } - } - return s -} - -// nodeMethods returns all single node methods, such as unicast and plain gRPC methods. -func nodeMethods(methods []*protogen.Method) (s []*protogen.Method) { - for _, method := range methods { - if !hasConfigurationCallType(method) { - s = append(s, method) - } - } - return s -} diff --git a/cmd/protoc-gen-gorums/gengorums/template_correctable.go b/cmd/protoc-gen-gorums/gengorums/template_correctable.go deleted file mode 100644 index f61c2505f..000000000 --- a/cmd/protoc-gen-gorums/gengorums/template_correctable.go +++ /dev/null @@ -1,68 +0,0 @@ -package gengorums - -var correctableCallComment = ` -{{$comments := .Method.Comments.Leading}} -{{if ne $comments ""}} -{{$comments -}} -{{else}} -{{if hasPerNodeArg .Method}} -// {{$method}} asynchronously invokes a correctable quorum call on each node -// in configuration c, with the argument returned by the provided function f -// and returns a {{$correctableOut}}, which can be used to inspect -// the reply and error when available. -// The provide per node function f takes the provided {{$in}} argument -// and returns an {{$out}} object to be passed to the given nodeID. -// The per node function f should be thread-safe. -{{else}} -// {{$method}} asynchronously invokes a correctable quorum call on each node -// in configuration c and returns a {{$correctableOut}}, which can be used -// to inspect any replies or errors when available. -{{if correctableStream .Method -}} -// This method supports server-side preliminary replies (correctable stream). -{{end -}} -{{end -}} -{{end -}} -` - -var correctableVar = ` -{{$correctableOut := outType .Method $customOut}} -{{$protoMessage := use "proto.Message" .GenFile}} -{{$callData := use "gorums.CorrectableCallData" .GenFile}} -{{$genFile := .GenFile}} -{{$unexportMethod := unexport .Method.GoName}} -{{$context := use "context.Context" .GenFile}} -` - -var correctableSignature = `func (c *Configuration) {{$method}}(` + - `ctx {{$context}}, in *{{$in}}` + - `{{perNodeFnType .GenFile .Method ", f"}}) ` + - `*{{$correctableOut}} {` - -var correctableBody = ` cd := {{$callData}}{ - Message: in, - Method: "{{$fullName}}", - ServerStream: {{correctableStream .Method}}, - } - cd.QuorumFunction = func(req {{$protoMessage}}, replies map[uint32]{{$protoMessage}}) ({{$protoMessage}}, int, bool) { - r := make(map[uint32]*{{$out}}, len(replies)) - for k, v := range replies { - r[k] = v.(*{{$out}}) - } - return c.qspec.{{$method}}QF(req.(*{{$in}}), r) - } -{{- if hasPerNodeArg .Method}} - cd.PerNodeArgFn = func(req {{$protoMessage}}, nid uint32) {{$protoMessage}} { - return f(req.(*{{$in}}), nid) - } -{{- end}} - - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &{{$correctableOut}}{corr} -} -` - -var correctableCall = commonVariables + - correctableVar + - correctableCallComment + - correctableSignature + - correctableBody diff --git a/cmd/protoc-gen-gorums/gengorums/template_datatypes.go b/cmd/protoc-gen-gorums/gengorums/template_datatypes.go index ccb1badba..23bb380ec 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_datatypes.go +++ b/cmd/protoc-gen-gorums/gengorums/template_datatypes.go @@ -1,66 +1,25 @@ package gengorums -var internalOutDataType = ` -{{range $intOut, $out := mapInternalOutType .GenFile .Services}} -type {{$intOut}} struct { - nid uint32 - reply *{{$out}} - err error -} -{{end}} -` - -// This struct and API functions are generated only once per return type -// for a async call type. That is, if multiple async calls use the same -// return type, this struct and associated methods are only generated once. +// This type alias is generated only once per return type for an async call type. +// That is, if multiple async calls use the same return type, this type alias +// is only generated once. var asyncDataType = ` {{$async := use "gorums.Async" .GenFile}} {{range $asyncOut, $customOut := mapAsyncOutType .GenFile .Services}} -{{$customOutField := field $customOut}} -// {{$asyncOut}} is a async object for processing replies. -type {{$asyncOut}} struct { - *{{$async}} -} - -// Get returns the reply and any error associated with the called method. -// The method blocks until a reply or error is available. -func (f *{{$asyncOut}}) Get() (*{{$customOut}}, error) { - resp, err := f.Async.Get() - if err != nil { - return nil, err - } - return resp.(*{{$customOut}}), err -} +// {{$asyncOut}} is a future for async quorum calls returning *{{$customOut}}. +type {{$asyncOut}} = *{{$async}}[*{{$customOut}}] {{end}} ` -// This struct and API functions are generated only once per return type -// for a correctable call type. That is, if multiple correctable calls use the same -// return type, this struct and associated methods are only generated once. +// This type alias is generated only once per return type for a correctable call type. +// That is, if multiple correctable calls use the same return type, this type alias +// is only generated once. var correctableDataType = ` {{$correctable := use "gorums.Correctable" .GenFile}} {{range $correctableOut, $customOut := mapCorrectableOutType .GenFile .Services}} -{{$customOutField := field $customOut}} -// {{$correctableOut}} is a correctable object for processing replies. -type {{$correctableOut}} struct { - *{{$correctable}} -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *{{$correctableOut}}) Get() (*{{$customOut}}, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*{{$customOut}}), level, err -} -{{- end -}} +// {{$correctableOut}} is a correctable object for quorum calls returning *{{$customOut}}. +type {{$correctableOut}} = *{{$correctable}}[*{{$customOut}}] +{{end}} ` -var dataTypes = internalOutDataType + - asyncDataType + - correctableDataType +var dataTypes = asyncDataType + correctableDataType diff --git a/cmd/protoc-gen-gorums/gengorums/template_multicast.go b/cmd/protoc-gen-gorums/gengorums/template_multicast.go index f433ea6a0..ccd5d957d 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_multicast.go +++ b/cmd/protoc-gen-gorums/gengorums/template_multicast.go @@ -1,44 +1,33 @@ package gengorums -var multicastRefImports = ` -{{if contains $out "."}} -// Reference imports to suppress errors if they are not otherwise used. -var _ {{$out}} -{{end}} -` - var mcVar = ` -{{$callData := use "gorums.QuorumCallData" .GenFile}} {{$genFile := .GenFile}} -{{$unexportMethod := unexport .Method.GoName}} -{{$context := use "context.Context" .GenFile}} +{{$configContext := use "gorums.ConfigContext" .GenFile}} +{{$multicast := use "gorums.Multicast" .GenFile}} {{$callOpt := use "gorums.CallOption" .GenFile}} ` -var multicastSignature = `func (c *Configuration) {{$method}}(` + - `ctx {{$context}}, in *{{$in}}` + - `{{perNodeFnType .GenFile .Method ", f"}},` + - `opts ...{{$callOpt}}) { +var multicastComment = ` +{{$comments := .Method.Comments.Leading}} +{{if ne $comments ""}} +{{$comments -}} +{{else}} +// {{$method}} is a multicast call invoked on all nodes in the configuration in ctx. +// Use gorums.MapRequest to send different messages to each node. No replies are collected. +{{end -}} ` -var multicastBody = ` cd := {{$callData}}{ - Message: in, - Method: "{{$fullName}}", - } -{{- if hasPerNodeArg .Method}} -{{$protoMessage := use "proto.Message" .GenFile}} - cd.PerNodeArgFn = func(req {{$protoMessage}}, nid uint32) {{$protoMessage}} { - return f(req.(*{{$in}}), nid) - } -{{- end}} +var multicastSignature = `func {{$method}}(` + + `ctx *{{$configContext}}, in *{{$in}}, ` + + `opts ...{{$callOpt}}) error { +` - c.RawConfiguration.Multicast(ctx, cd, opts...) +var multicastBody = ` return {{$multicast}}(ctx, in, "{{$fullName}}", opts...) } ` var multicastCall = commonVariables + mcVar + - multicastRefImports + - quorumCallComment + + multicastComment + multicastSignature + multicastBody diff --git a/cmd/protoc-gen-gorums/gengorums/template_qspec.go b/cmd/protoc-gen-gorums/gengorums/template_qspec.go deleted file mode 100644 index 3afa740cf..000000000 --- a/cmd/protoc-gen-gorums/gengorums/template_qspec.go +++ /dev/null @@ -1,59 +0,0 @@ -package gengorums - -import ( - "github.com/relab/gorums" - "google.golang.org/protobuf/compiler/protogen" -) - -var qspecInterface = ` -{{$genFile := .GenFile}} -{{$configOpt := use "gorums.ConfigOption" .GenFile}} -{{- $services := qspecServices .Services}} -{{- if eq (len $services) 0}} - // There are no quorum calls. - type QuorumSpec interface{} -{{- end}} -{{- range $services}} - // QuorumSpec is the interface of quorum functions for {{.GoName}}. - type QuorumSpec interface { - {{$configOpt}} - - {{range qspecMethods .Methods -}} - {{/* Below . is the method object */}} - {{$method := .GoName}} - {{$in := in $genFile .}} - {{$out := out $genFile .}} - {{$customOut := customOut $genFile .}} - // {{$method}}QF is the quorum function for the {{$method}} - // {{docName .}} call method. The in parameter is the request object - // supplied to the {{$method}} method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *{{$in}}'. - {{$method}}QF(in *{{$in}}, replies map[uint32]*{{$out}}) (*{{$customOut}}{{withCorrectable . ", int"}}, bool) - {{end}} - } -{{end}} -` - -// qspecMethods returns all Gorums methods that require -// a quorum function; that is, all except multicast, unicast and plain gRPC methods. -func qspecMethods(methods []*protogen.Method) (s []*protogen.Method) { - for _, method := range methods { - if hasMethodOption(method, gorums.E_Multicast, gorums.E_Unicast) || !hasGorumsCallType(method) { - // ignore multicast, unicast and non-Gorums methods - continue - } - s = append(s, method) - } - return s -} - -// qspecServices returns all services that have qspec methods. -func qspecServices(services []*protogen.Service) (s []*protogen.Service) { - for _, service := range services { - if len(qspecMethods(service.Methods)) > 0 { - s = append(s, service) - } - } - return s -} diff --git a/cmd/protoc-gen-gorums/gengorums/template_quorumcall.go b/cmd/protoc-gen-gorums/gengorums/template_quorumcall.go index 64eeede8b..8a1b24324 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_quorumcall.go +++ b/cmd/protoc-gen-gorums/gengorums/template_quorumcall.go @@ -6,9 +6,6 @@ var commonVariables = ` {{$method := .Method.GoName}} {{$in := in .GenFile .Method}} {{$out := out .GenFile .Method}} -{{$intOut := internalOut $out}} -{{$customOut := customOut .GenFile .Method}} -{{$customOutField := field $customOut}} {{$unexportOutput := unexport .Method.Output.GoIdent.GoName}} ` @@ -17,60 +14,68 @@ var quorumCallComment = ` {{if ne $comments ""}} {{$comments -}} {{else}} -{{if hasPerNodeArg .Method}} -// {{$method}} is a quorum call invoked on each node in configuration c, -// with the argument returned by the provided function f, and returns the combined result. -// The per node function f receives a copy of the {{$in}} request argument and -// returns a {{$in}} manipulated to be passed to the given nodeID. -// The function f must be thread-safe. -{{else}} -// {{$method}} is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -{{end -}} +// {{$method}} is a quorum call invoked on all nodes in the configuration, +// with the same argument in. Use terminal methods like Majority(), First(), +// or Threshold(n) to retrieve the aggregated result. +// +// Example: +// resp, err := {{$method}}(ctx, in).Majority() {{end -}} ` -var quorumCallSignature = `func (c *Configuration) {{$method}}(` + - `ctx {{$context}}, in *{{$in}}` + - `{{perNodeFnType .GenFile .Method ", f"}})` + - `(resp *{{$customOut}}, err error) { -` - -var qcVar = ` -{{$protoMessage := use "proto.Message" .GenFile}} -{{$callData := use "gorums.QuorumCallData" .GenFile}} +var quorumCallVariables = ` {{$genFile := .GenFile}} -{{$unexportMethod := unexport .Method.GoName}} -{{$context := use "context.Context" .GenFile}} +{{$configContext := use "gorums.ConfigContext" .GenFile}} +{{$quorumCall := use "gorums.QuorumCall" .GenFile}} +{{$quorumCallStream := use "gorums.QuorumCallStream" .GenFile}} +{{$responses := use "gorums.Responses" .GenFile}} +{{$callOption := use "gorums.CallOption" .GenFile}} ` -var quorumCallBody = ` cd := {{$callData}}{ - Message: in, - Method: "{{$fullName}}", - } - cd.QuorumFunction = func(req {{$protoMessage}}, replies map[uint32]{{$protoMessage}}) ({{$protoMessage}}, bool) { - r := make(map[uint32]*{{$out}}, len(replies)) - for k, v := range replies { - r[k] = v.(*{{$out}}) - } - return c.qspec.{{$method}}QF(req.(*{{$in}}), r) - } -{{- if hasPerNodeArg .Method}} - cd.PerNodeArgFn = func(req {{$protoMessage}}, nid uint32) {{$protoMessage}} { - return f(req.(*{{$in}}), nid) - } -{{- end}} +var quorumCallSignature = `func {{$method}}(` + + `ctx *{{$configContext}}, in *{{$in}}, ` + + `opts ...{{$callOption}})` + + ` *{{$responses}}[*{{$out}}] { +` - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*{{$customOut}}), err +var quorumCallBody = ` return {{$quorumCall}}[*{{$in}}, *{{$out}}]( + ctx, in, "{{$fullName}}", + opts..., + ) } ` var quorumCall = commonVariables + - qcVar + + quorumCallVariables + quorumCallComment + quorumCallSignature + quorumCallBody + +// Streaming quorum call template +var quorumCallStreamComment = ` +{{$comments := .Method.Comments.Leading}} +{{if ne $comments ""}} +{{$comments -}} +{{else}} +// {{$method}} is a streaming quorum call where the server can send multiple responses. +// The response iterator continues until the context is canceled. +// +// Example: +// corr := {{$method}}(ctx, in).Correctable(2) +// <-corr.Watch(2) +// resp, level, err := corr.Get() +{{end -}} +` + +var quorumCallStreamBody = ` return {{$quorumCallStream}}[*{{$in}}, *{{$out}}]( + ctx, in, "{{$fullName}}", + opts..., + ) +} +` + +var quorumCallStream = commonVariables + + quorumCallVariables + + quorumCallStreamComment + + quorumCallSignature + + quorumCallStreamBody diff --git a/cmd/protoc-gen-gorums/gengorums/template_rpc.go b/cmd/protoc-gen-gorums/gengorums/template_rpc.go index d5f8080dc..228eaad2f 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_rpc.go +++ b/cmd/protoc-gen-gorums/gengorums/template_rpc.go @@ -1,38 +1,35 @@ package gengorums -var rpcSignature = `func (n *Node) {{$method}}(` + - `ctx {{$context}}, in *{{$in}}` + - `{{perNodeFnType .GenFile .Method ", f"}}) (resp *{{$customOut}}, err error) { +var rpcComment = ` +{{$comments := .Method.Comments.Leading}} +{{if ne $comments ""}} +{{$comments -}} +{{else}} +// {{$method}} is an RPC call invoked on the node in ctx. +{{end -}} +` + +var rpcSignature = `func {{$method}}(` + + `ctx *{{$nodeContext}}, in *{{$in}}) (resp *{{$out}}, err error) { ` var rpcVar = ` -{{$callData := use "gorums.CallData" .GenFile}} {{$genFile := .GenFile}} -{{$unexportMethod := unexport .Method.GoName}} -{{$context := use "context.Context" .GenFile}} +{{$nodeContext := use "gorums.NodeContext" .GenFile}} +{{$rpcCall := use "gorums.RPCCall" .GenFile}} +{{$_ := use "gorums.EnforceVersion" .GenFile}} ` -var rpcBody = ` cd := {{$callData}}{ - Message: in, - Method: "{{$fullName}}", - } -{{- if hasPerNodeArg .Method}} - {{$protoMessage := use "proto.Message" $genFile}} - cd.PerNodeArgFn = func(req {{$protoMessage}}, nid uint32) {{$protoMessage}} { - return f(req.(*{{$in}}), nid) - } -{{- end}} - - res, err := n.RawNode.RPCCall(ctx, cd) +var rpcBody = ` res, err := {{$rpcCall}}(ctx, in, "{{$fullName}}") if err != nil { return nil, err } - return res.(*{{$customOut}}), err + return res.(*{{$out}}), err } ` var rpcCall = commonVariables + rpcVar + - quorumCallComment + + rpcComment + rpcSignature + rpcBody diff --git a/cmd/protoc-gen-gorums/gengorums/template_server.go b/cmd/protoc-gen-gorums/gengorums/template_server.go index 53a539b7b..17aee7596 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_server.go +++ b/cmd/protoc-gen-gorums/gengorums/template_server.go @@ -13,7 +13,7 @@ type {{$service}}Server interface { {{- range .Methods}} {{- if isOneway .}} {{.GoName}}(ctx {{$context}}, request *{{in $genFile .}}) - {{- else if correctableStream .}} + {{- else if isStreamingServer .}} {{.GoName}}(ctx {{$context}}, request *{{in $genFile .}}, send func(response *{{out $genFile .}}) error) error {{- else}} {{.GoName}}(ctx {{$context}}, request *{{in $genFile .}}) (response *{{out $genFile .}}, err error) @@ -37,7 +37,7 @@ func Register{{$service}}Server(srv *{{use "gorums.Server" $genFile}}, impl {{$s {{- if isOneway .}} impl.{{.GoName}}(ctx, req) return nil, nil - {{- else if correctableStream .}} + {{- else if isStreamingServer .}} err := impl.{{.GoName}}(ctx, req, func(resp *{{out $genFile .}}) error { // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg md := {{use "proto.CloneOf" $genFile}}(in.GetMetadata()) diff --git a/cmd/protoc-gen-gorums/gengorums/template_static.go b/cmd/protoc-gen-gorums/gengorums/template_static.go index 970445260..706028c39 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_static.go +++ b/cmd/protoc-gen-gorums/gengorums/template_static.go @@ -5,138 +5,61 @@ package gengorums // pkgIdentMap maps from package name to one of the package's identifiers. // These identifiers are used by the Gorums protoc plugin to generate import statements. -var pkgIdentMap = map[string]string{"fmt": "Errorf", "github.com/relab/gorums": "ConfigOption", "google.golang.org/grpc/encoding": "GetCodec"} +var pkgIdentMap = map[string]string{"github.com/relab/gorums": "Configuration"} // reservedIdents holds the set of Gorums reserved identifiers. // These identifiers cannot be used to define message types in a proto file. -var reservedIdents = []string{"Configuration", "Manager", "Node", "QuorumSpec"} +var reservedIdents = []string{"Configuration", "Manager", "Node"} -var staticCode = `// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} +var staticCode = `// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} - -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } ` diff --git a/cmd/protoc-gen-gorums/gengorums/template_unicast.go b/cmd/protoc-gen-gorums/gengorums/template_unicast.go index 89b2bb890..ff8a18aed 100644 --- a/cmd/protoc-gen-gorums/gengorums/template_unicast.go +++ b/cmd/protoc-gen-gorums/gengorums/template_unicast.go @@ -1,23 +1,32 @@ package gengorums -var unicastVar = rpcVar + `{{$callOpt := use "gorums.CallOption" .GenFile}}` +var unicastVar = ` +{{$genFile := .GenFile}} +{{$nodeContext := use "gorums.NodeContext" .GenFile}} +{{$unicast := use "gorums.Unicast" .GenFile}} +{{$callOpt := use "gorums.CallOption" .GenFile}} +` -var unicastSignature = `func (n *Node) {{$method}}(` + - `ctx {{$context}}, in *{{$in}}, opts ...{{$callOpt}}) { +var unicastComment = ` +{{$comments := .Method.Comments.Leading}} +{{if ne $comments ""}} +{{$comments -}} +{{else}} +// {{$method}} is a unicast call invoked on the node in ctx. +// No reply is returned to the client. +{{end -}} ` -var unicastBody = ` cd := {{$callData}}{ - Message: in, - Method: "{{$fullName}}", - } +var unicastSignature = `func {{$method}}(` + + `ctx *{{$nodeContext}}, in *{{$in}}, opts ...{{$callOpt}}) error { +` - n.RawNode.Unicast(ctx, cd, opts...) +var unicastBody = ` return {{$unicast}}(ctx, in, "{{$fullName}}", opts...) } ` var unicastCall = commonVariables + unicastVar + - multicastRefImports + - quorumCallComment + + unicastComment + unicastSignature + unicastBody diff --git a/cmd/protoc-gen-gorums/main.go b/cmd/protoc-gen-gorums/main.go index d7e447f48..c5a4a3e89 100644 --- a/cmd/protoc-gen-gorums/main.go +++ b/cmd/protoc-gen-gorums/main.go @@ -21,7 +21,7 @@ const ( genGoDocURL = "https://developers.google.com/protocol-buffers/docs/reference/go-generated" supportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL) | uint64(pluginpb.CodeGeneratorResponse_FEATURE_SUPPORTS_EDITIONS) supportedEditionsMinimum = descriptorpb.Edition_EDITION_PROTO2 - supportedEditionsMaximum = descriptorpb.Edition_EDITION_2023 + supportedEditionsMaximum = descriptorpb.Edition_EDITION_2024 ) func main() { diff --git a/config.go b/config.go index 274836bee..e495c3f85 100644 --- a/config.go +++ b/config.go @@ -1,28 +1,99 @@ package gorums -import "fmt" +import ( + "context" + "fmt" +) -// RawConfiguration represents a static set of nodes on which quorum calls may be invoked. +// ConfigContext is a context that carries a configuration for quorum calls. +// It embeds context.Context and provides access to the Configuration. // -// NOTE: mutating the configuration is not supported. +// Use [Configuration.Context] to create a ConfigContext from an existing context. +type ConfigContext struct { + context.Context + cfg Configuration +} + +// Configuration returns the Configuration associated with this context. +func (c ConfigContext) Configuration() Configuration { + return c.cfg +} + +// Configuration represents a static set of nodes on which quorum calls may be invoked. // -// This type is intended to be used by generated code. -// You should use the generated `Configuration` type instead. -type RawConfiguration []*RawNode +// Mutating the configuration is not supported; instead, use NewConfiguration to create +// a new configuration. +type Configuration []*Node -// NewRawConfiguration returns a configuration based on the provided list of nodes. +// Context creates a new ConfigContext from the given parent context +// and this configuration. +// +// Example: +// +// config, _ := gorums.NewConfiguration(mgr, gorums.WithNodeList(addrs)) +// cfgCtx := config.Context(context.Background()) +// resp, err := paxos.Prepare(cfgCtx, req) +func (cfg Configuration) Context(parent context.Context) *ConfigContext { + if len(cfg) == 0 { + panic("gorums: Context called with empty configuration") + } + return &ConfigContext{Context: parent, cfg: cfg} +} + +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func NewRawConfiguration(mgr *RawManager, opt NodeListOption) (nodes RawConfiguration, err error) { +func NewConfiguration(mgr *Manager, opt NodeListOption) (nodes Configuration, err error) { if opt == nil { return nil, fmt.Errorf("config: missing required node list") } return opt.newConfig(mgr) } +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...Option) (Configuration, error) { + var ( + managerOptions []ManagerOption + nodeListOption NodeListOption + ) + for _, opt := range opts { + switch o := opt.(type) { + case ManagerOption: + managerOptions = append(managerOptions, o) + case NodeListOption: + if nodeListOption != nil { + return nil, fmt.Errorf("gorums: multiple NodeListOptions provided") + } + nodeListOption = o + default: + return nil, fmt.Errorf("gorums: unknown option type: %T", opt) + } + } + if nodeListOption == nil { + return nil, fmt.Errorf("gorums: missing required NodeListOption") + } + mgr := NewManager(managerOptions...) + return NewConfiguration(mgr, nodeListOption) +} + // NodeIDs returns a slice of this configuration's Node IDs. -func (c RawConfiguration) NodeIDs() []uint32 { +func (c Configuration) NodeIDs() []uint32 { ids := make([]uint32, len(c)) for i, node := range c { ids[i] = node.ID() @@ -31,19 +102,17 @@ func (c RawConfiguration) NodeIDs() []uint32 { } // Nodes returns the nodes in this configuration. -// -// NOTE: mutating the returned slice is not supported. -func (c RawConfiguration) Nodes() []*RawNode { +func (c Configuration) Nodes() []*Node { return c } // Size returns the number of nodes in this configuration. -func (c RawConfiguration) Size() int { +func (c Configuration) Size() int { return len(c) } // Equal returns true if configurations b and c have the same set of nodes. -func (c RawConfiguration) Equal(b RawConfiguration) bool { +func (c Configuration) Equal(b Configuration) bool { if len(c) != len(b) { return false } @@ -55,6 +124,16 @@ func (c RawConfiguration) Equal(b RawConfiguration) bool { return true } -func (c RawConfiguration) getMsgID() uint64 { - return c[0].mgr.getMsgID() +// Manager returns the Manager that manages this configuration's nodes. +// Returns nil if the configuration is empty. +func (c Configuration) Manager() *Manager { + if len(c) == 0 { + return nil + } + return c[0].mgr +} + +// nextMsgID returns the next message ID from this client's manager. +func (c Configuration) nextMsgID() uint64 { + return c[0].msgIDGen() } diff --git a/config_opts.go b/config_opts.go index 66c3bb50e..41327c9dd 100644 --- a/config_opts.go +++ b/config_opts.go @@ -1,37 +1,34 @@ package gorums import ( + "errors" "fmt" ) -// ConfigOption is a marker interface for options to NewConfiguration. -type ConfigOption any - // NodeListOption must be implemented by node providers. type NodeListOption interface { - ConfigOption - newConfig(*RawManager) (RawConfiguration, error) + Option + newConfig(*Manager) (Configuration, error) } type nodeIDMap struct { idMap map[string]uint32 } -func (o nodeIDMap) newConfig(mgr *RawManager) (nodes RawConfiguration, err error) { +func (nodeIDMap) isOption() {} + +func (o nodeIDMap) newConfig(mgr *Manager) (nodes Configuration, err error) { if len(o.idMap) == 0 { return nil, fmt.Errorf("config: missing required node map") } - nodes = make(RawConfiguration, 0, len(o.idMap)) + nodes = make(Configuration, 0, len(o.idMap)) for naddr, id := range o.idMap { node, found := mgr.Node(id) if !found { - node, err = NewRawNodeWithID(naddr, id) + node, err = mgr.newNode(naddr, id) if err != nil { return nil, err } - if err = mgr.AddNode(node); err != nil { - return nil, err - } } nodes = append(nodes, node) } @@ -51,22 +48,24 @@ type nodeList struct { addrsList []string } -func (o nodeList) newConfig(mgr *RawManager) (nodes RawConfiguration, err error) { +func (nodeList) isOption() {} + +func (o nodeList) newConfig(mgr *Manager) (nodes Configuration, err error) { if len(o.addrsList) == 0 { return nil, fmt.Errorf("config: missing required node addresses") } - nodes = make(RawConfiguration, 0, len(o.addrsList)) + nodes = make(Configuration, 0, len(o.addrsList)) for _, naddr := range o.addrsList { - node, err := NewRawNode(naddr) + id, err := nodeID(naddr) if err != nil { return nil, err } - if n, found := mgr.Node(node.ID()); !found { - if err = mgr.AddNode(node); err != nil { + node, found := mgr.Node(id) + if !found { + node, err = mgr.newNode(naddr, id) + if err != nil { return nil, err } - } else { - node = n } nodes = append(nodes, node) } @@ -86,11 +85,13 @@ type nodeIDs struct { nodeIDs []uint32 } -func (o nodeIDs) newConfig(mgr *RawManager) (nodes RawConfiguration, err error) { +func (nodeIDs) isOption() {} + +func (o nodeIDs) newConfig(mgr *Manager) (nodes Configuration, err error) { if len(o.nodeIDs) == 0 { return nil, fmt.Errorf("config: missing required node IDs") } - nodes = make(RawConfiguration, 0, len(o.nodeIDs)) + nodes = make(Configuration, 0, len(o.nodeIDs)) for _, id := range o.nodeIDs { node, found := mgr.Node(id) if !found { @@ -112,11 +113,13 @@ func WithNodeIDs(ids []uint32) NodeListOption { } type addNodes struct { - old RawConfiguration + old Configuration new NodeListOption } -func (o addNodes) newConfig(mgr *RawManager) (nodes RawConfiguration, err error) { +func (addNodes) isOption() {} + +func (o addNodes) newConfig(mgr *Manager) (nodes Configuration, err error) { newNodes, err := o.new.newConfig(mgr) if err != nil { return nil, err @@ -127,17 +130,19 @@ func (o addNodes) newConfig(mgr *RawManager) (nodes RawConfiguration, err error) // WithNewNodes returns a NodeListOption that can be used to create a new configuration // combining c and the new nodes. -func (c RawConfiguration) WithNewNodes(new NodeListOption) NodeListOption { - return &addNodes{old: c, new: new} +func (c Configuration) WithNewNodes(newNodes NodeListOption) NodeListOption { + return &addNodes{old: c, new: newNodes} } type addConfig struct { - old RawConfiguration - add RawConfiguration + old Configuration + add Configuration } -func (o addConfig) newConfig(mgr *RawManager) (nodes RawConfiguration, err error) { - nodes = make(RawConfiguration, 0, len(o.old)+len(o.add)) +func (addConfig) isOption() {} + +func (o addConfig) newConfig(mgr *Manager) (nodes Configuration, err error) { + nodes = make(Configuration, 0, len(o.old)+len(o.add)) m := make(map[uint32]bool) for _, n := range append(o.old, o.add...) { if !m[n.id] { @@ -152,13 +157,13 @@ func (o addConfig) newConfig(mgr *RawManager) (nodes RawConfiguration, err error } // And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c RawConfiguration) And(d RawConfiguration) NodeListOption { +func (c Configuration) And(d Configuration) NodeListOption { return &addConfig{old: c, add: d} } // WithoutNodes returns a NodeListOption that can be used to create a new configuration // from c without the given node IDs. -func (c RawConfiguration) WithoutNodes(ids ...uint32) NodeListOption { +func (c Configuration) WithoutNodes(ids ...uint32) NodeListOption { rmIDs := make(map[uint32]bool) for _, id := range ids { rmIDs[id] = true @@ -174,7 +179,7 @@ func (c RawConfiguration) WithoutNodes(ids ...uint32) NodeListOption { // Except returns a NodeListOption that can be used to create a new configuration // from c without the nodes in rm. -func (c RawConfiguration) Except(rm RawConfiguration) NodeListOption { +func (c Configuration) Except(rm Configuration) NodeListOption { rmIDs := make(map[uint32]bool) for _, rmNode := range rm { rmIDs[rmNode.id] = true @@ -187,3 +192,38 @@ func (c RawConfiguration) Except(rm RawConfiguration) NodeListOption { } return &nodeIDs{nodeIDs: keepIDs} } + +// WithoutErrors returns a NodeListOption that creates a new configuration +// excluding nodes that failed in the given QuorumCallError. +// If specific error types are provided, only nodes whose errors match +// one of those types (using errors.Is) will be excluded. +// If no error types are provided, all failed nodes are excluded. +func (c Configuration) WithoutErrors(err QuorumCallError, errorTypes ...error) NodeListOption { + // Decide whether an error should exclude a node. + exclude := func(cause error) bool { + if len(errorTypes) == 0 { + return true // no filter => exclude all failed nodes + } + for _, t := range errorTypes { + if errors.Is(cause, t) { + return true // match found + } + } + return false + } + + // Build a map of node IDs to exclude + rm := make(map[uint32]bool, len(err.errors)) + for _, ne := range err.errors { + rm[ne.nodeID] = exclude(ne.cause) + } + + // Build the list of node IDs to keep + keepIDs := make([]uint32, 0, len(c)) + for _, node := range c { + if !rm[node.id] { + keepIDs = append(keepIDs, node.id) + } + } + return &nodeIDs{nodeIDs: keepIDs} +} diff --git a/config_test.go b/config_test.go index a0d4dfc58..86d31ed50 100644 --- a/config_test.go +++ b/config_test.go @@ -1,7 +1,6 @@ package gorums_test import ( - "context" "errors" "sync" "testing" @@ -19,10 +18,17 @@ func init() { } } +var ( + nodes = []string{"127.0.0.1:9080", "127.0.0.1:9081", "127.0.0.1:9082"} + nodeMap = map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4} +) + func TestNewConfigurationEmptyNodeList(t *testing.T) { wantErr := errors.New("config: missing required node addresses") - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - _, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList([]string{})) + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + _, err := gorums.NewConfiguration(mgr, gorums.WithNodeList([]string{})) if err == nil { t.Fatalf("Expected error, got: %v, want: %v", err, wantErr) } @@ -32,8 +38,10 @@ func TestNewConfigurationEmptyNodeList(t *testing.T) { } func TestNewConfigurationNodeList(t *testing.T) { - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - cfg, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList(nodes)) + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(nodes)) if err != nil { t.Fatal(err) } @@ -41,7 +49,7 @@ func TestNewConfigurationNodeList(t *testing.T) { t.Errorf("cfg.Size() = %d, expected %d", cfg.Size(), len(nodes)) } - contains := func(nodes []*gorums.RawNode, addr string) bool { + contains := func(nodes []*gorums.Node, addr string) bool { for _, node := range nodes { if addr == node.Address() { return true @@ -68,8 +76,10 @@ func TestNewConfigurationNodeList(t *testing.T) { } func TestNewConfigurationNodeMap(t *testing.T) { - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - cfg, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeMap(nodeMap)) + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeMap(nodeMap)) if err != nil { t.Fatal(err) } @@ -92,8 +102,10 @@ func TestNewConfigurationNodeMap(t *testing.T) { } func TestNewConfigurationNodeIDs(t *testing.T) { - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - c1, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList(nodes)) + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + c1, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(nodes)) if err != nil { t.Fatal(err) } @@ -103,7 +115,7 @@ func TestNewConfigurationNodeIDs(t *testing.T) { // Identical configurations c1 == c2 nodeIDs := c1.NodeIDs() - c2, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeIDs(nodeIDs)) + c2, err := gorums.NewConfiguration(mgr, gorums.WithNodeIDs(nodeIDs)) if err != nil { t.Fatal(err) } @@ -115,7 +127,7 @@ func TestNewConfigurationNodeIDs(t *testing.T) { } // Configuration with one less node |c3| == |c1| - 1 - c3, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeIDs(nodeIDs[:len(nodeIDs)-1])) + c3, err := gorums.NewConfiguration(mgr, gorums.WithNodeIDs(nodeIDs[:len(nodeIDs)-1])) if err != nil { t.Fatal(err) } @@ -128,20 +140,22 @@ func TestNewConfigurationNodeIDs(t *testing.T) { } func TestNewConfigurationAnd(t *testing.T) { - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - c1, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList(nodes)) + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + c1, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(nodes)) if err != nil { t.Fatal(err) } c2Nodes := []string{"127.0.0.1:8080"} - c2, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList(c2Nodes)) + c2, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(c2Nodes)) if err != nil { t.Fatal(err) } // Add newNodes to c1, giving a new c3 with a total of 3+2 nodes newNodes := []string{"127.0.0.1:9083", "127.0.0.1:9084"} - c3, err := gorums.NewRawConfiguration( + c3, err := gorums.NewConfiguration( mgr, c1.WithNewNodes(gorums.WithNodeList(newNodes)), ) @@ -153,7 +167,7 @@ func TestNewConfigurationAnd(t *testing.T) { } // Combine c2 to c1, giving a new c4 with a total of 3+1 nodes - c4, err := gorums.NewRawConfiguration( + c4, err := gorums.NewConfiguration( mgr, c1.And(c2), ) @@ -167,7 +181,7 @@ func TestNewConfigurationAnd(t *testing.T) { // Combine c2 to c4, giving a new c5 with a total of 4 nodes // c4 already contains all nodes from c2 (see above): c4 = c1+c2 // c5 should essentially just be a copy of c4 (ignoring duplicates from c2) - c5, err := gorums.NewRawConfiguration( + c5, err := gorums.NewConfiguration( mgr, c4.And(c2), ) @@ -180,12 +194,14 @@ func TestNewConfigurationAnd(t *testing.T) { } func TestNewConfigurationExcept(t *testing.T) { - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - c1, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList(nodes)) + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + c1, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(nodes)) if err != nil { t.Fatal(err) } - c2, err := gorums.NewRawConfiguration( + c2, err := gorums.NewConfiguration( mgr, c1.WithoutNodes(c1[0].ID()), ) @@ -197,14 +213,14 @@ func TestNewConfigurationExcept(t *testing.T) { } newNodes := []string{"127.0.0.1:9083", "127.0.0.1:9084"} - c3, err := gorums.NewRawConfiguration( + c3, err := gorums.NewConfiguration( mgr, c1.WithNewNodes(gorums.WithNodeList(newNodes)), ) if err != nil { t.Fatal(err) } - c4, err := gorums.NewRawConfiguration( + c4, err := gorums.NewConfiguration( mgr, c3.Except(c1), ) @@ -217,19 +233,13 @@ func TestNewConfigurationExcept(t *testing.T) { } func TestConfigConcurrentAccess(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 1, nil) - defer teardown() - - node := gorums.NewNode(t, addrs[0]) + node := gorums.TestNode(t, nil) errCh := make(chan error, 2) var wg sync.WaitGroup for range 2 { wg.Go(func() { - _, err := node.RPCCall(context.Background(), gorums.CallData{ - Message: pb.String(""), - Method: mock.TestMethod, - }) + _, err := gorums.RPCCall(node.Context(t.Context()), pb.String(""), mock.TestMethod) if err != nil { errCh <- err } @@ -241,3 +251,80 @@ func TestConfigConcurrentAccess(t *testing.T) { t.Error(err) } } + +func TestConfigurationWithoutErrors(t *testing.T) { + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeMap(nodeMap)) + if err != nil { + t.Fatal(err) + } + + timeoutErr := errors.New("timeout") + connRefusedErr := errors.New("connection refused") + otherErr := errors.New("other error") + differentErr := errors.New("different error") + + tests := []struct { + name string + qcErr gorums.QuorumCallError + errorTypes []error + wantExcluded []uint32 + }{ + { + name: "ExcludeAllFailedNodes", + qcErr: gorums.TestQuorumCallError(t, map[uint32]error{1: timeoutErr, 2: connRefusedErr}), + errorTypes: nil, + wantExcluded: []uint32{1, 2}, + }, + { + name: "ExcludeNodesWithSpecificError", + qcErr: gorums.TestQuorumCallError(t, map[uint32]error{1: timeoutErr, 2: connRefusedErr, 3: otherErr}), + errorTypes: []error{timeoutErr}, + wantExcluded: []uint32{1}, + }, + { + name: "ExcludeNodesWithMultipleErrorTypes", + qcErr: gorums.TestQuorumCallError(t, map[uint32]error{1: timeoutErr, 2: connRefusedErr, 3: otherErr}), + errorTypes: []error{timeoutErr, connRefusedErr}, + wantExcluded: []uint32{1, 2}, + }, + { + name: "NoMatchingErrors", + qcErr: gorums.TestQuorumCallError(t, map[uint32]error{1: timeoutErr, 2: connRefusedErr}), + errorTypes: []error{differentErr}, + wantExcluded: []uint32{}, + }, + { + name: "EmptyErrors", + qcErr: gorums.TestQuorumCallError(t, map[uint32]error{}), + errorTypes: nil, + wantExcluded: []uint32{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newCfg, err := gorums.NewConfiguration(mgr, cfg.WithoutErrors(tt.qcErr, tt.errorTypes...)) + if err != nil { + t.Fatal(err) + } + + // Check that excluded nodes are not in the new configuration + newNodeIDs := newCfg.NodeIDs() + for _, excludedID := range tt.wantExcluded { + for _, nodeID := range newNodeIDs { + if nodeID == excludedID { + t.Errorf("Expected node %d to be excluded, but found it in configuration", excludedID) + } + } + } + + // Check that all other nodes are still in the configuration + expectedSize := cfg.Size() - len(tt.wantExcluded) + if newCfg.Size() != expectedSize { + t.Errorf("newCfg.Size() = %d, expected %d", newCfg.Size(), expectedSize) + } + }) + } +} diff --git a/correctable.go b/correctable.go index edbecc113..2724ccb19 100644 --- a/correctable.go +++ b/correctable.go @@ -1,11 +1,7 @@ package gorums import ( - "context" "sync" - - "github.com/relab/gorums/ordering" - "google.golang.org/protobuf/proto" ) // LevelNotSet is the zero value level used to indicate that no level (and @@ -17,12 +13,14 @@ type watcher struct { ch chan struct{} } -// Correctable encapsulates the state of a correctable quorum call. +// Correctable is a generic type for correctable quorum calls. +// It encapsulates the state of a correctable call and provides methods +// for checking the status or waiting for completion at specific levels. // -// This struct should be used by generated code only. -type Correctable struct { +// Type parameter Resp is the response type from nodes. +type Correctable[Resp any] struct { mu sync.Mutex - reply proto.Message + reply Resp level int err error done bool @@ -30,20 +28,28 @@ type Correctable struct { donech chan struct{} } +// NewCorrectable creates a new Correctable object. +func NewCorrectable[Resp any]() *Correctable[Resp] { + return &Correctable[Resp]{ + level: LevelNotSet, + donech: make(chan struct{}, 1), + } +} + // Get returns the latest response, the current level, and the last error. -func (c *Correctable) Get() (proto.Message, int, error) { +func (c *Correctable[Resp]) Get() (Resp, int, error) { c.mu.Lock() defer c.mu.Unlock() return c.reply, c.level, c.err } // Done returns a channel that will close when the correctable call is completed. -func (c *Correctable) Done() <-chan struct{} { +func (c *Correctable[Resp]) Done() <-chan struct{} { return c.donech } // Watch returns a channel that will close when the correctable call has reached a specified level. -func (c *Correctable) Watch(level int) <-chan struct{} { +func (c *Correctable[Resp]) Watch(level int) <-chan struct{} { ch := make(chan struct{}) c.mu.Lock() defer c.mu.Unlock() @@ -55,11 +61,14 @@ func (c *Correctable) Watch(level int) <-chan struct{} { return ch } -func (c *Correctable) set(reply proto.Message, level int, err error, done bool) { +// update updates the current state of the correctable call. +// It updates the response, level, and error, and notifies any watchers. +// If done is true, the call is considered complete and the Done channel is closed. +func (c *Correctable[Resp]) update(reply Resp, level int, done bool, err error) { c.mu.Lock() defer c.mu.Unlock() if c.done { - panic("set(...) called on a done correctable") + panic("update(...) called on a done correctable") } c.reply, c.level, c.err, c.done = reply, level, err, done if done { @@ -79,98 +88,50 @@ func (c *Correctable) set(reply proto.Message, level int, err error, done bool) } } -// CorrectableCallData contains data for making a correctable quorum call. +// Correctable returns a Correctable that provides progressive updates +// as responses arrive. The level increases with each successful response. +// Use this for correctable quorum patterns where you want to observe +// intermediate states. // -// This struct should only be used by generated code. -type CorrectableCallData struct { - Message proto.Message - Method string - PerNodeArgFn func(proto.Message, uint32) proto.Message - QuorumFunction func(proto.Message, map[uint32]proto.Message) (proto.Message, int, bool) - ServerStream bool -} - -type correctableCallState struct { - md *ordering.Metadata - data CorrectableCallData - replyChan <-chan response - expectedReplies int -} - -// CorrectableCall starts a new correctable quorum call and returns a new Correctable object. +// Example: // -// This method should only be used by generated code. -func (c RawConfiguration) CorrectableCall(ctx context.Context, d CorrectableCallData) *Correctable { - expectedReplies := len(c) - md := ordering.NewGorumsMetadata(ctx, c.getMsgID(), d.Method) - - replyChan := make(chan response, expectedReplies) - for _, n := range c { - msg := d.Message - if d.PerNodeArgFn != nil { - msg = d.PerNodeArgFn(d.Message, n.id) - if !msg.ProtoReflect().IsValid() { - expectedReplies-- - continue // don't send if no msg - } - } - n.channel.enqueue(request{ctx: ctx, msg: NewRequestMessage(md, msg), streaming: d.ServerStream}, replyChan) +// corr := ReadQC(ctx, req).Correctable(2) +// // Wait for level 2 to be reached +// <-corr.Watch(2) +// resp, level, err := corr.Get() +func (r *Responses[Resp]) Correctable(threshold int) *Correctable[Resp] { + corr := &Correctable[Resp]{ + level: LevelNotSet, + donech: make(chan struct{}, 1), } - corr := &Correctable{donech: make(chan struct{}, 1)} - - go c.handleCorrectableCall(ctx, corr, correctableCallState{ - md: md, - data: d, - replyChan: replyChan, - expectedReplies: expectedReplies, - }) - - return corr -} + go func() { + var ( + lastResp Resp + count int + errs []nodeError + ) + + for result := range r.ResponseSeq { + if result.Err != nil { + errs = append(errs, nodeError{nodeID: result.NodeID, cause: result.Err}) + continue + } -func (c RawConfiguration) handleCorrectableCall(ctx context.Context, corr *Correctable, state correctableCallState) { - var ( - resp proto.Message - errs []nodeError - rlevel int - clevel = LevelNotSet - quorum bool - replies = make(map[uint32]proto.Message) - ) - - if state.data.ServerStream { - for _, n := range c { - defer n.channel.deleteRouter(state.md.GetMessageID()) - } - } + count++ + lastResp = result.Value - for { - select { - case r := <-state.replyChan: - if r.err != nil { - errs = append(errs, nodeError{nodeID: r.nid, cause: r.err}) - break - } - replies[r.nid] = r.msg - if resp, rlevel, quorum = state.data.QuorumFunction(state.data.Message, replies); quorum { - if quorum { - corr.set(r.msg, rlevel, nil, true) - return - } - if rlevel > clevel { - clevel = rlevel - corr.set(r.msg, rlevel, nil, false) - } + // Check if we have reached the threshold + done := count >= threshold + corr.update(lastResp, count, done, nil) + if done { + return } - case <-ctx.Done(): - corr.set(resp, clevel, QuorumCallError{cause: ctx.Err(), errors: errs, replies: len(replies)}, true) - return } - if (state.data.ServerStream && len(errs) == state.expectedReplies) || - (!state.data.ServerStream && len(errs)+len(replies) == state.expectedReplies) { - corr.set(resp, clevel, QuorumCallError{cause: Incomplete, errors: errs, replies: len(replies)}, true) - return - } - } + + // If we didn't reach the threshold, mark as done with error + corr.update(lastResp, count, true, QuorumCallError{cause: ErrIncomplete, errors: errs}) + }() + + return corr } diff --git a/correctable_test.go b/correctable_test.go new file mode 100644 index 000000000..9d434e012 --- /dev/null +++ b/correctable_test.go @@ -0,0 +1,230 @@ +package gorums_test + +import ( + "fmt" + "testing" + "time" + + "github.com/relab/gorums" + "github.com/relab/gorums/internal/testutils/mock" + pb "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestCorrectableQuorumCall(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + ) + + // Wait for 2 responses + corr := responses.Correctable(2) + <-corr.Watch(2) + + reply, level, err := corr.Get() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if level < 2 { + t.Errorf("Expected level >= 2, got %d", level) + } + if reply.GetValue() != "echo: test" { + t.Errorf("Expected 'echo: test', got '%s'", reply.GetValue()) + } +} + +func TestCorrectableQuorumCallStream(t *testing.T) { + tests := []struct { + name string + threshold int + wantLevel int + wantValue string + }{ + { + name: "Level1", + threshold: 1, + wantLevel: 1, + wantValue: "echo: test-1", + }, + { + name: "Level2", + threshold: 2, + wantLevel: 2, + wantValue: "echo: test-2", + }, + { + name: "Level3", + threshold: 3, + wantLevel: 3, + wantValue: "echo: test-3", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Using StreamServerFn which sends 3 responses: + // "echo: val-1", "echo: val-2", "echo: val-3" + config := gorums.TestConfiguration(t, 3, gorums.StreamServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + responses := gorums.QuorumCallStream[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.Stream, + ) + + corr := responses.Correctable(tt.threshold) + + select { + case <-corr.Watch(tt.threshold): + case <-time.After(1 * time.Second): + t.Fatalf("Timeout waiting for level %d", tt.threshold) + } + + reply, level, err := corr.Get() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if level < tt.threshold { + t.Errorf("Expected level >= %d, got %d", tt.threshold, level) + } + // Note: The value might be from a higher level if updates happened fast, + // but for this test we expect at least the threshold level's value or higher. + // With StreamServerFn, level N returns "echo: test-N". + // Since we wait for level N, we expect at least "echo: test-N". + // Actually, StreamServerFn sends 1, then 2, then 3. We check for non-empty response. + if reply.GetValue() == "" { + t.Error("Expected non-empty response") + } + }) + } +} + +func TestCorrectableWatch(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.StreamServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + + responses := gorums.QuorumCallStream[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.Stream, + ) + + corr := responses.Correctable(3) + + // Watch for level 1 + select { + case <-corr.Watch(1): + // Success + case <-time.After(1 * time.Second): + t.Fatal("Timeout waiting for level 1") + } + + // Watch for level 3 + select { + case <-corr.Watch(3): + // Success + case <-time.After(1 * time.Second): + t.Fatal("Timeout waiting for level 3") + } +} + +func BenchmarkCorrectable(b *testing.B) { // skipcq: GO-R1005 + for _, numNodes := range []int{3, 5, 7, 9} { + + b.Run(fmt.Sprintf("QuorumCall/%d", numNodes), func(b *testing.B) { + config := gorums.TestConfiguration(b, numNodes, gorums.EchoServerFn) + cfgCtx := config.Context(b.Context()) + threshold := numNodes/2 + 1 + b.ReportAllocs() + for b.Loop() { + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ) + corr := responses.Correctable(threshold) + <-corr.Watch(threshold) + _, _, err := corr.Get() + if err != nil { + b.Fatalf("Correctable error: %v", err) + } + } + }) + + b.Run(fmt.Sprintf("QuorumCallIterator/%d", numNodes), func(b *testing.B) { + config := gorums.TestConfiguration(b, numNodes, gorums.EchoServerFn) + cfgCtx := config.Context(b.Context()) + threshold := numNodes/2 + 1 + b.ReportAllocs() + for b.Loop() { + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ) + count := 0 + for resp := range responses.Seq() { + if resp.Err == nil { + count++ + if count >= threshold { + break + } + } + } + if count < threshold { + b.Fatalf("Iterator failed to reach threshold") + } + } + }) + + b.Run(fmt.Sprintf("QuorumCallStream/%d", numNodes), func(b *testing.B) { + config := gorums.TestConfiguration(b, numNodes, gorums.StreamBenchmarkServerFn) + cfgCtx := config.Context(b.Context()) + threshold := numNodes/2 + 1 + b.ReportAllocs() + for b.Loop() { + responses := gorums.QuorumCallStream[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.Stream, + ) + corr := responses.Correctable(threshold) + <-corr.Watch(threshold) + _, _, err := corr.Get() + if err != nil { + b.Fatalf("Correctable error: %v", err) + } + } + }) + + b.Run(fmt.Sprintf("QuorumCallStreamIterator/%d", numNodes), func(b *testing.B) { + config := gorums.TestConfiguration(b, numNodes, gorums.StreamBenchmarkServerFn) + cfgCtx := config.Context(b.Context()) + threshold := numNodes/2 + 1 + b.ReportAllocs() + for b.Loop() { + responses := gorums.QuorumCallStream[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.Stream, + ) + count := 0 + for resp := range responses.Seq() { + if resp.Err == nil { + count++ + if count >= threshold { + break + } + } + } + if count < threshold { + b.Fatalf("Iterator failed to reach threshold") + } + } + }) + } +} diff --git a/doc/method-options.md b/doc/method-options.md deleted file mode 100644 index 8c1cee86b..000000000 --- a/doc/method-options.md +++ /dev/null @@ -1,95 +0,0 @@ -# Available Options and Their Meaning - -To select a call type for a Protobuf service method, you must specify one or more boolean options. -Gorums currently supports the following call types (these cannot be combined): - -| Call type | Gorums option | Description | -| ------------------- | -------------------- | ------------------------------------------------------------------- | -| Ordered RPC | no option | FIFO-ordered synchronous RPC to a single node. | -| Unicast | `gorums.unicast` | FIFO-ordered one-way asynchronous unicast. | -| Multicast | `gorums.multicast` | FIFO-ordered one-way asynchronous multicast. | -| Quorum Call | `gorums.quorumcall` | FIFO-ordered synchronous quorum call on a configuration of nodes. | -| Correctable | `gorums.correctable` | Asynchronous quorum call with *incremental consistency guarantees*. | - -All call types rely on a streaming RPC call named `NodeStream` to send and receive messages to/from individual nodes. -However, the server API generated by Gorums does not expose the gRPC stream-based API. -Instead, the generated API is similar to unary gRPC, except for the handling of return values is a bit different. - -Each call type may also specify some advanced options: - -| Name | Gorums option | Type | Description | -| ------------------ | --------------------------- | ------ | ---------------------------------------------------------------------------------------------- | -| Custom return type | `gorums.custom_return_type` | string | Makes the quorum function return a different type than the RPC response type. | -| Per node arguments | `gorums.per_node_arg` | bool | Makes it possible to send different request objects to each node in a call. | -| Asynchronous | `gorums.async` | bool | Makes the quorum call run asynchronously and returns a "promise" object to fetch the result. | - -The following matrix shows which call types can be combined with the different options: - -| Option | Ordered RPC | Unicast | Multicast | Quorum Call | Correctable | -| ------------------ | ----------- | ------- | --------- | ----------- | ----------- | -| Custom return type | N/A | N/A | N/A | Yes | Yes | -| Per node arguments | N/A | N/A | Yes | Yes | Yes | -| Asynchronous | No | N/A | N/A | Yes | No | - -The correctable quorum call is an asynchronous call type that comes in two variants. -In the first variant, each node of the configuration sends only a single response. -In the second variant, each node may return multiple responses. -The second variant uses the gRPC server-side stream API and is indicated with the `stream` keyword associated with the return type. -See `ReadCorrectableStream` method below for an example. - -Note that the `gorums.correctable` option should not be combined with the `gorums.async` option. - -## Example Protobuf Definitions - -Below is shown a list of RPC service methods, illustrating how to configure the different variants. - -```proto -service Storage { - // ReadOrdered is a FIFO-ordered RPC. - rpc ReadOrdered(ReadRequest) returns (State) {} - - // ReadQC is a FIFO-ordered synchronous quorum call. - rpc ReadQC(ReadRequest) returns (State) { - option (gorums.quorumcall) = true; - } - - // ReadAsync is an asynchronous quorum call that - // returns a promise object for retrieving results. - rpc ReadAsync(ReadRequest) returns (State) { - option (gorums.async) = true; - option (gorums.quorumcall) = true; - } - - // ReadCustomReturn is a FIFO-ordered synchronous quorum call with a custom return type. - rpc ReadCustomReturn(ReadRequest) returns (State) { - option (gorums.quorumcall) = true; - option (gorums.custom_return_type) = "MyState"; - } - - // ReadCorrectable is an asynchronous correctable quorum call that - // returns a correctable object for retrieving results. - rpc ReadCorrectable(ReadRequest) returns (State) { - option (gorums.correctable) = true; - } - - // ReadCorrectableStream is an asynchronous correctable quorum call that - // returns a correctable object for retrieving results over a stream. - rpc ReadCorrectableStream(ReadRequest) returns (stream State) { - option (gorums.correctable) = true; - } - - // WriteMulticast is an asynchronous multicast to all nodes in a configuration. - // No replies are collected. - rpc WriteMulticast(State) returns (Empty) { - option (gorums.multicast) = true; - } - - // WritePerNode is a synchronous quorum call, where, - // for each node, a provided function is called to determine - // the argument to be sent to that node. - rpc WritePerNode(State) returns (WriteResponse) { - option (gorums.quorumcall) = true; - option (gorums.per_node_arg) = true; - } -} -``` diff --git a/doc/migration.md b/doc/migration.md new file mode 100644 index 000000000..55f35c919 --- /dev/null +++ b/doc/migration.md @@ -0,0 +1,910 @@ +# Migration Guide: Gorums v2 to v3 + +This guide helps you migrate from Gorums v2 (QuorumSpec-based design) to v3 (iterator-based design with generic functions). + +## Overview of Changes + +Gorums v3 introduces a fundamental redesign focused on: + +- **Type aliases** instead of generated concrete types +- **Generic functions** with terminal methods instead of QuorumSpec interface +- **Iterator-based response aggregation** for flexible custom logic +- **ConfigContext pattern** for passing configuration to generic functions +- **Custom return types** support for aggregation functions + +### Why These Changes? + +1. **Simpler generated code**: Type aliases reduce code duplication +2. **More flexibility**: Iterators enable progressive response handling +3. **Better Go generics usage**: Leverages Go 1.18+ generics properly +4. **Cleaner API**: Terminal methods (`.Majority()`, `.First()`, `.All()`) for common patterns +5. **Works around Go limitations**: Go doesn't support generic methods ([Go FAQ](https://go.dev/doc/faq#generic_methods)) + +## Quick Comparison + +### Legacy v2 Design + +```go +// QuorumSpec interface generated per service +type QuorumSpec interface { + ReadQF(req *ReadRequest, replies map[uint32]*State) (*State, bool) + WriteQF(req *WriteRequest, replies map[uint32]*WriteResponse) (*WriteResponse, bool) +} + +// User implements QuorumSpec +type QSpec struct{ quorumSize int } + +func (qs *QSpec) ReadQF(_ *ReadRequest, replies map[uint32]*State) (*State, bool) { + if len(replies) < qs.quorumSize { + return nil, false + } + return newestState(replies), true +} + +// Configuration created with QuorumSpec +cfg, _ := mgr.NewConfiguration(&QSpec{2}, gorums.WithNodeList(addrs)) + +// Method calls on Configuration +reply, err := cfg.Read(ctx, &ReadRequest{}) +``` + +### New v3 Design + +```go +// No QuorumSpec interface needed + +// Standalone aggregation function +func newestValue(responses *gorums.Responses[*ReadResponse]) (*ReadResponse, error) { + var newest *ReadResponse + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + if newest == nil || resp.Value.GetTime().After(newest.GetTime()) { + newest = resp.Value + } + } + if newest == nil { + return nil, gorums.ErrIncomplete + } + return newest, nil +} + +// Configuration created without QuorumSpec +config, _ := gorums.NewConfiguration(mgr, gorums.WithNodeList(addrs)) + +// Generic function with ConfigContext +cfgCtx := config.Context(ctx) +responses := ReadQC(cfgCtx, &ReadRequest{}) + +// Option 1: Use custom aggregation +reply, err := newestValue(responses) + +// Option 2: Use built-in terminal method +reply, err := ReadQC(cfgCtx, &ReadRequest{}).Majority() +``` + +## Migration Steps + +### Step 1: Regenerate Protocol Buffers + +Regenerate all `.proto` files with the new `protoc-gen-gorums` plugin: + +```bash +make genproto +``` + +This generates new `*_gorums.pb.go` files with: + +- Type aliases for `Manager`, `Configuration`, `Node` +- Generic functions for quorum calls (e.g., `ReadQC`, `WriteQC`) +- Terminal methods on `*gorums.Responses[T]` + +### Step 2: Update Configuration Creation + +Remove QuorumSpec from configuration creation. + +**Before:** + +```go +mgr := NewManager(gorums.WithDialOptions(...)) +cfg, err := mgr.NewConfiguration( + &QSpec{quorumSize: 2}, // ❌ Remove QuorumSpec + gorums.WithNodeList(addrs), +) +``` + +**After:** + +```go +mgr := gorums.NewManager(gorums.WithDialOptions(...)) +cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(addrs)) +``` + +Or use the convenience function that creates both: + +```go +cfg, err := gorums.NewConfig( + gorums.WithNodeList(addrs), + gorums.WithDialOptions(...), +) +``` + +### Step 3: Convert QuorumSpec Methods to Aggregation Functions + +Transform your QuorumSpec methods into standalone aggregation functions that work with iterators. + +**Before:** + +```go +type QSpec struct{ quorumSize int } + +func (qs *QSpec) ReadQF(_ *ReadRequest, replies map[uint32]*State) (*State, bool) { + if len(replies) < qs.quorumSize { + return nil, false + } + var newest *State + for _, s := range replies { + if newest == nil || s.GetTimestamp() > newest.GetTimestamp() { + newest = s + } + } + return newest, true +} +``` + +**After:** + +```go +func newestState(responses *gorums.Responses[*State]) (*State, error) { + var newest *State + for resp := range responses.Seq() { + if resp.Err != nil { + continue // Skip failed responses + } + if newest == nil || resp.Value.GetTimestamp() > newest.GetTimestamp() { + newest = resp.Value + } + } + if newest == nil { + return nil, gorums.ErrIncomplete + } + return newest, nil +} +``` + +**Key Changes:** + +- No longer a method on a struct +- Takes `*gorums.Responses[T]` instead of `map[uint32]T` +- Returns `(T, error)` instead of `(T, bool)` +- Uses iterator: `for resp := range responses.Seq()` +- Each response has `.Value` and `.Err` fields +- Return `gorums.ErrIncomplete` instead of `false` + +### Step 4: Update Call Sites + +Change from Configuration methods to generic functions with ConfigContext. + +**Before:** + +```go +ctx := context.Background() +reply, err := cfg.Read(ctx, &ReadRequest{Key: "x"}) +``` + +**After:** + +```go +// Option 1: With custom aggregation +ctx := context.Background() +cfgCtx := config.Context(ctx) +reply, err := newestState(ReadQC(cfgCtx, &ReadRequest{Key: "x"})) + +// Option 2: With terminal method +reply, err := ReadQC(cfgCtx, &ReadRequest{Key: "x"}).Majority() + +// Option 3: With threshold (e.g., wait for 3 responses) +reply, err := ReadQC(cfgCtx, &ReadRequest{Key: "x"}).Threshold(3) + +// Option 4: First successful response +reply, err := ReadQC(cfgCtx, &ReadRequest{Key: "x"}).First() + +// Option 5: Wait for all responses +reply, err := ReadQC(cfgCtx, &ReadRequest{Key: "x"}).All() +``` + +## Terminal Methods Reference + +The new design provides built-in aggregation patterns via terminal methods on `*gorums.Responses[T]`: + +| Method | Description | Returns When | +| --------------- | ------------------------- | ---------------------------- | +| `.First()` | First successful response | Any node responds | +| `.Majority()` | Majority of responses | ⌈(n+1)/2⌉ responses received | +| `.All()` | All responses | All n nodes responded | +| `.Threshold(n)` | At least n responses | n responses received | + +### Example Usage + +```go +// Fast reads: return first successful response +reply, err := ReadQC(cfgCtx, req).First() + +// Byzantine fault tolerance: require majority +reply, err := WriteQC(cfgCtx, req).Majority() + +// Wait for all nodes (e.g., for debugging) +reply, err := ReadQC(cfgCtx, req).All() + +// Custom threshold (e.g., f+1 for crash tolerance) +reply, err := ReadQC(cfgCtx, req).Threshold(f + 1) +``` + +**Error Handling:** + +Terminal methods return `gorums.ErrIncomplete` when: + +- Context is canceled before threshold is met +- Not enough successful responses received +- All nodes returned errors + +## Iterator-Based Custom Aggregation + +For complex aggregation logic, use the iterator API with `responses.Seq()`. + +### Basic Iterator Pattern + +```go +func customAggregation(responses *gorums.Responses[*Response]) (*Response, error) { + for resp := range responses.Seq() { + // resp.Value contains the response message + // resp.Err contains any error from that node + // resp.NodeID contains the node identifier + + if resp.Err != nil { + // Handle or skip error + continue + } + + // Process resp.Value + } + return result, nil +} +``` + +### Iterator Helpers + +Gorums provides helper methods for common iterator operations: + +#### `IgnoreErrors()` - Skip Failed Responses + +```go +func countSuccessful(responses *gorums.Responses[*Response]) int { + count := 0 + for resp := range responses.Seq().IgnoreErrors() { + count++ + // resp.Value is guaranteed to be non-nil + // resp.Err is guaranteed to be nil + } + return count +} +``` + +#### `Filter()` - Custom Filtering + +```go +func newestValue(responses *gorums.Responses[*ReadResponse]) (*ReadResponse, error) { + var newest *ReadResponse + + // Filter for responses with non-zero timestamps + filtered := responses.Seq().Filter(func(nr gorums.NodeResponse[*ReadResponse]) bool { + return nr.Err == nil && nr.Value.GetTimestamp() > 0 + }) + + for resp := range filtered { + if newest == nil || resp.Value.GetTimestamp() > newest.GetTimestamp() { + newest = resp.Value + } + } + + if newest == nil { + return nil, gorums.ErrIncomplete + } + return newest, nil +} +``` + +#### `CollectN()` - Collect First N Responses + +```go +func majorityWrite(responses *gorums.Responses[*WriteResponse]) (*WriteResponse, error) { + majority := (responses.Size() + 1) / 2 + replies := responses.Seq().IgnoreErrors().CollectN(majority) + + if len(replies) < majority { + return nil, gorums.ErrIncomplete + } + + // Process the map[uint32]*WriteResponse + return aggregateWrites(replies), nil +} +``` + +#### `CollectAll()` - Collect All Responses + +```go +// Collect all responses into a map +replies := responses.Seq().CollectAll() +// Returns map[uint32]*Response (may include failed nodes with nil values) + +// Collect only successful responses +replies := responses.Seq().IgnoreErrors().CollectAll() +// Returns map[uint32]*Response (only successful responses) +``` + +### Complete Example: Quorum with Verification + +```go +func verifiedQuorum(responses *gorums.Responses[*SignedResponse]) (*SignedResponse, error) { + const quorumSize = 3 + var verified []*SignedResponse + + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + + // Custom validation logic + if !verifySignature(resp.Value) { + continue + } + + verified = append(verified, resp.Value) + + // Early exit once quorum is reached + if len(verified) >= quorumSize { + return verified[0], nil + } + } + + if len(verified) < quorumSize { + return nil, gorums.ErrIncomplete + } + return verified[0], nil +} +``` + +## Async Calls + +Async calls remain similar but now use terminal methods to create futures. + +### Basic Async Pattern + +**Before:** + +```go +future := cfg.ReadAsync(ctx, req) +// Do other work... +reply, err := future.Get() +``` + +**After:** + +```go +// Create async future with terminal method +future := ReadQC(cfgCtx, req).AsyncMajority() +// Do other work... +reply, err := future.Get() +``` + +### Async Terminal Methods + +All terminal methods have async variants: + +```go +// Async majority +fut := ReadQC(cfgCtx, req).AsyncMajority() + +// Async first +fut := ReadQC(cfgCtx, req).AsyncFirst() + +// Async all +fut := ReadQC(cfgCtx, req).AsyncAll() + +// Async threshold +fut := ReadQC(cfgCtx, req).AsyncThreshold(3) +``` + +### Using Async with Custom Aggregation + +For custom aggregation with async, get responses first then apply aggregation: + +```go +// Start the quorum call +responses := ReadQC(cfgCtx, req) + +// Get async future for custom threshold +future := responses.AsyncThreshold(3) + +// Do other work... + +// Wait for completion +reply, err := future.Get() +if err != nil { + return err +} + +// Now apply custom aggregation to collected responses +result := customAggregation(reply) +``` + +## Correctable Calls + +Correctable calls provide progressive updates as more responses arrive. + +### Basic Correctable Pattern + +**Before:** + +```go +correctable := cfg.ReadCorrectable(ctx, req) +reply := correctable.Get() +<-correctable.Watch(level) +reply = correctable.Get() +``` + +**After:** + +```go +// Create correctable with initial threshold +corr := ReadQC(cfgCtx, req).Correctable(2) + +// Get initial result (blocks until threshold met) +reply, level, err := corr.Get() + +// Watch for higher levels +<-corr.Watch(3) +reply, level, err = corr.Get() + +// Or iterate through levels +for level := range corr.Levels() { + reply, _, _ := corr.Get() + fmt.Printf("Got reply at level %d\n", level) +} +``` + +### Progressive Response Handling + +```go +func progressiveRead(cfgCtx *gorums.ConfigContext, req *ReadRequest) { + corr := ReadQC(cfgCtx, req).Correctable(1) + + // Process each new level as it arrives + for level := range corr.Levels() { + reply, currentLevel, err := corr.Get() + if err != nil { + log.Printf("Error at level %d: %v", level, err) + continue + } + + log.Printf("Level %d: received %d replies", level, currentLevel) + processReply(reply) + + // Stop after reaching desired level + if currentLevel >= 3 { + corr.Stop() + break + } + } +} +``` + +## Custom Return Types + +The new design supports aggregation functions that return types different from the proto response type. + +### Example: Aggregating Multiple Responses + +Consider a `StopBenchmark` RPC that returns `*MemoryStat` from each node, but you want to return `*MemoryStatList` containing all stats: + +```go +// Proto definitions: +// rpc StopBenchmark(StopRequest) returns (MemoryStat) +// message MemoryStat { uint64 allocs = 1; uint64 memory = 2; } +// message MemoryStatList { repeated MemoryStat memory_stats = 1; } + +// Custom aggregation function with different return type +func StopBenchmarkQF(replies map[uint32]*MemoryStat) (*MemoryStatList, error) { + if len(replies) == 0 { + return nil, gorums.ErrIncomplete + } + return &MemoryStatList{ + MemoryStats: slices.Collect(maps.Values(replies)), + }, nil +} + +// Usage: +responses := StopBenchmark(cfgCtx, &StopRequest{}) +replies := responses.Seq().IgnoreErrors().CollectAll() // map[uint32]*MemoryStat +memStats, err := StopBenchmarkQF(replies) // Returns *MemoryStatList +``` + +### Example: Computing Aggregate Statistics + +```go +// Proto: rpc StopServerBenchmark(StopRequest) returns (Result) + +func StopServerBenchmarkQF(replies map[uint32]*Result) (*Result, error) { + if len(replies) == 0 { + return nil, gorums.ErrIncomplete + } + + // Combine results, calculating averages and pooled variance + resp := &Result{} + for _, reply := range replies { + resp.TotalOps += reply.TotalOps + resp.TotalTime += reply.TotalTime + resp.Throughput += reply.Throughput + resp.LatencyAvg += reply.LatencyAvg * float64(reply.TotalOps) + } + + // Calculate weighted average + resp.LatencyAvg /= float64(resp.TotalOps) + + // Calculate pooled variance + for _, reply := range replies { + resp.LatencyVar += float64(reply.TotalOps-1) * reply.LatencyVar + } + resp.LatencyVar /= (float64(resp.TotalOps) - float64(len(replies))) + + // Calculate averages + numNodes := len(replies) + resp.TotalOps /= uint64(numNodes) + resp.TotalTime /= int64(numNodes) + resp.Throughput /= float64(numNodes) + + return resp, nil +} +``` + +### Pattern: Two-Step Aggregation + +For custom return types, follow this pattern: + +1. **Collect responses** using `CollectAll()` or `CollectN()` +2. **Apply custom aggregation function** that transforms the collected map + +```go +// Step 1: Collect responses +responses := CustomQC(cfgCtx, req) +replies := responses.Seq().IgnoreErrors().CollectAll() // map[uint32]*ProtoResponse + +// Step 2: Apply custom aggregation +result, err := CustomAggregationQF(replies) // Returns *CustomType +``` + +## Configuration Manipulation + +Configuration manipulation APIs remain largely unchanged: + +```go +// Combine configurations +cfg3, _ := gorums.NewConfiguration(mgr, cfg1.And(cfg2)) + +// Exclude nodes +cfg4, _ := gorums.NewConfiguration(mgr, cfg1.Except(cfg2)) + +// Remove specific nodes by ID +cfg5, _ := gorums.NewConfiguration(mgr, cfg1.WithoutNodes(1, 2, 3)) +``` + +### New: Filter Out Failed Nodes + +The new design adds `WithoutErrors()` to create configurations excluding nodes that failed: + +```go +// Make a quorum call +reply, err := ReadQC(cfgCtx, req).Majority() + +// If there was a quorum error, filter out failed nodes +if qcErr, ok := err.(gorums.QuorumCallError); ok { + // Create new configuration without failed nodes + goodCfg, _ := gorums.NewConfiguration(mgr, config.WithoutErrors(qcErr)) + + // Retry with good nodes only + cfgCtx := goodCfg.Context(ctx) + reply, err = ReadQC(cfgCtx, req).Majority() +} +``` + +## Error Handling + +Error handling is similar but with some improvements. + +### QuorumCallError + +```go +reply, err := ReadQC(cfgCtx, req).Majority() +if err != nil { + // Check if it's a quorum call error + if qcErr, ok := err.(gorums.QuorumCallError); ok { + fmt.Printf("Quorum call failed: %v\n", qcErr) + fmt.Printf("Reason: %s\n", qcErr.Reason) + + // Access per-node errors + for nodeID, nodeErr := range qcErr.Errors { + fmt.Printf("Node %d error: %v\n", nodeID, nodeErr) + } + } +} +``` + +### Context Cancellation + +```go +ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +defer cancel() + +cfgCtx := config.Context(ctx) +reply, err := ReadQC(cfgCtx, req).Majority() +if err != nil { + if ctx.Err() != nil { + fmt.Println("Request timed out") + } +} +``` + +## Migration Checklist + +- [ ] Regenerate all `.proto` files: `make genproto` +- [ ] Remove all `QuorumSpec` interface implementations +- [ ] Convert QuorumSpec methods to standalone aggregation functions + - [ ] Change signature from `(req *Req, replies map[uint32]*Resp) (*Resp, bool)` to `(responses *gorums.Responses[*Resp]) (*Resp, error)` + - [ ] Use iterator: `for resp := range responses.Seq()` + - [ ] Return `gorums.ErrIncomplete` instead of `false` +- [ ] Update configuration creation + - [ ] Remove QuorumSpec from `NewConfiguration()` calls +- [ ] Update all quorum call sites + - [ ] Replace `cfg.Read(ctx, req)` with `ReadQC(cfgCtx, req).Majority()` + - [ ] Or use custom aggregation: `customAggregation(ReadQC(cfgCtx, req))` +- [ ] Update async calls to use async terminal methods + - [ ] Replace `cfg.ReadAsync(ctx, req)` with `ReadQC(cfgCtx, req).AsyncMajority()` +- [ ] Update correctable calls to use `.Correctable(threshold)` + - [ ] Replace `cfg.ReadCorrectable(ctx, req)` with `ReadQC(cfgCtx, req).Correctable(threshold)` +- [ ] Test thoroughly with your existing test suite +- [ ] Update documentation and examples + +## Common Migration Patterns + +### Pattern 1: Simple Majority Quorum + +**Before:** + +```go +type QSpec struct{} + +func (qs *QSpec) ReadQF(_ *ReadRequest, replies map[uint32]*State) (*State, bool) { + if len(replies) <= len(cfg.Nodes())/2 { + return nil, false + } + // Return first reply + for _, reply := range replies { + return reply, true + } + return nil, false +} + +cfg, _ := mgr.NewConfiguration(&QSpec{}, gorums.WithNodeList(addrs)) +reply, err := cfg.Read(ctx, req) +``` + +**After:** + +```go +// No QuorumSpec needed + +config, _ := gorums.NewConfiguration(mgr, gorums.WithNodeList(addrs)) +cfgCtx := config.Context(ctx) +reply, err := ReadQC(cfgCtx, req).Majority() +``` + +### Pattern 2: Custom Aggregation (Select Newest) + +**Before:** + +```go +func (qs *QSpec) ReadQF(_ *ReadRequest, replies map[uint32]*State) (*State, bool) { + if len(replies) < qs.quorumSize { + return nil, false + } + var newest *State + for _, s := range replies { + if newest == nil || s.Timestamp > newest.Timestamp { + newest = s + } + } + return newest, true +} +``` + +**After:** + +```go +func newestState(responses *gorums.Responses[*State]) (*State, error) { + var newest *State + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + if newest == nil || resp.Value.Timestamp > newest.Value.Timestamp { + newest = resp.Value + } + } + if newest == nil { + return nil, gorums.ErrIncomplete + } + return newest, nil +} + +// Usage +reply, err := newestState(ReadQC(cfgCtx, req)) +``` + +### Pattern 3: Counting Responses + +**Before:** + +```go +func (qs *QSpec) WriteQF(_ *WriteRequest, replies map[uint32]*WriteResponse) (*WriteResponse, bool) { + if len(replies) < qs.quorumSize { + return nil, false + } + count := 0 + for _, r := range replies { + if r.Success { + count++ + } + } + return &WriteResponse{Success: count >= qs.quorumSize}, true +} +``` + +**After:** + +```go +func countSuccessful(responses *gorums.Responses[*WriteResponse]) (*WriteResponse, error) { + count := 0 + quorumSize := (responses.Size() + 1) / 2 + + for resp := range responses.Seq().IgnoreErrors() { + if resp.Value.Success { + count++ + } + } + + if count < quorumSize { + return nil, gorums.ErrIncomplete + } + return &WriteResponse{Success: true}, nil +} + +// Usage +reply, err := countSuccessful(WriteQC(cfgCtx, req)) +``` + +### Pattern 4: Early Termination + +**Before:** + +```go +func (qs *QSpec) FastReadQF(_ *ReadRequest, replies map[uint32]*State) (*State, bool) { + if len(replies) >= 1 { + for _, reply := range replies { + return reply, true // Return first reply + } + } + return nil, false +} +``` + +**After:** + +```go +// Built-in terminal method +reply, err := ReadQC(cfgCtx, req).First() + +// Or custom implementation +func firstValid(responses *gorums.Responses[*State]) (*State, error) { + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + if isValid(resp.Value) { + return resp.Value, nil + } + } + return nil, gorums.ErrIncomplete +} +``` + +## Troubleshooting + +### "Cannot use cfg.Read: undefined" + +**Problem:** Configuration no longer has RPC methods. + +**Solution:** Use generic functions with ConfigContext: + +```go +cfgCtx := config.Context(ctx) +reply, err := ReadQC(cfgCtx, req).Majority() +``` + +### "QuorumSpec interface not found" + +**Problem:** QuorumSpec interface no longer generated. + +**Solution:** Remove QuorumSpec implementation and convert methods to standalone functions. + +### "responses.Seq() not working as expected" + +**Problem:** Iterator might be exhausted if used multiple times. + +**Solution:** Iterators can only be consumed once. Collect into a map if you need multiple passes: + +```go +replies := responses.Seq().CollectAll() +// Now you can iterate over replies multiple times +``` + +### "How do I wait for all responses?" + +**Solution:** Use `.All()` terminal method: + +```go +reply, err := ReadQC(cfgCtx, req).All() +``` + +### "How do I implement early termination?" + +**Solution:** Return from your aggregation function as soon as the condition is met: + +```go +func earlyTermination(responses *gorums.Responses[*Response]) (*Response, error) { + for resp := range responses.Seq() { + if resp.Err == nil && condition(resp.Value) { + return resp.Value, nil // Early return + } + } + return nil, gorums.ErrIncomplete +} +``` + +## Additional Resources + +- [User Guide](user-guide.md) - Complete guide to using Gorums +- [Developer Guide](dev-guide.md) - Contributing to Gorums + +- [Examples](../examples/) - Working examples demonstrating various patterns + +## Summary + +The v3 design provides more flexibility and better Go idioms: + +**Key Improvements:** + +- ✅ No more QuorumSpec interface boilerplate +- ✅ Built-in terminal methods for common patterns +- ✅ Flexible iterator-based aggregation +- ✅ Progressive response handling with correctables +- ✅ Custom return types support +- ✅ Better error filtering with `WithoutErrors()` + +**Migration Effort:** + +- Low: For code using simple majority/first patterns (switch to terminal methods) +- Medium: For code with custom QuorumSpec logic (convert to iterators) +- Benefits: Cleaner code, more flexibility, better performance + +Welcome to Gorums v3! 🎉 diff --git a/doc/ordering.md b/doc/ordering.md index 9e2a038e4..f9469404e 100644 --- a/doc/ordering.md +++ b/doc/ordering.md @@ -6,7 +6,7 @@ Many of the use cases for a framework like Gorums depend on messages arriving in Unfortunately, gRPC does not guarantee that unary RPCs invoked in order, will be received by the server in the same order. That is, according to the [gRPC docs](https://grpc.io/docs/what-is-grpc/core-concepts/): "*gRPC guarantees message ordering within an individual RPC call.*" -Further, as explained [here](https://github.com/grpc/grpc/issues/10853#issuecomment-297478862): +Further, as explained [in this issue comment](https://github.com/grpc/grpc/issues/10853#issuecomment-297478862): "*Separate requests are independent of each other, and there is no guarantee that they will be handled in any particular order. If you want to preserve the order of a set of messages between a single client and a single server, I would recommend using a streaming call. Messages within a single stream are guaranteed to be received in the order in which they are sent.*" This was a source of [problems](https://github.com/relab/gorums/issues/16) in early versions of Gorums. @@ -81,7 +81,7 @@ func (s *testSrv) AsyncHandler(ctx gorums.ServerCtx, req *Request) (resp *Respon // this code will run concurrently with other handlers // perform slow / async work here time.Sleep(10 * time.Millisecond) - // at some point later, the response passed back to Gorums through the `ret` function, + // at some point later, the response passed back to the Gorums runtime, // and gets sent back to the client. return response, nil } @@ -94,6 +94,6 @@ Hence, to preserve message ordering, the following rules must be adhered to: * (Client-side) Quorum calls cannot be started in separate goroutines, as the scheduling of goroutines is non-deterministic. -* (Client-side) To process replies from different quorum calls concurrently, use the `async` option. +* (Client-side) To process replies from different quorum calls concurrently, use the `Async*` terminal methods. * (Server-side) If the server must return replies in the same order as the client sent them, the server-side handler must also preserve ordering. diff --git a/doc/user-guide.md b/doc/user-guide.md index bf0aac2d8..f4bc5167d 100644 --- a/doc/user-guide.md +++ b/doc/user-guide.md @@ -39,10 +39,7 @@ There are a few tools that need to be installed first: ## Creating and Compiling a Protobuf Service Description into Gorums Code In this example we will create a simple storage service. -The storage can store a single `{string,timestamp}` tuple and has two methods: - -* `Read() State` -* `Write(State) Response` +The storage can store a single `{value, timestamp}` tuple with methods for reading and writing state. First, we define our storage as a gRPC service by using the protocol buffers interface definition language (IDL). Refer to the protocol buffers [language guide](https://developers.google.com/protocol-buffers/docs/proto3) to learn more about the Protobuf IDL. @@ -57,37 +54,78 @@ go mod init gorumsexample go get github.com/relab/gorums ``` -The file `storage.proto` should have the following content: +### Call Types + +Gorums offers several call types including synchronous quorum calls, and one-way `unicast` and `multicast` communication. +To select a call type for a Protobuf service method, specify one of the following options (they cannot be combined): + +| Call type | Gorums option | Description | +| ----------- | ------------------- | ----------------------------------------------------------------- | +| Ordered RPC | no option | FIFO-ordered synchronous RPC to a single node. | +| Unicast | `gorums.unicast` | FIFO-ordered one-way asynchronous unicast. | +| Multicast | `gorums.multicast` | FIFO-ordered one-way asynchronous multicast. | +| Quorum Call | `gorums.quorumcall` | FIFO-ordered synchronous quorum call on a configuration of nodes. | + +The generated API is similar to unary gRPC, unless the `stream` keyword is used in the proto definition. +Server streaming is only supported for quorum calls. +Client streaming is not supported. + +### Configuring Call Types in Protobuf + +The file `storage.proto` should have the following content, illustrating the different call types: + +```proto +edition = "2024"; -```protobuf -syntax = "proto3"; package gorumsexample; -option go_package = ".;gorumsexample"; +option go_package = "github.com/relab/gorums/examples/gorumsexample"; +option features.field_presence = IMPLICIT; + +import "google/protobuf/empty.proto"; +import "gorums.proto"; +// Storage service defines the RPCs for a simple key-value storage system. service Storage { - rpc Read(ReadRequest) returns (State) { } - rpc Write(State) returns (WriteResponse) { } -} + // ReadOrdered is a FIFO-ordered RPC to a single node. + rpc ReadOrdered(google.protobuf.Empty) returns (State) {} -message State { - string Value = 1; - int64 Timestamp = 2; -} + // WriteUnicast is an asynchronous unicast to a single node. + // No reply is collected. + rpc WriteUnicast(State) returns (google.protobuf.Empty) { + option (gorums.unicast) = true; + } + + // WriteMulticast is an asynchronous multicast to all nodes in a configuration. + // No replies are collected. + rpc WriteMulticast(State) returns (google.protobuf.Empty) { + option (gorums.multicast) = true; + } + + // ReadQC is a FIFO-ordered synchronous quorum call. + // Use terminal methods (.Majority(), .First(), etc.) to retrieve results. + // Use .AsyncMajority() for async variant, .Correctable(n) for correctable variant. + rpc ReadQC(google.protobuf.Empty) returns (State) { + option (gorums.quorumcall) = true; + } -message WriteResponse { - bool New = 1; + // ReadQCStream is a FIFO-ordered synchronous quorum call with server streaming. + // The stream keyword enables correctable calls where nodes can send multiple + // progressive updates. Use .Correctable(n) to watch for updates as they arrive. + rpc ReadQCStream(google.protobuf.Empty) returns (stream State) { + option (gorums.quorumcall) = true; + } } -message ReadRequest {} +// State represents the value and its timestamp stored in a node. +message State { + string value = 1; + int64 timestamp = 2; +} ``` -Every RPC method must take and return a single Protobuf message. -This is a requirement of the Protobuf IDL. -The `Read` method in this example, therefore, takes an empty `ReadRequest` as input since no information is needed by the method. +For the `unicast` and `multicast` call types, the response message type will be unused by Gorums. -**Note:** Gorums offers one-way communication through the `unicast` and `multicast` call types. -For these call types, the response message type will be unused by Gorums. -For a detailed overview of the available method options to control the call types, see the [method options](method-options.md) document. +### Compiling the Service Definition Next, we compile our service definition into Go code which includes: @@ -106,30 +144,41 @@ protoc -I="$(go list -m -f {{.Dir}} github.com/relab/gorums):." \ The above step should produce two files named `storage.pb.go` and `storage_gorums.pb.go` in your package directory. The former contains the Protobuf definitions of our messages. The latter contains the Gorums generated client and server interfaces. + +### Examining the Gorums Generated Code + Let us examine the `storage_gorums.pb.go` file to see the code generated from our Protobuf definitions. -Our two RPC methods have the following signatures: +The client functions below are generated by Gorums based on the call type specified in the Protobuf definition. +The first two functions are used to send requests to a single node determined by the `NodeContext`, while the last three are used to send requests to a configuration of nodes determined by the `ConfigContext`. +The last two functions return a `*gorums.Responses[*State]` object, which is a collection of responses from the nodes in the configuration. ```go -func (n *Node) Read(ctx context.Context, in *ReadRequest) (*State, error) -func (n *Node) Write(ctx context.Context, in *State) (*WriteResponse, error) +func ReadOrdered(ctx *gorums.NodeContext, in *emptypb.Empty) (resp *State, err error) +func WriteUnicast(ctx *gorums.NodeContext, in*State, opts ...gorums.CallOption) error +func WriteMulticast(ctx *gorums.ConfigContext, in *State, opts ...gorums.CallOption) error +func ReadQC(ctx *gorums.ConfigContext, in *emptypb.Empty, opts ...gorums.CallOption) *gorums.Responses[*State] +func ReadQCStream(ctx *gorums.ConfigContext, in *emptypb.Empty, opts ...gorums.CallOption) *gorums.Responses[*State] ``` And this is our server interface: ```go -type Storage interface { - Read(gorums.ServerCtx, *ReadRequest) (*State, error) - Write(gorums.ServerCtx, *State) (*WriteResponse, error) +type StorageServer interface { + ReadOrdered(ctx gorums.ServerCtx, request *emptypb.Empty) (response *State, err error) + WriteUnicast(ctx gorums.ServerCtx, request *State) + WriteMulticast(ctx gorums.ServerCtx, request *State) + ReadQC(ctx gorums.ServerCtx, request *emptypb.Empty) (response *State, err error) + ReadQCStream(ctx gorums.ServerCtx, request *emptypb.Empty, send func(response *State) error) error } ``` -**Note:** For a real use case, you may decide to keep the `.proto` file and the generated `.pb.go` files in a separate directory/package and import that package (the generated Gorums API) into your application. +**Note:** +You may decide to keep the `.proto` file and the generated `.pb.go` files in a separate directory/package and import that package (the generated Gorums API) into your application. We skip that here for the sake of simplicity. ## Implementing the StorageServer -We now describe how to use the generated Gorums API for the `Storage` service. -We begin by implementing the `Storage` server interface from above: +We now describe how to implement the `StorageServer` interface from above. ```go type storageSrv struct { @@ -137,22 +186,38 @@ type storageSrv struct { state *State } -func (srv *storageSrv) Read(_ gorums.ServerCtx, req *ReadRequest) (resp *State, err error) { +func (srv *storageSrv) ReadOrdered(_ gorums.ServerCtx, req *emptypb.Empty) (resp *State, err error) { srv.mut.Lock() defer srv.mut.Unlock() - fmt.Println("Got Read()") return srv.state, nil } -func (srv *storageSrv) Write(_ gorums.ServerCtx, req *State) (resp *WriteResponse, err error) { +func (srv *storageSrv) WriteUnicast(_ gorums.ServerCtx, req *State) { + srv.mut.Lock() + defer srv.mut.Unlock() + if srv.state.GetTimestamp() < req.GetTimestamp() { + srv.state = req + } +} + +func (srv *storageSrv) WriteMulticast(_ gorums.ServerCtx, req *State) { srv.mut.Lock() defer srv.mut.Unlock() - if srv.state.Timestamp < req.Timestamp { + if srv.state.GetTimestamp() < req.GetTimestamp() { srv.state = req - fmt.Println("Got Write(", req.Value, ")") - return &WriteResponse{New: true}, nil } - return &WriteResponse{New: false}, nil +} + +func (srv *storageSrv) ReadQC(_ gorums.ServerCtx, req *emptypb.Empty) (resp *State, err error) { + srv.mut.Lock() + defer srv.mut.Unlock() + return srv.state, nil +} + +func (srv *storageSrv) ReadQCStream(_ gorums.ServerCtx, req *emptypb.Empty, send func(response *State) error) error { + srv.mut.Lock() + defer srv.mut.Unlock() + return send(srv.state) } ``` @@ -160,22 +225,23 @@ There are some important things to note about implementing the server interfaces * The handlers run in the order messages are received. * Messages from the same sender are executed in FIFO order at all servers. + See the auxiliary documentation for more information about [message ordering](./ordering.md) in Gorums. * Messages from different senders may be received in a different order at the different servers. To guarantee messages from different senders are executed in-order at the different servers, you must use a total ordering protocol. +* Errors should be returned using the [`status` package](https://pkg.go.dev/google.golang.org/grpc/status?tab=doc). * Handlers run synchronously, and hence a long-running handler will prevent other messages from being handled. To help solve this problem, our `ServerCtx` objects have a `Release()` function that releases the handler's lock on the server, which allows the next request to be processed. After `ctx.Release()` has been called, the handler may run concurrently with the handlers for the next requests. The handler automatically calls `ctx.Release()` after returning. ```go - func (srv *storageSrv) Read(ctx gorums.ServerCtx, req *ReadRequest) (resp *State), err error { + func (srv *storageSrv) ReadOrdered(ctx gorums.ServerCtx, req *emptypb.Empty) (resp *State, err error) { // any code running before this will be executed in-order ctx.Release() // after Release() has been called, a new request handler may be started, // and thus it is not guaranteed that the replies will be sent back the same order. srv.mut.Lock() defer srv.mut.Unlock() - fmt.Println("Got Read()") return srv.state, nil } ``` @@ -183,11 +249,6 @@ There are some important things to note about implementing the server interfaces * The context passed to the handlers is the gRPC stream context of the underlying gRPC stream. This context can be used to retrieve [metadata](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) and [peer](https://godoc.org/google.golang.org/grpc/peer) information from the client. -* Errors should be returned using the [`status` package](https://pkg.go.dev/google.golang.org/grpc/status?tab=doc). -* It is currently not possible to send more than one reply message per request. - This may change with other call types in the future. - -For more information about message ordering and why we use channels instead of `return` in our handlers, read [ordering.md](./ordering.md). To start the server, we need to create a *listener* and a *GorumsServer*, and then register our server implementation: @@ -214,7 +275,7 @@ Nodes are added to the connection pool via new configurations, as shown below. The manager takes as arguments a set of optional manager options. We can forward gRPC dial options to the manager if needed. The manager will use these options when connecting to nodes. -Three different options are specified in the example below. +Below we use only a simple insecure connection option. ```go package gorumsexample @@ -222,13 +283,14 @@ package gorumsexample import ( "log" + "github.com/relab/gorums" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/insecure" ) func ExampleStorageClient() { mgr := NewManager( - gorums.WithGrpcDialOptions( + gorums.WithDialOptions( grpc.WithTransportCredentials(insecure.NewCredentials()), ), ) @@ -246,7 +308,7 @@ The code below shows how to create a configuration: "127.0.0.1:8082", } // Create a configuration including all nodes - allNodesConfig, err := mgr.NewConfiguration(gorums.WithNodeList(addrs)) + allNodesConfig, err := NewConfiguration(mgr, gorums.WithNodeList(addrs)) if err != nil { log.Fatalln("error creating read config:", err) } @@ -255,142 +317,220 @@ The code below shows how to create a configuration: The `Manager` and `Configuration` types also have few other available methods. Inspect the package documentation or source code for details. -We can now invoke the Write RPC on each `node` in the configuration: +We can now invoke the WriteUnicast RPC on each `node` in the configuration: ```go - // Test state state := &State{ - Value: "42", - Timestamp: time.Now().Unix(), + Value: proto.String("42"), + Timestamp: proto.Int64(time.Now().Unix()), } - // Invoke Write RPC on all nodes in config + // Invoke WriteUnicast RPC on all nodes in config + ctx := context.Background() for _, node := range allNodesConfig.Nodes() { - reply, err := node.Write(context.Background(), state) + nodeCtx := node.Context(ctx) + err := WriteUnicast(nodeCtx, state) if err != nil { - log.Fatalln("read rpc returned error:", err) - } else if !reply.New { - log.Println("state was not new.") + log.Fatalln("write rpc returned error:", err) } } ``` -While Gorums allows us to call RPCs on individual nodes as we did above, Gorums also provides a call type *quorum call* that allows us to invoke an RPC on all nodes in a configuration with a single invocation, as we show in the next section. +While Gorums allows us to call RPCs on individual nodes as we did above, Gorums also provides call types *multicast* and *quorum call* that allow us to invoke an RPC on all nodes in a configuration with a single invocation, as we show in the next section. ## Quorum Calls -Instead of invoking an RPC explicitly on all nodes in a configuration, Gorums allows users to invoke a *quorum call* via a method on the `Configuration` type. -If an RPC is invoked as a quorum call, Gorums will invoke the RPCs on all nodes in parallel and collect and process the replies. +Instead of invoking an RPC explicitly on all nodes in a configuration, Gorums allows users to invoke a *quorum call* that sends RPCs to all nodes in parallel and collects responses. -For the Gorums plugin to generate quorum calls we need to specify the `quorumcall` option for our RPC methods in the proto file, as shown below: +Specifying the `quorumcall` option for RPC methods: ```protobuf -import "gorums.proto"; - -service QCStorage { - rpc Read(ReadRequest) returns (State) { - option (gorums.quorumcall) = true; - } - rpc Write(State) returns (WriteResponse) { - option (gorums.quorumcall) = true; - } +rpc ReadQC(google.protobuf.Empty) returns (State) { + option (gorums.quorumcall) = true; } ``` -The generated methods have the following client-side interface: +The generated code provides a function for each quorum call method: ```go -func (c *Configuration) Read(ctx context.Context, in *ReadRequest) (*State, error) -func (c *Configuration) Write(ctx context.Context, in *State) (*WriteResponse, error) +func ReadQC(ctx *gorums.ConfigContext, in *emptypb.Empty, opts ...gorums.CallOption) *gorums.Responses[*State] ``` -## The QuorumSpec Interface with Quorum Functions +This function returns a `*gorums.Responses[*State]` object that provides several ways to aggregate and process responses. + +### Terminal Methods for Response Aggregation + +The `*gorums.Responses[T]` type provides built-in *terminal methods* for common aggregation patterns: -Gorums uses a *quorum function*, that takes as input a set of replies from individual servers, and computes the reply to be returned from the quorum call. -Such a quorum function has two responsibilities: +| Method | Description | Returns When | +| --------------- | ------------------------- | ------------------------------------ | +| `.First()` | First successful response | Any node responds successfully | +| `.Majority()` | Majority of responses | ⌈(n+1)/2⌉ nodes respond successfully | +| `.All()` | All responses | All n nodes respond | +| `.Threshold(n)` | At least n responses | n nodes respond successfully | -1. Report when a set of replies form a quorum. -2. Compute a single reply from a set of replies that form a quorum. +Each terminal method blocks until the threshold is met or the context is canceled, then returns a single aggregated response and an error. -Behind the scenes, the RPCs invoked as part of a quorum call return multiple replies. -Typically, only one of these replies should be returned to the end-user. -However, Gorums cannot provide a generic policy for selecting a single reply from many replies. -Instead, Gorums makes this an application-specific choice. -For example, it may be necessary to compare the content of several reply messages when deciding which reply to return to the client's quorum call, and sometimes several replies must be combined into a new one. -Gorums, therefore, generates a `QuorumSpec` interface that contains a quorum function for every quorum call. -The `QuorumSpec` generated for our example is as follows: +### Using Terminal Methods ```go -type QuorumSpec interface { - // ReadQF is the quorum function for the Read - // quorum call method. - ReadQF(req *ReadRequest, replies map[uint32]*State) (*State, bool) +func ExampleTerminalMethods(config *Configuration) { + ctx := context.Background() + cfgCtx := config.Context(ctx) + + // Fast reads: return first successful response + reply, err := ReadQC(cfgCtx, &emptypb.Empty{}).First() + + // Crash fault tolerance: require simple majority + reply, err = ReadQC(cfgCtx, &emptypb.Empty{}).Majority() - // WriteQF is the quorum function for the Write - // quorum call method. - WriteQF(req *WriteRequest, replies map[uint32]*WriteResponse) (*WriteResponse, bool) + // Wait for all nodes (useful for debugging) + reply, err = ReadQC(cfgCtx, &emptypb.Empty{}).All() + + // Custom threshold (e.g., f+1 for crash tolerance) + f := 1 + reply, err = ReadQC(cfgCtx, &emptypb.Empty{}).Threshold(f + 1) } ``` -A `QuorumSpec` implementation must be provided when creating a new configuration. -Each quorum function of the `QuorumSpec` must adhere to the following rules: +Terminal methods return `gorums.ErrIncomplete` when: + +* The context is canceled before the threshold is met +* Not enough successful responses are received +* All nodes return errors + +### Async and Correctable Variants + +Quorum calls support asynchronous and correctable variants through additional *terminal methods*. -* If too few replies have been received, a quorum function must return `false`, signaling to Gorums that the quorum call should wait for more replies. -* Once sufficiently many replies have been received, the quorum function must return `true` along with an appropriate return value, signaling that the quorum call can return the value to the client. +**Async variants** return a future (`*Async[T]`) that can be awaited later: -The example below shows an implementation of the `QuorumSpec` interface. -Here, `ReadQF` returns the `*State` with the highest timestamp and `true`, signaling that the quorum call can return. -The quorum call will return the `*State` chosen by the quorum function. +| Method | Description | Returns | +| -------------------- | -------------------- | ----------- | +| `.AsyncFirst()` | First response async | `*Async[T]` | +| `.AsyncMajority()` | Majority async | `*Async[T]` | +| `.AsyncAll()` | All responses async | `*Async[T]` | +| `.AsyncThreshold(n)` | Threshold async | `*Async[T]` | ```go -package gorumsexample +future := ReadQC(cfgCtx, &emptypb.Empty{}).AsyncMajority() +// Do other work... +reply, err := future.Get() +``` -import "sort" +**Correctable variant** provides progressive updates as more responses arrive: -type QSpec struct { - quorumSize int -} +| Method | Description | Returns | +| ----------------- | -------------------------- | ----------------- | +| `.Correctable(n)` | Progressive updates from n | `*Correctable[T]` | + +```go +corr := ReadQCStream(cfgCtx, &emptypb.Empty{}).Correctable(2) // Initial threshold +reply, level, err := corr.Get() +<-corr.Watch(3) // Wait for higher level +reply, level, err = corr.Get() +``` + +## Iterator-Based Custom Aggregation + +For complex aggregation logic beyond the built-in terminal methods, use the iterator API provided by `responses.Seq()`. +The iterator yields responses progressively as they arrive, allowing custom decision-making logic. + +### Basic Iterator Pattern -// ReadQF is the quorum function for the Read RPC method. -func (qs *QSpec) ReadQF(_ *ReadResponse, replies map[uint32]*State) (*State, bool) { - if len(replies) < qs.quorumSize { - return nil, false +```go +func newestValue(responses *gorums.Responses[*State]) (*State, error) { + var newest *State + for resp := range responses.Seq() { + // resp.Value contains the response message (may be nil if resp.Err is set) + // resp.Err contains any error from that node (nil if successful) + // resp.NodeID contains the node identifier + + if resp.Err != nil { + continue // Skip failed responses + } + + if newest == nil || resp.Value.GetTimestamp() > newest.GetTimestamp() { + newest = resp.Value + } } - return newestState(replies), true -} -// WriteQF is the quorum function for the Write RPC method. -func (qs *QSpec) WriteQF(_ *WriteRequest, replies map[uint32]*WriteResponse) (*WriteResponse, bool) { - if len(replies) < qs.quorumSize { - return nil, false + if newest == nil { + return nil, gorums.ErrIncomplete } - // return the first response we find - var reply *WriteResponse - for _, r := range replies { - reply = r - break + return newest, nil +} + +// Usage +cfgCtx := config.Context(ctx) +reply, err := newestValue(ReadQC(cfgCtx, &ReadRequest{})) +``` + +### Iterator Helper Methods + +The `ResponseSeq[T]` type provides helper methods for common operations: + +#### `IgnoreErrors()` - Skip Failed Responses + +```go +func countSuccessful(responses *gorums.Responses[*Response]) int { + count := 0 + for resp := range responses.Seq().IgnoreErrors() { + count++ + // resp.Value is guaranteed to be non-nil + // resp.Err is guaranteed to be nil } - return reply, true + return count } +``` -func newestState(replies map[uint32]*State) *State { - var newest *State - for _, s := range replies { - if s.GetTimestamp() >= newest.GetTimestamp() { - newest = s - } +#### `Filter()` - Custom Filtering + +```go +func validResponses(responses *gorums.Responses[*Response]) []*Response { + var valid []*Response + + // Only include responses that pass validation + filtered := responses.Seq().Filter(func(nr gorums.NodeResponse[*Response]) bool { + return nr.Err == nil && isValid(nr.Value) + }) + + for resp := range filtered { + valid = append(valid, resp.Value) } - return newest + return valid } ``` -## Invoking Quorum Calls on the Configuration in the StorageClient +#### `CollectN()` and `CollectAll()` - Collect Multiple Responses -In the following code snippet, we create a configuration, including an instance of the `QSpec` defined above, and invoke a quorum call. -The quorum call will return after receiving replies from two servers. -Gorums ignores any outstanding replies. -Note that Gorums does not cancel any outstanding RPCs, leaving it to the client to manage any cancellations through the `Context` argument to the quorum call. -However, canceling RPCs to a replicated server may not be the desired behavior, since then one or more servers may not have seen previous messages. +```go +func majorityWrite(responses *gorums.Responses[*WriteResponse]) (*WriteResponse, error) { + majority := (responses.Size() + 1) / 2 + + // Collect first 'majority' successful responses + replies := responses.Seq().IgnoreErrors().CollectN(majority) + + if len(replies) < majority { + return nil, gorums.ErrIncomplete + } + + // Process the map[uint32]*WriteResponse + return aggregateWrites(replies), nil +} + +// CollectAll waits for all responses +func allResponses(responses *gorums.Responses[*Response]) map[uint32]*Response { + return responses.Seq().CollectAll() +} + +// Combine with IgnoreErrors to collect only successful responses +func allSuccessful(responses *gorums.Responses[*Response]) map[uint32]*Response { + return responses.Seq().IgnoreErrors().CollectAll() +} +``` + +### Complete Example: Storage Client with Custom Aggregation ```go func ExampleStorageClient() { @@ -400,37 +540,536 @@ func ExampleStorageClient() { "127.0.0.1:8082", } - mgr := NewManager( - gorums.WithGrpcDialOptions( + mgr := gorums.NewManager( + gorums.WithDialOptions( grpc.WithTransportCredentials(insecure.NewCredentials()), ), ) - // Create a configuration including all nodes - allNodesConfig, err := mgr.NewConfiguration( - &QSpec{2}, - gorums.WithNodeList(addrs), - ) + + // Create a configuration with all nodes + cfg, err := NewConfiguration(mgr, gorums.WithNodeList(addrs)) if err != nil { - log.Fatalln("error creating read config:", err) + log.Fatalln("error creating configuration:", err) } - // Invoke read quorum call: - ctx, cancel := context.WithCancel(context.Background()) - reply, err := allNodesConfig.Read(ctx, &ReadRequest{}) + ctx := context.Background() + cfgCtx := config.Context(ctx) + + // Option 1: Use custom aggregation function + reply, err := newestValue(ReadQC(cfgCtx, &ReadRequest{Key: "x"})) if err != nil { - log.Fatalln("read rpc returned error:", err) + log.Fatalln("read quorum call returned error:", err) } - cancel() + fmt.Printf("Read value: %v\n", reply.GetValue()) + + // Option 2: Use built-in terminal method + reply, err = ReadQC(cfgCtx, &ReadRequest{Key: "x"}).Majority() + if err != nil { + log.Fatalln("read quorum call returned error:", err) + } + fmt.Printf("Read value: %v\n", reply.GetValue()) } + +// newestValue returns the response with the most recent timestamp +func newestValue(responses *gorums.Responses[*State]) (*State, error) { + var newest *State + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + if newest == nil || resp.Value.GetTimestamp() > newest.GetTimestamp() { + newest = resp.Value + } + } + if newest == nil { + return nil, gorums.ErrIncomplete + } + return newest, nil +} +``` + +### Early Termination with Iterators + +Iterators allow early termination once a condition is met: + +```go +func fastQuorum(responses *gorums.Responses[*Response]) (*Response, error) { + const threshold = 2 + count := 0 + + for resp := range responses.Seq().IgnoreErrors() { + count++ + if count >= threshold { + return resp.Value, nil // Return immediately after threshold + } + } + + return nil, gorums.ErrIncomplete +} +``` + +## Custom Return Types + +Gorums supports custom aggregation functions that return types different from the proto response type. +This is useful when you need to aggregate multiple responses into a summary, statistics object, or any other custom type. + +### Recommended Pattern: Functions Taking `*Responses[Resp]` + +The recommended approach is to define functions that accept `*Responses[Resp]` directly. +This gives you full access to all iterator methods (`IgnoreErrors()`, `Filter()`, `CollectN()`, `CollectAll()`) and the ability to return any type. + +```go +// Custom aggregation function that returns a different type +// Input: *Responses[*MemoryStat], Output: *MemoryStatList +func CollectStats(resp *gorums.Responses[*MemoryStat]) (*MemoryStatList, error) { + replies := resp.IgnoreErrors().CollectAll() + if len(replies) == 0 { + return nil, gorums.ErrIncomplete + } + return &MemoryStatList{ + MemoryStats: slices.Collect(maps.Values(replies)), + }, nil +} + +// Usage: Call the function directly, passing the Responses object +cfgCtx := config.Context(ctx) +memStats, err := CollectStats(StopBenchmark(cfgCtx, &StopRequest{})) +``` + +### Example: Same Type Aggregation + +When the return type matches the response type, you can still use this pattern for custom quorum logic: + +```go +// Custom majority quorum with validation +func ValidatedMajority(resp *gorums.Responses[*State]) (*State, error) { + replies := resp.IgnoreErrors().CollectN(resp.Size()/2 + 1) + if len(replies) < resp.Size()/2+1 { + return nil, gorums.ErrIncomplete + } + // Return the first valid reply + for _, r := range replies { + if isValid(r) { + return r, nil + } + } + return nil, gorums.ErrIncomplete +} + +// Usage +cfgCtx := config.Context(ctx) +state, err := ValidatedMajority(ReadQC(cfgCtx, &ReadRequest{})) ``` +### Example: Custom Return Type (Slice) + +```go +// Collect all string values from responses +func CollectAllValues(resp *gorums.Responses[*StringValue]) ([]string, error) { + replies := resp.IgnoreErrors().CollectAll() + if len(replies) == 0 { + return nil, gorums.ErrIncomplete + } + result := make([]string, 0, len(replies)) + for _, v := range replies { + result = append(result, v.GetValue()) + } + return result, nil +} + +// Usage: Returns []string instead of *StringValue +values, err := CollectAllValues(GetValues(cfgCtx, &Request{})) +``` + +### Example: Computing Aggregate Statistics + +```go +// Aggregate results from multiple nodes into a summary +func AggregateResults(resp *gorums.Responses[*Result]) (*Result, error) { + replies := resp.IgnoreErrors().CollectAll() + if len(replies) == 0 { + return nil, gorums.ErrIncomplete + } + + summary := &Result{} + for _, reply := range replies { + summary.TotalOps += reply.TotalOps + summary.TotalTime += reply.TotalTime + summary.Throughput += reply.Throughput + } + + // Calculate averages + n := uint64(len(replies)) + summary.TotalOps /= n + summary.TotalTime /= int64(n) + summary.Throughput /= float64(n) + + return summary, nil +} +``` + +### Example: Returning a Primitive Type + +```go +// Count responses from specific nodes +func CountFromPrimaryNodes(resp *gorums.Responses[*Response]) (int, error) { + count := 0 + for r := range resp.IgnoreErrors().Filter(func(nr gorums.NodeResponse[*Response]) bool { + return isPrimaryNode(nr.NodeID) + }) { + count++ + } + if count == 0 { + return 0, gorums.ErrIncomplete + } + return count, nil +} +``` + +### Example: Explicit Error Handling + +When you need to handle errors from individual nodes explicitly: + +```go +// Require all nodes to succeed +func RequireAllSuccess(resp *gorums.Responses[*Response]) (*Response, error) { + var first *Response + for r := range resp.Seq() { + if r.Err != nil { + return nil, r.Err // Fail fast on any error + } + if first == nil { + first = r.Value + } + } + if first == nil { + return nil, gorums.ErrIncomplete + } + return first, nil +} +``` + +## Interceptors for Request/Response Transformation + +Gorums provides interceptors to transform requests and responses on a per-node basis. +Interceptors are passed as call options and can be chained together. + +### MapRequest Interceptor + +Transform requests before sending to each node: + +```go +cfgCtx := config.Context(ctx) +resp, err := WriteQC(cfgCtx, req, + gorums.Interceptors( + gorums.MapRequest(func(req *WriteRequest, node *gorums.Node) *WriteRequest { + // Customize request for each node + return &WriteRequest{Value: fmt.Sprintf("%s-node-%d", req.Value, node.ID())} + }), + ), +).Majority() +``` + +### MapResponse Interceptor + +Transform responses received from each node: + +```go +resp, err := ReadQC(cfgCtx, req, + gorums.Interceptors( + gorums.MapResponse(func(resp *ReadResponse, node *gorums.Node) *ReadResponse { + // Transform response, e.g., add node ID + resp.NodeID = node.ID() + return resp + }), + ), +).Majority() +``` + +### Multicast/Unicast with MapRequest + +```go +// Send different messages to each node in a multicast +cfgCtx := config.Context(ctx) +WriteMulticast(cfgCtx, &State{}, + gorums.Interceptors( + gorums.MapRequest(func(msg *State, node *gorums.Node) *State { + return &State{Value: proto.String(fmt.Sprintf("node-%d", node.ID()))} + }), + ), +) +``` + +**Note:** If `MapRequest` returns `nil` for a node, the message will not be sent to that node. + +### Custom Interceptors + +Beyond the built-in `MapRequest` and `MapResponse` interceptors, you can create custom interceptors for logging, filtering, or other cross-cutting concerns. + +A custom interceptor has the signature: + +```go +func(ctx *gorums.ClientCtx[Req, Resp], next gorums.ResponseSeq[Resp]) gorums.ResponseSeq[Resp] +``` + +The interceptor receives: + +* `ctx` - the `ClientCtx` providing access to: + * `.Request()` - the original request + * `.Config()` - the configuration being used + * `.Method()` - the RPC method name + * `.Nodes()` - all nodes in the configuration + * `.Node(id)` - get a specific node by ID + * `.Size()` - number of nodes in the configuration +* `next` - the response iterator from the previous interceptor (or the default iterator) + +The interceptor returns a new `ResponseSeq` that wraps `next` with custom logic. + +#### Chaining Interceptors + +Multiple interceptors can be passed to `gorums.Interceptors()` and are executed in order: + +```go +cfgCtx := config.Context(ctx) +resp, err := ReadQC(cfgCtx, req, + gorums.Interceptors( + loggingInterceptor, + gorums.MapRequest(transformFunc), + filterInterceptor, + ), +).Majority() +``` + +#### Example: Logging Interceptor + +Create a logging interceptor that wraps the response iterator: + +```go +func LoggingInterceptor[Req, Resp proto.Message]( + ctx *gorums.ClientCtx[Req, Resp], + next gorums.ResponseSeq[Resp], +) gorums.ResponseSeq[Resp] { + startTime := time.Now() + log.Printf("[%s] Starting quorum call with request: %v", ctx.Method(), ctx.Request()) + + // Wrap the response iterator to log each response + return func(yield func(gorums.NodeResponse[Resp]) bool) { + count := 0 + for resp := range next { + count++ + if resp.Err != nil { + log.Printf("[%s] Node %d error: %v", ctx.Method(), resp.NodeID, resp.Err) + } else { + log.Printf("[%s] Node %d response received", ctx.Method(), resp.NodeID) + } + if !yield(resp) { + log.Printf("[%s] Iteration stopped after %d responses", ctx.Method(), count) + return + } + } + log.Printf("[%s] Completed: %d responses in %v", ctx.Method(), count, time.Since(startTime)) + } +} + +// Usage +resp, err := ReadQC(cfgCtx, req, + gorums.Interceptors(LoggingInterceptor[*ReadRequest, *ReadResponse]), +).Majority() +``` + +#### Example: Response Filtering Interceptor + +Filter out responses that don't meet certain criteria: + +```go +func FilterInterceptor[Req, Resp proto.Message]( + shouldInclude func(Resp) bool, +) gorums.QuorumInterceptor[Req, Resp] { + return func(ctx *gorums.ClientCtx[Req, Resp], next gorums.ResponseSeq[Resp]) gorums.ResponseSeq[Resp] { + return func(yield func(gorums.NodeResponse[Resp]) bool) { + for resp := range next { + // Skip responses that don't pass the filter + if resp.Err == nil && !shouldInclude(resp.Value) { + continue + } + if !yield(resp) { + return + } + } + } + } +} + +// Usage: only include responses with timestamp > threshold +threshold := time.Now().Add(-1 * time.Hour) +resp, err := ReadQC(cfgCtx, req, + gorums.Interceptors( + FilterInterceptor[*ReadRequest, *ReadResponse](func(r *ReadResponse) bool { + return r.GetTimestamp().AsTime().After(threshold) + }), + ), +).Majority() +``` + +#### Example: Counting Interceptor + +Count responses passing through the interceptor: + +```go +func CountingInterceptor[Req, Resp proto.Message]( + counter *int, +) gorums.QuorumInterceptor[Req, Resp] { + return func(_ *gorums.ClientCtx[Req, Resp], next gorums.ResponseSeq[Resp]) gorums.ResponseSeq[Resp] { + return func(yield func(gorums.NodeResponse[Resp]) bool) { + for resp := range next { + *counter++ + if !yield(resp) { + return + } + } + } + } +} +``` + +**Note:** Custom interceptors can be defined in any package. The `ClientCtx` type and `QuorumInterceptor` signature are exported from the gorums package. + +## Error Handling + +Gorums provides structured error types to help you understand and handle failures in quorum calls. +The primary error type is `QuorumCallError`, which contains detailed information about which nodes failed and why. + +### QuorumCallError + +A `QuorumCallError` is returned when a quorum call fails. +It provides the following methods: + +* **`Cause() error`** - Returns the underlying cause of the failure (e.g., `ErrIncomplete`, `ErrSendFailure`) +* **`NodeErrors() int`** - Returns the number of nodes that failed +* **`Unwrap() []error`** - Supports error unwrapping for use with `errors.Is` and `errors.As` + +The error implements Go's standard error unwrapping interface, allowing `errors.Is()` and `errors.As()` to check both the direct cause and any wrapped node-specific errors. + +#### Common Error Causes + +Gorums defines several sentinel errors that commonly appear as the cause of a `QuorumCallError`: + +* **`ErrIncomplete`** - The call could not be completed due to insufficient non-error replies to form a quorum +* **`ErrSendFailure`** - Message sending failed for one or more nodes + +### Error Handling Example + +Here's how to properly handle errors from a quorum call: + +```go +func handleQuorumCall(cfg *gorums.Configuration, req *ReadRequest) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cfgCtx := config.Context(ctx) + reply, err := ReadQC(cfgCtx, req).Majority() + + if err != nil { + // Check if it's a QuorumCallError + var qcErr gorums.QuorumCallError + if errors.As(err, &qcErr) { + log.Printf("Quorum call failed: %v", qcErr.Cause()) + log.Printf("Failed nodes: %d", qcErr.NodeErrors()) + + // Handle specific cause types + if errors.Is(err, gorums.ErrIncomplete) { + // Not enough replies to form a quorum + log.Println("Insufficient responses to reach quorum") + } + } + + // Check for context errors + if errors.Is(err, context.DeadlineExceeded) { + log.Println("Quorum call timed out") + } else if errors.Is(err, context.Canceled) { + log.Println("Quorum call was canceled") + } + return + } + + // Process successful reply + log.Printf("Read successful: %v", reply) +} +``` + +### Checking for Specific Error Types + +Use `errors.Is()` and `errors.As()` to check for specific error types, including both the direct cause and any node-specific errors: + +```go +// Check if the error is caused by insufficient responses +if errors.Is(err, gorums.ErrIncomplete) { + // Handle incomplete quorum +} + +// Check if any node returned a specific gRPC error +if errors.Is(err, status.Error(codes.Unavailable, "")) { + // Handle unavailable nodes +} + +// Extract custom error types from node failures +var customErr MyCustomError +if errors.As(err, &customErr) { + // Handle custom error from a node +} +``` + +## Handling Failed Nodes in Configurations + +When a quorum call fails, you can create a new configuration that excludes the failed nodes. +The `WithoutErrors` method allows you to filter nodes based on the errors they returned: + +```go +import ( + "context" + "errors" + "io" + + "github.com/relab/gorums" +) + +// Invoke a quorum call +cfgCtx := config.Context(ctx) +state, err := ReadQC(cfgCtx, &ReadRequest{}).Majority() +if err != nil { + var qcErr gorums.QuorumCallError + if errors.As(err, &qcErr) { + // Option 1: Exclude all failed nodes + newConfig, err := NewConfiguration(mgr, config.WithoutErrors(qcErr)) + + // Option 2: Exclude only nodes with specific error types + // For example, exclude only nodes that timed out + newConfig, err := NewConfiguration(mgr, config.WithoutErrors(qcErr, context.DeadlineExceeded)) + + // Option 3: Exclude nodes with multiple specific error types + newConfig, err := NewConfiguration(mgr, config.WithoutErrors(qcErr, + context.DeadlineExceeded, + context.Canceled, + io.EOF, + ), + ) + + // Retry the operation with the new configuration + newCfgCtx := newConfig.Context(ctx) + state, err = ReadQC(newCfgCtx, &ReadRequest{}).Majority() + } +} +``` + +The error type matching uses `errors.Is`, which properly handles wrapped errors. +This allows you to filter nodes based on the underlying cause of their failures, enabling fine-grained control over which nodes to exclude when creating new configurations. + ## Working with Configurations Below is an example demonstrating how to work with configurations. These configurations are viewed from the client's perspective, and to actually make quorum calls on these configurations, there must be server endpoints to connect to. -We ignore the construction of `mgr` and error handling. +We ignore the construction of `mgr` and error handling (except for the last configuration). -Depending on the application's requirements, the `QSpec` argument may depend on the resulting configuration's size. In the example below, we simply use fixed quorum sizes. ```go @@ -441,8 +1080,7 @@ func ExampleConfigClient() { "127.0.0.1:8082", } // Make configuration c1 from addrs, giving |c1| = |addrs| = 3 - c1, _ := mgr.NewConfiguration( - &QSpec{2}, + c1, _ := NewConfiguration(mgr, gorums.WithNodeList(addrs), ) @@ -451,33 +1089,47 @@ func ExampleConfigClient() { "127.0.0.1:9081", } // Make configuration c2 from newAddrs, giving |c2| = |newAddrs| = 2 - c2, _ := mgr.NewConfiguration( - &QSpec{1}, + c2, _ := NewConfiguration(mgr, gorums.WithNodeList(newAddrs), ) // Make new configuration c3 from c1 and newAddrs, giving |c3| = |c1| + |newAddrs| = 3+2=5 - c3, _ := mgr.NewConfiguration( - &QSpec{3}, + c3, _ := NewConfiguration(mgr, c1.WithNewNodes(gorums.WithNodeList(newAddrs)), ) // Make new configuration c4 from c1 and c2, giving |c4| = |c1| + |c2| = 3+2=5 - c4, _ := mgr.NewConfiguration( - &QSpec{3}, + c4, _ := NewConfiguration(mgr, c1.And(c2), ) // Make new configuration c5 from c1 except the first node from c1, giving |c5| = |c1| - 1 = 3-1 = 2 - c5, _ := mgr.NewConfiguration( - &QSpec{1}, + c5, _ := NewConfiguration(mgr, c1.WithoutNodes(c1.NodeIDs()[0]), ) // Make new configuration c6 from c3 except c1, giving |c6| = |c3| - |c1| = 5-3 = 2 - c6, _ := mgr.NewConfiguration( - &QSpec{1}, + c6, _ := NewConfiguration(mgr, c3.Except(c1), ) + + // Example: Handling quorum call failures and creating a new configuration + // without failed nodes + cfgCtx := c1.Context(ctx) + state, err := ReadQC(cfgCtx, &ReadRequest{}).Majority() + if err != nil { + var qcErr gorums.QuorumCallError + if errors.As(err, &qcErr) { + // Create a new configuration excluding all nodes that failed + c7, _ := NewConfiguration(mgr, + c1.WithoutErrors(qcErr), + ) + + // Or exclude only nodes with specific error types (e.g., timeout errors) + c8, _ := NewConfiguration(mgr, + c1.WithoutErrors(qcErr, context.DeadlineExceeded), + ) + } + } } ``` diff --git a/encoding.go b/encoding.go index 0ed8ca1ad..39e8b5e59 100644 --- a/encoding.go +++ b/encoding.go @@ -5,6 +5,7 @@ import ( "github.com/relab/gorums/ordering" "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" @@ -12,6 +13,10 @@ import ( "google.golang.org/protobuf/reflect/protoregistry" ) +func init() { + encoding.RegisterCodec(NewCodec()) +} + // ContentSubtype is the subtype used by gorums when sending messages via gRPC. const ContentSubtype = "gorums" @@ -93,7 +98,7 @@ func (m *Message) GetMessageID() uint64 { if m == nil { return 0 } - return m.metadata.GetMessageID() + return m.metadata.GetMessageSeqNo() } func (m *Message) GetStatus() *status.Status { diff --git a/errors.go b/errors.go index 2ac77f9b4..b22018584 100644 --- a/errors.go +++ b/errors.go @@ -6,15 +6,32 @@ import ( "strings" ) -// Incomplete is the error returned by a quorum call when the call cannot completed -// due insufficient non-error replies to form a quorum according to the quorum function. -var Incomplete = errors.New("incomplete call") +// ErrIncomplete is the error returned by a quorum call when the call cannot be completed +// due to insufficient non-error replies to form a quorum according to the quorum function. +var ErrIncomplete = errors.New("incomplete call") + +// ErrSendFailure is the error returned by a multicast call when message sending fails for one or more nodes. +var ErrSendFailure = errors.New("send failure") + +// ErrTypeMismatch is returned when a response cannot be cast to the expected type. +var ErrTypeMismatch = errors.New("response type mismatch") // QuorumCallError reports on a failed quorum call. +// It provides detailed information about which nodes failed. type QuorumCallError struct { - cause error - errors []nodeError - replies int + cause error + errors []nodeError +} + +// Cause returns the underlying cause of the quorum call failure. +// Common causes include ErrIncomplete and ErrSendFailure. +func (e QuorumCallError) Cause() error { + return e.cause +} + +// NodeErrors returns the number of nodes that failed during the quorum call. +func (e QuorumCallError) NodeErrors() int { + return len(e.errors) } // Is reports whether the target error is the same as the cause of the QuorumCallError. @@ -25,8 +42,17 @@ func (e QuorumCallError) Is(target error) bool { return e.cause == target } +// Unwrap returns all the underlying node errors as a slice. +// This allows the error to work with errors.Is and errors.As for any wrapped errors. +func (e QuorumCallError) Unwrap() (errs []error) { + for _, ne := range e.errors { + errs = append(errs, ne.cause) + } + return errs +} + func (e QuorumCallError) Error() string { - s := fmt.Sprintf("quorum call error: %s (errors: %d, replies: %d)", e.cause, len(e.errors), e.replies) + s := fmt.Sprintf("quorum call error: %s (errors: %d)", e.cause, len(e.errors)) var b strings.Builder b.WriteString(s) if len(e.errors) == 0 { @@ -41,7 +67,7 @@ func (e QuorumCallError) Error() string { return b.String() } -// nodeError reports on a failed RPC call. +// nodeError reports on a failed RPC call from a specific node. type nodeError struct { cause error nodeID uint32 diff --git a/errors_test.go b/errors_test.go index 8dceb6dd1..5b5f37656 100644 --- a/errors_test.go +++ b/errors_test.go @@ -4,6 +4,9 @@ import ( "context" "errors" "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestQuorumCallErrorIs(t *testing.T) { @@ -15,25 +18,25 @@ func TestQuorumCallErrorIs(t *testing.T) { }{ { name: "SameCauseError", - err: QuorumCallError{cause: Incomplete}, - target: Incomplete, + err: QuorumCallError{cause: ErrIncomplete}, + target: ErrIncomplete, want: true, }, { name: "SameCauseQCError", - err: QuorumCallError{cause: Incomplete}, - target: QuorumCallError{cause: Incomplete}, + err: QuorumCallError{cause: ErrIncomplete}, + target: QuorumCallError{cause: ErrIncomplete}, want: true, }, { name: "DifferentError", - err: QuorumCallError{cause: Incomplete}, + err: QuorumCallError{cause: ErrIncomplete}, target: errors.New("incomplete call"), want: false, }, { name: "DifferentQCError", - err: QuorumCallError{cause: Incomplete}, + err: QuorumCallError{cause: ErrIncomplete}, target: QuorumCallError{cause: errors.New("incomplete call")}, want: false, }, @@ -64,3 +67,148 @@ func TestQuorumCallErrorIs(t *testing.T) { }) } } + +func TestQuorumCallErrorAccessors(t *testing.T) { + tests := []struct { + name string + qcErr QuorumCallError + wantCause error + wantNodeErrors int + }{ + { + name: "NoErrors", + qcErr: QuorumCallError{ + cause: ErrIncomplete, + errors: nil, + }, + wantCause: ErrIncomplete, + wantNodeErrors: 0, + }, + { + name: "SingleError", + qcErr: QuorumCallError{ + cause: ErrIncomplete, + errors: []nodeError{ + {nodeID: 1, cause: status.Error(codes.Unavailable, "node down")}, + }, + }, + wantCause: ErrIncomplete, + wantNodeErrors: 1, + }, + { + name: "MultipleErrors", + qcErr: QuorumCallError{ + cause: ErrIncomplete, + errors: []nodeError{ + {nodeID: 1, cause: status.Error(codes.Unavailable, "node down")}, + {nodeID: 3, cause: status.Error(codes.DeadlineExceeded, "timeout")}, + {nodeID: 5, cause: status.Error(codes.Unavailable, "connection refused")}, + }, + }, + wantCause: ErrIncomplete, + wantNodeErrors: 3, + }, + { + name: "SendFailure", + qcErr: QuorumCallError{ + cause: ErrSendFailure, + errors: []nodeError{ + {nodeID: 2, cause: errors.New("send failed")}, + }, + }, + wantCause: ErrSendFailure, + wantNodeErrors: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.qcErr.Cause(); got != tt.wantCause { + t.Errorf("QuorumCallError.Cause() = %v, want %v", got, tt.wantCause) + } + if got := tt.qcErr.NodeErrors(); got != tt.wantNodeErrors { + t.Errorf("QuorumCallError.NodeErrors() = %d, want %d", got, tt.wantNodeErrors) + } + }) + } +} + +func TestQuorumCallErrorUnwrap(t *testing.T) { + unavailableErr := status.Error(codes.Unavailable, "node down") + timeoutErr := status.Error(codes.DeadlineExceeded, "timeout") + connectionErr := errors.New("connection refused") + + qcErr := QuorumCallError{ + cause: ErrIncomplete, + errors: []nodeError{ + {nodeID: 1, cause: unavailableErr}, + {nodeID: 3, cause: timeoutErr}, + {nodeID: 5, cause: connectionErr}, + }, + } + + // Test Unwrap returns all node error causes + unwrapped := qcErr.Unwrap() + if len(unwrapped) != 3 { + t.Fatalf("Unwrap() returned %d errors, want 3", len(unwrapped)) + } + + // Verify the unwrapped errors are the node error causes + if unwrapped[0] != unavailableErr { + t.Errorf("Unwrap()[0] = %v, want %v", unwrapped[0], unavailableErr) + } + if unwrapped[1] != timeoutErr { + t.Errorf("Unwrap()[1] = %v, want %v", unwrapped[1], timeoutErr) + } + if unwrapped[2] != connectionErr { + t.Errorf("Unwrap()[2] = %v, want %v", unwrapped[2], connectionErr) + } + + // Test errors.Is with the cause (handled by Is() method) + if !errors.Is(qcErr, ErrIncomplete) { + t.Error("errors.Is(qcErr, ErrIncomplete) = false, want true") + } + + // Test errors.Is with wrapped node errors (handled by Unwrap() method) + if !errors.Is(qcErr, unavailableErr) { + t.Error("errors.Is(qcErr, unavailableErr) = false, want true") + } + if !errors.Is(qcErr, timeoutErr) { + t.Error("errors.Is(qcErr, timeoutErr) = false, want true") + } + if !errors.Is(qcErr, connectionErr) { + t.Error("errors.Is(qcErr, connectionErr) = false, want true") + } + + // Test errors.Is with unrelated error + if errors.Is(qcErr, ErrSendFailure) { + t.Error("errors.Is(qcErr, ErrSendFailure) = true, want false") + } +} + +// customError is a custom error type for testing errors.As +type customError struct { + msg string +} + +func (e customError) Error() string { return e.msg } + +func TestQuorumCallErrorUnwrapWithAs(t *testing.T) { + customErr := customError{msg: "custom node error"} + qcErr := QuorumCallError{ + cause: ErrIncomplete, + errors: []nodeError{ + {nodeID: 1, cause: customErr}, + {nodeID: 2, cause: status.Error(codes.Unavailable, "down")}, + }, + } + + // Test errors.As can find the custom error in wrapped errors + var target customError + if !errors.As(qcErr, &target) { + t.Fatal("errors.As(qcErr, &customError) = false, want true") + } + if target.msg != "custom node error" { + t.Errorf("extracted customError.msg = %q, want %q", target.msg, "custom node error") + } +} diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index 333fe55de..000000000 --- a/examples/.gitignore +++ /dev/null @@ -1 +0,0 @@ -storage/storage diff --git a/examples/Makefile b/examples/Makefile index 3cb41db9e..a27c33520 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -4,17 +4,22 @@ proto_go := $(proto_src:%.proto=%.pb.go) gorums_go := $(proto_src:%.proto=%_gorums.pb.go) binaries := storage/storage -.PHONY: all debug protos download tools $(binaries) +.PHONY: all debug proto download tools $(binaries) all: $(binaries) +# Build binaries in debug mode: Tell the Go compiler to compile without +# optimizations (-N) or inlining (-l) so that debuggers, stack traces, +# and variable inspection behave exactly like the source code. debug: GCFLAGS += -gcflags='all=-N -l' debug: $(binaries) -$(binaries): protos +# Build binaries. +$(binaries): proto @go build -o ./$@ $(GCFLAGS) ./$(dir $@) -protos: tools download $(proto_go) $(gorums_go) +# Generate Go files from proto files. +proto: tools download $(proto_go) $(gorums_go) download: @go mod download diff --git a/examples/go.mod b/examples/go.mod index ece4467d4..ea1e71cdb 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,21 +1,22 @@ module github.com/relab/gorums/examples -go 1.25.4 +go 1.25.5 require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/relab/gorums v0.10.0 - golang.org/x/term v0.37.0 - google.golang.org/grpc v1.76.0 - google.golang.org/protobuf v1.36.10 + golang.org/x/term v0.38.0 + google.golang.org/grpc v1.77.0 + google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect - google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect + go.uber.org/goleak v1.3.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 // indirect ) replace github.com/relab/gorums => ../ diff --git a/examples/go.sum b/examples/go.sum index ee664b275..0d85c8621 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -1,3 +1,5 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -10,33 +12,41 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1/go.mod h1:5KF+wpkbTSbGcR9zteSqZV6fqFOWBl4Yde8En8MryZA= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 h1:6Al3kEFFP9VJhRz3DID6quisgPnTeZVr4lep9kkxdPA= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0/go.mod h1:QLvsjh0OIR0TYBeiu2bkWGTJBUNQ64st52iWj/yA93I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/gorumsexample/storage.proto b/examples/gorumsexample/storage.proto new file mode 100644 index 000000000..9a4015cff --- /dev/null +++ b/examples/gorumsexample/storage.proto @@ -0,0 +1,46 @@ +edition = "2024"; + +package gorumsexample; +option go_package = "github.com/relab/gorums/examples/gorumsexample"; +option features.field_presence = IMPLICIT; + +import "google/protobuf/empty.proto"; +import "gorums.proto"; + +// Storage service defines the RPCs for a simple key-value storage system. +service Storage { + // ReadOrdered is a FIFO-ordered RPC to a single node. + rpc ReadOrdered(google.protobuf.Empty) returns (State) {} + + // WriteUnicast is an asynchronous unicast to a single node. + // No reply is collected. + rpc WriteUnicast(State) returns (google.protobuf.Empty) { + option (gorums.unicast) = true; + } + + // WriteMulticast is an asynchronous multicast to all nodes in a configuration. + // No replies are collected. + rpc WriteMulticast(State) returns (google.protobuf.Empty) { + option (gorums.multicast) = true; + } + + // ReadQC is a FIFO-ordered synchronous quorum call. + // Use terminal methods (.Majority(), .First(), etc.) to retrieve results. + // Use .AsyncMajority() for async variant, .Correctable(n) for correctable variant. + rpc ReadQC(google.protobuf.Empty) returns (State) { + option (gorums.quorumcall) = true; + } + + // ReadQCStream is a FIFO-ordered synchronous quorum call with server streaming. + // The stream keyword enables correctable calls where nodes can send multiple + // progressive updates. Use .Correctable(n) to watch for updates as they arrive. + rpc ReadQCStream(google.protobuf.Empty) returns (stream State) { + option (gorums.quorumcall) = true; + } +} + +// State represents the value and its timestamp stored in a node. +message State { + string value = 1; + int64 timestamp = 2; +} diff --git a/examples/storage/client.go b/examples/storage/client.go index cde90ebfe..cc8686519 100644 --- a/examples/storage/client.go +++ b/examples/storage/client.go @@ -15,13 +15,13 @@ func runClient(addresses []string) error { } // init gorums manager - mgr := proto.NewManager( - gorums.WithGrpcDialOptions( + mgr := gorums.NewManager( + gorums.WithDialOptions( grpc.WithTransportCredentials(insecure.NewCredentials()), // disable TLS ), ) // create configuration containing all nodes - cfg, err := mgr.NewConfiguration(&qspec{cfgSize: len(addresses)}, gorums.WithNodeList(addresses)) + cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeList(addresses)) if err != nil { log.Fatal(err) } @@ -29,63 +29,40 @@ func runClient(addresses []string) error { return Repl(mgr, cfg) } -type qspec struct { - cfgSize int -} - -// ReadQCQF is the quorum function for the ReadQC -// ordered quorum call method. The in parameter is the request object -// supplied to the ReadQC method at call time, and may or may not -// be used by the quorum function. If the in parameter is not needed -// you should implement your quorum function with '_ *ReadRequest'. -func (q qspec) ReadQCQF(_ *proto.ReadRequest, replies map[uint32]*proto.ReadResponse) (*proto.ReadResponse, bool) { - // wait until at least half of the replicas have responded - if len(replies) <= q.cfgSize/2 { - return nil, false - } - // return the value with the most recent timestamp - return newestValue(replies), true -} - -// WriteQCQF is the quorum function for the WriteQC -// ordered quorum call method. The in parameter is the request object -// supplied to the WriteQC method at call time, and may or may not -// be used by the quorum function. If the in parameter is not needed -// you should implement your quorum function with '_ *WriteRequest'. -func (q qspec) WriteQCQF(in *proto.WriteRequest, replies map[uint32]*proto.WriteResponse) (*proto.WriteResponse, bool) { - // wait until at least half of the replicas have responded and have updated their value - if numUpdated(replies) > q.cfgSize/2 { - return proto.WriteResponse_builder{New: true}.Build(), true - } - // if all replicas have responded, there must have been another write before ours - // that had a newer timestamp - if len(replies) == q.cfgSize { - return proto.WriteResponse_builder{New: false}.Build(), true - } - return nil, false -} - -// newestValue returns the reply that had the most recent timestamp -func newestValue(values map[uint32]*proto.ReadResponse) *proto.ReadResponse { - if len(values) < 1 { - return nil - } +// newestValue processes responses from a ReadQC call and returns the reply +// with the most recent timestamp. +func newestValue(responses *gorums.Responses[*proto.ReadResponse]) (*proto.ReadResponse, error) { var newest *proto.ReadResponse - for _, v := range values { - if v.GetTime().AsTime().After(newest.GetTime().AsTime()) { - newest = v + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + if newest == nil || resp.Value.GetTime().AsTime().After(newest.GetTime().AsTime()) { + newest = resp.Value } } - return newest + if newest == nil { + return nil, gorums.ErrIncomplete + } + return newest, nil } -// numUpdated returns the number of replicas that updated their value -func numUpdated(replies map[uint32]*proto.WriteResponse) int { - count := 0 - for _, r := range replies { - if r.GetNew() { +// numUpdated processes responses from a WriteQC call and returns true if +// a majority of nodes updated their value. +func numUpdated(responses *gorums.Responses[*proto.WriteResponse]) (*proto.WriteResponse, error) { + var count int + size := responses.Size() + for resp := range responses.Seq() { + if resp.Err != nil { + continue + } + if resp.Value.GetNew() { count++ } } - return count + // We need a majority of nodes to have updated the value + if count > size/2 { + return proto.WriteResponse_builder{New: true}.Build(), nil + } + return proto.WriteResponse_builder{New: false}.Build(), nil } diff --git a/examples/storage/main.go b/examples/storage/main.go index 5726b53ac..c881b88e1 100644 --- a/examples/storage/main.go +++ b/examples/storage/main.go @@ -24,7 +24,7 @@ func main() { if len(addrs) == 1 && addrs[0] == "" { addrs = nil srvs := make([]*gorums.Server, 0, 4) - for i := 0; i < 4; i++ { + for range 4 { srv, addr := startServer("127.0.0.1:0") srvs = append(srvs, srv) addrs = append(addrs, addr) diff --git a/examples/storage/proto/storage.pb.go b/examples/storage/proto/storage.pb.go index 0527fcce5..e4d5f7f90 100644 --- a/examples/storage/proto/storage.pb.go +++ b/examples/storage/proto/storage.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: storage/proto/storage.proto package proto diff --git a/examples/storage/proto/storage.proto b/examples/storage/proto/storage.proto index 86372f294..16e2c5bd4 100644 --- a/examples/storage/proto/storage.proto +++ b/examples/storage/proto/storage.proto @@ -1,52 +1,63 @@ -edition = "2023"; - -package storage; +edition = "2024"; +package proto; option go_package = "github.com/relab/gorums/examples/storage/proto"; - option features.field_presence = IMPLICIT; -import "gorums.proto"; -import "google/protobuf/timestamp.proto"; import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +import "gorums.proto"; +// Storage is a simple key-value storage service service Storage { - // ReadRPC executes the Read RPC on a single Node - rpc ReadRPC(ReadRequest) returns (ReadResponse) {} - // WriteRPC executes the Write RPC on a single Node - rpc WriteRPC(WriteRequest) returns (WriteResponse) {} - // ReadQC executes the Read Quorum Call on a configuration - // of Nodes and returns the most recent value. - rpc ReadQC(ReadRequest) returns (ReadResponse) { - option (gorums.quorumcall) = true; - } - // WriteQC executes the Write Quorum Call on a configuration - // of Nodes and returns true if a majority of Nodes were updated. - rpc WriteQC(WriteRequest) returns (WriteResponse) { - option (gorums.quorumcall) = true; - } - - rpc WriteMulticast(WriteRequest) returns (google.protobuf.Empty) { - option (gorums.multicast) = true; - } + // ReadRPC executes a Read RPC on a single node and + // returns the value for the provided key. + rpc ReadRPC(ReadRequest) returns (ReadResponse) {} + + // WriteRPC executes a Write RPC on a single node and + // returns true if the value was updated. + rpc WriteRPC(WriteRequest) returns (WriteResponse) {} + + // ReadQC executes a Read quorum call on a configuration of nodes and + // returns the most recent value. + rpc ReadQC(ReadRequest) returns (ReadResponse) { + option (gorums.quorumcall) = true; + } + + // WriteQC executes a Write quorum call on a configuration of nodes and + // returns true if a majority of nodes were updated. + rpc WriteQC(WriteRequest) returns (WriteResponse) { + option (gorums.quorumcall) = true; + } + + // WriteMulticast executes a Write multicast call on a configuration of nodes. + // It does not wait for any responses. + rpc WriteMulticast(WriteRequest) returns (google.protobuf.Empty) { + option (gorums.multicast) = true; + } } +// ReadRequest is the request message for Read RPCs and Read quorum calls. message ReadRequest { - string Key = 1; + string key = 1; } +// ReadResponse is the response message for Read RPCs and Read quorum calls. message ReadResponse { - bool OK = 1; - string Value = 2; - google.protobuf.Timestamp Time = 3; + bool ok = 1; + string value = 2; + google.protobuf.Timestamp time = 3; } +// WriteRequest is the request message for Write RPCs, Write quorum calls, and Write multicast calls. message WriteRequest { - string Key = 1; - string Value = 2; - google.protobuf.Timestamp Time = 3; + string key = 1; + string value = 2; + google.protobuf.Timestamp time = 3; } +// WriteResponse is the response message for Write RPCs and Write quorum calls. message WriteResponse { - bool New = 1; + bool new = 1; } diff --git a/examples/storage/proto/storage_gorums.pb.go b/examples/storage/proto/storage_gorums.pb.go index a44212f9f..fa46f9970 100644 --- a/examples/storage/proto/storage_gorums.pb.go +++ b/examples/storage/proto/storage_gorums.pb.go @@ -1,17 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: storage/proto/storage.proto package proto import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" - proto "google.golang.org/protobuf/proto" emptypb "google.golang.org/protobuf/types/known/emptypb" ) @@ -22,256 +18,112 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// StorageClient is the client interface for the Storage service. -type StorageClient interface { - ReadQC(ctx context.Context, in *ReadRequest) (resp *ReadResponse, err error) - WriteQC(ctx context.Context, in *WriteRequest) (resp *WriteResponse, err error) - WriteMulticast(ctx context.Context, in *WriteRequest, opts ...gorums.CallOption) -} +// AsyncReadResponse is a future for async quorum calls returning *ReadResponse. +type AsyncReadResponse = *gorums.Async[*ReadResponse] -// enforce interface compliance -var _ StorageClient = (*Configuration)(nil) +// AsyncWriteResponse is a future for async quorum calls returning *WriteResponse. +type AsyncWriteResponse = *gorums.Async[*WriteResponse] -// StorageNodeClient is the single node client interface for the Storage service. -type StorageNodeClient interface { - ReadRPC(ctx context.Context, in *ReadRequest) (resp *ReadResponse, err error) - WriteRPC(ctx context.Context, in *WriteRequest) (resp *WriteResponse, err error) -} +// CorrectableReadResponse is a correctable object for quorum calls returning *ReadResponse. +type CorrectableReadResponse = *gorums.Correctable[*ReadResponse] -// enforce interface compliance -var _ StorageNodeClient = (*Node)(nil) +// CorrectableWriteResponse is a correctable object for quorum calls returning *WriteResponse. +type CorrectableWriteResponse = *gorums.Correctable[*WriteResponse] // Reference imports to suppress errors if they are not otherwise used. var _ emptypb.Empty -// WriteMulticast is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) WriteMulticast(ctx context.Context, in *WriteRequest, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "storage.Storage.WriteMulticast", - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) -} - -// QuorumSpec is the interface of quorum functions for Storage. -type QuorumSpec interface { - gorums.ConfigOption - - // ReadQCQF is the quorum function for the ReadQC - // quorum call method. The in parameter is the request object - // supplied to the ReadQC method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *ReadRequest'. - ReadQCQF(in *ReadRequest, replies map[uint32]*ReadResponse) (*ReadResponse, bool) - - // WriteQCQF is the quorum function for the WriteQC - // quorum call method. The in parameter is the request object - // supplied to the WriteQC method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *WriteRequest'. - WriteQCQF(in *WriteRequest, replies map[uint32]*WriteResponse) (*WriteResponse, bool) -} - -// ReadQC executes the Read Quorum Call on a configuration -// of Nodes and returns the most recent value. -func (c *Configuration) ReadQC(ctx context.Context, in *ReadRequest) (resp *ReadResponse, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "storage.Storage.ReadQC", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*ReadResponse, len(replies)) - for k, v := range replies { - r[k] = v.(*ReadResponse) - } - return c.qspec.ReadQCQF(req.(*ReadRequest), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) +// ReadRPC executes the Read RPC on a single Node +func ReadRPC(ctx *gorums.NodeContext, in *ReadRequest) (resp *ReadResponse, err error) { + res, err := gorums.RPCCall(ctx, in, "storage.Storage.ReadRPC") if err != nil { return nil, err } return res.(*ReadResponse), err } -// WriteQC executes the Write Quorum Call on a configuration -// of Nodes and returns true if a majority of Nodes were updated. -func (c *Configuration) WriteQC(ctx context.Context, in *WriteRequest) (resp *WriteResponse, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "storage.Storage.WriteQC", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*WriteResponse, len(replies)) - for k, v := range replies { - r[k] = v.(*WriteResponse) - } - return c.qspec.WriteQCQF(req.(*WriteRequest), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) +// WriteRPC executes the Write RPC on a single Node +func WriteRPC(ctx *gorums.NodeContext, in *WriteRequest) (resp *WriteResponse, err error) { + res, err := gorums.RPCCall(ctx, in, "storage.Storage.WriteRPC") if err != nil { return nil, err } return res.(*WriteResponse), err } -// ReadRPC executes the Read RPC on a single Node -func (n *Node) ReadRPC(ctx context.Context, in *ReadRequest) (resp *ReadResponse, err error) { - cd := gorums.CallData{ - Message: in, - Method: "storage.Storage.ReadRPC", - } - - res, err := n.RawNode.RPCCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*ReadResponse), err +// ReadQC executes the Read Quorum Call on a configuration +// of Nodes and returns the most recent value. +func ReadQC(ctx *gorums.ConfigContext, in *ReadRequest, opts ...gorums.CallOption) *gorums.Responses[*ReadResponse] { + return gorums.QuorumCall[*ReadRequest, *ReadResponse]( + ctx, in, "storage.Storage.ReadQC", + opts..., + ) } -// WriteRPC executes the Write RPC on a single Node -func (n *Node) WriteRPC(ctx context.Context, in *WriteRequest) (resp *WriteResponse, err error) { - cd := gorums.CallData{ - Message: in, - Method: "storage.Storage.WriteRPC", - } +// WriteQC executes the Write Quorum Call on a configuration +// of Nodes and returns true if a majority of Nodes were updated. +func WriteQC(ctx *gorums.ConfigContext, in *WriteRequest, opts ...gorums.CallOption) *gorums.Responses[*WriteResponse] { + return gorums.QuorumCall[*WriteRequest, *WriteResponse]( + ctx, in, "storage.Storage.WriteQC", + opts..., + ) +} - res, err := n.RawNode.RPCCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*WriteResponse), err +// WriteMulticast is a multicast call invoked on all nodes in the configuration in ctx. +// Use gorums.MapRequest to send different messages to each node. No replies are collected. +func WriteMulticast(ctx *gorums.ConfigContext, in *WriteRequest, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "storage.Storage.WriteMulticast", opts...) } // Storage is the server-side API for the Storage Service @@ -310,15 +162,3 @@ func RegisterStorageServer(srv *gorums.Server, impl StorageServer) { return nil, nil }) } - -type internalReadResponse struct { - nid uint32 - reply *ReadResponse - err error -} - -type internalWriteResponse struct { - nid uint32 - reply *WriteResponse - err error -} diff --git a/examples/storage/repl.go b/examples/storage/repl.go index a939013c3..c278a0f81 100644 --- a/examples/storage/repl.go +++ b/examples/storage/repl.go @@ -53,11 +53,11 @@ The command performs the write quorum call on node 0 and 2 type repl struct { mgr *pb.Manager - cfg *pb.Configuration + cfg pb.Configuration term *term.Terminal } -func newRepl(mgr *pb.Manager, cfg *pb.Configuration) *repl { +func newRepl(mgr *pb.Manager, cfg pb.Configuration) *repl { return &repl{ mgr: mgr, cfg: cfg, @@ -90,7 +90,7 @@ func (r repl) ReadLine() (string, error) { // Repl runs an interactive Read-eval-print loop, that allows users to run commands that perform // RPCs and quorum calls using the manager and configuration. -func Repl(mgr *pb.Manager, defaultCfg *pb.Configuration) error { +func Repl(mgr *pb.Manager, defaultCfg pb.Configuration) error { r := newRepl(mgr, defaultCfg) fmt.Println(help) @@ -157,7 +157,7 @@ func (r repl) rpc(args []string) { return } - node := r.cfg.Nodes()[index] + node := r.cfg[index] switch args[1] { case "read": @@ -174,7 +174,8 @@ func (r repl) multicast(args []string) { } ctx, cancel := context.WithTimeout(context.Background(), time.Second) - r.cfg.WriteMulticast(ctx, pb.WriteRequest_builder{Key: args[0], Value: args[1]}.Build()) + cfgCtx := r.cfg.Context(ctx) + pb.WriteMulticast(cfgCtx, pb.WriteRequest_builder{Key: args[0], Value: args[1]}.Build()) cancel() fmt.Println("Multicast OK: (server output not synchronized)") } @@ -216,7 +217,8 @@ func (repl) readRPC(args []string, node *pb.Node) { return } ctx, cancel := context.WithTimeout(context.Background(), time.Second) - resp, err := node.ReadRPC(ctx, pb.ReadRequest_builder{Key: args[0]}.Build()) + nodeCtx := node.Context(ctx) + resp, err := pb.ReadRPC(nodeCtx, pb.ReadRequest_builder{Key: args[0]}.Build()) cancel() if err != nil { fmt.Printf("Read RPC finished with error: %v\n", err) @@ -235,7 +237,8 @@ func (repl) writeRPC(args []string, node *pb.Node) { return } ctx, cancel := context.WithTimeout(context.Background(), time.Second) - resp, err := node.WriteRPC(ctx, pb.WriteRequest_builder{Key: args[0], Value: args[1], Time: timestamppb.Now()}.Build()) + nodeCtx := node.Context(ctx) + resp, err := pb.WriteRPC(nodeCtx, pb.WriteRequest_builder{Key: args[0], Value: args[1], Time: timestamppb.Now()}.Build()) cancel() if err != nil { fmt.Printf("Write RPC finished with error: %v\n", err) @@ -248,13 +251,15 @@ func (repl) writeRPC(args []string, node *pb.Node) { fmt.Println("Write OK") } -func (repl) readQC(args []string, cfg *pb.Configuration) { +func (repl) readQC(args []string, config pb.Configuration) { if len(args) < 1 { fmt.Println("Read requires a key to read.") return } ctx, cancel := context.WithTimeout(context.Background(), time.Second) - resp, err := cfg.ReadQC(ctx, pb.ReadRequest_builder{Key: args[0]}.Build()) + cfgCtx := config.Context(ctx) + // Use the responses iterator to find the newest value + resp, err := newestValue(pb.ReadQC(cfgCtx, pb.ReadRequest_builder{Key: args[0]}.Build())) cancel() if err != nil { fmt.Printf("Read RPC finished with error: %v\n", err) @@ -267,13 +272,15 @@ func (repl) readQC(args []string, cfg *pb.Configuration) { fmt.Printf("%s = %s\n", args[0], resp.GetValue()) } -func (repl) writeQC(args []string, cfg *pb.Configuration) { +func (repl) writeQC(args []string, config pb.Configuration) { if len(args) < 2 { fmt.Println("Write requires a key and a value to write.") return } ctx, cancel := context.WithTimeout(context.Background(), time.Second) - resp, err := cfg.WriteQC(ctx, pb.WriteRequest_builder{Key: args[0], Value: args[1], Time: timestamppb.Now()}.Build()) + cfgCtx := config.Context(ctx) + // Use the responses iterator to count successful updates + resp, err := numUpdated(pb.WriteQC(cfgCtx, pb.WriteRequest_builder{Key: args[0], Value: args[1], Time: timestamppb.Now()}.Build())) cancel() if err != nil { fmt.Printf("Write RPC finished with error: %v\n", err) @@ -286,7 +293,7 @@ func (repl) writeQC(args []string, cfg *pb.Configuration) { fmt.Println("Write OK") } -func (r repl) parseConfiguration(cfgStr string) (cfg *pb.Configuration) { +func (r repl) parseConfiguration(cfgStr string) (cfg pb.Configuration) { // configuration using range syntax if i := strings.Index(cfgStr, ":"); i > -1 { var start, stop int @@ -318,7 +325,7 @@ func (r repl) parseConfiguration(cfgStr string) (cfg *pb.Configuration) { for _, node := range r.mgr.Nodes()[start:stop] { nodes = append(nodes, node.Address()) } - cfg, err = r.mgr.NewConfiguration(&qspec{cfgSize: stop - start}, gorums.WithNodeList(nodes)) + cfg, err = gorums.NewConfiguration(r.mgr, gorums.WithNodeList(nodes)) if err != nil { fmt.Printf("Failed to create configuration: %v\n", err) return nil @@ -341,7 +348,7 @@ func (r repl) parseConfiguration(cfgStr string) (cfg *pb.Configuration) { } selectedNodes = append(selectedNodes, nodes[i].Address()) } - cfg, err := r.mgr.NewConfiguration(&qspec{cfgSize: len(selectedNodes)}, gorums.WithNodeList(selectedNodes)) + cfg, err := gorums.NewConfiguration(r.mgr, gorums.WithNodeList(selectedNodes)) if err != nil { fmt.Printf("Failed to create configuration: %v\n", err) return nil diff --git a/examples/storage/server.go b/examples/storage/server.go index d4775af29..d3da361d0 100644 --- a/examples/storage/server.go +++ b/examples/storage/server.go @@ -67,7 +67,7 @@ type state struct { // storageServer is an implementation of pb.Storage type storageServer struct { storage map[string]state - mut sync.RWMutex + mut sync.Mutex logger *log.Logger } @@ -107,8 +107,8 @@ func (s *storageServer) WriteMulticast(_ gorums.ServerCtx, req *pb.WriteRequest) // Read reads a value from storage func (s *storageServer) Read(req *pb.ReadRequest) (*pb.ReadResponse, error) { s.logger.Printf("Read '%s'\n", req.GetKey()) - s.mut.RLock() - defer s.mut.RUnlock() + s.mut.Lock() + defer s.mut.Unlock() state, ok := s.storage[req.GetKey()] if !ok { return pb.ReadResponse_builder{OK: false}.Build(), nil diff --git a/go.mod b/go.mod index cf64a722e..bafe0da3d 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,24 @@ module github.com/relab/gorums -go 1.25.4 +go 1.25.5 require ( github.com/google/go-cmp v0.7.0 go.uber.org/goleak v1.3.0 - golang.org/x/sync v0.18.0 - golang.org/x/tools v0.39.0 - google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba - google.golang.org/grpc v1.76.0 - google.golang.org/protobuf v1.36.10 + golang.org/x/sync v0.19.0 + golang.org/x/tools v0.40.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 + google.golang.org/grpc v1.77.0 + google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect + golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 // indirect ) tool ( diff --git a/go.sum b/go.sum index ba9ad1fd2..a53624ec6 100644 --- a/go.sum +++ b/go.sum @@ -14,47 +14,47 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0= -golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1/go.mod h1:5KF+wpkbTSbGcR9zteSqZV6fqFOWBl4Yde8En8MryZA= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 h1:6Al3kEFFP9VJhRz3DID6quisgPnTeZVr4lep9kkxdPA= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0/go.mod h1:QLvsjh0OIR0TYBeiu2bkWGTJBUNQ64st52iWj/yA93I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work index 6a9e8c6ad..ca1b32944 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.25.4 +go 1.25.5 use ( . diff --git a/gorums.pb.go b/gorums.pb.go index ddd17f978..8e35f17d5 100644 --- a/gorums.pb.go +++ b/gorums.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: gorums.proto package gorums @@ -54,38 +54,6 @@ var file_gorums_proto_extTypes = []protoimpl.ExtensionInfo{ Tag: "varint,50004,opt,name=quorumcall", Filename: "gorums.proto", }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*bool)(nil), - Field: 50005, - Name: "gorums.correctable", - Tag: "varint,50005,opt,name=correctable", - Filename: "gorums.proto", - }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*bool)(nil), - Field: 50010, - Name: "gorums.async", - Tag: "varint,50010,opt,name=async", - Filename: "gorums.proto", - }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*bool)(nil), - Field: 50020, - Name: "gorums.per_node_arg", - Tag: "varint,50020,opt,name=per_node_arg", - Filename: "gorums.proto", - }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*string)(nil), - Field: 50030, - Name: "gorums.custom_return_type", - Tag: "bytes,50030,opt,name=custom_return_type", - Filename: "gorums.proto", - }, } // Extension fields to descriptorpb.MethodOptions. @@ -100,16 +68,6 @@ var ( E_Multicast = &file_gorums_proto_extTypes[2] // optional bool quorumcall = 50004; E_Quorumcall = &file_gorums_proto_extTypes[3] - // optional bool correctable = 50005; - E_Correctable = &file_gorums_proto_extTypes[4] - // options for call types - // - // optional bool async = 50010; - E_Async = &file_gorums_proto_extTypes[5] - // optional bool per_node_arg = 50020; - E_PerNodeArg = &file_gorums_proto_extTypes[6] - // optional string custom_return_type = 50030; - E_CustomReturnType = &file_gorums_proto_extTypes[7] ) var File_gorums_proto protoreflect.FileDescriptor @@ -122,12 +80,7 @@ const file_gorums_proto_rawDesc = "" + "\tmulticast\x12\x1e.google.protobuf.MethodOptions\x18ӆ\x03 \x01(\bR\tmulticast:@\n" + "\n" + "quorumcall\x12\x1e.google.protobuf.MethodOptions\x18Ԇ\x03 \x01(\bR\n" + - "quorumcall:B\n" + - "\vcorrectable\x12\x1e.google.protobuf.MethodOptions\x18Ն\x03 \x01(\bR\vcorrectable:6\n" + - "\x05async\x12\x1e.google.protobuf.MethodOptions\x18چ\x03 \x01(\bR\x05async:B\n" + - "\fper_node_arg\x12\x1e.google.protobuf.MethodOptions\x18\xe4\x86\x03 \x01(\bR\n" + - "perNodeArg:N\n" + - "\x12custom_return_type\x12\x1e.google.protobuf.MethodOptions\x18\xee\x86\x03 \x01(\tR\x10customReturnTypeB\x1eZ\x17github.com/relab/gorums\x92\x03\x02\b\x02b\beditionsp\xe8\a" + "quorumcallB\x1eZ\x17github.com/relab/gorums\x92\x03\x02\b\x02b\beditionsp\xe8\a" var file_gorums_proto_goTypes = []any{ (*descriptorpb.MethodOptions)(nil), // 0: google.protobuf.MethodOptions @@ -137,14 +90,10 @@ var file_gorums_proto_depIdxs = []int32{ 0, // 1: gorums.unicast:extendee -> google.protobuf.MethodOptions 0, // 2: gorums.multicast:extendee -> google.protobuf.MethodOptions 0, // 3: gorums.quorumcall:extendee -> google.protobuf.MethodOptions - 0, // 4: gorums.correctable:extendee -> google.protobuf.MethodOptions - 0, // 5: gorums.async:extendee -> google.protobuf.MethodOptions - 0, // 6: gorums.per_node_arg:extendee -> google.protobuf.MethodOptions - 0, // 7: gorums.custom_return_type:extendee -> google.protobuf.MethodOptions - 8, // [8:8] is the sub-list for method output_type - 8, // [8:8] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 0, // [0:8] is the sub-list for extension extendee + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 0, // [0:4] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name } @@ -160,7 +109,7 @@ func file_gorums_proto_init() { RawDescriptor: unsafe.Slice(unsafe.StringData(file_gorums_proto_rawDesc), len(file_gorums_proto_rawDesc)), NumEnums: 0, NumMessages: 0, - NumExtensions: 8, + NumExtensions: 4, NumServices: 0, }, GoTypes: file_gorums_proto_goTypes, diff --git a/gorums.proto b/gorums.proto index 45d9f19dd..c78497d15 100644 --- a/gorums.proto +++ b/gorums.proto @@ -9,13 +9,8 @@ import "google/protobuf/descriptor.proto"; extend google.protobuf.MethodOptions { // call types - bool rpc = 50001; - bool unicast = 50002; - bool multicast = 50003; - bool quorumcall = 50004; - bool correctable = 50005; - // options for call types - bool async = 50010; - bool per_node_arg = 50020; - string custom_return_type = 50030; + bool rpc = 50001; + bool unicast = 50002; + bool multicast = 50003; + bool quorumcall = 50004; } diff --git a/internal/correctable/opts.pb.go b/internal/correctable/opts.pb.go deleted file mode 100644 index a1e4b4f85..000000000 --- a/internal/correctable/opts.pb.go +++ /dev/null @@ -1,94 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 -// source: internal/correctable/opts.proto - -package correctable - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - descriptorpb "google.golang.org/protobuf/types/descriptorpb" - reflect "reflect" - unsafe "unsafe" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -var file_internal_correctable_opts_proto_extTypes = []protoimpl.ExtensionInfo{ - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*bool)(nil), - Field: 6200, - Name: "correctable.correctable", - Tag: "varint,6200,opt,name=correctable", - Filename: "internal/correctable/opts.proto", - }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*bool)(nil), - Field: 6201, - Name: "correctable.correctable_stream", - Tag: "varint,6201,opt,name=correctable_stream", - Filename: "internal/correctable/opts.proto", - }, -} - -// Extension fields to descriptorpb.MethodOptions. -var ( - // optional bool correctable = 6200; - E_Correctable = &file_internal_correctable_opts_proto_extTypes[0] - // optional bool correctable_stream = 6201; - E_CorrectableStream = &file_internal_correctable_opts_proto_extTypes[1] -) - -var File_internal_correctable_opts_proto protoreflect.FileDescriptor - -const file_internal_correctable_opts_proto_rawDesc = "" + - "\n" + - "\x1finternal/correctable/opts.proto\x12\vcorrectable\x1a google/protobuf/descriptor.proto:A\n" + - "\vcorrectable\x12\x1e.google.protobuf.MethodOptions\x18\xb80 \x01(\bR\vcorrectable:N\n" + - "\x12correctable_stream\x12\x1e.google.protobuf.MethodOptions\x18\xb90 \x01(\bR\x11correctableStreamB3Z,github.com/relab/gorums/internal/correctable\x92\x03\x02\b\x02b\beditionsp\xe8\a" - -var file_internal_correctable_opts_proto_goTypes = []any{ - (*descriptorpb.MethodOptions)(nil), // 0: google.protobuf.MethodOptions -} -var file_internal_correctable_opts_proto_depIdxs = []int32{ - 0, // 0: correctable.correctable:extendee -> google.protobuf.MethodOptions - 0, // 1: correctable.correctable_stream:extendee -> google.protobuf.MethodOptions - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 0, // [0:2] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_internal_correctable_opts_proto_init() } -func file_internal_correctable_opts_proto_init() { - if File_internal_correctable_opts_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_correctable_opts_proto_rawDesc), len(file_internal_correctable_opts_proto_rawDesc)), - NumEnums: 0, - NumMessages: 0, - NumExtensions: 2, - NumServices: 0, - }, - GoTypes: file_internal_correctable_opts_proto_goTypes, - DependencyIndexes: file_internal_correctable_opts_proto_depIdxs, - ExtensionInfos: file_internal_correctable_opts_proto_extTypes, - }.Build() - File_internal_correctable_opts_proto = out.File - file_internal_correctable_opts_proto_goTypes = nil - file_internal_correctable_opts_proto_depIdxs = nil -} diff --git a/internal/correctable/opts.proto b/internal/correctable/opts.proto deleted file mode 100644 index 6c10375fa..000000000 --- a/internal/correctable/opts.proto +++ /dev/null @@ -1,15 +0,0 @@ -edition = "2023"; - -package correctable; -option go_package = "github.com/relab/gorums/internal/correctable"; - -option features.field_presence = IMPLICIT; - -import "google/protobuf/descriptor.proto"; - -// These fields are used by the gorums plugin to describe -// correctable types. They are not meant to be used elsewhere. -extend google.protobuf.MethodOptions { - bool correctable = 6200; - bool correctable_stream = 6201; -} diff --git a/internal/strconv/atoi.go b/internal/strconv/atoi.go new file mode 100644 index 000000000..37f093fc6 --- /dev/null +++ b/internal/strconv/atoi.go @@ -0,0 +1,35 @@ +package strconv + +import ( + "reflect" + "strconv" +) + +type Integer interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +func signed[T Integer]() bool { + var x T + x = ^x // -1 if T is signed, 0xff..ff if T is unsigned + return x == x>>1 +} + +// ParseInteger parses a string s in the given base and returns +// a value of type T. T must be an integer type (signed or unsigned). +// The base argument must be between 2 and 36, or be 0. +// +// Inspired by discussion in issue https://github.com/golang/go/issues/76223. +func ParseInteger[T Integer](s string, base int) (res T, err error) { + rv := reflect.ValueOf(&res).Elem() + if signed[T]() { + var i int64 + i, err = strconv.ParseInt(s, base, rv.Type().Bits()) + res = T(i) + } else { + var u uint64 + u, err = strconv.ParseUint(s, base, rv.Type().Bits()) + res = T(u) + } + return +} diff --git a/internal/testprotos/calltypes/async/async.proto b/internal/testprotos/calltypes/async/async.proto deleted file mode 100644 index 784ee150c..000000000 --- a/internal/testprotos/calltypes/async/async.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Test that Gorums can generate a single asynchronous quorum call method. - -edition = "2023"; - -package gorums.testprotos.calltypes.async; -option go_package = "github.com/relab/gorums/internal/testprotos/calltypes/async"; - -option features.field_presence = IMPLICIT; - -import "gorums.proto"; - -service AsyncQuorumCall { - rpc AsyncQuorumCall(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - } -} - -message Request { - string Value = 1; -} -message Response { - int64 Result = 1; -} diff --git a/internal/testprotos/calltypes/zorums/zorums.proto b/internal/testprotos/calltypes/zorums/zorums.proto index e8823953c..2e4acfcc0 100644 --- a/internal/testprotos/calltypes/zorums/zorums.proto +++ b/internal/testprotos/calltypes/zorums/zorums.proto @@ -16,9 +16,6 @@ message Request { message Response { int64 Result = 1; } -message MyResponse { - string Value = 1; -} // ZorumsService is not meant to be implemented. // The zorums generated code is placed alongside Gorums static code files @@ -41,22 +38,6 @@ service ZorumsService { rpc QuorumCall(Request) returns (Response) { option (gorums.quorumcall) = true; } - // QuorumCall with per_node_arg option. - rpc QuorumCallPerNodeArg(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.per_node_arg) = true; - } - // QuorumCall with custom_return_type option. - rpc QuorumCallCustomReturnType(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // QuorumCallCombo with all supported options. - rpc QuorumCallCombo(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } // QuorumCallEmpty for testing imported message type. rpc QuorumCallEmpty(google.protobuf.Empty) returns (Response) { option (gorums.quorumcall) = true; @@ -68,20 +49,12 @@ service ZorumsService { // ------------------------------------------------------------------ // Multicast variants - // The multicast call type does not support the 'custom_return_type' - // option since multicast is one-way and so it doesn't make sense to - // operate on response types. // ------------------------------------------------------------------ // Multicast plain. Response type is not needed here. rpc Multicast(Request) returns (Response) { option (gorums.multicast) = true; } - // MulticastPerNodeArg with per_node_arg option. - rpc MulticastPerNodeArg(Request) returns (Response) { - option (gorums.multicast) = true; - option (gorums.per_node_arg) = true; - } // Multicast2 is testing whether multiple streams work. rpc Multicast2(Request) returns (Response) { option (gorums.multicast) = true; @@ -96,116 +69,21 @@ service ZorumsService { } // --------------------------------------------------------------- - // Async variants + // Streaming quorum call variants // --------------------------------------------------------------- - // QuorumCallAsync plain. - rpc QuorumCallAsync(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - } - // QuorumCallAsyncPerNodeArg with per_node_arg option. - rpc QuorumCallAsyncPerNodeArg(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - option (gorums.per_node_arg) = true; - } - // QuorumCallAsyncCustomReturnType with custom_return_type option. - rpc QuorumCallAsyncCustomReturnType(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // QuorumCallAsyncCombo with all supported options. - rpc QuorumCallAsyncCombo(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // QuorumCallAsync2 plain; with same return type: Response. - rpc QuorumCallAsync2(Request) returns (Response) { + // QuorumCallStream plain. + rpc QuorumCallStream(Request) returns (stream Response) { option (gorums.quorumcall) = true; - option (gorums.async) = true; } - // QuorumCallAsyncEmpty for testing imported message type. - rpc QuorumCallAsyncEmpty(Request) returns (google.protobuf.Empty) { + // QuorumCallStreamEmpty for testing imported message type. + rpc QuorumCallStreamEmpty(Request) returns (stream google.protobuf.Empty) { option (gorums.quorumcall) = true; - option (gorums.async) = true; } - // QuorumCallAsyncEmpty2 for testing imported message type; with same return - // type as QuorumCallAsync: Response. - rpc QuorumCallAsyncEmpty2(google.protobuf.Empty) returns (Response) { + // QuorumCallStreamEmpty2 for testing imported message type; with same return + // type as QuorumCallStream: Response. + rpc QuorumCallStreamEmpty2(google.protobuf.Empty) returns (stream Response) { option (gorums.quorumcall) = true; - option (gorums.async) = true; - } - - // --------------------------------------------------------------- - // Correctable variants - // --------------------------------------------------------------- - - // Correctable plain. - rpc Correctable(Request) returns (Response) { - option (gorums.correctable) = true; - } - // CorrectablePerNodeArg with per_node_arg option. - rpc CorrectablePerNodeArg(Request) returns (Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - } - // CorrectableCustomReturnType with custom_return_type option. - rpc CorrectableCustomReturnType(Request) returns (Response) { - option (gorums.correctable) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableCombo with all supported options. - rpc CorrectableCombo(Request) returns (Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableEmpty for testing imported message type. - rpc CorrectableEmpty(Request) returns (google.protobuf.Empty) { - option (gorums.correctable) = true; - } - // CorrectableEmpty2 for testing imported message type; with same return - // type as Correctable: Response. - rpc CorrectableEmpty2(google.protobuf.Empty) returns (Response) { - option (gorums.correctable) = true; - } - - // --------------------------------------------------------------- - // CorrectableStream variants - // --------------------------------------------------------------- - - // CorrectableStream plain. - rpc CorrectableStream(Request) returns (stream Response) { - option (gorums.correctable) = true; - } - // CorrectablePerNodeArg with per_node_arg option. - rpc CorrectableStreamPerNodeArg(Request) returns (stream Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - } - // CorrectableCustomReturnType with custom_return_type option. - rpc CorrectableStreamCustomReturnType(Request) returns (stream Response) { - option (gorums.correctable) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableCombo with all supported options. - rpc CorrectableStreamCombo(Request) returns (stream Response) { - option (gorums.correctable) = true; - option (gorums.per_node_arg) = true; - option (gorums.custom_return_type) = "MyResponse"; - } - // CorrectableEmpty for testing imported message type. - rpc CorrectableStreamEmpty(Request) returns (stream google.protobuf.Empty) { - option (gorums.correctable) = true; - } - // CorrectableEmpty2 for testing imported message type; with same return - // type as Correctable: Response. - rpc CorrectableStreamEmpty2(google.protobuf.Empty) returns (stream Response) { - option (gorums.correctable) = true; } // --------------------------------------------------------------- diff --git a/internal/tests/Makefile b/internal/tests/Makefile index b9552c9e3..e10777fbe 100644 --- a/internal/tests/Makefile +++ b/internal/tests/Makefile @@ -1,12 +1,10 @@ # Tests that should be run each time -TESTS := qf ordering metadata tls unresponsive oneway config correctable +TESTS := ordering metadata tls unresponsive oneway config correctable -.PHONY: all clean qf ordering metadata tls unresponsive oneway config correctable +.PHONY: all clean ordering metadata tls unresponsive oneway config correctable all: $(TESTS) -qf: qf/qf.pb.go qf/qf_gorums.pb.go - ordering: ordering/order.pb.go ordering/order_gorums.pb.go metadata: metadata/metadata.pb.go metadata/metadata_gorums.pb.go diff --git a/internal/tests/config/config.pb.go b/internal/tests/config/config.pb.go index f2bca7e33..3031b7373 100644 --- a/internal/tests/config/config.pb.go +++ b/internal/tests/config/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: config/config.proto package config diff --git a/internal/tests/config/config_gorums.pb.go b/internal/tests/config/config_gorums.pb.go index dbe7d4abe..36c8cfc65 100644 --- a/internal/tests/config/config_gorums.pb.go +++ b/internal/tests/config/config_gorums.pb.go @@ -1,17 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: config/config.proto package config import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" - proto "google.golang.org/protobuf/proto" ) const ( @@ -21,174 +17,75 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// ConfigTestClient is the client interface for the ConfigTest service. -type ConfigTestClient interface { - Config(ctx context.Context, in *Request) (resp *Response, err error) +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ ConfigTestClient = (*Configuration)(nil) +// AsyncResponse is a future for async quorum calls returning *Response. +type AsyncResponse = *gorums.Async[*Response] -// QuorumSpec is the interface of quorum functions for ConfigTest. -type QuorumSpec interface { - gorums.ConfigOption +// CorrectableResponse is a correctable object for quorum calls returning *Response. +type CorrectableResponse = *gorums.Correctable[*Response] - // ConfigQF is the quorum function for the Config - // quorum call method. The in parameter is the request object - // supplied to the Config method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - ConfigQF(in *Request, replies map[uint32]*Response) (*Response, bool) -} - -// Config is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) Config(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "config.ConfigTest.Config", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.ConfigQF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err +// Config is a quorum call invoked on all nodes in the configuration, +// with the same argument in. Use terminal methods like Majority(), First(), +// or Threshold(n) to retrieve the aggregated result. +// +// Example: +// +// resp, err := Config(ctx, in).Majority() +func Config(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCall[*Request, *Response]( + ctx, in, "config.ConfigTest.Config", + opts..., + ) } // ConfigTest is the server-side API for the ConfigTest Service @@ -203,9 +100,3 @@ func RegisterConfigTestServer(srv *gorums.Server, impl ConfigTestServer) { return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) } - -type internalResponse struct { - nid uint32 - reply *Response - err error -} diff --git a/internal/tests/config/config_test.go b/internal/tests/config/config_test.go index fa40cc356..72b9936de 100644 --- a/internal/tests/config/config_test.go +++ b/internal/tests/config/config_test.go @@ -2,80 +2,35 @@ package config import ( "context" - fmt "fmt" + "fmt" "testing" gorums "github.com/relab/gorums" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) -type ( - cfgSrv struct { - name string - } - cfgQSpec struct { - quorum int - } -) +type cfgSrv struct{} -func (srv cfgSrv) Config(ctx gorums.ServerCtx, req *Request) (resp *Response, err error) { +func (cfgSrv) Config(_ gorums.ServerCtx, req *Request) (resp *Response, err error) { return Response_builder{ - Name: srv.name, - Num: req.GetNum(), + Num: req.GetNum(), }.Build(), nil } -func newQSpec(cfgSize int) *cfgQSpec { - return &cfgQSpec{quorum: cfgSize/2 + 1} -} - -func (q cfgQSpec) ConfigQF(_ *Request, replies map[uint32]*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - var reply *Response - for _, r := range replies { - reply = r - break - } - return reply, true -} - -// setup returns a new configuration of cfgSize and a corresponding teardown function. -// Calling setup multiple times will return a different configuration with different -// sets of nodes. -func setup(t *testing.T, mgr *Manager, cfgSize int) (cfg *Configuration, teardown func()) { - t.Helper() - srvs := make([]*cfgSrv, cfgSize) - for i := range srvs { - srvs[i] = &cfgSrv{} - } - addrs, closeServers := gorums.TestSetup(t, cfgSize, func(i int) gorums.ServerIface { - srv := gorums.NewServer() - RegisterConfigTestServer(srv, srvs[i]) - return srv - }) - for i := range srvs { - srvs[i].name = addrs[i] - } - cfg, err := mgr.NewConfiguration(newQSpec(cfgSize), gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } - teardown = func() { - mgr.Close() - closeServers() - } - return cfg, teardown +func serverFn(_ int) gorums.ServerIface { + srv := gorums.NewServer() + RegisterConfigTestServer(srv, &cfgSrv{}) + return srv } // TestConfig creates and combines multiple configurations and invokes the Config RPC // method on the different configurations created below. func TestConfig(t *testing.T) { - callRPC := func(cfg *Configuration) { + callRPC := func(config gorums.Configuration) { + cfgCtx := config.Context(context.Background()) for i := range 5 { - resp, err := cfg.Config(context.Background(), Request_builder{Num: uint64(i)}.Build()) + // Use the new terminal method API - wait for a majority + resp, err := Config(cfgCtx, + Request_builder{Num: uint64(i)}.Build()).Majority() if err != nil { t.Fatal(err) } @@ -84,37 +39,28 @@ func TestConfig(t *testing.T) { } } } - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - c1, teardown1 := setup(t, mgr, 4) - defer teardown1() + + c1 := gorums.TestConfiguration(t, 4, serverFn) fmt.Println("--- c1 ", c1.Nodes()) callRPC(c1) - c2, teardown2 := setup(t, mgr, 2) - defer teardown2() + // Create a new configuration c2 with 2 new nodes not in c1, using the same manager as c1. + c2 := gorums.TestConfiguration(t, 2, serverFn, gorums.WithManager(t, c1.Manager())) fmt.Println("--- c2 ", c2.Nodes()) callRPC(c2) + // Create c3 = c1 ∪ c2, using the same manager as c1 (and c2). newNodeList := c1.And(c2) - c3, err := mgr.NewConfiguration( - newQSpec(c1.Size()+c2.Size()), - newNodeList, - ) + c3, err := gorums.NewConfiguration(c1.Manager(), newNodeList) if err != nil { t.Fatal(err) } fmt.Println("--- c3 ", c3.Nodes()) callRPC(c3) + // Create c4 = c3 \ c1, using the same manager as c1 (and c2, c3). rmNodeList := c3.Except(c1) - c4, err := mgr.NewConfiguration( - newQSpec(c2.Size()), - rmNodeList, - ) + c4, err := gorums.NewConfiguration(c1.Manager(), rmNodeList) if err != nil { t.Fatal(err) } diff --git a/internal/tests/correctable/correctable.pb.go b/internal/tests/correctable/correctable.pb.go index bbf6dbd3b..d6001b276 100644 --- a/internal/tests/correctable/correctable.pb.go +++ b/internal/tests/correctable/correctable.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: correctable/correctable.proto package correctable @@ -21,26 +21,26 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type CorrectableRequest struct { +type Request struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *CorrectableRequest) Reset() { - *x = CorrectableRequest{} +func (x *Request) Reset() { + *x = Request{} mi := &file_correctable_correctable_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *CorrectableRequest) String() string { +func (x *Request) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CorrectableRequest) ProtoMessage() {} +func (*Request) ProtoMessage() {} -func (x *CorrectableRequest) ProtoReflect() protoreflect.Message { +func (x *Request) ProtoReflect() protoreflect.Message { mi := &file_correctable_correctable_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -52,39 +52,39 @@ func (x *CorrectableRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -type CorrectableRequest_builder struct { +type Request_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. } -func (b0 CorrectableRequest_builder) Build() *CorrectableRequest { - m0 := &CorrectableRequest{} +func (b0 Request_builder) Build() *Request { + m0 := &Request{} b, x := &b0, m0 _, _ = b, x return m0 } -type CorrectableResponse struct { +type Response struct { state protoimpl.MessageState `protogen:"opaque.v1"` xxx_hidden_Level int32 `protobuf:"varint,1,opt,name=Level"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *CorrectableResponse) Reset() { - *x = CorrectableResponse{} +func (x *Response) Reset() { + *x = Response{} mi := &file_correctable_correctable_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *CorrectableResponse) String() string { +func (x *Response) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CorrectableResponse) ProtoMessage() {} +func (*Response) ProtoMessage() {} -func (x *CorrectableResponse) ProtoReflect() protoreflect.Message { +func (x *Response) ProtoReflect() protoreflect.Message { mi := &file_correctable_correctable_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -96,25 +96,25 @@ func (x *CorrectableResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *CorrectableResponse) GetLevel() int32 { +func (x *Response) GetLevel() int32 { if x != nil { return x.xxx_hidden_Level } return 0 } -func (x *CorrectableResponse) SetLevel(v int32) { +func (x *Response) SetLevel(v int32) { x.xxx_hidden_Level = v } -type CorrectableResponse_builder struct { +type Response_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. Level int32 } -func (b0 CorrectableResponse_builder) Build() *CorrectableResponse { - m0 := &CorrectableResponse{} +func (b0 Response_builder) Build() *Response { + m0 := &Response{} b, x := &b0, m0 _, _ = b, x x.xxx_hidden_Level = b.Level @@ -125,24 +125,24 @@ var File_correctable_correctable_proto protoreflect.FileDescriptor const file_correctable_correctable_proto_rawDesc = "" + "\n" + - "\x1dcorrectable/correctable.proto\x12\vcorrectable\x1a\fgorums.proto\"\x14\n" + - "\x12CorrectableRequest\"+\n" + - "\x13CorrectableResponse\x12\x14\n" + - "\x05Level\x18\x01 \x01(\x05R\x05Level2\xc9\x01\n" + - "\x0fCorrectableTest\x12V\n" + - "\vCorrectable\x12\x1f.correctable.CorrectableRequest\x1a .correctable.CorrectableResponse\"\x04\xa8\xb5\x18\x01\x12^\n" + - "\x11CorrectableStream\x12\x1f.correctable.CorrectableRequest\x1a .correctable.CorrectableResponse\"\x04\xa8\xb5\x18\x010\x01B0Z)github.com/relab/gorums/tests/correctable\x92\x03\x02\b\x02b\beditionsp\xe8\a" + "\x1dcorrectable/correctable.proto\x12\vcorrectable\x1a\fgorums.proto\"\t\n" + + "\aRequest\" \n" + + "\bResponse\x12\x14\n" + + "\x05Level\x18\x01 \x01(\x05R\x05Level2\x9d\x01\n" + + "\x0fCorrectableTest\x12@\n" + + "\vCorrectable\x12\x14.correctable.Request\x1a\x15.correctable.Response\"\x04\xa0\xb5\x18\x01\x12H\n" + + "\x11CorrectableStream\x12\x14.correctable.Request\x1a\x15.correctable.Response\"\x04\xa0\xb5\x18\x010\x01B0Z)github.com/relab/gorums/tests/correctable\x92\x03\x02\b\x02b\beditionsp\xe8\a" var file_correctable_correctable_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_correctable_correctable_proto_goTypes = []any{ - (*CorrectableRequest)(nil), // 0: correctable.CorrectableRequest - (*CorrectableResponse)(nil), // 1: correctable.CorrectableResponse + (*Request)(nil), // 0: correctable.Request + (*Response)(nil), // 1: correctable.Response } var file_correctable_correctable_proto_depIdxs = []int32{ - 0, // 0: correctable.CorrectableTest.Correctable:input_type -> correctable.CorrectableRequest - 0, // 1: correctable.CorrectableTest.CorrectableStream:input_type -> correctable.CorrectableRequest - 1, // 2: correctable.CorrectableTest.Correctable:output_type -> correctable.CorrectableResponse - 1, // 3: correctable.CorrectableTest.CorrectableStream:output_type -> correctable.CorrectableResponse + 0, // 0: correctable.CorrectableTest.Correctable:input_type -> correctable.Request + 0, // 1: correctable.CorrectableTest.CorrectableStream:input_type -> correctable.Request + 1, // 2: correctable.CorrectableTest.Correctable:output_type -> correctable.Response + 1, // 3: correctable.CorrectableTest.CorrectableStream:output_type -> correctable.Response 2, // [2:4] is the sub-list for method output_type 0, // [0:2] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name diff --git a/internal/tests/correctable/correctable.proto b/internal/tests/correctable/correctable.proto index 6bae791f7..36910c70c 100644 --- a/internal/tests/correctable/correctable.proto +++ b/internal/tests/correctable/correctable.proto @@ -1,24 +1,22 @@ edition = "2023"; package correctable; - -option go_package = "github.com/relab/gorums/tests/correctable"; - +option go_package = "github.com/relab/gorums/tests/correctable"; option features.field_presence = IMPLICIT; import "gorums.proto"; service CorrectableTest { - rpc Correctable(CorrectableRequest) returns (CorrectableResponse) { - option (gorums.correctable) = true; + rpc Correctable(Request) returns (Response) { + option (gorums.quorumcall) = true; } - rpc CorrectableStream(CorrectableRequest) returns (stream CorrectableResponse) { - option (gorums.correctable) = true; + rpc CorrectableStream(Request) returns (stream Response) { + option (gorums.quorumcall) = true; } } -message CorrectableRequest {} +message Request {} -message CorrectableResponse { +message Response { int32 Level = 1; } diff --git a/internal/tests/correctable/correctable_gorums.pb.go b/internal/tests/correctable/correctable_gorums.pb.go index 849a33e26..3e0e7ad73 100644 --- a/internal/tests/correctable/correctable_gorums.pb.go +++ b/internal/tests/correctable/correctable_gorums.pb.go @@ -1,16 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: correctable/correctable.proto package correctable import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" proto "google.golang.org/protobuf/proto" ) @@ -21,220 +18,107 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} - -// CorrectableTestClient is the client interface for the CorrectableTest service. -type CorrectableTestClient interface { - Correctable(ctx context.Context, in *CorrectableRequest) *CorrectableCorrectableResponse - CorrectableStream(ctx context.Context, in *CorrectableRequest) *CorrectableStreamCorrectableResponse +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ CorrectableTestClient = (*Configuration)(nil) - -// Correctable asynchronously invokes a correctable quorum call on each node -// in configuration c and returns a CorrectableCorrectableResponse, which can be used -// to inspect any replies or errors when available. -func (c *Configuration) Correctable(ctx context.Context, in *CorrectableRequest) *CorrectableCorrectableResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "correctable.CorrectableTest.Correctable", - ServerStream: false, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*CorrectableResponse, len(replies)) - for k, v := range replies { - r[k] = v.(*CorrectableResponse) - } - return c.qspec.CorrectableQF(req.(*CorrectableRequest), r) - } +// AsyncResponse is a future for async quorum calls returning *Response. +type AsyncResponse = *gorums.Async[*Response] - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableCorrectableResponse{corr} -} - -// CorrectableStream asynchronously invokes a correctable quorum call on each node -// in configuration c and returns a CorrectableStreamCorrectableResponse, which can be used -// to inspect any replies or errors when available. -// This method supports server-side preliminary replies (correctable stream). -func (c *Configuration) CorrectableStream(ctx context.Context, in *CorrectableRequest) *CorrectableStreamCorrectableResponse { - cd := gorums.CorrectableCallData{ - Message: in, - Method: "correctable.CorrectableTest.CorrectableStream", - ServerStream: true, - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, int, bool) { - r := make(map[uint32]*CorrectableResponse, len(replies)) - for k, v := range replies { - r[k] = v.(*CorrectableResponse) - } - return c.qspec.CorrectableStreamQF(req.(*CorrectableRequest), r) - } +// CorrectableResponse is a correctable object for quorum calls returning *Response. +type CorrectableResponse = *gorums.Correctable[*Response] - corr := c.RawConfiguration.CorrectableCall(ctx, cd) - return &CorrectableStreamCorrectableResponse{corr} +// Correctable is a quorum call invoked on all nodes in the configuration, +// with the same argument in. Use terminal methods like Majority(), First(), +// or Threshold(n) to retrieve the aggregated result. +// +// Example: +// +// resp, err := Correctable(ctx, in).Majority() +func Correctable(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCall[*Request, *Response]( + ctx, in, "correctable.CorrectableTest.Correctable", + opts..., + ) } -// QuorumSpec is the interface of quorum functions for CorrectableTest. -type QuorumSpec interface { - gorums.ConfigOption - - // CorrectableQF is the quorum function for the Correctable - // correctable quorum call method. The in parameter is the request object - // supplied to the Correctable method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *CorrectableRequest'. - CorrectableQF(in *CorrectableRequest, replies map[uint32]*CorrectableResponse) (*CorrectableResponse, int, bool) - - // CorrectableStreamQF is the quorum function for the CorrectableStream - // correctable stream quorum call method. The in parameter is the request object - // supplied to the CorrectableStream method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *CorrectableRequest'. - CorrectableStreamQF(in *CorrectableRequest, replies map[uint32]*CorrectableResponse) (*CorrectableResponse, int, bool) +// CorrectableStream is a streaming quorum call where the server can send multiple responses. +// The response iterator continues until the context is canceled. +// +// Example: +// +// corr := CorrectableStream(ctx, in).Correctable(2) +// <-corr.Watch(2) +// resp, level, err := corr.Get() +func CorrectableStream(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCallStream[*Request, *Response]( + ctx, in, "correctable.CorrectableTest.CorrectableStream", + opts..., + ) } // CorrectableTest is the server-side API for the CorrectableTest Service type CorrectableTestServer interface { - Correctable(ctx gorums.ServerCtx, request *CorrectableRequest) (response *CorrectableResponse, err error) - CorrectableStream(ctx gorums.ServerCtx, request *CorrectableRequest, send func(response *CorrectableResponse) error) error + Correctable(ctx gorums.ServerCtx, request *Request) (response *Response, err error) + CorrectableStream(ctx gorums.ServerCtx, request *Request, send func(response *Response) error) error } func RegisterCorrectableTestServer(srv *gorums.Server, impl CorrectableTestServer) { srv.RegisterHandler("correctable.CorrectableTest.Correctable", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*CorrectableRequest](in) + req := gorums.AsProto[*Request](in) resp, err := impl.Correctable(ctx, req) return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) srv.RegisterHandler("correctable.CorrectableTest.CorrectableStream", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*CorrectableRequest](in) - err := impl.CorrectableStream(ctx, req, func(resp *CorrectableResponse) error { + req := gorums.AsProto[*Request](in) + err := impl.CorrectableStream(ctx, req, func(resp *Response) error { // create a copy of the metadata, to avoid a data race between NewResponseMessage and SendMsg md := proto.CloneOf(in.GetMetadata()) return ctx.SendMessage(gorums.NewResponseMessage(md, resp)) @@ -242,45 +126,3 @@ func RegisterCorrectableTestServer(srv *gorums.Server, impl CorrectableTestServe return nil, err }) } - -type internalCorrectableResponse struct { - nid uint32 - reply *CorrectableResponse - err error -} - -// CorrectableCorrectableResponse is a correctable object for processing replies. -type CorrectableCorrectableResponse struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableCorrectableResponse) Get() (*CorrectableResponse, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*CorrectableResponse), level, err -} - -// CorrectableStreamCorrectableResponse is a correctable object for processing replies. -type CorrectableStreamCorrectableResponse struct { - *gorums.Correctable -} - -// Get returns the reply, level and any error associated with the -// called method. The method does not block until a (possibly -// intermediate) reply or error is available. Level is set to LevelNotSet if no -// reply has yet been received. The Done or Watch methods should be used to -// ensure that a reply is available. -func (c *CorrectableStreamCorrectableResponse) Get() (*CorrectableResponse, int, error) { - resp, level, err := c.Correctable.Get() - if err != nil { - return nil, level, err - } - return resp.(*CorrectableResponse), level, err -} diff --git a/internal/tests/correctable/correctable_test.go b/internal/tests/correctable/correctable_test.go index 086f06f24..d821e790d 100644 --- a/internal/tests/correctable/correctable_test.go +++ b/internal/tests/correctable/correctable_test.go @@ -1,119 +1,115 @@ package correctable import ( - "context" "testing" "time" "github.com/relab/gorums" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) // run a test on a correctable call. -// n is the number of replicas, and div is a divider. -// the target level is n, and the level is calculated by the quorum function -// by dividing the sum of levels from the servers with the divider. -func run(t *testing.T, n int, div int, corr func(context.Context, *Configuration) *gorums.Correctable) { - addrs, teardown := gorums.TestSetup(t, n, func(i int) gorums.ServerIface { +// n is the number of replicas. +// the target level is n (quorum size). +func run(t testing.TB, n int, corr func(*gorums.ConfigContext, int) CorrectableResponse) { + t.Helper() + config := gorums.TestConfiguration(t, n, func(_ int) gorums.ServerIface { gorumsSrv := gorums.NewServer() RegisterCorrectableTestServer(gorumsSrv, &testSrv{n}) return gorumsSrv }) - defer teardown() - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - - cfg, err := mgr.NewConfiguration(qspec{div, n}, gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - res := corr(ctx, cfg) - - donech := res.Done() + ctx := gorums.TestContext(t, 100*time.Millisecond) + configCtx := config.Context(ctx) + res := corr(configCtx, n) + done := res.Done() for i := 1; i <= n; i++ { select { case <-res.Watch(i): - case <-donech: + case <-done: } } - - <-donech - - _, _, err = res.Get() - if err != nil { + <-done + if _, _, err := res.Get(); err != nil { t.Error(err) } } func TestCorrectable(t *testing.T) { - run(t, 4, 1, func(ctx context.Context, c *Configuration) *gorums.Correctable { - corr := c.Correctable(ctx, &CorrectableRequest{}) - return corr.Correctable + run(t, 4, func(ctx *gorums.ConfigContext, n int) CorrectableResponse { + // Correctable returns *Responses, user calls Correctable to get *Correctable + return Correctable(ctx, &Request{}).Correctable(n) }) } func TestCorrectableStream(t *testing.T) { - run(t, 4, 4, func(ctx context.Context, c *Configuration) *gorums.Correctable { - corr := c.CorrectableStream(ctx, &CorrectableRequest{}) - return corr.Correctable + run(t, 4, func(ctx *gorums.ConfigContext, n int) CorrectableResponse { + // CorrectableStream returns *Responses, user calls Correctable to get *Correctable + return CorrectableStream(ctx, &Request{}).Correctable(n) }) } -type qspec struct { - div int - doneLevel int -} +// TestCorrectableWithWatch tests progressive level watching using the type alias +func TestCorrectableWithWatch(t *testing.T) { + n := 4 + config := gorums.TestConfiguration(t, n, func(_ int) gorums.ServerIface { + gorumsSrv := gorums.NewServer() + RegisterCorrectableTestServer(gorumsSrv, &testSrv{n}) + return gorumsSrv + }) -func (q qspec) q(replies map[uint32]*CorrectableResponse) (*CorrectableResponse, int, bool) { - sum := 0 - for _, reply := range replies { - sum += int(reply.GetLevel()) - } - level := sum / q.div - return CorrectableResponse_builder{Level: int32(level)}.Build(), level, level >= q.doneLevel -} + ctx := gorums.TestContext(t, 100*time.Millisecond) + configCtx := config.Context(ctx) -// CorrectableStreamQF is the quorum function for the CorrectableStream -// correctable stream quorum call method. The in parameter is the request object -// supplied to the CorrectableStream method at call time, and may or may not -// be used by the quorum function. If the in parameter is not needed -// you should implement your quorum function with '_ *CorrectableRequest'. -func (q qspec) CorrectableStreamQF(_ *CorrectableRequest, replies map[uint32]*CorrectableResponse) (*CorrectableResponse, int, bool) { - return q.q(replies) -} + // Use the type alias for the correctable result + corr := CorrectableStream(configCtx, &Request{}).Correctable(n) -// CorrectableQF is the quorum function for the Correctable -// correctable quorum call method. The in parameter is the request object -// supplied to the Correctable method at call time, and may or may not -// be used by the quorum function. If the in parameter is not needed -// you should implement your quorum function with '_ *CorrectableRequest'. -func (q qspec) CorrectableQF(_ *CorrectableRequest, replies map[uint32]*CorrectableResponse) (*CorrectableResponse, int, bool) { - return q.q(replies) + // Watch for each level progressively + for level := 1; level <= n; level++ { + select { + case <-corr.Watch(level): + resp, gotLevel, err := corr.Get() + if err != nil { + t.Errorf("level %d: unexpected error: %v", level, err) + } + if gotLevel < level { + t.Errorf("level %d: expected level >= %d, got %d", level, level, gotLevel) + } + if resp == nil { + t.Errorf("level %d: expected non-nil response", level) + } + case <-corr.Done(): + // Done is also acceptable + } + } + + <-corr.Done() + resp, finalLevel, err := corr.Get() + if err != nil { + t.Errorf("final: unexpected error: %v", err) + } + if finalLevel != n { + t.Errorf("final: expected level %d, got %d", n, finalLevel) + } + if resp == nil { + t.Error("final: expected non-nil response") + } } type testSrv struct { n int } -func (srv testSrv) CorrectableStream(_ gorums.ServerCtx, request *CorrectableRequest, send func(response *CorrectableResponse) error) error { +func (testSrv) Correctable(_ gorums.ServerCtx, _ *Request) (*Response, error) { + return Response_builder{Level: 1}.Build(), nil +} + +func (srv testSrv) CorrectableStream(_ gorums.ServerCtx, _ *Request, send func(response *Response) error) error { for i := range srv.n { - err := send(CorrectableResponse_builder{Level: int32(i + 1)}.Build()) + err := send(Response_builder{Level: int32(i + 1)}.Build()) if err != nil { return err } } return nil } - -func (srv testSrv) Correctable(_ gorums.ServerCtx, request *CorrectableRequest) (response *CorrectableResponse, err error) { - return CorrectableResponse_builder{Level: 1}.Build(), nil -} diff --git a/internal/tests/metadata/metadata.pb.go b/internal/tests/metadata/metadata.pb.go index 65b9e8f78..f5a78ae67 100644 --- a/internal/tests/metadata/metadata.pb.go +++ b/internal/tests/metadata/metadata.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: metadata/metadata.proto package metadata diff --git a/internal/tests/metadata/metadata_gorums.pb.go b/internal/tests/metadata/metadata_gorums.pb.go index eb7e37028..a3bfd1375 100644 --- a/internal/tests/metadata/metadata_gorums.pb.go +++ b/internal/tests/metadata/metadata_gorums.pb.go @@ -1,16 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: metadata/metadata.proto package metadata import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" emptypb "google.golang.org/protobuf/types/known/emptypb" ) @@ -21,154 +18,60 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// MetadataTestNodeClient is the single node client interface for the MetadataTest service. -type MetadataTestNodeClient interface { - IDFromMD(ctx context.Context, in *emptypb.Empty) (resp *NodeID, err error) - WhatIP(ctx context.Context, in *emptypb.Empty) (resp *IPAddr, err error) +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ MetadataTestNodeClient = (*Node)(nil) - -// There are no quorum calls. -type QuorumSpec interface{} - // IDFromMD returns the 'id' field from the metadata. -func (n *Node) IDFromMD(ctx context.Context, in *emptypb.Empty) (resp *NodeID, err error) { - cd := gorums.CallData{ - Message: in, - Method: "metadata.MetadataTest.IDFromMD", - } - - res, err := n.RawNode.RPCCall(ctx, cd) +func IDFromMD(ctx *gorums.NodeContext, in *emptypb.Empty) (resp *NodeID, err error) { + res, err := gorums.RPCCall(ctx, in, "metadata.MetadataTest.IDFromMD") if err != nil { return nil, err } @@ -176,13 +79,8 @@ func (n *Node) IDFromMD(ctx context.Context, in *emptypb.Empty) (resp *NodeID, e } // WhatIP returns the address of the client that calls it. -func (n *Node) WhatIP(ctx context.Context, in *emptypb.Empty) (resp *IPAddr, err error) { - cd := gorums.CallData{ - Message: in, - Method: "metadata.MetadataTest.WhatIP", - } - - res, err := n.RawNode.RPCCall(ctx, cd) +func WhatIP(ctx *gorums.NodeContext, in *emptypb.Empty) (resp *IPAddr, err error) { + res, err := gorums.RPCCall(ctx, in, "metadata.MetadataTest.WhatIP") if err != nil { return nil, err } diff --git a/internal/tests/metadata/metadata_test.go b/internal/tests/metadata/metadata_test.go index dde393456..a8eeb5fc4 100644 --- a/internal/tests/metadata/metadata_test.go +++ b/internal/tests/metadata/metadata_test.go @@ -1,15 +1,12 @@ package metadata import ( - "context" "fmt" - "strconv" "testing" "github.com/relab/gorums" - "google.golang.org/grpc" + "github.com/relab/gorums/internal/strconv" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" @@ -18,7 +15,7 @@ import ( type testSrv struct{} -func (srv testSrv) IDFromMD(ctx gorums.ServerCtx, _ *emptypb.Empty) (resp *NodeID, err error) { +func (testSrv) IDFromMD(ctx gorums.ServerCtx, _ *emptypb.Empty) (resp *NodeID, err error) { md, ok := metadata.FromIncomingContext(ctx) if !ok { return nil, status.Error(codes.NotFound, "metadata unavailable") @@ -27,14 +24,14 @@ func (srv testSrv) IDFromMD(ctx gorums.ServerCtx, _ *emptypb.Empty) (resp *NodeI if len(v) < 1 { return nil, status.Error(codes.NotFound, "missing metadata field: id") } - id, err := strconv.Atoi(v[0]) + id, err := strconv.ParseInteger[uint32](v[0], 10) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "value of id field: %q is not a number: %v", v[0], err) } - return NodeID_builder{ID: uint32(id)}.Build(), nil + return NodeID_builder{ID: id}.Build(), nil } -func (srv testSrv) WhatIP(ctx gorums.ServerCtx, _ *emptypb.Empty) (resp *IPAddr, err error) { +func (testSrv) WhatIP(ctx gorums.ServerCtx, _ *emptypb.Empty) (resp *IPAddr, err error) { peerInfo, ok := peer.FromContext(ctx) if !ok { return nil, status.Error(codes.NotFound, "Peer info unavailable") @@ -42,7 +39,7 @@ func (srv testSrv) WhatIP(ctx gorums.ServerCtx, _ *emptypb.Empty) (resp *IPAddr, return IPAddr_builder{Addr: peerInfo.Addr.String()}.Build(), nil } -func initServer() *gorums.Server { +func serverFn(_ int) gorums.ServerIface { srv := gorums.NewServer() RegisterMetadataTestServer(srv, &testSrv{}) return srv @@ -50,27 +47,13 @@ func initServer() *gorums.Server { func TestMetadata(t *testing.T) { want := uint32(1) - - addrs, teardown := gorums.TestSetup(t, 1, func(_ int) gorums.ServerIface { return initServer() }) - defer teardown() - md := metadata.New(map[string]string{ "id": fmt.Sprint(want), }) - mgr := NewManager( - gorums.WithMetadata(md), - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - _, err := mgr.NewConfiguration(gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } - - node := mgr.Nodes()[0] - resp, err := node.IDFromMD(context.Background(), &emptypb.Empty{}) + node := gorums.TestNode(t, serverFn, gorums.WithMetadata(md)) + nodeCtx := node.Context(t.Context()) + resp, err := IDFromMD(nodeCtx, &emptypb.Empty{}) if err != nil { t.Fatalf("RPC error: %v", err) } @@ -81,28 +64,15 @@ func TestMetadata(t *testing.T) { } func TestPerMessageMetadata(t *testing.T) { - want := uint32(1) - - addrs, teardown := gorums.TestSetup(t, 1, func(_ int) gorums.ServerIface { return initServer() }) - defer teardown() - - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - _, err := mgr.NewConfiguration(gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } - - node := mgr.Nodes()[0] + node := gorums.TestNode(t, serverFn) + want := uint32(1) md := metadata.New(map[string]string{ "id": fmt.Sprint(want), }) - ctx := metadata.NewOutgoingContext(context.Background(), md) - resp, err := node.IDFromMD(ctx, &emptypb.Empty{}) + ctx := metadata.NewOutgoingContext(t.Context(), md) + nodeCtx := node.Context(ctx) + resp, err := IDFromMD(nodeCtx, &emptypb.Empty{}) if err != nil { t.Fatalf("RPC error: %v", err) } @@ -113,28 +83,17 @@ func TestPerMessageMetadata(t *testing.T) { } func TestPerNodeMetadata(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 2, func(_ int) gorums.ServerIface { return initServer() }) - defer teardown() - perNodeMD := func(nid uint32) metadata.MD { return metadata.New(map[string]string{ "id": fmt.Sprintf("%d", nid), }) } - mgr := NewManager( - gorums.WithPerNodeMetadata(perNodeMD), - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - _, err := mgr.NewConfiguration(gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } + cfg := gorums.TestConfiguration(t, 2, serverFn, gorums.WithPerNodeMetadata(perNodeMD)) - for _, node := range mgr.Nodes() { - resp, err := node.IDFromMD(context.Background(), &emptypb.Empty{}) + for _, node := range cfg { + nodeCtx := node.Context(t.Context()) + resp, err := IDFromMD(nodeCtx, &emptypb.Empty{}) if err != nil { t.Fatalf("RPC error: %v", err) } @@ -146,21 +105,9 @@ func TestPerNodeMetadata(t *testing.T) { } func TestCanGetPeerInfo(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 1, func(_ int) gorums.ServerIface { return initServer() }) - defer teardown() - - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - _, err := mgr.NewConfiguration(gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } - - node := mgr.Nodes()[0] - ip, err := node.WhatIP(context.Background(), &emptypb.Empty{}) + node := gorums.TestNode(t, serverFn) + nodeCtx := node.Context(t.Context()) + ip, err := WhatIP(nodeCtx, &emptypb.Empty{}) if err != nil { t.Fatalf("RPC error: %v", err) } diff --git a/internal/tests/oneway/oneway.pb.go b/internal/tests/oneway/oneway.pb.go index 74c8a18c5..e4e9d2440 100644 --- a/internal/tests/oneway/oneway.pb.go +++ b/internal/tests/oneway/oneway.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: oneway/oneway.proto package oneway @@ -128,12 +128,11 @@ const file_oneway_oneway_proto_rawDesc = "" + "\x13oneway/oneway.proto\x12\x06oneway\x1a\fgorums.proto\"\x1b\n" + "\aRequest\x12\x10\n" + "\x03Num\x18\x01 \x01(\x04R\x03Num\"\a\n" + - "\x05Empty2\xae\x01\n" + + "\x05Empty2p\n" + "\n" + "OnewayTest\x12/\n" + "\aUnicast\x12\x0f.oneway.Request\x1a\r.oneway.Empty\"\x04\x90\xb5\x18\x01\x121\n" + - "\tMulticast\x12\x0f.oneway.Request\x1a\r.oneway.Empty\"\x04\x98\xb5\x18\x01\x12<\n" + - "\x10MulticastPerNode\x12\x0f.oneway.Request\x1a\r.oneway.Empty\"\b\x98\xb5\x18\x01\xa0\xb6\x18\x01B+Z$github.com/relab/gorums/tests/oneway\x92\x03\x02\b\x02b\beditionsp\xe8\a" + "\tMulticast\x12\x0f.oneway.Request\x1a\r.oneway.Empty\"\x04\x98\xb5\x18\x01B+Z$github.com/relab/gorums/tests/oneway\x92\x03\x02\b\x02b\beditionsp\xe8\a" var file_oneway_oneway_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_oneway_oneway_proto_goTypes = []any{ @@ -143,12 +142,10 @@ var file_oneway_oneway_proto_goTypes = []any{ var file_oneway_oneway_proto_depIdxs = []int32{ 0, // 0: oneway.OnewayTest.Unicast:input_type -> oneway.Request 0, // 1: oneway.OnewayTest.Multicast:input_type -> oneway.Request - 0, // 2: oneway.OnewayTest.MulticastPerNode:input_type -> oneway.Request - 1, // 3: oneway.OnewayTest.Unicast:output_type -> oneway.Empty - 1, // 4: oneway.OnewayTest.Multicast:output_type -> oneway.Empty - 1, // 5: oneway.OnewayTest.MulticastPerNode:output_type -> oneway.Empty - 3, // [3:6] is the sub-list for method output_type - 0, // [0:3] is the sub-list for method input_type + 1, // 2: oneway.OnewayTest.Unicast:output_type -> oneway.Empty + 1, // 3: oneway.OnewayTest.Multicast:output_type -> oneway.Empty + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/internal/tests/oneway/oneway.proto b/internal/tests/oneway/oneway.proto index 4522ec09a..346c87578 100644 --- a/internal/tests/oneway/oneway.proto +++ b/internal/tests/oneway/oneway.proto @@ -14,10 +14,6 @@ service OnewayTest { rpc Multicast(Request) returns (Empty) { option (gorums.multicast) = true; } - rpc MulticastPerNode(Request) returns (Empty) { - option (gorums.multicast) = true; - option (gorums.per_node_arg) = true; - } } message Request { diff --git a/internal/tests/oneway/oneway_gorums.pb.go b/internal/tests/oneway/oneway_gorums.pb.go index 2bc3c1661..0bf1cf08f 100644 --- a/internal/tests/oneway/oneway_gorums.pb.go +++ b/internal/tests/oneway/oneway_gorums.pb.go @@ -1,17 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: oneway/oneway.proto package oneway import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" - proto "google.golang.org/protobuf/proto" ) const ( @@ -21,188 +17,73 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} - -// OnewayTestClient is the client interface for the OnewayTest service. -type OnewayTestClient interface { - Multicast(ctx context.Context, in *Request, opts ...gorums.CallOption) - MulticastPerNode(ctx context.Context, in *Request, f func(*Request, uint32) *Request, opts ...gorums.CallOption) +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ OnewayTestClient = (*Configuration)(nil) - -// OnewayTestNodeClient is the single node client interface for the OnewayTest service. -type OnewayTestNodeClient interface { - Unicast(ctx context.Context, in *Request, opts ...gorums.CallOption) +// Unicast is a unicast call invoked on the node in ctx. +// No reply is returned to the client. +func Unicast(ctx *gorums.NodeContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Unicast(ctx, in, "oneway.OnewayTest.Unicast", opts...) } -// enforce interface compliance -var _ OnewayTestNodeClient = (*Node)(nil) - -// Multicast is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) Multicast(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "oneway.OnewayTest.Multicast", - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) +// Multicast is a multicast call invoked on all nodes in the configuration in ctx. +// Use gorums.MapRequest to send different messages to each node. No replies are collected. +func Multicast(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) error { + return gorums.Multicast(ctx, in, "oneway.OnewayTest.Multicast", opts...) } -// MulticastPerNode is a quorum call invoked on each node in configuration c, -// with the argument returned by the provided function f, and returns the combined result. -// The per node function f receives a copy of the Request request argument and -// returns a Request manipulated to be passed to the given nodeID. -// The function f must be thread-safe. -func (c *Configuration) MulticastPerNode(ctx context.Context, in *Request, f func(*Request, uint32) *Request, opts ...gorums.CallOption) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "oneway.OnewayTest.MulticastPerNode", - } - - cd.PerNodeArgFn = func(req proto.Message, nid uint32) proto.Message { - return f(req.(*Request), nid) - } - - c.RawConfiguration.Multicast(ctx, cd, opts...) -} - -// There are no quorum calls. -type QuorumSpec interface{} - // OnewayTest is the server-side API for the OnewayTest Service type OnewayTestServer interface { Unicast(ctx gorums.ServerCtx, request *Request) Multicast(ctx gorums.ServerCtx, request *Request) - MulticastPerNode(ctx gorums.ServerCtx, request *Request) } func RegisterOnewayTestServer(srv *gorums.Server, impl OnewayTestServer) { @@ -216,20 +97,4 @@ func RegisterOnewayTestServer(srv *gorums.Server, impl OnewayTestServer) { impl.Multicast(ctx, req) return nil, nil }) - srv.RegisterHandler("oneway.OnewayTest.MulticastPerNode", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - impl.MulticastPerNode(ctx, req) - return nil, nil - }) -} - -// Unicast is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (n *Node) Unicast(ctx context.Context, in *Request, opts ...gorums.CallOption) { - cd := gorums.CallData{ - Message: in, - Method: "oneway.OnewayTest.Unicast", - } - - n.RawNode.Unicast(ctx, cd, opts...) } diff --git a/internal/tests/oneway/oneway_test.go b/internal/tests/oneway/oneway_test.go index 91c6de5be..8b1c96d50 100644 --- a/internal/tests/oneway/oneway_test.go +++ b/internal/tests/oneway/oneway_test.go @@ -8,25 +8,18 @@ import ( "github.com/relab/gorums" "github.com/relab/gorums/internal/tests/oneway" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/emptypb" ) +const numCalls = 50 + type onewaySrv struct { benchmark bool wg sync.WaitGroup received chan *oneway.Request } -func (s *onewaySrv) Unicast(ctx gorums.ServerCtx, r *oneway.Request) { - if s.benchmark { - return - } - s.received <- r - s.wg.Done() -} - -func (s *onewaySrv) Multicast(ctx gorums.ServerCtx, r *oneway.Request) { +func (s *onewaySrv) Unicast(_ gorums.ServerCtx, r *oneway.Request) { if s.benchmark { return } @@ -34,7 +27,7 @@ func (s *onewaySrv) Multicast(ctx gorums.ServerCtx, r *oneway.Request) { s.wg.Done() } -func (s *onewaySrv) MulticastPerNode(ctx gorums.ServerCtx, r *oneway.Request) { +func (s *onewaySrv) Multicast(_ gorums.ServerCtx, r *oneway.Request) { if s.benchmark { return } @@ -42,43 +35,24 @@ func (s *onewaySrv) MulticastPerNode(ctx gorums.ServerCtx, r *oneway.Request) { s.wg.Done() } -type testQSpec struct{} - -func setup(t testing.TB, cfgSize int) (cfg *oneway.Configuration, srvs []*onewaySrv, teardown func()) { +// setupWithNodeMap sets up servers and configuration with sequential node IDs +// (0, 1, 2, ...) matching the server array indices. This is needed for tests like +// TestMulticastPerNode that verify per-node message transformations based on node ID. +func setupWithNodeMap(t testing.TB, cfgSize int) (cfg gorums.Configuration, srvs []*onewaySrv) { t.Helper() srvs = make([]*onewaySrv, cfgSize) for i := range cfgSize { srvs[i] = &onewaySrv{received: make(chan *oneway.Request, numCalls)} } - addrs, closeServers := gorums.TestSetup(t, cfgSize, func(i int) gorums.ServerIface { + cfg = gorums.TestConfiguration(t, cfgSize, func(i int) gorums.ServerIface { srv := gorums.NewServer() oneway.RegisterOnewayTestServer(srv, srvs[i]) return srv }) - nodeMap := make(map[string]uint32) - for i, addr := range addrs { - nodeMap[addr] = uint32(i) - } - - mgr := oneway.NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - cfg, err := mgr.NewConfiguration(&testQSpec{}, gorums.WithNodeMap(nodeMap)) - if err != nil { - t.Fatal(err) - } - teardown = func() { - mgr.Close() - closeServers() - } - return cfg, srvs, teardown + return cfg, srvs } -const numCalls = 50 - func TestOnewayCalls(t *testing.T) { tests := []struct { name string @@ -96,26 +70,38 @@ func TestOnewayCalls(t *testing.T) { {name: "MulticastNoSendWaiting", calls: numCalls, servers: 9, sendWait: false}, } - f := func(c *oneway.Configuration) func(context.Context, *oneway.Request, ...gorums.CallOption) { - if c.Size() == 1 { - return c.Nodes()[0].Unicast - } - return c.Multicast - } - for _, test := range tests { t.Run(fmt.Sprintf("%s/Servers=%d", test.name, test.servers), func(t *testing.T) { - cfg, srvs, teardown := setup(t, test.servers) + config, srvs := setupWithNodeMap(t, test.servers) for i := range srvs { srvs[i].wg.Add(test.calls) } for c := 1; c <= test.calls; c++ { in := oneway.Request_builder{Num: uint64(c)}.Build() - if test.sendWait { - f(cfg)(context.Background(), in) + if config.Size() == 1 { + node := config[0] + nodeCtx := node.Context(context.Background()) + if test.sendWait { + if err := oneway.Unicast(nodeCtx, in); err != nil { + t.Error(err) + } + } else { + if err := oneway.Unicast(nodeCtx, in, gorums.IgnoreErrors()); err != nil { + t.Error(err) + } + } } else { - f(cfg)(context.Background(), in, gorums.WithNoSendWaiting()) + cfgCtx := config.Context(context.Background()) + if test.sendWait { + if err := oneway.Multicast(cfgCtx, in); err != nil { + t.Error(err) + } + } else { + if err := oneway.Multicast(cfgCtx, in, gorums.IgnoreErrors()); err != nil { + t.Error(err) + } + } } } @@ -131,7 +117,6 @@ func TestOnewayCalls(t *testing.T) { expected++ } } - teardown() }) } } @@ -139,23 +124,26 @@ func TestOnewayCalls(t *testing.T) { func TestMulticastPerNode(t *testing.T) { add := func(n uint64, id uint32) uint64 { return n + uint64(id) } - // simple transformation function - f := func(msg *oneway.Request, id uint32) *oneway.Request { - return oneway.Request_builder{Num: add(msg.GetNum(), id)}.Build() + // transformation function that uses the MapRequest interceptor + // to add the msg ID + node ID to the Num field + f := func(msg *oneway.Request, node *gorums.Node) *oneway.Request { + return oneway.Request_builder{Num: add(msg.GetNum(), node.ID())}.Build() } - ignoreNodes := []int{} + // the ignoreNodes slice is updated in each test case below; it is a hack + // to avoid having to pass it through to the ignore() function. + ignoreNodes := []int{} // skipcq: GO-W1027 ignore := func(id uint32) bool { for _, ignore := range ignoreNodes { return id == uint32(ignore) } return false } - // transformation of all except some nodes - g := func(msg *oneway.Request, id uint32) *oneway.Request { - if ignore(id) { + // transformation for all except some nodes that are ignored + g := func(msg *oneway.Request, node *gorums.Node) *oneway.Request { + if ignore(node.ID()) { return nil } - return oneway.Request_builder{Num: add(msg.GetNum(), id)}.Build() + return oneway.Request_builder{Num: add(msg.GetNum(), node.ID())}.Build() } tests := []struct { name string @@ -163,28 +151,28 @@ func TestMulticastPerNode(t *testing.T) { servers int sendWait bool ignoreNodes []int - f func(*oneway.Request, uint32) *oneway.Request + mapFunc func(*oneway.Request, *gorums.Node) *oneway.Request }{ - {name: "MulticastPerNodeNoSendWaiting", calls: numCalls, servers: 1, sendWait: false, f: f}, - {name: "MulticastPerNodeNoSendWaiting", calls: numCalls, servers: 3, sendWait: false, f: f}, - {name: "MulticastPerNodeNoSendWaiting", calls: numCalls, servers: 9, sendWait: false, f: f}, - {name: "MulticastPerNodeSendWaiting", calls: numCalls, servers: 1, sendWait: true, f: f}, - {name: "MulticastPerNodeSendWaiting", calls: numCalls, servers: 3, sendWait: true, f: f}, - {name: "MulticastPerNodeSendWaiting", calls: numCalls, servers: 9, sendWait: true, f: f}, - {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, f: g, ignoreNodes: []int{0}}, - {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, f: g, ignoreNodes: []int{1}}, - {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, f: g, ignoreNodes: []int{0, 1}}, - {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, f: g, ignoreNodes: []int{0, 1, 2}}, - {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, f: g, ignoreNodes: []int{0}}, - {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, f: g, ignoreNodes: []int{1}}, - {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, f: g, ignoreNodes: []int{0, 1}}, - {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, f: g, ignoreNodes: []int{0, 1, 2}}, + {name: "MulticastPerNodeNoSendWaiting", calls: numCalls, servers: 1, sendWait: false, mapFunc: f}, + {name: "MulticastPerNodeNoSendWaiting", calls: numCalls, servers: 3, sendWait: false, mapFunc: f}, + {name: "MulticastPerNodeNoSendWaiting", calls: numCalls, servers: 9, sendWait: false, mapFunc: f}, + {name: "MulticastPerNodeSendWaiting", calls: numCalls, servers: 1, sendWait: true, mapFunc: f}, + {name: "MulticastPerNodeSendWaiting", calls: numCalls, servers: 3, sendWait: true, mapFunc: f}, + {name: "MulticastPerNodeSendWaiting", calls: numCalls, servers: 9, sendWait: true, mapFunc: f}, + {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, mapFunc: g, ignoreNodes: []int{0}}, + {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, mapFunc: g, ignoreNodes: []int{1}}, + {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, mapFunc: g, ignoreNodes: []int{0, 1}}, + {name: "MulticastPerNodeNoSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: false, mapFunc: g, ignoreNodes: []int{0, 1, 2}}, + {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, mapFunc: g, ignoreNodes: []int{0}}, + {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, mapFunc: g, ignoreNodes: []int{1}}, + {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, mapFunc: g, ignoreNodes: []int{0, 1}}, + {name: "MulticastPerNodeSendWaitingIgnoreNodes", calls: numCalls, servers: 3, sendWait: true, mapFunc: g, ignoreNodes: []int{0, 1, 2}}, } for _, test := range tests { t.Run(fmt.Sprintf("%s/Servers=%d/IgnoredNodes=%v", test.name, test.servers, test.ignoreNodes), func(t *testing.T) { - cfg, srvs, teardown := setup(t, test.servers) - nodeIDs := cfg.NodeIDs() + config, srvs := setupWithNodeMap(t, test.servers) + nodeIDs := config.NodeIDs() // set the ignoreNodes variable used by the ignore() function ignoreNodes = test.ignoreNodes @@ -197,10 +185,21 @@ func TestMulticastPerNode(t *testing.T) { for c := 1; c <= test.calls; c++ { in := oneway.Request_builder{Num: uint64(c)}.Build() + cfgCtx := config.Context(context.Background()) + mapInterceptor := gorums.MapRequest[*oneway.Request, *emptypb.Empty](test.mapFunc) if test.sendWait { - cfg.MulticastPerNode(context.Background(), in, test.f) + if err := oneway.Multicast(cfgCtx, in, + gorums.Interceptors(mapInterceptor), + ); err != nil { + t.Error(err) + } } else { - cfg.MulticastPerNode(context.Background(), in, test.f, gorums.WithNoSendWaiting()) + if err := oneway.Multicast(cfgCtx, in, + gorums.Interceptors(mapInterceptor), + gorums.IgnoreErrors(), + ); err != nil { + t.Error(err) + } } } @@ -219,35 +218,35 @@ func TestMulticastPerNode(t *testing.T) { expected++ } } - teardown() }) } } func BenchmarkUnicast(b *testing.B) { - cfg, srvs, teardown := setup(b, 1) + cfg, srvs := setupWithNodeMap(b, 1) for _, srv := range srvs { srv.benchmark = true } - node := cfg.Nodes()[0] + node := cfg[0] in := oneway.Request_builder{Num: 0}.Build() b.Run("UnicastSendWaiting__", func(b *testing.B) { for c := 1; c <= b.N; c++ { in.SetNum(uint64(c)) - node.Unicast(context.Background(), in) + nodeCtx := node.Context(context.Background()) + oneway.Unicast(nodeCtx, in) } }) b.Run("UnicastNoSendWaiting", func(b *testing.B) { for c := 1; c <= b.N; c++ { in.SetNum(uint64(c)) - node.Unicast(context.Background(), in, gorums.WithNoSendWaiting()) + nodeCtx := node.Context(context.Background()) + oneway.Unicast(nodeCtx, in, gorums.IgnoreErrors()) } }) - teardown() } func BenchmarkMulticast(b *testing.B) { - cfg, srvs, teardown := setup(b, 3) + config, srvs := setupWithNodeMap(b, 3) for _, srv := range srvs { srv.benchmark = true } @@ -255,14 +254,15 @@ func BenchmarkMulticast(b *testing.B) { b.Run("MulticastSendWaiting__", func(b *testing.B) { for c := 1; c <= b.N; c++ { in.SetNum(uint64(c)) - cfg.Multicast(context.Background(), in) + cfgCtx := config.Context(context.Background()) + oneway.Multicast(cfgCtx, in) } }) b.Run("MulticastNoSendWaiting", func(b *testing.B) { for c := 1; c <= b.N; c++ { in.SetNum(uint64(c)) - cfg.Multicast(context.Background(), in, gorums.WithNoSendWaiting()) + cfgCtx := config.Context(context.Background()) + oneway.Multicast(cfgCtx, in, gorums.IgnoreErrors()) } }) - teardown() } diff --git a/internal/tests/ordering/order.pb.go b/internal/tests/ordering/order.pb.go index ae7a1edae..817d974c3 100644 --- a/internal/tests/ordering/order.pb.go +++ b/internal/tests/ordering/order.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: ordering/order.proto package ordering @@ -143,11 +143,11 @@ const file_ordering_order_proto_rawDesc = "" + "\aRequest\x12\x10\n" + "\x03Num\x18\x01 \x01(\x04R\x03Num\"$\n" + "\bResponse\x12\x18\n" + - "\aInOrder\x18\x01 \x01(\bR\aInOrder2\xb0\x01\n" + + "\aInOrder\x18\x01 \x01(\bR\aInOrder2|\n" + "\n" + - "GorumsTest\x121\n" + - "\x02QC\x12\x11.ordering.Request\x1a\x12.ordering.Response\"\x04\xa0\xb5\x18\x01\x12:\n" + - "\aQCAsync\x12\x11.ordering.Request\x1a\x12.ordering.Response\"\b\xa0\xb5\x18\x01е\x18\x01\x123\n" + + "GorumsTest\x129\n" + + "\n" + + "QuorumCall\x12\x11.ordering.Request\x1a\x12.ordering.Response\"\x04\xa0\xb5\x18\x01\x123\n" + "\bUnaryRPC\x12\x11.ordering.Request\x1a\x12.ordering.Response\"\x00B-Z&github.com/relab/gorums/tests/ordering\x92\x03\x02\b\x02b\beditionsp\xe8\a" var file_ordering_order_proto_msgTypes = make([]protoimpl.MessageInfo, 2) @@ -156,14 +156,12 @@ var file_ordering_order_proto_goTypes = []any{ (*Response)(nil), // 1: ordering.Response } var file_ordering_order_proto_depIdxs = []int32{ - 0, // 0: ordering.GorumsTest.QC:input_type -> ordering.Request - 0, // 1: ordering.GorumsTest.QCAsync:input_type -> ordering.Request - 0, // 2: ordering.GorumsTest.UnaryRPC:input_type -> ordering.Request - 1, // 3: ordering.GorumsTest.QC:output_type -> ordering.Response - 1, // 4: ordering.GorumsTest.QCAsync:output_type -> ordering.Response - 1, // 5: ordering.GorumsTest.UnaryRPC:output_type -> ordering.Response - 3, // [3:6] is the sub-list for method output_type - 0, // [0:3] is the sub-list for method input_type + 0, // 0: ordering.GorumsTest.QuorumCall:input_type -> ordering.Request + 0, // 1: ordering.GorumsTest.UnaryRPC:input_type -> ordering.Request + 1, // 2: ordering.GorumsTest.QuorumCall:output_type -> ordering.Response + 1, // 3: ordering.GorumsTest.UnaryRPC:output_type -> ordering.Response + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/internal/tests/ordering/order.proto b/internal/tests/ordering/order.proto index b22c6542d..e03328038 100644 --- a/internal/tests/ordering/order.proto +++ b/internal/tests/ordering/order.proto @@ -8,15 +8,10 @@ option features.field_presence = IMPLICIT; import "gorums.proto"; service GorumsTest { - rpc QC(Request) returns (Response) { + rpc QuorumCall(Request) returns (Response) { option (gorums.quorumcall) = true; } - rpc QCAsync(Request) returns (Response) { - option (gorums.quorumcall) = true; - option (gorums.async) = true; - } - rpc UnaryRPC(Request) returns (Response) {} } diff --git a/internal/tests/ordering/order_gorums.pb.go b/internal/tests/ordering/order_gorums.pb.go index 71762717b..9b4f25e1f 100644 --- a/internal/tests/ordering/order_gorums.pb.go +++ b/internal/tests/ordering/order_gorums.pb.go @@ -1,17 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: ordering/order.proto package ordering import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" - proto "google.golang.org/protobuf/proto" ) const ( @@ -21,221 +17,80 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} - -// QCAsync asynchronously invokes a quorum call on configuration c -// and returns a AsyncResponse, which can be used to inspect the quorum call -// reply and error when available. -func (c *Configuration) QCAsync(ctx context.Context, in *Request) *AsyncResponse { - cd := gorums.QuorumCallData{ - Message: in, - Method: "ordering.GorumsTest.QCAsync", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QCAsyncQF(req.(*Request), r) - } - - fut := c.RawConfiguration.AsyncCall(ctx, cd) - return &AsyncResponse{fut} +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// GorumsTestClient is the client interface for the GorumsTest service. -type GorumsTestClient interface { - QC(ctx context.Context, in *Request) (resp *Response, err error) - QCAsync(ctx context.Context, in *Request) *AsyncResponse -} - -// enforce interface compliance -var _ GorumsTestClient = (*Configuration)(nil) - -// GorumsTestNodeClient is the single node client interface for the GorumsTest service. -type GorumsTestNodeClient interface { - UnaryRPC(ctx context.Context, in *Request) (resp *Response, err error) +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ GorumsTestNodeClient = (*Node)(nil) - -// QuorumSpec is the interface of quorum functions for GorumsTest. -type QuorumSpec interface { - gorums.ConfigOption +// AsyncResponse is a future for async quorum calls returning *Response. +type AsyncResponse = *gorums.Async[*Response] - // QCQF is the quorum function for the QC - // quorum call method. The in parameter is the request object - // supplied to the QC method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QCQF(in *Request, replies map[uint32]*Response) (*Response, bool) +// CorrectableResponse is a correctable object for quorum calls returning *Response. +type CorrectableResponse = *gorums.Correctable[*Response] - // QCAsyncQF is the quorum function for the QCAsync - // asynchronous quorum call method. The in parameter is the request object - // supplied to the QCAsync method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - QCAsyncQF(in *Request, replies map[uint32]*Response) (*Response, bool) -} - -// QC is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) QC(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "ordering.GorumsTest.QC", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.QCQF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err +// QuorumCall is a quorum call invoked on all nodes in the configuration, +// with the same argument in. Use terminal methods like Majority(), First(), +// or Threshold(n) to retrieve the aggregated result. +// +// Example: +// +// resp, err := QuorumCall(ctx, in).Majority() +func QuorumCall(ctx *gorums.ConfigContext, in *Request, opts ...gorums.CallOption) *gorums.Responses[*Response] { + return gorums.QuorumCall[*Request, *Response]( + ctx, in, "ordering.GorumsTest.QuorumCall", + opts..., + ) } -// UnaryRPC is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (n *Node) UnaryRPC(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.CallData{ - Message: in, - Method: "ordering.GorumsTest.UnaryRPC", - } - - res, err := n.RawNode.RPCCall(ctx, cd) +// UnaryRPC is an RPC call invoked on the node in ctx. +func UnaryRPC(ctx *gorums.NodeContext, in *Request) (resp *Response, err error) { + res, err := gorums.RPCCall(ctx, in, "ordering.GorumsTest.UnaryRPC") if err != nil { return nil, err } @@ -244,20 +99,14 @@ func (n *Node) UnaryRPC(ctx context.Context, in *Request) (resp *Response, err e // GorumsTest is the server-side API for the GorumsTest Service type GorumsTestServer interface { - QC(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - QCAsync(ctx gorums.ServerCtx, request *Request) (response *Response, err error) + QuorumCall(ctx gorums.ServerCtx, request *Request) (response *Response, err error) UnaryRPC(ctx gorums.ServerCtx, request *Request) (response *Response, err error) } func RegisterGorumsTestServer(srv *gorums.Server, impl GorumsTestServer) { - srv.RegisterHandler("ordering.GorumsTest.QC", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + srv.RegisterHandler("ordering.GorumsTest.QuorumCall", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { req := gorums.AsProto[*Request](in) - resp, err := impl.QC(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("ordering.GorumsTest.QCAsync", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.QCAsync(ctx, req) + resp, err := impl.QuorumCall(ctx, req) return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) srv.RegisterHandler("ordering.GorumsTest.UnaryRPC", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { @@ -266,24 +115,3 @@ func RegisterGorumsTestServer(srv *gorums.Server, impl GorumsTestServer) { return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) } - -type internalResponse struct { - nid uint32 - reply *Response - err error -} - -// AsyncResponse is a async object for processing replies. -type AsyncResponse struct { - *gorums.Async -} - -// Get returns the reply and any error associated with the called method. -// The method blocks until a reply or error is available. -func (f *AsyncResponse) Get() (*Response, error) { - resp, err := f.Async.Get() - if err != nil { - return nil, err - } - return resp.(*Response), err -} diff --git a/internal/tests/ordering/order_stress_test.go b/internal/tests/ordering/order_stress_test.go new file mode 100644 index 000000000..91fe682e2 --- /dev/null +++ b/internal/tests/ordering/order_stress_test.go @@ -0,0 +1,7 @@ +//go:build stress + +package ordering + +func init() { + stressMode = true +} diff --git a/internal/tests/ordering/order_test.go b/internal/tests/ordering/order_test.go index 33b72173d..14af2bfda 100644 --- a/internal/tests/ordering/order_test.go +++ b/internal/tests/ordering/order_test.go @@ -3,16 +3,47 @@ package ordering import ( "context" "errors" + "iter" "sync" "testing" "time" "github.com/relab/gorums" - "go.uber.org/goleak" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) +// stressMode controls whether tests run in stress mode (time-based) or normal mode (iteration-based). +// This is set to true in order_stress_test.go via the stress build tag. +var stressMode = false + +// testIterations is the number of iterations for ordering tests in normal mode. +const testIterations = 100 + +// stressDuration is the duration for stress tests when stressMode is true. +const stressDuration = 5 * time.Second + +// iterations returns an iterator that yields sequential integers starting from 1. +// In normal mode, it yields testIterations values. +// In stress mode, it yields values for stressDuration. +func iterations() iter.Seq[int] { + if stressMode { + return func(yield func(int) bool) { + stopTime := time.Now().Add(stressDuration) + for i := 1; time.Now().Before(stopTime); i++ { + if !yield(i) { + return + } + } + } + } + return func(yield func(int) bool) { + for i := range testIterations { + if !yield(i + 1) { + return + } + } + } +} + type testSrv struct { sync.Mutex lastNum uint64 @@ -28,13 +59,7 @@ func (s *testSrv) isInOrder(num uint64) bool { return false } -func (s *testSrv) QC(_ gorums.ServerCtx, req *Request) (resp *Response, err error) { - return Response_builder{ - InOrder: s.isInOrder(req.GetNum()), - }.Build(), nil -} - -func (s *testSrv) QCAsync(_ gorums.ServerCtx, req *Request) (resp *Response, err error) { +func (s *testSrv) QuorumCall(_ gorums.ServerCtx, req *Request) (resp *Response, err error) { return Response_builder{ InOrder: s.isInOrder(req.GetNum()), }.Build(), nil @@ -46,73 +71,20 @@ func (s *testSrv) UnaryRPC(_ gorums.ServerCtx, req *Request) (resp *Response, er }.Build(), nil } -type testQSpec struct { - quorum int -} - -func (q testQSpec) qf(replies map[uint32]*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - for _, reply := range replies { - if !reply.GetInOrder() { - return reply, true - } - } - var reply *Response - for _, r := range replies { - reply = r - break - } - return reply, true -} - -func (q testQSpec) QCQF(_ *Request, replies map[uint32]*Response) (*Response, bool) { - return q.qf(replies) -} - -func (q testQSpec) QCAsyncQF(_ *Request, replies map[uint32]*Response) (*Response, bool) { - return q.qf(replies) -} - -func (q testQSpec) AsyncHandlerQF(_ *Request, replies map[uint32]*Response) (*Response, bool) { - return q.qf(replies) -} - -func setup(t *testing.T, cfgSize int) (cfg *Configuration, teardown func()) { - t.Helper() - addrs, closeServers := gorums.TestSetup(t, cfgSize, func(_ int) gorums.ServerIface { - srv := gorums.NewServer() - RegisterGorumsTestServer(srv, &testSrv{}) - return srv - }) - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - cfg, err := mgr.NewConfiguration(&testQSpec{cfgSize}, gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } - teardown = func() { - mgr.Close() - closeServers() - } - return cfg, teardown +// serverFn creates a new server with an independent testSrv instance. +// Each server needs its own testSrv to track ordering independently. +func serverFn(_ int) gorums.ServerIface { + srv := gorums.NewServer() + RegisterGorumsTestServer(srv, &testSrv{}) + return srv } func TestUnaryRPCOrdering(t *testing.T) { - defer goleak.VerifyNone(t) - cfg, teardown := setup(t, 1) - defer teardown() - node := cfg.Nodes()[0] - // begin test - stopTime := time.Now().Add(5 * time.Second) - i := 1 - for time.Now().Before(stopTime) { - i++ - resp, err := node.UnaryRPC(context.Background(), Request_builder{Num: uint64(i)}.Build()) + node := gorums.TestNode(t, serverFn) + + for i := range iterations() { + nodeCtx := node.Context(t.Context()) + resp, err := UnaryRPC(nodeCtx, Request_builder{Num: uint64(i)}.Build()) if err != nil { t.Fatalf("RPC error: %v", err) } @@ -125,47 +97,40 @@ func TestUnaryRPCOrdering(t *testing.T) { } } -func TestQCOrdering(t *testing.T) { - defer goleak.VerifyNone(t) - cfg, teardown := setup(t, 4) - defer teardown() - // begin test - stopTime := time.Now().Add(5 * time.Second) - i := 1 - for time.Now().Before(stopTime) { - i++ - resp, err := cfg.QC(context.Background(), Request_builder{Num: uint64(i)}.Build()) - if err != nil { - t.Fatalf("QC error: %v", err) - } - if resp == nil { - t.Fatal("Got nil response") +func TestQuorumCallOrdering(t *testing.T) { + config := gorums.TestConfiguration(t, 4, serverFn) + cfgCtx := config.Context(t.Context()) + + for i := range iterations() { + // Use CollectAll to get all responses and check for ordering + responses := QuorumCall(cfgCtx, Request_builder{Num: uint64(i)}.Build()) + replies := responses.CollectAll() + if len(replies) < config.Size() { + t.Fatalf("incomplete call: %d replies", len(replies)) } - if !resp.GetInOrder() { - t.Fatalf("Message received out of order.") + for _, reply := range replies { + if !reply.GetInOrder() { + t.Fatalf("Message received out of order.") + } } } } -func TestQCAsyncOrdering(t *testing.T) { - defer goleak.VerifyNone(t) - cfg, teardown := setup(t, 4) - defer teardown() - ctx, cancel := context.WithCancel(context.Background()) - // begin test +func TestQuorumCallAsyncOrdering(t *testing.T) { + config := gorums.TestConfiguration(t, 4, serverFn) + cfgCtx := config.Context(t.Context()) + var wg sync.WaitGroup - stopTime := time.Now().Add(5 * time.Second) - i := 1 - for time.Now().Before(stopTime) { - i++ - promise := cfg.QCAsync(ctx, Request_builder{Num: uint64(i)}.Build()) + for i := range iterations() { + // QuorumCall returns Responses; use .AsyncMajority() to get an Async future + promise := QuorumCall(cfgCtx, Request_builder{Num: uint64(i)}.Build()).AsyncMajority() wg.Go(func() { resp, err := promise.Get() if err != nil { if errors.Is(err, context.Canceled) { return } - t.Errorf("QC error: %v", err) + t.Errorf("QuorumCall error: %v", err) } if resp == nil { t.Errorf("Got nil response") @@ -176,33 +141,29 @@ func TestQCAsyncOrdering(t *testing.T) { }) } wg.Wait() - cancel() } func TestMixedOrdering(t *testing.T) { - defer goleak.VerifyNone(t) - cfg, teardown := setup(t, 4) - defer teardown() - nodes := cfg.Nodes() - // begin test - stopTime := time.Now().Add(5 * time.Second) - i := 1 - for time.Now().Before(stopTime) { - resp, err := cfg.QC(context.Background(), Request_builder{Num: uint64(i)}.Build()) - i++ - if err != nil { - t.Fatalf("QC error: %v", err) + config := gorums.TestConfiguration(t, 4, serverFn) + cfgCtx := config.Context(t.Context()) + + for i := range iterations() { + // Use CollectAll to get all responses and check for ordering + responses := QuorumCall(cfgCtx, Request_builder{Num: uint64(2*i - 1)}.Build()) + replies := responses.CollectAll() + if len(replies) < config.Size() { + t.Fatalf("incomplete call: %d replies", len(replies)) } - if resp == nil { - t.Fatal("Got nil response") - } - if !resp.GetInOrder() { - t.Fatalf("Message received out of order.") + for _, reply := range replies { + if !reply.GetInOrder() { + t.Fatalf("Message received out of order.") + } } var wg sync.WaitGroup - for _, node := range nodes { + for _, node := range config.Nodes() { wg.Go(func() { - resp, err := node.UnaryRPC(context.Background(), Request_builder{Num: uint64(i)}.Build()) + nodeCtx := node.Context(t.Context()) + resp, err := UnaryRPC(nodeCtx, Request_builder{Num: uint64(2 * i)}.Build()) if err != nil { t.Errorf("RPC error: %v", err) return @@ -217,6 +178,5 @@ func TestMixedOrdering(t *testing.T) { }) } wg.Wait() - i++ } } diff --git a/internal/tests/qf/plots/benchgraph-run-map-and-slice.html b/internal/tests/qf/plots/benchgraph-run-map-and-slice.html deleted file mode 100644 index 9ad1ad06f..000000000 --- a/internal/tests/qf/plots/benchgraph-run-map-and-slice.html +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - Quorum Function Evaluation - - - - - - - - - - - - -
- - - - \ No newline at end of file diff --git a/internal/tests/qf/plots/benchgraph-run1.html b/internal/tests/qf/plots/benchgraph-run1.html deleted file mode 100644 index a655e4b46..000000000 --- a/internal/tests/qf/plots/benchgraph-run1.html +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - Quorum Function Evaluation - - - - - - - - - - - - -
- - - - \ No newline at end of file diff --git a/internal/tests/qf/plots/benchgraph-run2.html b/internal/tests/qf/plots/benchgraph-run2.html deleted file mode 100644 index 13b7d2bf9..000000000 --- a/internal/tests/qf/plots/benchgraph-run2.html +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - Quorum Function Evaluation - - - - - - - - - - - - -
- - - - \ No newline at end of file diff --git a/internal/tests/qf/plots/benchgraph-run3.html b/internal/tests/qf/plots/benchgraph-run3.html deleted file mode 100644 index e5ade8042..000000000 --- a/internal/tests/qf/plots/benchgraph-run3.html +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - Quorum Function Evaluation - - - - - - - - - - - - -
- - - - \ No newline at end of file diff --git a/internal/tests/qf/qf.pb.go b/internal/tests/qf/qf.pb.go deleted file mode 100644 index 414aa267b..000000000 --- a/internal/tests/qf/qf.pb.go +++ /dev/null @@ -1,192 +0,0 @@ -// Test to benchmark quorum functions with and without the request parameter. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 -// source: qf/qf.proto - -package qf - -import ( - _ "github.com/relab/gorums" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - unsafe "unsafe" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Request struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Value int64 `protobuf:"varint,1,opt,name=Value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *Request) Reset() { - *x = Request{} - mi := &file_qf_qf_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *Request) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Request) ProtoMessage() {} - -func (x *Request) ProtoReflect() protoreflect.Message { - mi := &file_qf_qf_proto_msgTypes[0] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -func (x *Request) GetValue() int64 { - if x != nil { - return x.xxx_hidden_Value - } - return 0 -} - -func (x *Request) SetValue(v int64) { - x.xxx_hidden_Value = v -} - -type Request_builder struct { - _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - - Value int64 -} - -func (b0 Request_builder) Build() *Request { - m0 := &Request{} - b, x := &b0, m0 - _, _ = b, x - x.xxx_hidden_Value = b.Value - return m0 -} - -type Response struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Result int64 `protobuf:"varint,1,opt,name=Result"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *Response) Reset() { - *x = Response{} - mi := &file_qf_qf_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *Response) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Response) ProtoMessage() {} - -func (x *Response) ProtoReflect() protoreflect.Message { - mi := &file_qf_qf_proto_msgTypes[1] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -func (x *Response) GetResult() int64 { - if x != nil { - return x.xxx_hidden_Result - } - return 0 -} - -func (x *Response) SetResult(v int64) { - x.xxx_hidden_Result = v -} - -type Response_builder struct { - _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - - Result int64 -} - -func (b0 Response_builder) Build() *Response { - m0 := &Response{} - b, x := &b0, m0 - _, _ = b, x - x.xxx_hidden_Result = b.Result - return m0 -} - -var File_qf_qf_proto protoreflect.FileDescriptor - -const file_qf_qf_proto_rawDesc = "" + - "\n" + - "\vqf/qf.proto\x12\x02qf\x1a\fgorums.proto\"\x1f\n" + - "\aRequest\x12\x14\n" + - "\x05Value\x18\x01 \x01(\x03R\x05Value\"\"\n" + - "\bResponse\x12\x16\n" + - "\x06Result\x18\x01 \x01(\x03R\x06Result2i\n" + - "\x0eQuorumFunction\x12)\n" + - "\x06UseReq\x12\v.qf.Request\x1a\f.qf.Response\"\x04\xa0\xb5\x18\x01\x12,\n" + - "\tIgnoreReq\x12\v.qf.Request\x1a\f.qf.Response\"\x04\xa0\xb5\x18\x01B'Z github.com/relab/gorums/tests/qf\x92\x03\x02\b\x02b\beditionsp\xe8\a" - -var file_qf_qf_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_qf_qf_proto_goTypes = []any{ - (*Request)(nil), // 0: qf.Request - (*Response)(nil), // 1: qf.Response -} -var file_qf_qf_proto_depIdxs = []int32{ - 0, // 0: qf.QuorumFunction.UseReq:input_type -> qf.Request - 0, // 1: qf.QuorumFunction.IgnoreReq:input_type -> qf.Request - 1, // 2: qf.QuorumFunction.UseReq:output_type -> qf.Response - 1, // 3: qf.QuorumFunction.IgnoreReq:output_type -> qf.Response - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_qf_qf_proto_init() } -func file_qf_qf_proto_init() { - if File_qf_qf_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_qf_qf_proto_rawDesc), len(file_qf_qf_proto_rawDesc)), - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_qf_qf_proto_goTypes, - DependencyIndexes: file_qf_qf_proto_depIdxs, - MessageInfos: file_qf_qf_proto_msgTypes, - }.Build() - File_qf_qf_proto = out.File - file_qf_qf_proto_goTypes = nil - file_qf_qf_proto_depIdxs = nil -} diff --git a/internal/tests/qf/qf.proto b/internal/tests/qf/qf.proto deleted file mode 100644 index a49b101da..000000000 --- a/internal/tests/qf/qf.proto +++ /dev/null @@ -1,25 +0,0 @@ -// Test to benchmark quorum functions with and without the request parameter. -edition = "2023"; - -package qf; -option go_package = "github.com/relab/gorums/tests/qf"; - -option features.field_presence = IMPLICIT; - -import "gorums.proto"; - -service QuorumFunction { - rpc UseReq(Request) returns (Response) { - option (gorums.quorumcall) = true; - } - rpc IgnoreReq(Request) returns (Response) { - option (gorums.quorumcall) = true; - } -} - -message Request { - int64 Value = 1; -} -message Response { - int64 Result = 1; -} diff --git a/internal/tests/qf/qf_gorums.pb.go b/internal/tests/qf/qf_gorums.pb.go deleted file mode 100644 index ee2394548..000000000 --- a/internal/tests/qf/qf_gorums.pb.go +++ /dev/null @@ -1,247 +0,0 @@ -// Code generated by protoc-gen-gorums. DO NOT EDIT. -// versions: -// protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 -// source: qf/qf.proto - -package qf - -import ( - context "context" - fmt "fmt" - gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" - proto "google.golang.org/protobuf/proto" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = gorums.EnforceVersion(10 - gorums.MinVersion) - // Verify that the gorums runtime is sufficiently up-to-date. - _ = gorums.EnforceVersion(gorums.MaxVersion - 10) -) - -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} - -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} - -// NewManager returns a new Manager for managing connection to nodes added -// to the manager. This function accepts manager options used to configure -// various aspects of the manager. -func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } -} - -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. -// Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. -// A new configuration can also be created from an existing configuration, -// using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes -} - -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} - -// QuorumFunctionClient is the client interface for the QuorumFunction service. -type QuorumFunctionClient interface { - UseReq(ctx context.Context, in *Request) (resp *Response, err error) - IgnoreReq(ctx context.Context, in *Request) (resp *Response, err error) -} - -// enforce interface compliance -var _ QuorumFunctionClient = (*Configuration)(nil) - -// QuorumSpec is the interface of quorum functions for QuorumFunction. -type QuorumSpec interface { - gorums.ConfigOption - - // UseReqQF is the quorum function for the UseReq - // quorum call method. The in parameter is the request object - // supplied to the UseReq method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - UseReqQF(in *Request, replies map[uint32]*Response) (*Response, bool) - - // IgnoreReqQF is the quorum function for the IgnoreReq - // quorum call method. The in parameter is the request object - // supplied to the IgnoreReq method at call time, and may or may not - // be used by the quorum function. If the in parameter is not needed - // you should implement your quorum function with '_ *Request'. - IgnoreReqQF(in *Request, replies map[uint32]*Response) (*Response, bool) -} - -// UseReq is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) UseReq(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "qf.QuorumFunction.UseReq", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.UseReqQF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err -} - -// IgnoreReq is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (c *Configuration) IgnoreReq(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.QuorumCallData{ - Message: in, - Method: "qf.QuorumFunction.IgnoreReq", - } - cd.QuorumFunction = func(req proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - r := make(map[uint32]*Response, len(replies)) - for k, v := range replies { - r[k] = v.(*Response) - } - return c.qspec.IgnoreReqQF(req.(*Request), r) - } - - res, err := c.RawConfiguration.QuorumCall(ctx, cd) - if err != nil { - return nil, err - } - return res.(*Response), err -} - -// QuorumFunction is the server-side API for the QuorumFunction Service -type QuorumFunctionServer interface { - UseReq(ctx gorums.ServerCtx, request *Request) (response *Response, err error) - IgnoreReq(ctx gorums.ServerCtx, request *Request) (response *Response, err error) -} - -func RegisterQuorumFunctionServer(srv *gorums.Server, impl QuorumFunctionServer) { - srv.RegisterHandler("qf.QuorumFunction.UseReq", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.UseReq(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) - srv.RegisterHandler("qf.QuorumFunction.IgnoreReq", func(ctx gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { - req := gorums.AsProto[*Request](in) - resp, err := impl.IgnoreReq(ctx, req) - return gorums.NewResponseMessage(in.GetMetadata(), resp), err - }) -} - -type internalResponse struct { - nid uint32 - reply *Response - err error -} diff --git a/internal/tests/qf/qf_test.go b/internal/tests/qf/qf_test.go deleted file mode 100644 index 3589dd8d9..000000000 --- a/internal/tests/qf/qf_test.go +++ /dev/null @@ -1,236 +0,0 @@ -package qf - -import ( - "context" - "fmt" - "testing" - - "github.com/relab/gorums" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -const requestValue = 0x1001_1001 - -type testQSpec struct { - quorum int -} - -func (q testQSpec) SliceUseReqQF(in *Request, replies []*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - expected := in.GetValue() - for _, reply := range replies { - if expected != reply.GetResult() { - return reply, true - } - } - return replies[0], true -} - -func (q testQSpec) SliceIgnoreReqQF(_ *Request, replies []*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - expected := int64(requestValue) - for _, reply := range replies { - if expected != reply.GetResult() { - return reply, true - } - } - return replies[0], true -} - -func (q testQSpec) SliceWithoutReqQF(replies []*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - expected := int64(requestValue) - for _, reply := range replies { - if expected != reply.GetResult() { - return reply, true - } - } - return replies[0], true -} - -func (q testQSpec) UseReqQF(in *Request, replies map[uint32]*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - _ = in.GetValue() - var reply *Response - for _, r := range replies { - reply = r - break - } - return reply, true -} - -func (q testQSpec) IgnoreReqQF(_ *Request, replies map[uint32]*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - var reply *Response - for _, r := range replies { - reply = r - break - } - return reply, true -} - -func (q testQSpec) WithoutReqQF(replies map[uint32]*Response) (*Response, bool) { - if len(replies) < q.quorum { - return nil, false - } - var reply *Response - for _, r := range replies { - reply = r - break - } - return reply, true -} - -func BenchmarkQF(b *testing.B) { - for n := 3; n < 20; n += 2 { - quorum := n / 2 - mapSize := n << 1 - qspec := &testQSpec{quorum: quorum} - request := Request_builder{Value: 1}.Build() - - b.Run(fmt.Sprintf("UseReq_%d", n), func(b *testing.B) { - for b.Loop() { - replies := make(map[uint32]*Response, mapSize) - for j := range n { - replies[uint32(j)] = Response_builder{Result: request.GetValue()}.Build() - resp, q := qspec.UseReqQF(request, replies) - if q { - _ = resp.GetResult() - } - } - } - }) - b.Run(fmt.Sprintf("IgnoreReq_%d", n), func(b *testing.B) { - for b.Loop() { - replies := make(map[uint32]*Response, mapSize) - for j := range n { - replies[uint32(j)] = Response_builder{Result: request.GetValue()}.Build() - resp, q := qspec.IgnoreReqQF(request, replies) - if q { - _ = resp.GetResult() - } - } - } - }) - b.Run(fmt.Sprintf("WithoutReq_%d", n), func(b *testing.B) { - for b.Loop() { - replies := make(map[uint32]*Response, mapSize) - for j := range n { - replies[uint32(j)] = Response_builder{Result: request.GetValue()}.Build() - resp, q := qspec.WithoutReqQF(replies) - if q { - _ = resp.GetResult() - } - } - } - }) - - // Slice versions - - b.Run(fmt.Sprintf("SliceUseReq_%d", n), func(b *testing.B) { - for b.Loop() { - replies := make([]*Response, n) - for j := range n { - replies[uint32(j)] = Response_builder{Result: request.GetValue()}.Build() - resp, q := qspec.SliceUseReqQF(request, replies) - if q { - _ = resp.GetResult() - } - } - } - }) - b.Run(fmt.Sprintf("SliceIgnoreReq_%d", n), func(b *testing.B) { - for b.Loop() { - replies := make([]*Response, n) - for j := range n { - replies[uint32(j)] = Response_builder{Result: request.GetValue()}.Build() - resp, q := qspec.SliceIgnoreReqQF(request, replies) - if q { - _ = resp.GetResult() - } - } - } - }) - b.Run(fmt.Sprintf("SliceWithoutReq_%d", n), func(b *testing.B) { - for b.Loop() { - replies := make([]*Response, n) - for j := range n { - replies[uint32(j)] = Response_builder{Result: request.GetValue()}.Build() - resp, q := qspec.SliceWithoutReqQF(replies) - if q { - _ = resp.GetResult() - } - } - } - }) - } -} - -type testSrv struct{} - -func (s testSrv) UseReq(_ gorums.ServerCtx, req *Request) (resp *Response, err error) { - return Response_builder{Result: req.GetValue()}.Build(), nil -} - -func (s testSrv) IgnoreReq(_ gorums.ServerCtx, req *Request) (resp *Response, err error) { - return Response_builder{Result: req.GetValue()}.Build(), nil -} - -func BenchmarkFullStackQF(b *testing.B) { - for n := 3; n < 20; n += 2 { - _, stop := gorums.TestSetup(b, n, func(_ int) gorums.ServerIface { - srv := gorums.NewServer() - RegisterQuorumFunctionServer(srv, &testSrv{}) - return srv - }) - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - c, err := mgr.NewConfiguration( - &testQSpec{quorum: n / 2}, - gorums.WithNodeList([]string{"127.0.0.1:9080", "127.0.0.1:9081", "127.0.0.1:9082"}), // dummy node list; won't actually be used in test - ) - if err != nil { - b.Fatal(err) - } - - b.ReportAllocs() - b.ResetTimer() - - b.Run(fmt.Sprintf("UseReq_%d", n), func(b *testing.B) { - for b.Loop() { - resp, err := c.UseReq(context.Background(), Request_builder{Value: int64(requestValue)}.Build()) - if err != nil { - b.Fatalf("UseReq error: %v", err) - } - _ = resp.GetResult() - } - }) - b.Run(fmt.Sprintf("IgnoreReq_%d", n), func(b *testing.B) { - for b.Loop() { - resp, err := c.IgnoreReq(context.Background(), Request_builder{Value: int64(requestValue)}.Build()) - if err != nil { - b.Fatalf("IgnoreReq error: %v", err) - } - _ = resp.GetResult() - } - }) - // close manager and stop gRPC servers; - // must be done for each iteration - mgr.Close() - stop() - } -} diff --git a/internal/tests/qf/readme.md b/internal/tests/qf/readme.md deleted file mode 100644 index 1d2edfc4e..000000000 --- a/internal/tests/qf/readme.md +++ /dev/null @@ -1,46 +0,0 @@ -# Benchmarks for quorum functions with and without the request object - -The question we want to answer in this document is: -_Will Gorums be faster if we generate quorum functions without the `request` parameter, when the `request` is not needed by the quorum function?_ - -Below are three runs of the benchmark for three variants of using the quorum function. - -- The `UseReqQF` method actually receive and use the `request` object passed to the quorum function. If your quorum function needs the request object, you anyway have to use this approach. -- The `IgnoreReqQF` method has the same function signature as `UseReqQF`, but ignores the `request` object by using a `_` in the implementation. This would be the recommended approach if we make the quorum function signature always require the request object. -- The `WithoutReqQF` method has a function signature without the `request` object, and as such it should be the fastest, since it does not need to pass the request object on the stack at all. - -All three benchmark methods has the same number of allocations, and so we exclude those details from the results below. - -The conclusion from the benchmarks below is that `UseReqQF` is always slower, which is to be expected. However, from three runs of the benchmark on the same machine (a mid-2013 MacPro), it is not possible to observe any relevant difference between `WithoutReqQF` and `IgnoreReqQF`. In conclusion, it does not appear to be any value keeping the `qf_with_req` option. - -## Running the benchmarks - -First compile the qf proto file. - -```sh -cd tests -make qf -``` - -To plot graphs based on the benchmarks, you will need benchgraph: - -TODO(meling): fork this repo and fix the go.mod file - -```sh -go get -v github.com/AntonioSun/benchgraph -``` - -To run the benchmarks: - -```sh -cd qf -go test -bench=BenchmarkQF | benchgraph -title "Quorum Function Evaluation" -go test -bench=BenchmarkFullStackQF | benchgraph -title "Quorum Function Evaluation (full stack)" -``` - -To view plots for three separate runs with slice implementation and one with both map and slice implementations: - -- [Run 1](plots/benchgraph-run1.html) -- [Run 2](plots/benchgraph-run2.html) -- [Run 3](plots/benchgraph-run3.html) -- [New Run with both slice and map comparison](plots/benchgraph-run-map-and-slice.html) diff --git a/internal/tests/tls/tls.pb.go b/internal/tests/tls/tls.pb.go index 20ed1dd0f..e9d5668a9 100644 --- a/internal/tests/tls/tls.pb.go +++ b/internal/tests/tls/tls.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: tls/tls.proto package tls diff --git a/internal/tests/tls/tls_gorums.pb.go b/internal/tests/tls/tls_gorums.pb.go index 1ac541609..f0761df1c 100644 --- a/internal/tests/tls/tls_gorums.pb.go +++ b/internal/tests/tls/tls_gorums.pb.go @@ -1,16 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: tls/tls.proto package tls import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" ) const ( @@ -20,154 +17,60 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} - -// TLSNodeClient is the single node client interface for the TLS service. -type TLSNodeClient interface { - TestTLS(ctx context.Context, in *Request) (resp *Response, err error) +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ TLSNodeClient = (*Node)(nil) - -// There are no quorum calls. -type QuorumSpec interface{} - -// TestTLS is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (n *Node) TestTLS(ctx context.Context, in *Request) (resp *Response, err error) { - cd := gorums.CallData{ - Message: in, - Method: "tls.TLS.TestTLS", - } - - res, err := n.RawNode.RPCCall(ctx, cd) +// TestTLS is an RPC call invoked on the node in ctx. +func TestTLS(ctx *gorums.NodeContext, in *Request) (resp *Response, err error) { + res, err := gorums.RPCCall(ctx, in, "tls.TLS.TestTLS") if err != nil { return nil, err } diff --git a/internal/tests/tls/tls_test.go b/internal/tests/tls/tls_test.go index cd34f0aed..d4863f05e 100644 --- a/internal/tests/tls/tls_test.go +++ b/internal/tests/tls/tls_test.go @@ -14,7 +14,7 @@ import ( type testSrv struct{} -func (t testSrv) TestTLS(ctx gorums.ServerCtx, in *Request) (resp *Response, err error) { +func (testSrv) TestTLS(ctx gorums.ServerCtx, _ *Request) (resp *Response, err error) { peerInfo, ok := peer.FromContext(ctx) if !ok || peerInfo.AuthInfo.AuthType() != "tls" { return Response_builder{OK: false}.Build(), nil @@ -22,7 +22,7 @@ func (t testSrv) TestTLS(ctx gorums.ServerCtx, in *Request) (resp *Response, err return Response_builder{OK: true}.Build(), nil } -func TestTLS(t *testing.T) { +func TestTLSConnection(t *testing.T) { cert, key, err := generateCert() if err != nil { t.Errorf("Failed to generate certificate: %v", err) @@ -38,25 +38,16 @@ func TestTLS(t *testing.T) { t.Errorf("Failed to parse cert: %v", err) } - addrs, teardown := gorums.TestSetup(t, 1, func(_ int) gorums.ServerIface { + srvFn := func(_ int) gorums.ServerIface { srv := gorums.NewServer(gorums.WithGRPCServerOptions(grpc.Creds(credentials.NewServerTLSFromCert(&tlsCert)))) RegisterTLSServer(srv, &testSrv{}) return srv - }) - defer teardown() - - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(cp, "")), - ), - ) - _, err = mgr.NewConfiguration(gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) } + node := gorums.TestNode(t, srvFn, gorums.WithDialOptions( + grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(cp, "")), + )) - node := mgr.Nodes()[0] - resp, err := node.TestTLS(context.Background(), &Request{}) + resp, err := TestTLS(node.Context(context.Background()), &Request{}) if err != nil { t.Fatalf("RPC error: %v", err) } diff --git a/internal/tests/unresponsive/unreponsive_test.go b/internal/tests/unresponsive/unreponsive_test.go index 71eb0b3a4..f79a15f93 100644 --- a/internal/tests/unresponsive/unreponsive_test.go +++ b/internal/tests/unresponsive/unreponsive_test.go @@ -7,42 +7,28 @@ import ( "time" "github.com/relab/gorums" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type testSrv struct{} -func (srv testSrv) TestUnresponsive(ctx gorums.ServerCtx, _ *Empty) (resp *Empty, err error) { +func (testSrv) TestUnresponsive(ctx gorums.ServerCtx, _ *Empty) (resp *Empty, err error) { <-ctx.Done() return nil, nil } -// TestUnresponsive checks that the client is not blocked when the server is not receiving messages -func TestUnresponsive(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 1, func(_ int) gorums.ServerIface { - gorumsSrv := gorums.NewServer() - srv := &testSrv{} - RegisterUnresponsiveServer(gorumsSrv, srv) - return gorumsSrv - }) - defer teardown() - - mgr := NewManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - _, err := mgr.NewConfiguration(gorums.WithNodeList(addrs)) - if err != nil { - t.Fatal(err) - } +func serverFn(_ int) gorums.ServerIface { + gorumsSrv := gorums.NewServer() + RegisterUnresponsiveServer(gorumsSrv, &testSrv{}) + return gorumsSrv +} - node := mgr.Nodes()[0] +// TestUnresponsiveServer checks that the client is not blocked when the server is not receiving messages +func TestUnresponsiveServer(t *testing.T) { + node := gorums.TestNode(t, serverFn) - for range 1000 { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - _, err = node.TestUnresponsive(ctx, &Empty{}) + for range 100 { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + _, err := TestUnresponsive(node.Context(ctx), &Empty{}) if err != nil && errors.Is(err, context.Canceled) { t.Error(err) } diff --git a/internal/tests/unresponsive/unresponsive.pb.go b/internal/tests/unresponsive/unresponsive.pb.go index 1152ad734..f1e9fe1cd 100644 --- a/internal/tests/unresponsive/unresponsive.pb.go +++ b/internal/tests/unresponsive/unresponsive.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: unresponsive/unresponsive.proto package unresponsive diff --git a/internal/tests/unresponsive/unresponsive_gorums.pb.go b/internal/tests/unresponsive/unresponsive_gorums.pb.go index 5af991c0b..0b12df894 100644 --- a/internal/tests/unresponsive/unresponsive_gorums.pb.go +++ b/internal/tests/unresponsive/unresponsive_gorums.pb.go @@ -1,16 +1,13 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.10.0-devel -// protoc v6.33.0 +// protoc v6.33.2 // source: unresponsive/unresponsive.proto package unresponsive import ( - context "context" - fmt "fmt" gorums "github.com/relab/gorums" - encoding "google.golang.org/grpc/encoding" ) const ( @@ -20,154 +17,60 @@ const ( _ = gorums.EnforceVersion(gorums.MaxVersion - 10) ) -// A Configuration represents a static set of nodes on which quorum remote -// procedure calls may be invoked. -type Configuration struct { - gorums.RawConfiguration - qspec QuorumSpec - nodes []*Node -} - -// ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. -// -// This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) -func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) (*Configuration, error) { - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - newCfg := &Configuration{ - RawConfiguration: rawCfg, - qspec: qspec, - } - // initialize the nodes slice - newCfg.nodes = make([]*Node, newCfg.Size()) - for i, n := range rawCfg { - newCfg.nodes[i] = &Node{n} - } - return newCfg, nil -} - -// Nodes returns a slice of each available node. IDs are returned in the same -// order as they were provided in the creation of the Manager. -// -// NOTE: mutating the returned slice is not supported. -func (c *Configuration) Nodes() []*Node { - return c.nodes -} - -// And returns a NodeListOption that can be used to create a new configuration combining c and d. -func (c Configuration) And(d *Configuration) gorums.NodeListOption { - return c.RawConfiguration.And(d.RawConfiguration) -} - -// Except returns a NodeListOption that can be used to create a new configuration -// from c without the nodes in rm. -func (c Configuration) Except(rm *Configuration) gorums.NodeListOption { - return c.RawConfiguration.Except(rm.RawConfiguration) -} - -func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) - } -} +// Type aliases for important Gorums types to make them more accessible +// from user code already interacting with the generated code. +type ( + Configuration = gorums.Configuration + Manager = gorums.Manager + Node = gorums.Node +) -// Manager maintains a connection pool of nodes on -// which quorum calls can be performed. -type Manager struct { - *gorums.RawManager -} +// Use the aliased types to add them to the reserved identifiers list. +// This prevents users from defining message types with these names. +var ( + _ = (*Configuration)(nil) + _ = (*Manager)(nil) + _ = (*Node)(nil) +) // NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure // various aspects of the manager. func NewManager(opts ...gorums.ManagerOption) *Manager { - return &Manager{ - RawManager: gorums.NewRawManager(opts...), - } + return gorums.NewManager(opts...) } -// NewConfiguration returns a configuration based on the provided list of nodes (required) -// and an optional quorum specification. The QuorumSpec is necessary for call types that -// must process replies. For configurations only used for unicast or multicast call types, -// a QuorumSpec is not needed. The QuorumSpec interface is also a ConfigOption. +// NewConfiguration returns a configuration based on the provided list of nodes. // Nodes can be supplied using WithNodeMap or WithNodeList, or WithNodeIDs. // A new configuration can also be created from an existing configuration, // using the And, WithNewNodes, Except, and WithoutNodes methods. -func (m *Manager) NewConfiguration(opts ...gorums.ConfigOption) (c *Configuration, err error) { - if len(opts) < 1 || len(opts) > 2 { - return nil, fmt.Errorf("config: wrong number of options: %d", len(opts)) - } - c = &Configuration{} - for _, opt := range opts { - switch v := opt.(type) { - case gorums.NodeListOption: - c.RawConfiguration, err = gorums.NewRawConfiguration(m.RawManager, v) - if err != nil { - return nil, err - } - case QuorumSpec: - // Must be last since v may match QuorumSpec if it is interface{} - c.qspec = v - default: - return nil, fmt.Errorf("config: unknown option type: %v", v) - } - } - // return an error if the QuorumSpec interface is not empty and no implementation was provided. - var test any = struct{}{} - if _, empty := test.(QuorumSpec); !empty && c.qspec == nil { - return nil, fmt.Errorf("config: missing required QuorumSpec") - } - // initialize the nodes slice - c.nodes = make([]*Node, c.Size()) - for i, n := range c.RawConfiguration { - c.nodes[i] = &Node{n} - } - return c, nil -} - -// Nodes returns a slice of available nodes on this manager. -// IDs are returned in the order they were added at creation of the manager. -func (m *Manager) Nodes() []*Node { - gorumsNodes := m.RawManager.Nodes() - nodes := make([]*Node, len(gorumsNodes)) - for i, n := range gorumsNodes { - nodes[i] = &Node{n} - } - return nodes +func NewConfiguration(mgr *Manager, opt gorums.NodeListOption) (Configuration, error) { + return gorums.NewConfiguration(mgr, opt) } -// Node encapsulates the state of a node on which a remote procedure call -// can be performed. -type Node struct { - *gorums.RawNode -} - -// UnresponsiveNodeClient is the single node client interface for the Unresponsive service. -type UnresponsiveNodeClient interface { - TestUnresponsive(ctx context.Context, in *Empty) (resp *Empty, err error) +// NewConfig returns a new [Configuration] based on the provided [gorums.Option]s. +// It accepts exactly one [gorums.NodeListOption] and multiple [gorums.ManagerOption]s. +// You may use this function to create the initial configuration for a new manager. +// +// Example: +// +// cfg, err := NewConfig( +// gorums.WithNodeList([]string{"localhost:8080", "localhost:8081", "localhost:8082"}), +// gorums.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), +// ) +// +// This is a convenience function for creating a configuration without explicitly +// creating a manager first. However, the manager can be accessed using the +// [Configuration.Manager] method. This method should only be used once since it +// creates a new manager; if a manager already exists, use [NewConfiguration] +// instead, and provide the existing manager as the first argument. +func NewConfig(opts ...gorums.Option) (Configuration, error) { + return gorums.NewConfig(opts...) } -// enforce interface compliance -var _ UnresponsiveNodeClient = (*Node)(nil) - -// There are no quorum calls. -type QuorumSpec interface{} - -// TestUnresponsive is a quorum call invoked on all nodes in configuration c, -// with the same argument in, and returns a combined result. -func (n *Node) TestUnresponsive(ctx context.Context, in *Empty) (resp *Empty, err error) { - cd := gorums.CallData{ - Message: in, - Method: "unresponsive.Unresponsive.TestUnresponsive", - } - - res, err := n.RawNode.RPCCall(ctx, cd) +// TestUnresponsive is an RPC call invoked on the node in ctx. +func TestUnresponsive(ctx *gorums.NodeContext, in *Empty) (resp *Empty, err error) { + res, err := gorums.RPCCall(ctx, in, "unresponsive.Unresponsive.TestUnresponsive") if err != nil { return nil, err } diff --git a/internal/testutils/mock/mock.go b/internal/testutils/mock/mock.go index 2dd1cd464..128050e61 100644 --- a/internal/testutils/mock/mock.go +++ b/internal/testutils/mock/mock.go @@ -8,13 +8,30 @@ import ( "google.golang.org/protobuf/reflect/protodesc" "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/descriptorpb" - "google.golang.org/protobuf/types/known/wrapperspb" + pb "google.golang.org/protobuf/types/known/wrapperspb" ) +func init() { + err := RegisterServices([]Service{ + { + Name: "mock.MockService", + Methods: []Method{ + {Name: "Test", Input: &pb.StringValue{}, Output: &pb.StringValue{}}, + {Name: "GetValue", Input: &pb.Int32Value{}, Output: &pb.Int32Value{}}, + {Name: "Stream", Input: &pb.StringValue{}, Output: &pb.StringValue{}}, + }, + }, + }) + if err != nil { + panic(err) + } +} + // TestMethod and GetValueMethod are the methods supported by the mock package. const ( TestMethod = "mock.MockService.Test" GetValueMethod = "mock.MockService.GetValue" + Stream = "mock.MockService.Stream" ) // Service represents a service to be registered. @@ -117,7 +134,7 @@ func GetVal(msg proto.Message) string { if msg == nil { return "" } - if m, ok := msg.(*wrapperspb.StringValue); ok { + if m, ok := msg.(*pb.StringValue); ok { return m.Value } return "" @@ -127,7 +144,7 @@ func SetVal(msg proto.Message, val string) { if msg == nil { return } - if m, ok := msg.(*wrapperspb.StringValue); ok { + if m, ok := msg.(*pb.StringValue); ok { m.Value = val } } diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 09263b5d5..000000000 --- a/main_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package gorums - -import ( - "os" - "testing" - - "github.com/relab/gorums/internal/testutils/mock" - pb "google.golang.org/protobuf/types/known/wrapperspb" -) - -func TestMain(m *testing.M) { - // Register the default mock services for integration tests. - if err := mock.RegisterServices([]mock.Service{ - { - Name: "mock.MockService", - Methods: []mock.Method{ - { - Name: "Test", - Input: &pb.StringValue{}, - Output: &pb.StringValue{}, - }, - { - Name: "GetValue", - Input: &pb.Int32Value{}, - Output: &pb.Int32Value{}, - }, - }, - }, - }); err != nil { - panic(err) - } - - os.Exit(m.Run()) -} diff --git a/mgr.go b/mgr.go index ffd15ebb8..3f407a149 100644 --- a/mgr.go +++ b/mgr.go @@ -1,6 +1,7 @@ package gorums import ( + "errors" "fmt" "log" "sync" @@ -10,28 +11,24 @@ import ( "google.golang.org/grpc/backoff" ) -// RawManager maintains a connection pool of nodes on +// Manager maintains a connection pool of nodes on // which quorum calls can be performed. -// -// This struct is intended to be used by generated code. -// You should use the generated `Manager` struct instead. -type RawManager struct { +type Manager struct { mu sync.Mutex - nodes []*RawNode - lookup map[uint32]*RawNode + nodes []*Node + lookup map[uint32]*Node closeOnce sync.Once logger *log.Logger opts managerOptions nextMsgID uint64 } -// NewRawManager returns a new RawManager for managing connection to nodes added +// NewManager returns a new Manager for managing connection to nodes added // to the manager. This function accepts manager options used to configure -// various aspects of the manager. This function is meant for internal use. -// You should use the `NewManager` function in the generated code instead. -func NewRawManager(opts ...ManagerOption) *RawManager { - m := &RawManager{ - lookup: make(map[uint32]*RawNode), +// various aspects of the manager. +func NewManager(opts ...ManagerOption) *Manager { + m := &Manager{ + lookup: make(map[uint32]*Node), opts: newManagerOptions(), } for _, opt := range opts { @@ -54,28 +51,20 @@ func NewRawManager(opts ...ManagerOption) *RawManager { return m } -func (m *RawManager) closeNodeConns() { - for _, node := range m.nodes { - err := node.close() - if err != nil && m.logger != nil { - m.logger.Printf("error closing: %v", err) - } - } -} - // Close closes all node connections and any client streams. -func (m *RawManager) Close() { +func (m *Manager) Close() error { + var err error m.closeOnce.Do(func() { - if m.logger != nil { - m.logger.Printf("closing") + for _, node := range m.nodes { + err = errors.Join(err, node.close()) } - m.closeNodeConns() }) + return err } // NodeIDs returns the identifier of each available node. IDs are returned in // the same order as they were provided in the creation of the Manager. -func (m *RawManager) NodeIDs() []uint32 { +func (m *Manager) NodeIDs() []uint32 { m.mu.Lock() defer m.mu.Unlock() ids := make([]uint32, 0, len(m.nodes)) @@ -86,7 +75,7 @@ func (m *RawManager) NodeIDs() []uint32 { } // Node returns the node with the given identifier if present. -func (m *RawManager) Node(id uint32) (node *RawNode, found bool) { +func (m *Manager) Node(id uint32) (node *Node, found bool) { m.mu.Lock() defer m.mu.Unlock() node, found = m.lookup[id] @@ -95,43 +84,48 @@ func (m *RawManager) Node(id uint32) (node *RawNode, found bool) { // Nodes returns a slice of each available node. IDs are returned in the same // order as they were provided in the creation of the Manager. -func (m *RawManager) Nodes() []*RawNode { +func (m *Manager) Nodes() []*Node { m.mu.Lock() defer m.mu.Unlock() return m.nodes } // Size returns the number of nodes in the Manager. -func (m *RawManager) Size() (nodes int) { +func (m *Manager) Size() (nodes int) { m.mu.Lock() defer m.mu.Unlock() return len(m.nodes) } -// AddNode adds the node to the manager's node pool -// and establishes a connection to the node. -func (m *RawManager) AddNode(node *RawNode) error { - if _, found := m.Node(node.ID()); found { - // Node IDs must be unique - return fmt.Errorf("config: node %d (%s) already exists", node.ID(), node.Address()) - } - if m.logger != nil { - m.logger.Printf("Connecting to %s with id %d\n", node, node.id) - } - if err := node.connect(m); err != nil { - if m.logger != nil { - m.logger.Printf("Failed to connect to %s: %v (retrying)", node, err) - } - } - +func (m *Manager) addNode(node *Node) { m.mu.Lock() defer m.mu.Unlock() m.lookup[node.id] = node m.nodes = append(m.nodes, node) - return nil } -// getMsgID returns a unique message ID. -func (m *RawManager) getMsgID() uint64 { +func (m *Manager) newNode(addr string, id uint32) (*Node, error) { + if _, found := m.Node(id); found { + return nil, fmt.Errorf("node %d already exists", id) + } + opts := nodeOptions{ + ID: id, + SendBufferSize: m.opts.sendBuffer, + MsgIDGen: m.getMsgID, + Metadata: m.opts.metadata, + PerNodeMD: m.opts.perNodeMD, + DialOpts: m.opts.grpcDialOpts, + Manager: m, + } + n, err := newNode(addr, opts) + if err != nil { + return nil, err + } + m.addNode(n) + return n, nil +} + +// getMsgID returns a unique message ID for a new RPC from this client's manager. +func (m *Manager) getMsgID() uint64 { return atomic.AddUint64(&m.nextMsgID, 1) } diff --git a/mgr_test.go b/mgr_test.go index e1f5da376..5fde7a5b5 100644 --- a/mgr_test.go +++ b/mgr_test.go @@ -1,24 +1,19 @@ -package gorums_test +package gorums import ( "bytes" "log" + "strings" "testing" - "github.com/relab/gorums" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/encoding" ) -var ( - nodes = []string{"127.0.0.1:9080", "127.0.0.1:9081", "127.0.0.1:9082"} - nodeMap = map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4} -) +var nodeMap = map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4} func init() { - if encoding.GetCodec(gorums.ContentSubtype) == nil { - encoding.RegisterCodec(gorums.NewCodec()) + if encoding.GetCodec(ContentSubtype) == nil { + encoding.RegisterCodec(NewCodec()) } } @@ -27,17 +22,20 @@ func TestManagerLogging(t *testing.T) { buf bytes.Buffer logger = log.New(&buf, "logger: ", log.Lshortfile) ) - buf.WriteString("\n") - _ = gorums.NewRawManager( - gorums.WithNoConnect(), - gorums.WithLogger(logger), - ) - t.Log(buf.String()) + mgr := NewManager(InsecureDialOptions(t), WithLogger(logger)) + t.Cleanup(Closer(t, mgr)) + + want := "logger: mgr.go:49: ready" + if strings.TrimSpace(buf.String()) != want { + t.Errorf("logger: got %q, want %q", buf.String(), want) + } } -func TestManagerAddNode(t *testing.T) { - mgr := gorums.NewRawManager(gorums.WithNoConnect()) - _, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeMap(nodeMap)) +func TestManagerNewNode(t *testing.T) { + mgr := NewManager(InsecureDialOptions(t)) + t.Cleanup(Closer(t, mgr)) + + _, err := NewConfiguration(mgr, WithNodeMap(nodeMap)) if err != nil { t.Fatal(err) } @@ -46,34 +44,28 @@ func TestManagerAddNode(t *testing.T) { id uint32 err string }{ - {"127.0.1.1:1234", 1, "config: node 1 (127.0.1.1:1234) already exists"}, + {"127.0.1.1:1234", 4, "node 4 already exists"}, {"127.0.1.1:1234", 5, ""}, {"127.0.1.1:1234", 6, ""}, // The same addr:port can have different IDs - {"127.0.1.1:1234", 2, "config: node 2 (127.0.1.1:1234) already exists"}, } for _, test := range tests { - node, err := gorums.NewRawNodeWithID(test.addr, test.id) + _, err := mgr.newNode(test.addr, test.id) if err != nil { - t.Fatal(err) - } - err = mgr.AddNode(node) - if err != nil && err.Error() != test.err { - t.Errorf("mgr.AddNode(Node(%s, %d)) = %s, expected %s", test.addr, test.id, err.Error(), test.err) + if err.Error() == test.err { + continue + } + t.Errorf("mgr.addNode(%s, %d) = %q, want %q", test.addr, test.id, err.Error(), test.err) } } } -func TestManagerAddNodeWithConn(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 3, nil) - defer teardown() - mgr := gorums.NewRawManager( - gorums.WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - ) - defer mgr.Close() +func TestManagerNewNodeWithConn(t *testing.T) { + addrs := TestServers(t, 3, nil) + mgr := NewManager(InsecureDialOptions(t)) + t.Cleanup(Closer(t, mgr)) - _, err := gorums.NewRawConfiguration(mgr, gorums.WithNodeList(addrs[:2])) + // Create configuration with only first 2 nodes + _, err := NewConfiguration(mgr, WithNodeList(addrs[:2])) if err != nil { t.Fatal(err) } @@ -81,13 +73,14 @@ func TestManagerAddNodeWithConn(t *testing.T) { t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(addrs)-1) } - node, err := gorums.NewRawNode(addrs[2]) + // Obtain ID for the 3rd node + id, err := nodeID(addrs[2]) if err != nil { t.Fatal(err) } - err = mgr.AddNode(node) - if err != nil { - t.Errorf("mgr.AddNode(%s) = %q, expected %q", addrs[2], err.Error(), "") + // Create the 3rd node and add it to the manager + if _, err := mgr.newNode(addrs[2], id); err != nil { + t.Errorf("mgr.newNode(%s) = %q, expected no error", addrs[2], err.Error()) } if mgr.Size() != len(addrs) { t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(addrs)) diff --git a/multicast.go b/multicast.go index d563f5320..58b0ee0ab 100644 --- a/multicast.go +++ b/multicast.go @@ -1,9 +1,8 @@ package gorums import ( - "context" - - "github.com/relab/gorums/ordering" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" ) // Multicast is a one-way call; no replies are returned to the client. @@ -11,39 +10,41 @@ import ( // By default, this method blocks until messages have been sent to all nodes. // This ensures that send operations complete before the caller proceeds, which can // be useful for observing context cancellation or for pacing message sends. +// If the sending fails, the error is returned to the caller. // -// With the WithNoSendWaiting call option, the method returns immediately after +// With the IgnoreErrors call option, the method returns nil immediately after // enqueueing messages to all nodes (fire-and-forget semantics). // +// Multicast supports request transformation interceptors via the gorums.Interceptors +// option. Use gorums.MapRequest to transform requests per-node. +// // This method should be used by generated code only. -func (c RawConfiguration) Multicast(ctx context.Context, d QuorumCallData, opts ...CallOption) { - o := getCallOptions(E_Multicast, opts) - md := ordering.NewGorumsMetadata(ctx, c.getMsgID(), d.Method) - sentMsgs := 0 +func Multicast[Req proto.Message](ctx *ConfigContext, msg Req, method string, opts ...CallOption) error { + callOpts := getCallOptions(E_Multicast, opts...) + waitSendDone := callOpts.mustWaitSendDone() - var replyChan chan response - if o.waitSendDone { - replyChan = make(chan response, len(c)) - } - for _, n := range c { - msg := d.Message - if d.PerNodeArgFn != nil { - msg = d.PerNodeArgFn(d.Message, n.id) - if !msg.ProtoReflect().IsValid() { - continue // don't send if no msg - } - } - n.channel.enqueue(request{ctx: ctx, msg: NewRequestMessage(md, msg), opts: o}, replyChan) - sentMsgs++ - } + clientCtx := newClientCtxBuilder[Req, *emptypb.Empty](ctx, msg, method).WithWaitSendDone(waitSendDone).Build() + clientCtx.applyInterceptors(callOpts.interceptors) - // Fire-and-forget: return immediately without waiting for send completion - if !o.waitSendDone { - return - } + // Send messages immediately (multicast doesn't use lazy sending) + clientCtx.sendOnce.Do(clientCtx.send) - // Default: block until all sends complete - for ; sentMsgs > 0; sentMsgs-- { - <-replyChan + // If waiting for send completion, drain the reply channel and return the first error. + if waitSendDone { + var errs []nodeError + for range clientCtx.expectedReplies { + select { + case r := <-clientCtx.replyChan: + if r.Err != nil { + errs = append(errs, nodeError{cause: r.Err, nodeID: r.NodeID}) + } + case <-ctx.Done(): + return ctx.Err() + } + } + if len(errs) > 0 { + return QuorumCallError{cause: ErrSendFailure, errors: errs} + } } + return nil } diff --git a/node.go b/node.go index 89c078253..6cefce96e 100644 --- a/node.go +++ b/node.go @@ -15,102 +15,122 @@ import ( const nilAngleString = "" -// RawNode encapsulates the state of a node on which a remote procedure call -// can be performed. +// NodeContext is a context that carries a node for unicast and RPC calls. +// It embeds context.Context and provides access to the Node. // -// This struct is intended to be used by generated code. -// You should use the generated `Node` struct instead. -type RawNode struct { +// Use [Node.Context] to create a NodeContext from an existing context. +type NodeContext struct { + context.Context + node *Node +} + +// Node returns the Node associated with this context. +func (c NodeContext) Node() *Node { + return c.node +} + +// enqueue enqueues a request to this node's channel. +func (c NodeContext) enqueue(req request) { + c.node.channel.enqueue(req) +} + +// nextMsgID returns the next message ID from this client's manager. +func (c NodeContext) nextMsgID() uint64 { + return c.node.msgIDGen() +} + +// Node encapsulates the state of a node on which a remote procedure call +// can be performed. +type Node struct { // Only assigned at creation. - id uint32 - addr string - conn *grpc.ClientConn - cancel func() - mgr *RawManager + id uint32 + addr string + mgr *Manager // only used for backward compatibility to allow Configuration.Manager() - // the default channel - channel *channel + msgIDGen func() uint64 + channel *channel } -// NewRawNode returns a new node for the provided address. -func NewRawNode(addr string) (*RawNode, error) { - tcpAddr, err := net.ResolveTCPAddr("tcp", addr) - if err != nil { - return nil, err +// Context creates a new NodeContext from the given parent context +// and this node. +// +// Example: +// +// nodeCtx := node.Context(context.Background()) +// resp, err := service.GRPCCall(nodeCtx, req) +func (n *Node) Context(parent context.Context) *NodeContext { + if n == nil { + panic("gorums: Context called with nil node") } - h := fnv.New32a() - _, _ = h.Write([]byte(tcpAddr.String())) - return &RawNode{ - id: h.Sum32(), - addr: tcpAddr.String(), - }, nil + return &NodeContext{Context: parent, node: n} +} + +// nodeOptions contains configuration options for creating a new Node. +type nodeOptions struct { + ID uint32 + SendBufferSize uint + MsgIDGen func() uint64 + Metadata metadata.MD + PerNodeMD func(uint32) metadata.MD + DialOpts []grpc.DialOption + Manager *Manager // only used for backward compatibility to allow Configuration.Manager() } -// NewRawNodeWithID returns a new node for the provided address and id. -func NewRawNodeWithID(addr string, id uint32) (*RawNode, error) { +// newNode creates a new node using the provided options. +// It establishes the connection (lazy dial) and initializes the channel. +func newNode(addr string, opts nodeOptions) (*Node, error) { tcpAddr, err := net.ResolveTCPAddr("tcp", addr) if err != nil { return nil, err } - return &RawNode{ - id: id, - addr: tcpAddr.String(), - }, nil -} -// connect to this node and associate it with the manager. -func (n *RawNode) connect(mgr *RawManager) (err error) { - n.mgr = mgr - if n.mgr.opts.noConnect { - return nil + n := &Node{ + id: opts.ID, + addr: tcpAddr.String(), + mgr: opts.Manager, + msgIDGen: opts.MsgIDGen, } - // Create gRPC connection - n.conn, err = grpc.NewClient(n.addr, n.mgr.opts.grpcDialOpts...) + // Create gRPC connection to the node without connecting (lazy dial). + conn, err := grpc.NewClient(n.addr, opts.DialOpts...) if err != nil { - return nodeError{nodeID: n.id, cause: err} + return nil, nodeError{nodeID: n.id, cause: err} } - // Create channel and establish gRPC node stream - n.channel = newChannel(n) - if err := n.channel.ensureStream(); err != nil { - return nodeError{nodeID: n.id, cause: err} + // Create outgoing context with metadata for this node's stream. + md := opts.Metadata.Copy() + if opts.PerNodeMD != nil { + md = metadata.Join(md, opts.PerNodeMD(n.id)) } - return nil + ctx := metadata.NewOutgoingContext(context.Background(), md) + + // Create channel and establish gRPC node stream + n.channel = newChannel(ctx, conn, n.id, opts.SendBufferSize) + return n, nil } -// newContext returns a new context for this node's channel. -// This context is used by the channel implementation to stop -// all goroutines and the NodeStream, when the context is canceled. -// -// This method must be called for each connection to ensure -// fresh contexts. Reusing contexts could result in reusing -// a cancelled context. -func (n *RawNode) newContext() context.Context { - md := n.mgr.opts.metadata.Copy() - if n.mgr.opts.perNodeMD != nil { - md = metadata.Join(md, n.mgr.opts.perNodeMD(n.id)) +// nodeID returns the ID for the provided address. +// It resolves the address to ensure the ID is consistent. +func nodeID(addr string) (uint32, error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + return 0, err } - var ctx context.Context - ctx, n.cancel = context.WithCancel(context.Background()) - return metadata.NewOutgoingContext(ctx, md) + h := fnv.New32a() + _, _ = h.Write([]byte(tcpAddr.String())) + return h.Sum32(), nil } // close this node. -func (n *RawNode) close() error { - // important to cancel first to stop goroutines - n.cancel() - if n.conn == nil { - return nil - } - if err := n.conn.Close(); err != nil { - return nodeError{nodeID: n.id, cause: err} +func (n *Node) close() error { + if n.channel != nil { + return n.channel.close() } return nil } // ID returns the ID of n. -func (n *RawNode) ID() uint32 { +func (n *Node) ID() uint32 { if n != nil { return n.id } @@ -118,7 +138,7 @@ func (n *RawNode) ID() uint32 { } // Address returns network address of n. -func (n *RawNode) Address() string { +func (n *Node) Address() string { if n != nil { return n.addr } @@ -126,7 +146,7 @@ func (n *RawNode) Address() string { } // Host returns the network host of n. -func (n *RawNode) Host() string { +func (n *Node) Host() string { if n == nil { return nilAngleString } @@ -135,7 +155,7 @@ func (n *RawNode) Host() string { } // Port returns network port of n. -func (n *RawNode) Port() string { +func (n *Node) Port() string { if n != nil { _, port, _ := net.SplitHostPort(n.addr) return port @@ -143,7 +163,7 @@ func (n *RawNode) Port() string { return nilAngleString } -func (n *RawNode) String() string { +func (n *Node) String() string { if n != nil { return fmt.Sprintf("addr: %s", n.addr) } @@ -152,7 +172,7 @@ func (n *RawNode) String() string { // FullString returns a more descriptive string representation of n that // includes id, network address and latency information. -func (n *RawNode) FullString() string { +func (n *Node) FullString() string { if n != nil { return fmt.Sprintf("node %d | addr: %s", n.id, n.addr) } @@ -160,26 +180,26 @@ func (n *RawNode) FullString() string { } // LastErr returns the last error encountered (if any) for this node. -func (n *RawNode) LastErr() error { +func (n *Node) LastErr() error { return n.channel.lastErr() } // Latency returns the latency between the client and this node. -func (n *RawNode) Latency() time.Duration { +func (n *Node) Latency() time.Duration { return n.channel.channelLatency() } -type lessFunc func(n1, n2 *RawNode) bool +type lessFunc func(n1, n2 *Node) bool // MultiSorter implements the Sort interface, sorting the nodes within. type MultiSorter struct { - nodes []*RawNode + nodes []*Node less []lessFunc } // Sort sorts the argument slice according to the less functions passed to // OrderedBy. -func (ms *MultiSorter) Sort(nodes []*RawNode) { +func (ms *MultiSorter) Sort(nodes []*Node) { ms.nodes = nodes sort.Sort(ms) } @@ -229,13 +249,13 @@ func (ms *MultiSorter) Less(i, j int) bool { } // ID sorts nodes by their identifier in increasing order. -var ID = func(n1, n2 *RawNode) bool { +var ID = func(n1, n2 *Node) bool { return n1.id < n2.id } // Port sorts nodes by their port number in increasing order. // Warning: This function may be removed in the future. -var Port = func(n1, n2 *RawNode) bool { +var Port = func(n1, n2 *Node) bool { p1, _ := strconv.Atoi(n1.Port()) p2, _ := strconv.Atoi(n2.Port()) return p1 < p2 @@ -243,7 +263,7 @@ var Port = func(n1, n2 *RawNode) bool { // LastNodeError sorts nodes by their LastErr() status in increasing order. A // node with LastErr() != nil is larger than a node with LastErr() == nil. -var LastNodeError = func(n1, n2 *RawNode) bool { +var LastNodeError = func(n1, n2 *Node) bool { if n1.channel.lastErr() != nil && n2.channel.lastErr() == nil { return false } diff --git a/node_test.go b/node_test.go index 33cd31b9f..3a2604116 100644 --- a/node_test.go +++ b/node_test.go @@ -8,7 +8,7 @@ import ( ) func TestNodeSort(t *testing.T) { - nodes := []*RawNode{ + nodes := []*Node{ { id: 100, channel: &channel{ @@ -58,7 +58,7 @@ func TestNodeSort(t *testing.T) { } } -func printNodes(t *testing.T, nodes []*RawNode) { +func printNodes(t *testing.T, nodes []*Node) { t.Helper() for i, n := range nodes { nodeStr := fmt.Sprintf( diff --git a/opts.go b/opts.go index 689c140dc..bc68e19de 100644 --- a/opts.go +++ b/opts.go @@ -8,10 +8,19 @@ import ( "google.golang.org/grpc/metadata" ) +// Option is a marker interface for options to NewConfig. +type Option interface { + isOption() +} + +// ManagerOption provides a way to set different options on a new Manager. +type ManagerOption func(*managerOptions) + +func (ManagerOption) isOption() {} + type managerOptions struct { grpcDialOpts []grpc.DialOption logger *log.Logger - noConnect bool backoff backoff.Config sendBuffer uint metadata metadata.MD @@ -25,12 +34,9 @@ func newManagerOptions() managerOptions { } } -// ManagerOption provides a way to set different options on a new Manager. -type ManagerOption func(*managerOptions) - -// WithGrpcDialOptions returns a ManagerOption which sets any gRPC dial options +// WithDialOptions returns a ManagerOption which sets any gRPC dial options // the Manager should use when initially connecting to each node in its pool. -func WithGrpcDialOptions(opts ...grpc.DialOption) ManagerOption { +func WithDialOptions(opts ...grpc.DialOption) ManagerOption { return func(o *managerOptions) { o.grpcDialOpts = append(o.grpcDialOpts, opts...) } @@ -44,14 +50,6 @@ func WithLogger(logger *log.Logger) ManagerOption { } } -// WithNoConnect returns a ManagerOption which instructs the Manager not to -// connect to any of its nodes. Mainly used for testing purposes. -func WithNoConnect() ManagerOption { - return func(o *managerOptions) { - o.noConnect = true - } -} - // WithBackoff allows for changing the backoff delays used by Gorums. func WithBackoff(backoff backoff.Config) ManagerOption { return func(o *managerOptions) { diff --git a/ordering/gorums_metadata.go b/ordering/gorums_metadata.go index cde158e94..0a4716cb3 100644 --- a/ordering/gorums_metadata.go +++ b/ordering/gorums_metadata.go @@ -13,7 +13,7 @@ import ( // This is used to pass client-specific metadata to the server via Gorums. // This method should be used by generated code only. func NewGorumsMetadata(ctx context.Context, msgID uint64, method string) *Metadata { - gorumsMetadata := Metadata_builder{MessageID: msgID, Method: method} + gorumsMetadata := Metadata_builder{MessageSeqNo: msgID, Method: method} md, _ := metadata.FromOutgoingContext(ctx) for k, vv := range md { for _, v := range vv { diff --git a/ordering/ordering.pb.go b/ordering/ordering.pb.go index a74efbd4e..e4f7cc858 100644 --- a/ordering/ordering.pb.go +++ b/ordering/ordering.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.33.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: ordering/ordering.proto package ordering @@ -24,13 +24,13 @@ const ( // Metadata is sent together with application-specific message types, // and contains information necessary for Gorums to handle the messages. type Metadata struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_MessageID uint64 `protobuf:"varint,1,opt,name=MessageID"` - xxx_hidden_Method string `protobuf:"bytes,2,opt,name=Method"` - xxx_hidden_Status *status.Status `protobuf:"bytes,3,opt,name=Status"` - xxx_hidden_Entry *[]*MetadataEntry `protobuf:"bytes,4,rep,name=Entry"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_MessageSeqNo uint64 `protobuf:"varint,1,opt,name=message_seq_no,json=messageSeqNo"` + xxx_hidden_Method string `protobuf:"bytes,2,opt,name=method"` + xxx_hidden_Status *status.Status `protobuf:"bytes,3,opt,name=status"` + xxx_hidden_Entry *[]*MetadataEntry `protobuf:"bytes,4,rep,name=entry"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Metadata) Reset() { @@ -58,9 +58,9 @@ func (x *Metadata) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *Metadata) GetMessageID() uint64 { +func (x *Metadata) GetMessageSeqNo() uint64 { if x != nil { - return x.xxx_hidden_MessageID + return x.xxx_hidden_MessageSeqNo } return 0 } @@ -88,8 +88,8 @@ func (x *Metadata) GetEntry() []*MetadataEntry { return nil } -func (x *Metadata) SetMessageID(v uint64) { - x.xxx_hidden_MessageID = v +func (x *Metadata) SetMessageSeqNo(v uint64) { + x.xxx_hidden_MessageSeqNo = v } func (x *Metadata) SetMethod(v string) { @@ -118,28 +118,28 @@ func (x *Metadata) ClearStatus() { type Metadata_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - MessageID uint64 - Method string - Status *status.Status - // per message client-generated metadata - Entry []*MetadataEntry + MessageSeqNo uint64 + Method string + Status *status.Status + Entry []*MetadataEntry } func (b0 Metadata_builder) Build() *Metadata { m0 := &Metadata{} b, x := &b0, m0 _, _ = b, x - x.xxx_hidden_MessageID = b.MessageID + x.xxx_hidden_MessageSeqNo = b.MessageSeqNo x.xxx_hidden_Method = b.Method x.xxx_hidden_Status = b.Status x.xxx_hidden_Entry = &b.Entry return m0 } +// MetadataEntry is a key-value pair for Metadata entries. type MetadataEntry struct { state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Key string `protobuf:"bytes,1,opt,name=Key"` - xxx_hidden_Value string `protobuf:"bytes,2,opt,name=Value"` + xxx_hidden_Key string `protobuf:"bytes,1,opt,name=key"` + xxx_hidden_Value string `protobuf:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -211,15 +211,15 @@ var File_ordering_ordering_proto protoreflect.FileDescriptor const file_ordering_ordering_proto_rawDesc = "" + "\n" + - "\x17ordering/ordering.proto\x12\bordering\x1a\x17google/rpc/status.proto\"\x9b\x01\n" + - "\bMetadata\x12\x1c\n" + - "\tMessageID\x18\x01 \x01(\x04R\tMessageID\x12\x16\n" + - "\x06Method\x18\x02 \x01(\tR\x06Method\x12*\n" + - "\x06Status\x18\x03 \x01(\v2\x12.google.rpc.StatusR\x06Status\x12-\n" + - "\x05Entry\x18\x04 \x03(\v2\x17.ordering.MetadataEntryR\x05Entry\"7\n" + + "\x17ordering/ordering.proto\x12\bordering\x1a\x17google/rpc/status.proto\"\xa3\x01\n" + + "\bMetadata\x12$\n" + + "\x0emessage_seq_no\x18\x01 \x01(\x04R\fmessageSeqNo\x12\x16\n" + + "\x06method\x18\x02 \x01(\tR\x06method\x12*\n" + + "\x06status\x18\x03 \x01(\v2\x12.google.rpc.StatusR\x06status\x12-\n" + + "\x05entry\x18\x04 \x03(\v2\x17.ordering.MetadataEntryR\x05entry\"7\n" + "\rMetadataEntry\x12\x10\n" + - "\x03Key\x18\x01 \x01(\tR\x03Key\x12\x14\n" + - "\x05Value\x18\x02 \x01(\tR\x05Value2B\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value2B\n" + "\x06Gorums\x128\n" + "\n" + "NodeStream\x12\x12.ordering.Metadata\x1a\x12.ordering.Metadata(\x010\x01B'Z github.com/relab/gorums/ordering\x92\x03\x02\b\x02b\beditionsp\xe8\a" @@ -231,8 +231,8 @@ var file_ordering_ordering_proto_goTypes = []any{ (*status.Status)(nil), // 2: google.rpc.Status } var file_ordering_ordering_proto_depIdxs = []int32{ - 2, // 0: ordering.Metadata.Status:type_name -> google.rpc.Status - 1, // 1: ordering.Metadata.Entry:type_name -> ordering.MetadataEntry + 2, // 0: ordering.Metadata.status:type_name -> google.rpc.Status + 1, // 1: ordering.Metadata.entry:type_name -> ordering.MetadataEntry 0, // 2: ordering.Gorums.NodeStream:input_type -> ordering.Metadata 0, // 3: ordering.Gorums.NodeStream:output_type -> ordering.Metadata 3, // [3:4] is the sub-list for method output_type diff --git a/ordering/ordering.proto b/ordering/ordering.proto index 531fbe02a..34447afac 100644 --- a/ordering/ordering.proto +++ b/ordering/ordering.proto @@ -2,29 +2,29 @@ edition = "2023"; package ordering; option go_package = "github.com/relab/gorums/ordering"; - option features.field_presence = IMPLICIT; import "google/rpc/status.proto"; +// Gorums is the main service for Gorums' per-node stream communication. service Gorums { // NodeStream is a stream that connects a client to a Node. // The messages that are sent on the stream contain both Metadata // and an application-specific message. - rpc NodeStream(stream Metadata) returns (stream Metadata); + rpc NodeStream(stream Metadata) returns (stream Metadata); } // Metadata is sent together with application-specific message types, // and contains information necessary for Gorums to handle the messages. message Metadata { - uint64 MessageID = 1; - string Method = 2; - google.rpc.Status Status = 3; - // per message client-generated metadata - repeated MetadataEntry Entry = 4; + uint64 message_seq_no = 1; // sequence number for this message + string method = 2; // method name to invoke on the server + google.rpc.Status status = 3; // status of an invocation (for responses) + repeated MetadataEntry entry = 4; // per message client-generated metadata } +// MetadataEntry is a key-value pair for Metadata entries. message MetadataEntry { - string Key = 1; - string Value = 2; + string key = 1; + string value = 2; } diff --git a/ordering/ordering_grpc.pb.go b/ordering/ordering_grpc.pb.go index a18fe9a48..4c3c63841 100644 --- a/ordering/ordering_grpc.pb.go +++ b/ordering/ordering_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.5.1 -// - protoc v6.33.0 +// - protoc-gen-go-grpc v1.6.0 +// - protoc v6.33.2 // source: ordering/ordering.proto package ordering @@ -25,6 +25,8 @@ const ( // GorumsClient is the client API for Gorums service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Gorums is the main service for Gorums' per-node stream communication. type GorumsClient interface { // NodeStream is a stream that connects a client to a Node. // The messages that are sent on the stream contain both Metadata @@ -56,6 +58,8 @@ type Gorums_NodeStreamClient = grpc.BidiStreamingClient[Metadata, Metadata] // GorumsServer is the server API for Gorums service. // All implementations must embed UnimplementedGorumsServer // for forward compatibility. +// +// Gorums is the main service for Gorums' per-node stream communication. type GorumsServer interface { // NodeStream is a stream that connects a client to a Node. // The messages that are sent on the stream contain both Metadata @@ -72,7 +76,7 @@ type GorumsServer interface { type UnimplementedGorumsServer struct{} func (UnimplementedGorumsServer) NodeStream(grpc.BidiStreamingServer[Metadata, Metadata]) error { - return status.Errorf(codes.Unimplemented, "method NodeStream not implemented") + return status.Error(codes.Unimplemented, "method NodeStream not implemented") } func (UnimplementedGorumsServer) mustEmbedUnimplementedGorumsServer() {} func (UnimplementedGorumsServer) testEmbeddedByValue() {} @@ -85,7 +89,7 @@ type UnsafeGorumsServer interface { } func RegisterGorumsServer(s grpc.ServiceRegistrar, srv GorumsServer) { - // If the following call pancis, it indicates UnimplementedGorumsServer was + // If the following call panics, it indicates UnimplementedGorumsServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. diff --git a/quorumcall.go b/quorumcall.go index 58a815fd6..afbc91ac8 100644 --- a/quorumcall.go +++ b/quorumcall.go @@ -1,66 +1,61 @@ package gorums -import ( - "context" - - "github.com/relab/gorums/ordering" - "google.golang.org/protobuf/proto" -) - -// QuorumCallData holds the message, destination nodes, method identifier, -// and other information necessary to perform the various quorum call types -// supported by Gorums. +// QuorumCall performs a quorum call and returns a Responses object +// that provides access to node responses via terminal methods and fluent iteration. +// +// Type parameters: +// - Req: The request message type +// - Resp: The response message type from individual nodes +// +// The opts parameter accepts CallOption values such as Interceptors. +// Interceptors are applied in the order they are provided via Interceptors, +// modifying the clientCtx before the user calls a terminal method. +// +// Note: Messages are not sent to nodes until a terminal method (like Majority, First) +// or iterator method (like Seq) is called, applying any registered request transformations. +// This lazy sending is necessary to allow interceptors to register transformations prior to dispatch. // -// This struct should be used by generated code only. -type QuorumCallData struct { - Message proto.Message - Method string - PerNodeArgFn func(proto.Message, uint32) proto.Message - QuorumFunction func(proto.Message, map[uint32]proto.Message) (proto.Message, bool) +// This function should be used by generated code only. +func QuorumCall[Req, Resp msg]( + ctx *ConfigContext, + req Req, + method string, + opts ...CallOption, +) *Responses[Resp] { + return invokeQuorumCall[Req, Resp](ctx, req, method, false, opts...) } -// QuorumCall performs a quorum call on the configuration. +// QuorumCallStream performs a streaming quorum call and returns a Responses object. +// This is used for correctable stream methods where the server sends multiple responses. // -// This method should be used by generated code only. -func (c RawConfiguration) QuorumCall(ctx context.Context, d QuorumCallData) (resp proto.Message, err error) { - expectedReplies := len(c) - md := ordering.NewGorumsMetadata(ctx, c.getMsgID(), d.Method) +// In streaming mode, the response iterator continues indefinitely until the context +// is canceled, allowing the server to send multiple responses over time. +// +// This function should be used by generated code only. +func QuorumCallStream[Req, Resp msg]( + ctx *ConfigContext, + req Req, + method string, + opts ...CallOption, +) *Responses[Resp] { + return invokeQuorumCall[Req, Resp](ctx, req, method, true, opts...) +} - replyChan := make(chan response, expectedReplies) - for _, n := range c { - msg := d.Message - if d.PerNodeArgFn != nil { - msg = d.PerNodeArgFn(d.Message, n.id) - if !msg.ProtoReflect().IsValid() { - expectedReplies-- - continue // don't send if no msg - } - } - n.channel.enqueue(request{ctx: ctx, msg: NewRequestMessage(md, msg)}, replyChan) +// invokeQuorumCall is the internal implementation shared by QuorumCall and QuorumCallStream. +func invokeQuorumCall[Req, Resp msg]( + ctx *ConfigContext, + req Req, + method string, + streaming bool, + opts ...CallOption, +) *Responses[Resp] { + callOpts := getCallOptions(E_Quorumcall, opts...) + builder := newClientCtxBuilder[Req, Resp](ctx, req, method) + if streaming { + builder = builder.WithStreaming() } + clientCtx := builder.Build() + clientCtx.applyInterceptors(callOpts.interceptors) - var ( - errs []nodeError - quorum bool - replies = make(map[uint32]proto.Message) - ) - - for { - select { - case r := <-replyChan: - if r.err != nil { - errs = append(errs, nodeError{nodeID: r.nid, cause: r.err}) - break - } - replies[r.nid] = r.msg - if resp, quorum = d.QuorumFunction(d.Message, replies); quorum { - return resp, nil - } - case <-ctx.Done(): - return resp, QuorumCallError{cause: ctx.Err(), errors: errs, replies: len(replies)} - } - if len(errs)+len(replies) == expectedReplies { - return resp, QuorumCallError{cause: Incomplete, errors: errs, replies: len(replies)} - } - } + return NewResponses(clientCtx) } diff --git a/quorumcall_test.go b/quorumcall_test.go index 8cb320fa0..11bd80e93 100644 --- a/quorumcall_test.go +++ b/quorumcall_test.go @@ -2,45 +2,390 @@ package gorums_test import ( "context" + "errors" + "fmt" + "slices" "testing" "time" "github.com/relab/gorums" "github.com/relab/gorums/internal/testutils/mock" - "google.golang.org/protobuf/proto" pb "google.golang.org/protobuf/types/known/wrapperspb" ) -func TestQuorumCallSuccess(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 3, nil) - t.Cleanup(teardown) - - cfg := gorums.NewConfig(t, addrs) - - cd := gorums.QuorumCallData{ - Message: pb.String(""), - Method: mock.TestMethod, - QuorumFunction: func(_ proto.Message, replies map[uint32]proto.Message) (proto.Message, bool) { - t.Logf("Received %d replies: %v", len(replies), replies) - if len(replies) > 2 { - for _, reply := range replies { - if reply == nil { - continue +// checkQuorumCall returns true if the quorum call was successful. +// It returns false if an error occurred or the context timed out. +func checkQuorumCall(t *testing.T, ctxErr, err error) bool { + t.Helper() + if errors.Is(ctxErr, context.DeadlineExceeded) { + t.Error(ctxErr) + return false + } + if err != nil { + t.Errorf("QuorumCall failed: %v", err) + return false + } + return true +} + +func TestQuorumCall(t *testing.T) { + // type alias short hand for the responses type + type respType = *gorums.Responses[*pb.StringValue] + tests := []struct { + name string + call func(respType) (*pb.StringValue, error) + numNodes int + wantValue string + }{ + { + name: "Majority", + call: respType.Majority, + numNodes: 3, + wantValue: "echo: test", + }, + { + name: "First", + call: respType.First, + numNodes: 3, + wantValue: "echo: test", + }, + { + name: "All", + call: respType.All, + numNodes: 3, + wantValue: "echo: test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := gorums.TestConfiguration(t, tt.numNodes, gorums.EchoServerFn) + ctx := gorums.TestContext(t, 2*time.Second) + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + ) + + result, err := tt.call(responses) + + if !checkQuorumCall(t, ctx.Err(), err) { + return + } + + if result.GetValue() != tt.wantValue { + t.Errorf("Expected %q, got %q", tt.wantValue, result.GetValue()) + } + }) + } +} + +func TestPartialFailures(t *testing.T) { + // Function type for the call to test + type callInfo struct { + name string + callFunc func(*gorums.ConfigContext, *pb.StringValue) error + } + + const numServers = 3 + + type respType = *gorums.Responses[*pb.StringValue] + + // Helper to create QuorumCall variants + quorumcall := func(name string, aggregateFunc func(respType) (*pb.StringValue, error)) callInfo { + return callInfo{ + name: "QuorumCall/" + name, + callFunc: func(ctx *gorums.ConfigContext, req *pb.StringValue) error { + _, err := aggregateFunc(gorums.QuorumCall[*pb.StringValue, *pb.StringValue](ctx, req, mock.TestMethod)) + return err + }, + } + } + + // Helper to create Multicast variants + multicast := func(name string, opts ...gorums.CallOption) callInfo { + return callInfo{ + name: "Multicast/" + name, + callFunc: func(ctx *gorums.ConfigContext, req *pb.StringValue) error { + return gorums.Multicast(ctx, req, mock.TestMethod, opts...) + }, + } + } + + tests := []struct { + call callInfo + failing int // number of servers to stop (0 to 3) + wantErr error + }{ + // Multicast: Fails if ANY node fails + {multicast("Wait"), 0, nil}, + {multicast("Wait"), 1, gorums.ErrSendFailure}, + {multicast("Wait"), 2, gorums.ErrSendFailure}, + {multicast("Wait"), 3, gorums.ErrSendFailure}, + + // Multicast with IgnoreErrors: Should not return error even if nodes fail + {multicast("IgnoreErrors", gorums.IgnoreErrors()), 0, nil}, + {multicast("IgnoreErrors", gorums.IgnoreErrors()), 3, nil}, + + // QuorumCall Majority (2/3): Tolerates 1 failure + {quorumcall("Majority", respType.Majority), 0, nil}, + {quorumcall("Majority", respType.Majority), 1, nil}, + {quorumcall("Majority", respType.Majority), 2, gorums.ErrIncomplete}, + {quorumcall("Majority", respType.Majority), 3, gorums.ErrIncomplete}, + + // QuorumCall First (1/3): Tolerates 2 failures + {quorumcall("First", respType.First), 0, nil}, + {quorumcall("First", respType.First), 1, nil}, + {quorumcall("First", respType.First), 2, nil}, + {quorumcall("First", respType.First), 3, gorums.ErrIncomplete}, + + // QuorumCall All (3/3): Fails if any node fails + {quorumcall("All", respType.All), 0, nil}, + {quorumcall("All", respType.All), 1, gorums.ErrIncomplete}, + {quorumcall("All", respType.All), 2, gorums.ErrIncomplete}, + {quorumcall("All", respType.All), 3, gorums.ErrIncomplete}, + } + for _, tt := range tests { + testName := fmt.Sprintf("%s/fail=%d", tt.call.name, tt.failing) + t.Run(testName, func(t *testing.T) { + var stopNodes func(...int) + config := gorums.TestConfiguration(t, numServers, nil, gorums.WithStopFunc(t, &stopNodes)) + ctx := config.Context(t.Context()) + req := pb.String("test") + + // Warmup to ensure connections + if err := tt.call.callFunc(ctx, req); err != nil { + t.Fatalf("Warmup failed: %v", err) + } + + if tt.failing > 0 { + // Collect indices to stop + stopNodes(slices.Collect(gorums.Range(tt.failing))...) + time.Sleep(50 * time.Millisecond) + } + + // Execute test calls after stopping nodes, if any + var err error + for range 5 { + err = tt.call.callFunc(ctx, req) + if tt.wantErr != nil && err != nil { + break // failed as expected + } + if tt.wantErr == nil && err != nil { + break // failed unexpected + } + time.Sleep(10 * time.Millisecond) + } + + if tt.wantErr != nil { + if err == nil { + t.Errorf("Expected error %v, got nil", tt.wantErr) + } else if !errors.Is(err, tt.wantErr) { + t.Errorf("Expected error %v, got %v", tt.wantErr, err) + } else { + var qcErr gorums.QuorumCallError + if errors.As(err, &qcErr) { + if qcErr.NodeErrors() != tt.failing { + t.Errorf("Expected %d node errors, got %d", tt.failing, qcErr.NodeErrors()) + } } - return reply, true + } + } else { + if err != nil { + t.Errorf("Expected success, got error: %v", err) } } - return nil, false - }, + }) } +} - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - response, err := cfg.QuorumCall(ctx, cd) - if err != nil { - t.Fatalf("Unexpected error, got: %v, want: %v", err, nil) +// TestQuorumCallCustomAggregation tests custom response aggregation +func TestQuorumCallCustomAggregation(t *testing.T) { + config := gorums.TestConfiguration(t, 3, nil) // uses default server that returns (i+1)*10 + + ctx := gorums.TestContext(t, 2*time.Second) + responses := gorums.QuorumCall[*pb.Int32Value, *pb.Int32Value]( + config.Context(ctx), + pb.Int32(0), + mock.GetValueMethod, + ) + + // Custom aggregation: sum all responses + // Default server returns 10, 20, 30 for nodes 0, 1, 2 + var sum int32 + for r := range responses.Seq().IgnoreErrors() { + sum += r.Value.GetValue() + } + + if sum != 60 { // 10 + 20 + 30 = 60 + t.Errorf("Expected sum 60, got %d", sum) + } +} + +// TestQuorumCallCollectAll tests collecting all responses +func TestQuorumCallCollectAll(t *testing.T) { + config := gorums.TestConfiguration(t, 3, gorums.EchoServerFn) + + ctx := gorums.TestContext(t, 2*time.Second) + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + config.Context(ctx), + pb.String("test"), + mock.TestMethod, + ) + + collected := responses.CollectAll() + if len(collected) != 3 { + t.Errorf("Expected 3 responses, got %d", len(collected)) } - if response == nil { - t.Fatalf("Unexpected response, got: %v, want: %v", response, pb.String("")) +} + +// BenchmarkQuorumCallTerminalMethods benchmarks the built-in terminal methods with real servers. +func BenchmarkQuorumCallTerminalMethods(b *testing.B) { + for _, numNodes := range []int{3, 5, 7, 9, 13, 17, 19} { + config := gorums.TestConfiguration(b, numNodes, gorums.EchoServerFn) + cfgCtx := config.Context(b.Context()) + + b.Run(fmt.Sprintf("Majority/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + resp, err := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ).Majority() + if err != nil { + b.Fatalf("QuorumCall error: %v", err) + } + _ = resp.GetValue() + } + }) + + b.Run(fmt.Sprintf("Threshold/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + threshold := numNodes/2 + 1 + for b.Loop() { + resp, err := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ).Threshold(threshold) + if err != nil { + b.Fatalf("QuorumCall error: %v", err) + } + _ = resp.GetValue() + } + }) + + b.Run(fmt.Sprintf("First/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + resp, err := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ).First() + if err != nil { + b.Fatalf("QuorumCall error: %v", err) + } + _ = resp.GetValue() + } + }) + + b.Run(fmt.Sprintf("All/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + resp, err := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ).All() + if err != nil { + b.Fatalf("QuorumCall error: %v", err) + } + _ = resp.GetValue() + } + }) + } +} + +// BenchmarkQuorumCall benchmarks custom aggregation using different iterator patterns. +func BenchmarkQuorumCall(b *testing.B) { + for _, numNodes := range []int{3, 5, 7, 9, 13, 17, 19} { + config := gorums.TestConfiguration(b, numNodes, gorums.EchoServerFn) + cfgCtx := config.Context(b.Context()) + + // Using CollectAll and then checking quorum + b.Run(fmt.Sprintf("CollectAllThenCheck/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + quorum := numNodes/2 + 1 + for b.Loop() { + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ) + replies := responses.CollectAll() + if len(replies) < quorum { + b.Fatalf("not enough replies: %d < %d", len(replies), quorum) + } + // Return first reply + for _, r := range replies { + _ = r.GetValue() + break + } + } + }) + + // Using CollectN for early termination + b.Run(fmt.Sprintf("CollectN/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + quorum := numNodes/2 + 1 + for b.Loop() { + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ) + replies := responses.CollectN(quorum) + if len(replies) < quorum { + b.Fatalf("not enough replies: %d < %d", len(replies), quorum) + } + // Return first reply + for _, r := range replies { + _ = r.GetValue() + break + } + } + }) + + // Using manual iteration with early return + b.Run(fmt.Sprintf("Iterator/%d", numNodes), func(b *testing.B) { + b.ReportAllocs() + quorum := numNodes/2 + 1 + for b.Loop() { + responses := gorums.QuorumCall[*pb.StringValue, *pb.StringValue]( + cfgCtx, + pb.String("test"), + mock.TestMethod, + ) + var firstResp *pb.StringValue + var count int + for result := range responses.Seq() { + if result.Err == nil { + count++ + if firstResp == nil { + firstResp = result.Value + } + if count >= quorum { + break + } + } + } + if count < quorum { + b.Fatalf("not enough responses: %d < %d", count, quorum) + } + _ = firstResp.GetValue() + } + }) } } diff --git a/responses.go b/responses.go new file mode 100644 index 000000000..856e4b7ac --- /dev/null +++ b/responses.go @@ -0,0 +1,210 @@ +package gorums + +import ( + "iter" + + "google.golang.org/protobuf/proto" +) + +// msg is a type alias for proto.Message intended to be used as a type parameter. +type msg = proto.Message + +// ------------------------------------------------------------------------- +// Iterator Helpers +// ------------------------------------------------------------------------- + +// ResponseSeq is an iterator that yields NodeResponse[T] values from a quorum call. +type ResponseSeq[T msg] iter.Seq[NodeResponse[T]] + +// IgnoreErrors returns an iterator that yields only successful responses, +// discarding any responses with errors. This is useful when you want to process +// only valid responses from nodes. +// +// Example: +// +// responses := QuorumCall(ctx, Request_builder{Num: uint64(42)}.Build()) +// var sum int32 +// for resp := range responses.IgnoreErrors() { +// // resp is guaranteed to be a successful response +// sum += resp.Value.GetValue() +// } +func (seq ResponseSeq[Resp]) IgnoreErrors() ResponseSeq[Resp] { + return func(yield func(NodeResponse[Resp]) bool) { + for result := range seq { + if result.Err == nil { + if !yield(result) { + return + } + } + } + } +} + +// Filter returns an iterator that yields only the responses for which the +// provided keep function returns true. This is useful for verifying or filtering +// responses from servers before further processing. +// +// Example: +// +// responses := QuorumCall(ctx, req) +// // Filter to only responses from a specific node +// for resp := range responses.Filter(func(r NodeResponse[Resp]) bool { +// return r.NodeID == 1 +// }) { +// // process resp +// } +func (seq ResponseSeq[Resp]) Filter(keep func(NodeResponse[Resp]) bool) ResponseSeq[Resp] { + return func(yield func(NodeResponse[Resp]) bool) { + for result := range seq { + if keep(result) { + if !yield(result) { + return + } + } + } + } +} + +// CollectN collects up to n responses, including errors, from the iterator +// into a map by node ID. It returns early if n responses are collected or +// the iterator is exhausted. +// +// Example: +// +// responses := QuorumCall(ctx, req) +// // Collect the first 2 responses (including errors) +// replies := responses.CollectN(2) +// // or collect 2 successful responses +// replies = responses.IgnoreErrors().CollectN(2) +func (seq ResponseSeq[Resp]) CollectN(n int) map[uint32]Resp { + replies := make(map[uint32]Resp, n) + for result := range seq { + replies[result.NodeID] = result.Value + if len(replies) >= n { + break + } + } + return replies +} + +// CollectAll collects all responses, including errors, from the iterator +// into a map by node ID. +// +// Example: +// +// responses := QuorumCall(ctx, req) +// // Collect all responses (including errors) +// replies := responses.CollectAll() +// // or collect all successful responses +// replies = responses.IgnoreErrors().CollectAll() +func (seq ResponseSeq[Resp]) CollectAll() map[uint32]Resp { + replies := make(map[uint32]Resp) + for result := range seq { + replies[result.NodeID] = result.Value + } + return replies +} + +// ------------------------------------------------------------------------- +// Response Methods +// ------------------------------------------------------------------------- + +// Responses provides access to quorum call responses and terminal methods. +// It is returned by quorum call functions and allows fluent-style API usage: +// +// resp, err := ReadQuorumCall(ctx, req).Majority() +// // or +// resp, err := ReadQuorumCall(ctx, req).First() +// // or +// replies := ReadQuorumCall(ctx, req).IgnoreErrors().CollectAll() +// +// Type parameter: +// - Resp: The response message type +type Responses[Resp msg] struct { + ResponseSeq[Resp] + size int + sendNow func() // sendNow triggers immediate sending of requests +} + +func NewResponses[Req, Resp msg](ctx *ClientCtx[Req, Resp]) *Responses[Resp] { + return &Responses[Resp]{ + ResponseSeq: ctx.responseSeq, + size: ctx.Size(), + sendNow: func() { ctx.sendOnce.Do(ctx.send) }, + } +} + +// Size returns the number of nodes in the configuration. +func (r *Responses[Resp]) Size() int { + return r.size +} + +// Seq returns the underlying response iterator that yields node responses as they arrive. +// It returns a single-use iterator. Users can use this to implement custom aggregation logic. +// This method triggers lazy sending of requests. +// +// The iterator will: +// - Yield responses as they arrive from nodes +// - Continue until the context is canceled or all expected responses have been received +// - Allow early termination by breaking from the range loop +// +// Example usage: +// +// for result := range ReadQuorumCall(ctx, req).Seq() { +// if result.Err != nil { +// // Handle node error +// continue +// } +// // Process result.Value +// } +func (r *Responses[Resp]) Seq() ResponseSeq[Resp] { + return r.ResponseSeq +} + +// ------------------------------------------------------------------------- +// Terminal Methods (Aggregators) +// ------------------------------------------------------------------------- + +// First returns the first successful response received from any node. +// This is useful for read-any patterns where any single response is sufficient. +func (r *Responses[Resp]) First() (Resp, error) { + return r.Threshold(1) +} + +// Majority returns the first response once a simple majority (⌈(n+1)/2⌉) +// of successful responses are received. +func (r *Responses[Resp]) Majority() (Resp, error) { + quorumSize := r.size/2 + 1 + return r.Threshold(quorumSize) +} + +// All returns the first response once all nodes have responded successfully. +// If any node fails, it returns an error. +func (r *Responses[Resp]) All() (Resp, error) { + return r.Threshold(r.size) +} + +// Threshold waits for a threshold number of successful responses. +// It returns the first response once the threshold is reached. +func (r *Responses[Resp]) Threshold(threshold int) (resp Resp, err error) { + var ( + count int + errs []nodeError + ) + for result := range r.ResponseSeq { + if result.Err != nil { + errs = append(errs, nodeError{nodeID: result.NodeID, cause: result.Err}) + continue + } + if count == 0 { + resp = result.Value + } + count++ + + // Check if we have reached the threshold + if count >= threshold { + return resp, nil + } + } + return resp, QuorumCallError{cause: ErrIncomplete, errors: errs} +} diff --git a/responses_test.go b/responses_test.go new file mode 100644 index 000000000..30db3d2c8 --- /dev/null +++ b/responses_test.go @@ -0,0 +1,442 @@ +package gorums + +import ( + "errors" + "testing" + + "google.golang.org/protobuf/proto" + pb "google.golang.org/protobuf/types/known/wrapperspb" +) + +// makeClientCtx is a helper to create a ClientCtx with mock responses for unit tests. +// It creates a channel with the provided responses and returns a ClientCtx. +func makeClientCtx[Req, Resp proto.Message](t *testing.T, numNodes int, responses []NodeResponse[proto.Message]) *ClientCtx[Req, Resp] { + t.Helper() + + resultChan := make(chan NodeResponse[proto.Message], len(responses)) + for _, r := range responses { + resultChan <- r + } + close(resultChan) + + config := make(Configuration, numNodes) + for i := range numNodes { + config[i] = &Node{id: uint32(i + 1)} + } + + c := &ClientCtx[Req, Resp]{ + Context: t.Context(), + config: config, + replyChan: resultChan, + expectedReplies: numNodes, + } + // Mark sendOnce as done since test responses are already in the channel + c.sendOnce.Do(func() {}) + c.responseSeq = c.defaultResponseSeq() + return c +} + +// checkError returns true if the error matches the expected error. +func checkError(t *testing.T, wantErr bool, err, wantErrType error) bool { + t.Helper() + if wantErr { + if err == nil { + t.Error("Expected error, got nil") + return false + } + if wantErrType != nil && !errors.Is(err, wantErrType) { + t.Errorf("Expected error type %v, got %v", wantErrType, err) + return false + } + return true + } + if err != nil { + t.Errorf("Expected no error, got %v", err) + return false + } + return true +} + +// ------------------------------------------------------------------------- +// Terminal Method Tests +// ------------------------------------------------------------------------- + +// TestTerminalMethods tests the terminal methods on Responses +func TestTerminalMethods(t *testing.T) { + type respType = *Responses[*pb.StringValue] + tests := []struct { + name string + numNodes int + responses []NodeResponse[proto.Message] + call func(resp respType) (*pb.StringValue, error) + wantValue string + wantErr bool + wantErrType error + }{ + // First tests + { + name: "First_Success", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + }, + call: respType.First, + wantValue: "response1", + }, + { + name: "First_Error", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: nil, Err: errors.New("node error")}, + {NodeID: 2, Value: nil, Err: errors.New("node error")}, + {NodeID: 3, Value: nil, Err: errors.New("node error")}, + }, + call: respType.First, + wantErr: true, + wantErrType: ErrIncomplete, + }, + // Majority tests + { + name: "Majority_Success_3Nodes", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + }, + call: respType.Majority, + wantValue: "response1", + }, + { + name: "Majority_Insufficient", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: nil, Err: errors.New("node error")}, + {NodeID: 3, Value: nil, Err: errors.New("node error")}, + }, + call: respType.Majority, + wantErr: true, + wantErrType: ErrIncomplete, + }, + { + name: "Majority_Even_Success", + numNodes: 4, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + }, + call: respType.Majority, + wantValue: "response1", + }, + // All tests + { + name: "All_Success", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + }, + call: respType.All, + wantValue: "response1", + }, + { + name: "All_PartialFailure", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: nil, Err: errors.New("node error")}, + }, + call: respType.All, + wantErr: true, + wantErrType: ErrIncomplete, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, tt.numNodes, tt.responses) + responses := NewResponses(clientCtx) + + result, err := tt.call(responses) + + if !checkError(t, tt.wantErr, err, tt.wantErrType) { + return + } + if !tt.wantErr && result.GetValue() != tt.wantValue { + t.Errorf("Expected value %q, got %q", tt.wantValue, result.GetValue()) + } + }) + } +} + +func TestTerminalMethodsThreshold(t *testing.T) { + type respType = *Responses[*pb.StringValue] + tests := []struct { + name string + numNodes int + responses []NodeResponse[proto.Message] + call func(resp respType, threshold int) (*pb.StringValue, error) + threshold int + wantValue string + wantErr bool + wantErrType error + }{ + { + name: "Threshold_Success", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + }, + call: respType.Threshold, + threshold: 2, + wantValue: "response1", + }, + { + name: "Threshold_Insufficient", + numNodes: 3, + responses: []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: nil, Err: errors.New("node error")}, + {NodeID: 3, Value: nil, Err: errors.New("node error")}, + }, + call: respType.Threshold, + threshold: 2, + wantErr: true, + wantErrType: ErrIncomplete, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, tt.numNodes, tt.responses) + responses := NewResponses(clientCtx) + + result, err := tt.call(responses, tt.threshold) + + if !checkError(t, tt.wantErr, err, tt.wantErrType) { + return + } + if !tt.wantErr && result.GetValue() != tt.wantValue { + t.Errorf("Expected value %q, got %q", tt.wantValue, result.GetValue()) + } + }) + } +} + +// ------------------------------------------------------------------------- +// Iterator Method Tests +// ------------------------------------------------------------------------- + +// TestIteratorMethods tests the iterator helper methods +func TestIteratorMethods(t *testing.T) { + t.Run("IgnoreErrors", func(t *testing.T) { + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: nil, Err: errors.New("node error")}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + var count int + for range r.Seq().IgnoreErrors() { + count++ + } + if count != 2 { + t.Errorf("Expected 2 successful responses, got %d", count) + } + }) + + t.Run("Filter", func(t *testing.T) { + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + // Filter to only node 2 + var count int + for resp := range r.Seq().Filter(func(resp NodeResponse[*pb.StringValue]) bool { + return resp.NodeID == 2 + }) { + count++ + if resp.Value.GetValue() != "response2" { + t.Errorf("Expected 'response2', got '%s'", resp.Value.GetValue()) + } + } + if count != 1 { + t.Errorf("Expected 1 filtered response, got %d", count) + } + }) + + t.Run("CollectN", func(t *testing.T) { + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + collected := r.CollectN(2) + if len(collected) != 2 { + t.Errorf("Expected 2 collected responses, got %d", len(collected)) + } + }) + + t.Run("CollectAll", func(t *testing.T) { + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + collected := r.CollectAll() + if len(collected) != 3 { + t.Errorf("Expected 3 collected responses, got %d", len(collected)) + } + }) +} + +// ------------------------------------------------------------------------- +// Custom Aggregation Pattern Tests +// ------------------------------------------------------------------------- + +// TestCustomAggregation demonstrates how users can define custom aggregation +// functions that operate on *Responses and return custom types. +func TestCustomAggregation(t *testing.T) { + t.Run("SameTypeAggregation", func(t *testing.T) { + // Aggregation function that returns the same type (Resp -> Resp) + majorityQF := func(resp *Responses[*pb.StringValue]) (*pb.StringValue, error) { + replies := resp.IgnoreErrors().CollectN(2) + if len(replies) < 2 { + return nil, ErrIncomplete + } + for _, v := range replies { + return v, nil + } + return nil, ErrIncomplete + } + + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + // Call the aggregation function directly + result, err := majorityQF(r) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if result.GetValue() != "response1" && result.GetValue() != "response2" { + t.Errorf("Expected response1 or response2, got %s", result.GetValue()) + } + }) + + t.Run("CustomReturnType", func(t *testing.T) { + // Aggregation function that returns a different type (Resp -> []string) + // This demonstrates the key benefit: Out can differ from In + collectAllValues := func(resp *Responses[*pb.StringValue]) ([]string, error) { + replies := resp.IgnoreErrors().CollectAll() + if len(replies) == 0 { + return nil, ErrIncomplete + } + result := make([]string, 0, len(replies)) + for _, v := range replies { + result = append(result, v.GetValue()) + } + return result, nil + } + + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("alpha"), Err: nil}, + {NodeID: 2, Value: pb.String("beta"), Err: nil}, + {NodeID: 3, Value: pb.String("gamma"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + // Call the aggregation function directly - returns []string from *Responses[*pb.StringValue] + result, err := collectAllValues(r) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(result) != 3 { + t.Errorf("Expected 3 values, got %d", len(result)) + } + }) + + t.Run("WithFiltering", func(t *testing.T) { + // Aggregation function that uses filtering and custom logic + filterAndCount := func(resp *Responses[*pb.StringValue]) (int, error) { + count := 0 + for range resp.IgnoreErrors().Filter(func(r NodeResponse[*pb.StringValue]) bool { + return r.NodeID > 1 // Only nodes 2 and 3 + }) { + count++ + } + if count == 0 { + return 0, ErrIncomplete + } + return count, nil + } + + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: pb.String("response2"), Err: nil}, + {NodeID: 3, Value: pb.String("response3"), Err: nil}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 3, responses) + r := NewResponses(clientCtx) + + // Call the aggregation function directly + count, err := filterAndCount(r) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if count != 2 { + t.Errorf("Expected 2 filtered responses, got %d", count) + } + }) + + t.Run("ErrorHandling", func(t *testing.T) { + // Aggregation function that handles errors explicitly + requireAllSuccess := func(resp *Responses[*pb.StringValue]) (*pb.StringValue, error) { + var first *pb.StringValue + for r := range resp.Seq() { + if r.Err != nil { + return nil, r.Err + } + if first == nil { + first = r.Value + } + } + if first == nil { + return nil, ErrIncomplete + } + return first, nil + } + + responses := []NodeResponse[proto.Message]{ + {NodeID: 1, Value: pb.String("response1"), Err: nil}, + {NodeID: 2, Value: nil, Err: errors.New("node 2 failed")}, + } + clientCtx := makeClientCtx[*pb.StringValue, *pb.StringValue](t, 2, responses) + r := NewResponses(clientCtx) + + // Call the aggregation function directly + _, err := requireAllSuccess(r) + if err == nil { + t.Error("Expected error, got nil") + } + }) +} diff --git a/rpc.go b/rpc.go index 446370fa8..d7b45902b 100644 --- a/rpc.go +++ b/rpc.go @@ -1,31 +1,21 @@ package gorums import ( - "context" - "github.com/relab/gorums/ordering" "google.golang.org/protobuf/proto" ) -// CallData contains data needed to make a remote procedure call. -// -// This struct should be used by generated code only. -type CallData struct { - Message proto.Message - Method string -} - // RPCCall executes a remote procedure call on the node. // // This method should be used by generated code only. -func (n *RawNode) RPCCall(ctx context.Context, d CallData) (proto.Message, error) { - md := ordering.NewGorumsMetadata(ctx, n.mgr.getMsgID(), d.Method) - replyChan := make(chan response, 1) - n.channel.enqueue(request{ctx: ctx, msg: NewRequestMessage(md, d.Message)}, replyChan) +func RPCCall(ctx *NodeContext, msg proto.Message, method string) (proto.Message, error) { + md := ordering.NewGorumsMetadata(ctx, ctx.nextMsgID(), method) + replyChan := make(chan NodeResponse[proto.Message], 1) + ctx.enqueue(request{ctx: ctx, msg: NewRequestMessage(md, msg), responseChan: replyChan}) select { case r := <-replyChan: - return r.msg, r.err + return r.Value, r.Err case <-ctx.Done(): return nil, ctx.Err() } diff --git a/rpc_test.go b/rpc_test.go index c3e4ebab4..d3055131e 100644 --- a/rpc_test.go +++ b/rpc_test.go @@ -19,17 +19,11 @@ func init() { } func TestRPCCallSuccess(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 1, nil) - defer teardown() + node := gorums.TestNode(t, nil) - node := gorums.NewNode(t, addrs[0]) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - response, err := node.RPCCall(ctx, gorums.CallData{ - Message: pb.String(""), - Method: mock.TestMethod, - }) + ctx := gorums.TestContext(t, 5*time.Second) + nodeCtx := node.Context(ctx) + response, err := gorums.RPCCall(nodeCtx, pb.String(""), mock.TestMethod) if err != nil { t.Fatalf("Unexpected error, got: %v, want: %v", err, nil) } @@ -39,17 +33,14 @@ func TestRPCCallSuccess(t *testing.T) { } func TestRPCCallDownedNode(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 1, nil) - node := gorums.NewNode(t, addrs[0]) + node := gorums.TestNode(t, nil, gorums.WithPreConnect(t, func(stopServers func()) { + stopServers() + time.Sleep(300 * time.Millisecond) // wait for servers to fully stop + })) - teardown() // stop all servers on purpose - time.Sleep(300 * time.Millisecond) // servers are not stopped immediately - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - response, err := node.RPCCall(ctx, gorums.CallData{ - Message: pb.String(""), - Method: mock.TestMethod, - }) + ctx := gorums.TestContext(t, 5*time.Second) + nodeCtx := node.Context(ctx) + response, err := gorums.RPCCall(nodeCtx, pb.String(""), mock.TestMethod) if err == nil { t.Fatalf("Expected error, got: %v, want: %v", err, fmt.Errorf("rpc error: code = Unavailable desc = stream is down")) } @@ -59,18 +50,13 @@ func TestRPCCallDownedNode(t *testing.T) { } func TestRPCCallTimedOut(t *testing.T) { - addrs, teardown := gorums.TestSetup(t, 1, nil) - defer teardown() - - node := gorums.NewNode(t, addrs[0]) + node := gorums.TestNode(t, nil) - ctx, cancel := context.WithTimeout(context.Background(), 0*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 0*time.Second) time.Sleep(50 * time.Millisecond) defer cancel() - response, err := node.RPCCall(ctx, gorums.CallData{ - Message: pb.String(""), - Method: mock.TestMethod, - }) + nodeCtx := node.Context(ctx) + response, err := gorums.RPCCall(nodeCtx, pb.String(""), mock.TestMethod) if err == nil { t.Fatalf("Expected error, got: %v, want: %v", err, fmt.Errorf("context deadline exceeded")) } diff --git a/server_test.go b/server_test.go index 75ceecd7c..49626797c 100644 --- a/server_test.go +++ b/server_test.go @@ -24,25 +24,17 @@ func TestServerCallback(t *testing.T) { var message string signal := make(chan struct{}) - srv := gorums.NewServer(gorums.WithConnectCallback(func(ctx context.Context) { + srvOption := gorums.WithConnectCallback(func(ctx context.Context) { m, ok := metadata.FromIncomingContext(ctx) if !ok { return } message = m.Get("message")[0] signal <- struct{}{} - })) - - lis, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - - go func() { _ = srv.Serve(lis) }() - defer srv.Stop() + }) + mgrOption := gorums.WithMetadata(metadata.New(map[string]string{"message": "hello"})) - md := metadata.New(map[string]string{"message": "hello"}) - gorums.NewNode(t, lis.Addr().String(), gorums.WithMetadata(md)) + gorums.TestNode(t, nil, srvOption, mgrOption) select { case <-time.After(100 * time.Millisecond): @@ -80,7 +72,7 @@ func (interceptorSrv) Test(_ gorums.ServerCtx, req proto.Message) (proto.Message func TestServerInterceptorsChain(t *testing.T) { // set up a server with two interceptors: i1, i2 - addrs, teardown := gorums.TestSetup(t, 1, func(_ int) gorums.ServerIface { + interceptorServerFn := func(_ int) gorums.ServerIface { interceptorSrv := &interceptorSrv{} s := gorums.NewServer(gorums.WithInterceptors( appendStringInterceptor("i1in-", "i1out"), @@ -92,18 +84,12 @@ func TestServerInterceptorsChain(t *testing.T) { return gorums.NewResponseMessage(in.GetMetadata(), resp), err }) return s - }) - defer teardown() - - node := gorums.NewNode(t, addrs[0]) + } + node := gorums.TestNode(t, interceptorServerFn) - // call the RPC - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - res, err := node.RPCCall(ctx, gorums.CallData{ - Message: pb.String("client-"), - Method: mock.TestMethod, - }) + ctx := gorums.TestContext(t, 5*time.Second) + nodeCtx := node.Context(ctx) + res, err := gorums.RPCCall(nodeCtx, pb.String("client-"), mock.TestMethod) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -115,3 +101,82 @@ func TestServerInterceptorsChain(t *testing.T) { t.Fatalf("unexpected response value: got %q, want %q", mock.GetVal(res), want) } } + +// TestTCPReconnection verifies that a node can reconnect after the +// underlying TCP connection is broken. +func TestTCPReconnection(t *testing.T) { + srv := gorums.NewServer() + srv.RegisterHandler(mock.TestMethod, func(_ gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + return gorums.NewResponseMessage(in.GetMetadata(), in.GetProtoMessage()), nil + }) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + addr := lis.Addr().String() + + go func() { + _ = srv.Serve(lis) + }() + + mgr := gorums.NewManager(gorums.InsecureDialOptions(t)) + t.Cleanup(gorums.Closer(t, mgr)) + + cfg, err := gorums.NewConfiguration(mgr, gorums.WithNodeList([]string{addr})) + if err != nil { + t.Fatalf("NewConfiguration failed: %v", err) + } + node := cfg.Nodes()[0] + + // Send first message + ctx := gorums.TestContext(t, time.Second) + nodeCtx := node.Context(ctx) + _, err = gorums.RPCCall(nodeCtx, pb.String("1"), mock.TestMethod) + if err != nil { + t.Fatalf("First call failed: %v", err) + } + + // Stop server + srv.Stop() + lis.Close() + + // Wait a bit + time.Sleep(100 * time.Millisecond) + + // Sending now should fail or timeout + ctx2 := gorums.TestContext(t, 200*time.Millisecond) + nodeCtx2 := node.Context(ctx2) + _, err = gorums.RPCCall(nodeCtx2, pb.String("2"), mock.TestMethod) + if err == nil { + // It might succeed if it just queued it? But we wait for response. + } else { + t.Logf("Got expected error during downtime: %v", err) + } + + // Restart server + lis2, err := net.Listen("tcp", addr) + if err != nil { + t.Skipf("Could not re-bind to %s: %v", addr, err) + } + + srv2 := gorums.NewServer() + srv2.RegisterHandler(mock.TestMethod, func(_ gorums.ServerCtx, in *gorums.Message) (*gorums.Message, error) { + return gorums.NewResponseMessage(in.GetMetadata(), in.GetProtoMessage()), nil + }) + go func() { + _ = srv2.Serve(lis2) + }() + defer srv2.Stop() + + // Wait for client backoff/reconnect + time.Sleep(2 * time.Second) + + // Send message again + ctx3 := gorums.TestContext(t, 2*time.Second) + nodeCtx3 := node.Context(ctx3) + _, err = gorums.RPCCall(nodeCtx3, pb.String("3"), mock.TestMethod) + if err != nil { + t.Errorf("Call after reconnection failed: %v", err) + } +} diff --git a/system.go b/system.go new file mode 100644 index 000000000..33454b82e --- /dev/null +++ b/system.go @@ -0,0 +1,61 @@ +package gorums + +import ( + "errors" + "io" + "net" +) + +// System encapsulates the state of a Gorums system, including the server, +// listener, and any registered closers (e.g. managers). +type System struct { + closers []io.Closer + srv *Server + lis net.Listener +} + +// NewSystem creates a new Gorums System with the provided listener and server options. +func NewSystem(lis net.Listener, opts ...ServerOption) *System { + return &System{ + srv: NewServer(opts...), + lis: lis, + } +} + +// RegisterService registers the service with the server using the provided register function. +// The closer is added to the list of closers to be closed when the system is stopped. +// +// Example usage: +// +// gs := NewSystem(lis) +// mgr := NewManager(...) +// impl := &srvImpl{} +// gs.RegisterService(mgr, func(srv *Server) { +// pb.RegisterMultiPaxosServer(srv, impl) +// }) +func (s *System) RegisterService(closer io.Closer, registerFunc func(*Server)) { + if closer != nil { + s.closers = append(s.closers, closer) + } + registerFunc(s.srv) +} + +// Serve starts the server. +func (s *System) Serve() error { + return s.srv.Serve(s.lis) +} + +// Stop stops the Gorums server and closes all registered closers. +// It immediately closes all open connections and listeners. It cancels +// all active RPCs on the server side and the corresponding pending RPCs +// on the client side will get notified by connection errors. +func (s *System) Stop() (errs error) { + // We cannot use graceful stop here since multicast methods does not + // respond to the client, and thus would block indefinitely. + // The server's listener is closed by s.srv.Stop(). + s.srv.Stop() + for _, closer := range s.closers { + errs = errors.Join(errs, closer.Close()) + } + return errs +} diff --git a/system_test.go b/system_test.go new file mode 100644 index 000000000..613c64af8 --- /dev/null +++ b/system_test.go @@ -0,0 +1,87 @@ +package gorums_test + +import ( + "errors" + "net" + "testing" + "time" + + "github.com/relab/gorums" +) + +type mockCloser struct { + closed bool + err error +} + +func (m *mockCloser) Close() error { + m.closed = true + return m.err +} + +func TestSystemLifecycle(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + + sys := gorums.NewSystem(lis) + if sys == nil { + t.Fatal("NewSystem returned nil") + } + + closer1 := &mockCloser{} + closer2 := &mockCloser{} + + sys.RegisterService(closer1, func(srv *gorums.Server) { + // In a real scenario, we would register a Gorums service here. + }) + sys.RegisterService(closer2, func(srv *gorums.Server) { + // Register another service or just use the callback. + }) + + go func() { + // Serve acts as a blocking call, so run in goroutine + if err := sys.Serve(); err != nil { + // Serve returns error on Stop usually (or net closed) + t.Logf("Serve returned: %v", err) + } + }() + + // Give it a moment to start + time.Sleep(10 * time.Millisecond) + + // Stop the system + if err := sys.Stop(); err != nil { + t.Errorf("Stop returned error: %v", err) + } + + if !closer1.closed { + t.Error("closer1 was not closed") + } + if !closer2.closed { + t.Error("closer2 was not closed") + } +} + +func TestSystemStopError(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + + sys := gorums.NewSystem(lis) + errCloser := &mockCloser{err: errors.New("closer error")} + + sys.RegisterService(errCloser, func(srv *gorums.Server) {}) + + go func() { + _ = sys.Serve() + }() + time.Sleep(10 * time.Millisecond) + + err = sys.Stop() + if err == nil { + t.Error("expected error from Stop, got nil") + } +} diff --git a/testing_gorums.go b/testing_gorums.go index a28251fe1..8d0930b82 100644 --- a/testing_gorums.go +++ b/testing_gorums.go @@ -1,138 +1,325 @@ package gorums import ( + "context" + "fmt" + "io" + "iter" + "maps" "net" + "slices" "sync" "testing" + "time" "github.com/relab/gorums/internal/testutils/mock" + "go.uber.org/goleak" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/proto" pb "google.golang.org/protobuf/types/known/wrapperspb" ) -// ServerIface is the interface that must be implemented by a server in order to support the TestSetup function. -type ServerIface interface { - Serve(net.Listener) error - Stop() +// TestContext creates a context with timeout for testing. +// It uses t.Context() as the parent and automatically cancels on cleanup. +func TestContext(t testing.TB, timeout time.Duration) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), timeout) + t.Cleanup(cancel) + return ctx } -// NewNode creates a node for the given server address and adds it to a new manager. -// The manager is automatically closed when the test finishes. -func NewNode(t testing.TB, srvAddr string, opts ...ManagerOption) *RawNode { - t.Helper() - mgrOpts := []ManagerOption{ - WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), - } - mgrOpts = append(mgrOpts, opts...) - mgr := NewRawManager(mgrOpts...) - t.Cleanup(mgr.Close) - node, err := NewRawNode(srvAddr) - if err != nil { - t.Fatal(err) - } - if err = mgr.AddNode(node); err != nil { - t.Fatal(err) +// InsecureDialOptions returns the default insecure gRPC dial options for testing. +func InsecureDialOptions(_ testing.TB) ManagerOption { + return WithDialOptions( + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) +} + +// TestQuorumCallError creates a QuorumCallError for testing. +// The nodeErrors map contains node IDs and their corresponding errors. +func TestQuorumCallError(_ testing.TB, nodeErrors map[uint32]error) QuorumCallError { + errs := make([]nodeError, 0, len(nodeErrors)) + for nodeID, err := range nodeErrors { + errs = append(errs, nodeError{cause: err, nodeID: nodeID}) } - return node + return QuorumCallError{cause: ErrIncomplete, errors: errs} } -// NewConfig creates a configuration for the given node addresses and adds it to a new manager. -// The manager is automatically closed when the test finishes. -func NewConfig(t testing.TB, addrs []string, opts ...ManagerOption) RawConfiguration { +// TestConfiguration creates servers and a configuration for testing. +// Both server and manager cleanup are handled via t.Cleanup in the correct order: +// manager is closed first, then servers are stopped. +// +// The provided srvFn is used to create and register the server handlers. +// If srvFn is nil, a default mock server implementation is used. +// +// Optional TestOptions can be provided to customize the manager, server, or configuration. +// +// By default, nodes are assigned sequential IDs (0, 1, 2, ...) matching the server +// creation order. This can be overridden by providing a NodeListOption. +// +// This is the recommended way to set up tests that need both servers and a configuration. +// It ensures proper cleanup and detects goroutine leaks. +func TestConfiguration(t testing.TB, numServers int, srvFn func(i int) ServerIface, opts ...TestOption) Configuration { t.Helper() - mgrOpts := []ManagerOption{ - WithGrpcDialOptions( - grpc.WithTransportCredentials(insecure.NewCredentials()), - ), + + testOpts := extractTestOptions(opts) + + // Register goleak check FIRST so it runs LAST (LIFO order) + // Only register if not reusing an existing manager (to avoid duplicate checks) + if _, ok := t.(*testing.B); !ok && !testOpts.hasManager() { + t.Cleanup(func() { goleak.VerifyNone(t) }) } - mgrOpts = append(mgrOpts, opts...) - mgr := NewRawManager(mgrOpts...) - t.Cleanup(mgr.Close) - for _, addr := range addrs { - node, err := NewRawNode(addr) - if err != nil { - t.Fatalf("Failed to create node: %v", err) - } - if err = mgr.AddNode(node); err != nil { - t.Fatalf("Failed to add node: %v", err) - } + // Start servers and register cleanup + addrs, stopFn := testSetupServers(t, numServers, testOpts.serverFunc(srvFn)) + stopAllFn := func() { stopFn() } // wrap to call without arguments to stop all servers + t.Cleanup(stopAllFn) + + // Capture the provided stop function to stop individual servers later + if testOpts.stopFuncPtr != nil { + *testOpts.stopFuncPtr = stopFn } - cfg, err := NewRawConfiguration(mgr, WithNodeList(addrs)) + + // Call preConnect hook if set (before connecting to servers) + if testOpts.preConnectHook != nil { + testOpts.preConnectHook(stopAllFn) + } + + mgr := testOpts.getOrCreateManager(t) + cfg, err := NewConfiguration(mgr, testOpts.nodeListOption(addrs)) if err != nil { t.Fatal(err) } return cfg } -// TestSetup starts numServers gRPC servers using the given registration -// function, and returns the server addresses along with a stop function -// that should be called to shut down the test. The stop function will block -// until all servers have stopped. The provided srvFn is used to register -// the server handlers. The service, method, and message types must be registered -// in the global protobuf registry before calling this function. See TestMain -// for an example of how to register the required information. If srvFn is nil, -// a default mock server implementation is used. +// TestNode creates a single server and returns the node for testing. +// Both server and manager cleanup are handled via t.Cleanup in the correct order. +// +// The provided srvFn is used to create and register the server handler. +// If srvFn is nil, a default mock server implementation is used. +// +// Optional TestOptions can be provided to customize the manager, server, or configuration. +// +// This is the recommended way to set up tests that need only a single server node. +// It ensures proper cleanup and detects goroutine leaks. +func TestNode(t testing.TB, srvFn func(i int) ServerIface, opts ...TestOption) *Node { + t.Helper() + return TestConfiguration(t, 1, srvFn, opts...).Nodes()[0] +} + +// TestServers starts numServers gRPC servers using the given registration +// function. Servers are automatically stopped when the test finishes via t.Cleanup. +// The cleanup is registered first, so it runs after any subsequently registered +// cleanups (e.g., manager.Close()), ensuring proper shutdown ordering. +// +// Goroutine leak detection via goleak is automatically enabled and runs after +// all other cleanup functions complete. +// +// The provided srvFn is used to create and register the server handlers. +// If srvFn is nil, a default mock server implementation is used. +// +// Example usage: +// +// addrs := gorums.TestServers(t, 3, serverFn) +// mgr := gorums.NewManager(gorums.InsecureGrpcDialOptions(t)) +// t.Cleanup(mgr.Close) +// ... // // This function can be used by other packages for testing purposes, as long as -// the required types are registered in the global protobuf registry. -func TestSetup(t testing.TB, numServers int, srvFn func(i int) ServerIface) ([]string, func()) { +// the required service, method, and message types are registered in the global +// protobuf registry before calling this function. +func TestServers(t testing.TB, numServers int, srvFn func(i int) ServerIface) []string { + t.Helper() + // Skip goleak check for benchmarks + if _, ok := t.(*testing.B); !ok { + // Register goleak check FIRST so it runs LAST (after all other cleanup) + t.Cleanup(func() { goleak.VerifyNone(t) }) + } + addrs, stopFn := testSetupServers(t, numServers, srvFn) + // Register server cleanup SECOND so it runs BEFORE goleak check + t.Cleanup(func() { stopFn() }) // wrap to call without arguments to stop all servers + return addrs +} + +// ServerIface is the interface that must be implemented by a server in order to support the TestSetup function. +type ServerIface interface { + Serve(net.Listener) error + Stop() +} + +// Closer returns a cleanup function that closes the given io.Closer. +func Closer(t testing.TB, c io.Closer) func() { t.Helper() - servers := make([]ServerIface, numServers) - listeners := make([]net.Listener, numServers) + return func() { + if err := c.Close(); err != nil { + t.Errorf("c.Close() = %q, expected no error", err.Error()) + } + } +} + +type serverState struct { + srv ServerIface + lis net.Listener + stopped chan struct{} +} + +func (s *serverState) start(_ testing.TB) { + _ = s.srv.Serve(s.lis) + s.stopped <- struct{}{} +} + +func (s *serverState) stop(t testing.TB) { + t.Helper() + if err := s.lis.Close(); err != nil { + t.Errorf("Failed to close listener: %v", err) + } + s.srv.Stop() + <-s.stopped +} + +// testSetupServers is the internal implementation of server setup. +// It starts servers and returns addresses and a variadic stop function. +// Call with no arguments to stop all servers, or with specific indices to stop those servers. +func testSetupServers(t testing.TB, numServers int, srvFn func(i int) ServerIface) ([]string, func(...int)) { + t.Helper() + addrs := make([]string, numServers) - srvStopped := make(chan struct{}, numServers) + muActive := &sync.Mutex{} + active := make(map[int]*serverState) + for i := range numServers { var srv ServerIface if srvFn != nil { srv = srvFn(i) } else { - srv = initServer() + srv = initServer(i) } - // listen on any available port lis, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("Failed to listen on port: %v", err) } - listeners[i] = lis addrs[i] = lis.Addr().String() - servers[i] = srv - go func() { - _ = srv.Serve(lis) // blocks until the server is stopped - srvStopped <- struct{}{} - }() + state := &serverState{srv: srv, lis: lis, stopped: make(chan struct{})} + active[i] = state + + go state.start(t) } - stopFn := sync.OnceFunc(func() { - for i := range numServers { - err := listeners[i].Close() - if err != nil { - t.Errorf("failed to close listener: %v", err) + + stopNodesFn := func(indices ...int) { + if len(indices) == 0 { + // Stop all active servers + indices = slices.Collect(Range(numServers)) + } + // Stop specific servers + toStop := make([]*serverState, 0, len(indices)) + muActive.Lock() + maps.DeleteFunc(active, func(k int, v *serverState) bool { + found := slices.Contains(indices, k) + if found { + toStop = append(toStop, v) } - servers[i].Stop() + return found + }) + muActive.Unlock() + + // Stop and wait for each server + for _, state := range toStop { + state.stop(t) } - // wait for all servers to stop - for range numServers { - <-srvStopped + } + return addrs, stopNodesFn +} + +func Range(n int) iter.Seq[int] { + return func(yield func(int) bool) { + for i := range n { + if !yield(i) { + return + } } - }) - return addrs, stopFn + } } -func initServer() *Server { +func initServer(i int) *Server { srv := NewServer() + ts := testSrv{val: int32((i + 1) * 10)} srv.RegisterHandler(mock.TestMethod, func(ctx ServerCtx, in *Message) (*Message, error) { - resp, err := (&testSrv{}).Test(ctx, in.GetProtoMessage()) + resp, err := ts.Test(ctx, in.GetProtoMessage()) + return NewResponseMessage(in.GetMetadata(), resp), err + }) + srv.RegisterHandler(mock.GetValueMethod, func(ctx ServerCtx, in *Message) (*Message, error) { + resp, err := ts.GetValue(ctx, in.GetProtoMessage()) return NewResponseMessage(in.GetMetadata(), resp), err }) return srv } -type testSrv struct{} +type testSrv struct { + val int32 +} func (testSrv) Test(_ ServerCtx, _ proto.Message) (proto.Message, error) { return pb.String(""), nil } + +func (ts testSrv) GetValue(_ ServerCtx, _ proto.Message) (proto.Message, error) { + return pb.Int32(ts.val), nil +} + +func EchoServerFn(_ int) ServerIface { + srv := NewServer() + srv.RegisterHandler(mock.TestMethod, func(ctx ServerCtx, in *Message) (*Message, error) { + resp, err := echoSrv{}.Test(ctx, in.GetProtoMessage()) + return NewResponseMessage(in.GetMetadata(), resp), err + }) + + return srv +} + +// echoSrv implements a simple echo server handler for testing +type echoSrv struct{} + +func (echoSrv) Test(_ ServerCtx, req proto.Message) (proto.Message, error) { + return pb.String("echo: " + mock.GetVal(req)), nil +} + +func StreamServerFn(_ int) ServerIface { + srv := NewServer() + srv.RegisterHandler(mock.Stream, func(ctx ServerCtx, in *Message) (*Message, error) { + req := in.GetProtoMessage() + val := mock.GetVal(req) + + // Send 3 responses + for i := 1; i <= 3; i++ { + resp := pb.String(fmt.Sprintf("echo: %s-%d", val, i)) + msg := NewResponseMessage(in.GetMetadata(), resp) + if err := ctx.SendMessage(msg); err != nil { + return nil, err + } + time.Sleep(10 * time.Millisecond) + } + return nil, nil + }) + return srv +} + +func StreamBenchmarkServerFn(_ int) ServerIface { + srv := NewServer() + srv.RegisterHandler(mock.Stream, func(ctx ServerCtx, in *Message) (*Message, error) { + req := in.GetProtoMessage() + val := mock.GetVal(req) + + // Send 3 responses + for i := 1; i <= 3; i++ { + resp := pb.String(fmt.Sprintf("echo: %s-%d", val, i)) + msg := NewResponseMessage(in.GetMetadata(), resp) + if err := ctx.SendMessage(msg); err != nil { + return nil, err + } + } + return nil, nil + }) + return srv +} diff --git a/testopts.go b/testopts.go new file mode 100644 index 000000000..e7fddb070 --- /dev/null +++ b/testopts.go @@ -0,0 +1,168 @@ +package gorums + +import "testing" + +// TestOption is a marker interface that can hold ManagerOption, +// ServerOption, or NodeListOption. This allows test helpers to accept +// a single variadic parameter that can be filtered and passed to the +// appropriate constructors (NewManager, NewServer, NewConfiguration). +// +// Each option type (ManagerOption, ServerOption, NodeListOption) embeds +// this interface, so they can be passed directly without wrapping: +// +// SetupConfiguration(t, 3, nil, +// WithBackoff(...), // ManagerOption +// WithReceiveBufferSize(10), // ServerOption +// WithNodeMap(...), // NodeListOption +// ) +type TestOption any + +// testOptions holds extracted options from a slice of TestOption. +type testOptions struct { + managerOpts []ManagerOption + serverOpts []ServerOption + nodeListOpts []NodeListOption + existingMgr *Manager + stopFuncPtr *func(...int) // pointer to capture the variadic stop function + preConnectHook func(stopFn func()) // called before connecting to servers +} + +// hasManager returns true if an existing manager was provided. +func (to *testOptions) hasManager() bool { + return to.existingMgr != nil +} + +// serverFunc returns a server creation function based on the server options. +// If no server options are provided, it returns nil (use default server). +func (to *testOptions) serverFunc(srvFn func(i int) ServerIface) func(i int) ServerIface { + if srvFn == nil && len(to.serverOpts) > 0 { + return func(_ int) ServerIface { + return NewServer(to.serverOpts...) + } + } + return srvFn +} + +// getOrCreateManager returns the existing manager or creates a new one. +// If a new manager is created, its cleanup is registered via t.Cleanup. +func (to *testOptions) getOrCreateManager(t testing.TB) *Manager { + if to.existingMgr != nil { + // Don't register cleanup - caller is responsible for closing the manager + return to.existingMgr + } + // Create manager and register its cleanup LAST so it runs FIRST (LIFO) + mgrOpts := append([]ManagerOption{InsecureDialOptions(t)}, to.managerOpts...) + mgr := NewManager(mgrOpts...) + t.Cleanup(Closer(t, mgr)) + return mgr +} + +// nodeListOption returns the appropriate NodeListOption for the configuration. +// It uses provided options if available, or generates defaults based on whether +// an existing manager is being reused. +func (to *testOptions) nodeListOption(addrs []string) NodeListOption { + if len(to.nodeListOpts) > 0 { + // Use the last provided NodeListOption (allows overriding) + return to.nodeListOpts[len(to.nodeListOpts)-1] + } + if to.existingMgr != nil { + // When reusing a manager, use WithNodeList to auto-generate unique IDs + // based on addresses, avoiding conflicts with existing nodes + return WithNodeList(addrs) + } + // Default for new manager: build nodeMap with sequential IDs (0, 1, 2, ...) + nodeMap := make(map[string]uint32, len(addrs)) + for i, addr := range addrs { + nodeMap[addr] = uint32(i) + } + return WithNodeMap(nodeMap) +} + +// extractTestOptions separates a slice of TestOption into their specific types. +func extractTestOptions(opts []TestOption) testOptions { + var result testOptions + for _, opt := range opts { + switch o := opt.(type) { + case ManagerOption: + result.managerOpts = append(result.managerOpts, o) + case ServerOption: + result.serverOpts = append(result.serverOpts, o) + case NodeListOption: + result.nodeListOpts = append(result.nodeListOpts, o) + case *Manager: + result.existingMgr = o + case stopFuncProvider: + result.stopFuncPtr = o.stopFunc + case preConnectProvider: + result.preConnectHook = o.hook + } + } + return result +} + +// WithManager returns a TestOption that provides an existing manager to use +// instead of creating a new one. This is useful when creating multiple +// configurations that should share the same manager. +// +// When using WithManager, the caller is responsible for closing the manager. +// SetupConfiguration will NOT register a cleanup function for the manager. +// +// This option is intended for testing purposes only. +func WithManager(_ testing.TB, mgr *Manager) TestOption { + if mgr == nil { + panic("gorums: WithManager called with nil manager") + } + return mgr +} + +// stopFuncProvider is a TestOption that captures the server stop function. +type stopFuncProvider struct { + stopFunc *func(...int) +} + +// WithStopFunc returns a TestOption that captures the variadic server stop function, +// allowing tests to stop servers at any point during test execution. +// Call with no arguments to stop all servers, or with specific indices to stop those servers. +// This is useful for testing server failure scenarios. +// +// Usage: +// +// var stopServers func(...int) +// config := gorums.TestConfiguration(t, 3, nil, gorums.WithStopFunc(t, &stopServers)) +// // ... send some messages ... +// stopServers() // stop all servers +// // OR +// stopServers(0, 2) // stop servers at indices 0 and 2 +// // ... verify error handling ... +// +// This option is intended for testing purposes only. +func WithStopFunc(_ testing.TB, fn *func(...int)) TestOption { + if fn == nil { + panic("gorums: WithStopFunc called with nil pointer") + } + return stopFuncProvider{stopFunc: fn} +} + +// preConnectProvider is a TestOption that registers a pre-connect hook. +type preConnectProvider struct { + hook func(stopFn func()) +} + +// WithPreConnect returns a TestOption that registers a function to be called +// after servers are started but before nodes attempt to connect. The function +// receives a stopServers callback that can be used to stop the test servers. +// +// This is useful for testing error handling when servers are unavailable: +// +// node := gorums.TestNode(t, nil, gorums.WithPreConnect(t, func(stopServers func()) { +// stopServers() +// time.Sleep(300 * time.Millisecond) // wait for server to fully stop +// })) +// +// This option is intended for testing purposes only. +func WithPreConnect(_ testing.TB, fn func(stopServers func())) TestOption { + if fn == nil { + panic("gorums: WithPreConnect called with nil function") + } + return preConnectProvider{hook: fn} +} diff --git a/unicast.go b/unicast.go index b84aa6d28..55d17633d 100644 --- a/unicast.go +++ b/unicast.go @@ -1,9 +1,8 @@ package gorums import ( - "context" - "github.com/relab/gorums/ordering" + "google.golang.org/protobuf/proto" ) // Unicast is a one-way call; no replies are returned to the client. @@ -11,24 +10,35 @@ import ( // By default, this method blocks until the message has been sent to the node. // This ensures that send operations complete before the caller proceeds, which can // be useful for observing context cancellation or for pacing message sends. +// If the sending fails, the error is returned to the caller. // -// With the WithNoSendWaiting call option, the method returns immediately after +// With the IgnoreErrors call option, the method returns nil immediately after // enqueueing the message (fire-and-forget semantics). // // This method should be used by generated code only. -func (n *RawNode) Unicast(ctx context.Context, d CallData, opts ...CallOption) { - o := getCallOptions(E_Unicast, opts) - - md := ordering.NewGorumsMetadata(ctx, n.mgr.getMsgID(), d.Method) - req := request{ctx: ctx, msg: NewRequestMessage(md, d.Message), opts: o} +func Unicast[Req proto.Message](ctx *NodeContext, req Req, method string, opts ...CallOption) error { + callOpts := getCallOptions(E_Unicast, opts...) + md := ordering.NewGorumsMetadata(ctx, ctx.nextMsgID(), method) + msg := NewRequestMessage(md, req) - if !o.waitSendDone { - n.channel.enqueue(req, nil) - return // fire-and-forget: don't wait for send completion + waitSendDone := callOpts.mustWaitSendDone() + if !waitSendDone { + // Fire-and-forget: enqueue and return immediately + ctx.enqueue(request{ctx: ctx, msg: msg}) + return nil } // Default: block until send completes - replyChan := make(chan response, 1) - n.channel.enqueue(req, replyChan) - <-replyChan + replyChan := make(chan NodeResponse[proto.Message], 1) + ctx.enqueue(request{ctx: ctx, msg: msg, waitSendDone: true, responseChan: replyChan}) + + // Wait for send confirmation + select { + case r := <-replyChan: + // Unicast doesn't expect replies, but we still + // want to report errors from the send attempt. + return r.Err + case <-ctx.Done(): + return ctx.Err() + } }