diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9ed1290..97979a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,7 @@ on: - main - release-* env: - GO_VERSION: 1.26.6 + GO_VERSION: 1.26.9 ARTIFACT_DIR: ./artifacts LINT_DIR: ./.github/linters COV_DIR: ./artifacts/cov @@ -183,5 +183,5 @@ jobs: if: always() id: check-coverage run: |- - go install github.com/vladopajic/go-test-coverage/v2@latest + go install github.com/vladopajic/go-test-coverage/v2@v2.19.0 go-test-coverage --config=${{ env.LINT_DIR }}/testcoverage.yml diff --git a/README.md b/README.md index c7acf06..cffbfc7 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,13 @@ go build -o phorcys . ## Running +Ensure the following options are configured within RabbitMQ: + +```bash +log.exchange = true +log.exchange.level = warning +``` + ```bash export PHORCYS_AMQP_URL="amqp://admin:secret@rabbitmq:5672/" export PHORCYS_MGMT_URL="http://rabbitmq:15672" @@ -81,6 +88,10 @@ Phorcys runs until it receives `SIGINT` or `SIGTERM`. - RabbitMQ 3.8+ with quorum queues enabled - The process must run on the **same host** as the RabbitMQ node (for filesystem access to Raft data and `rabbitmqctl` fallback eviction) +- The user configured by PHORCYS_MGMT_USER and the user embedded in + PHORCYS_AMQP_URL must have full(configure, read, write) access + to all Virtual Host within RabbitMQ. +- The RabbitMQ System log exchange must be enabled. - Go 1.21+ to build from source ## Project layout diff --git a/internal/amqpx/amqptest/amqptest.go b/internal/amqpx/amqptest/amqptest.go index ebce222..8b5ac2b 100644 --- a/internal/amqpx/amqptest/amqptest.go +++ b/internal/amqpx/amqptest/amqptest.go @@ -40,6 +40,17 @@ type Binding struct { Args amqp.Table } +// DeclaredExchange records an ExchangeDeclare call. +type DeclaredExchange struct { + Name string + Kind string + Durable bool + AutoDelete bool + Internal bool + NoWait bool + Args amqp.Table +} + // Consumer records a Consume call. type Consumer struct { Queue string @@ -68,10 +79,11 @@ type Publication struct { // overridden per zero-based publish index via NackAt, NoConfirmAt, or // CloseConfirmsAt. type Channel struct { - ConfirmErr error - DeclareErr error - BindErr error - ConsumeErr error + ConfirmErr error + ExchangeDeclareErr error + DeclareErr error + BindErr error + ConsumeErr error // PublishErrAt fails the publish at the given index; the message is not recorded. PublishErrAt map[int]error // NackAt negatively acknowledges the publish at the given index. @@ -89,6 +101,7 @@ type Channel struct { confirmMode bool confirmNoWait bool confirms chan amqp.Confirmation + exchanges []DeclaredExchange declared []DeclaredQueue bindings []Binding consumers []Consumer @@ -121,6 +134,14 @@ func (c *Channel) NotifyPublish(confirm chan amqp.Confirmation) chan amqp.Confir return confirm } +// ExchangeDeclare implements amqpx.Channel. +func (c *Channel) ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp.Table) error { + c.mu.Lock() + defer c.mu.Unlock() + c.exchanges = append(c.exchanges, DeclaredExchange{name, kind, durable, autoDelete, internal, noWait, args}) + return c.ExchangeDeclareErr +} + // QueueDeclare implements amqpx.Channel. func (c *Channel) QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) (amqp.Queue, error) { c.mu.Lock() @@ -207,6 +228,13 @@ func (c *Channel) ConfirmNoWait() bool { return c.confirmNoWait } +// DeclaredExchanges returns all recorded ExchangeDeclare calls. +func (c *Channel) DeclaredExchanges() []DeclaredExchange { + c.mu.Lock() + defer c.mu.Unlock() + return append([]DeclaredExchange(nil), c.exchanges...) +} + // Declared returns all recorded QueueDeclare calls. func (c *Channel) Declared() []DeclaredQueue { c.mu.Lock() diff --git a/internal/amqpx/amqpx.go b/internal/amqpx/amqpx.go index bb55c38..a1978f1 100644 --- a/internal/amqpx/amqpx.go +++ b/internal/amqpx/amqpx.go @@ -15,6 +15,7 @@ import ( // Channel is the subset of *amqp.Channel used by Phorcys. type Channel interface { Confirm(noWait bool) error + ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp.Table) error NotifyPublish(confirm chan amqp.Confirmation) chan amqp.Confirmation QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) (amqp.Queue, error) QueueBind(name, key, exchange string, noWait bool, args amqp.Table) error diff --git a/internal/broker/diagnostics_test.go b/internal/broker/diagnostics_test.go index b955e19..b45c887 100644 --- a/internal/broker/diagnostics_test.go +++ b/internal/broker/diagnostics_test.go @@ -148,6 +148,129 @@ func TestFetchSpecificQueue_NonOKNon500ReturnsError(t *testing.T) { assert.False(t, is500, "is500 must be false for 404") } +// --------------------------------------------------------------------------- +// ResolveQuorumBasePath +// --------------------------------------------------------------------------- + +func TestResolveQuorumBasePath_UsesFirstNodeName(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewEncoder(w).Encode([]RabbitNode{{Name: "rabbit@n1"}, {Name: "rabbit@n2"}})) + })) + defer server.Close() + + got := newTestManager(server).ResolveQuorumBasePath(context.Background()) + + assert.Equal(t, "/var/lib/rabbitmq/mnesia/rabbit@n1/quorum/rabbit@n1", got) +} + +func TestResolveQuorumBasePath_FallsBackOnAPIError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + got := newTestManager(server).ResolveQuorumBasePath(context.Background()) + + assert.Equal(t, "/var/lib/rabbitmq/mnesia/rabbit@localhost/quorum/rabbit@localhost", got) +} + +func TestResolveQuorumBasePath_FallsBackOnEmptyNodeList(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("[]")) + })) + defer server.Close() + + got := newTestManager(server).ResolveQuorumBasePath(context.Background()) + + assert.Equal(t, "/var/lib/rabbitmq/mnesia/rabbit@localhost/quorum/rabbit@localhost", got) +} + +// --------------------------------------------------------------------------- +// probeQueueGet +// --------------------------------------------------------------------------- + +func TestProbeQueueGet(t *testing.T) { + tests := []struct { + name string + status int + body string + want bool + }{ + {"healthy 200", http.StatusOK, "[]", false}, + {"500 with noproc", http.StatusInternalServerError, `{"reason":"noproc"}`, true}, + {"400 with noproc", http.StatusBadRequest, `{"reason":"noproc"}`, true}, + {"500 without noproc", http.StatusInternalServerError, "boom", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, "/api/queues/%2F/q1/get", r.URL.EscapedPath()) + w.WriteHeader(tt.status) + _, _ = w.Write([]byte(tt.body)) + })) + defer server.Close() + + assert.Equal(t, tt.want, newTestManager(server).probeQueueGet(context.Background(), "/", "q1")) + }) + } +} + +func TestProbeQueueGet_TransportAndRequestErrorsReturnFalse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + dm := newTestManager(server) + server.Close() // connection refused + + assert.False(t, dm.probeQueueGet(context.Background(), "/", "q1")) + + dm.APIURL = "http://bad host/" // invalid URL -> NewRequest error + assert.False(t, dm.probeQueueGet(context.Background(), "/", "q1")) +} + +// --------------------------------------------------------------------------- +// request / transport / decode error paths +// --------------------------------------------------------------------------- + +func TestFetchers_InvalidJSONReturnsError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("not json")) + })) + defer server.Close() + dm := newTestManager(server) + + _, err := dm.fetchNodes(context.Background()) + require.Error(t, err) + _, err = dm.FetchAllQueues(context.Background()) + require.Error(t, err) + _, _, err = dm.fetchSpecificQueue(context.Background(), "/", "q") + require.Error(t, err) +} + +func TestFetchers_TransportErrorReturnsError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + dm := newTestManager(server) + server.Close() + + _, err := dm.fetchNodes(context.Background()) + require.Error(t, err) + _, err = dm.FetchAllQueues(context.Background()) + require.Error(t, err) + _, is500, err := dm.fetchSpecificQueue(context.Background(), "/", "q") + require.Error(t, err) + assert.False(t, is500) +} + +func TestFetchers_InvalidURLReturnsError(t *testing.T) { + dm := &DiagnosticsManager{APIURL: "http://bad host/", Client: http.DefaultClient} + + _, err := dm.fetchNodes(context.Background()) + require.Error(t, err) + _, err = dm.FetchAllQueues(context.Background()) + require.Error(t, err) + _, _, err = dm.fetchSpecificQueue(context.Background(), "/", "q") + require.Error(t, err) +} + func newTestManager(server *httptest.Server) *DiagnosticsManager { return &DiagnosticsManager{ APIURL: server.URL, diff --git a/internal/recovery/consensus.go b/internal/recovery/consensus.go new file mode 100644 index 0000000..69fec00 --- /dev/null +++ b/internal/recovery/consensus.go @@ -0,0 +1,78 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package recovery + +import ( + "context" + "encoding/json" + "fmt" + "log" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// ConsensusExchange is the fanout exchange every Phorcys instance publishes +// RecoveryAnnouncements to. Fanout ensures every node subscribed to it — i.e. +// every other node hosting a replica of the announced queue — receives every +// announcement regardless of routing key. +const ConsensusExchange = "phorcys.consensus" + +// RecoveryAnnouncement is what a node publishes to ConsensusExchange after +// Prepare backs up a queue, so other nodes hosting that queue can compare +// Term/Index (via RaftPosition.After) and agree on which one calls Execute. +type RecoveryAnnouncement struct { + Hostname string `json:"hostname"` + VHost string `json:"vhost"` + Queue string `json:"queue"` + Timestamp time.Time `json:"timestamp"` + Term uint64 `json:"term"` + Index uint64 `json:"index"` +} + +// PublishRecoveryAnnouncement declares ConsensusExchange and publishes ann to +// it as JSON. +func PublishRecoveryAnnouncement(ctx context.Context, amqpURL string, ann RecoveryAnnouncement) error { + conn, err := dialAMQP(amqpURL) + if err != nil { + return fmt.Errorf("failed to connect to broker: %w", err) + } + defer func() { _ = conn.Close() }() + + ch, err := conn.Channel() + if err != nil { + return fmt.Errorf("failed to open channel: %w", err) + } + defer func() { _ = ch.Close() }() + + if err := ch.ExchangeDeclare(ConsensusExchange, "fanout", true, false, false, false, nil); err != nil { + return fmt.Errorf("failed to declare consensus exchange: %w", err) + } + + body, err := json.Marshal(ann) + if err != nil { + return fmt.Errorf("failed to marshal announcement: %w", err) + } + + pubCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + err = ch.PublishWithContext(pubCtx, + ConsensusExchange, + "", // fanout ignores routing key + false, + false, + amqp.Publishing{ + ContentType: "application/json", + Timestamp: ann.Timestamp, + Body: body, + }, + ) + if err != nil { + return fmt.Errorf("failed to publish announcement: %w", err) + } + + log.Printf("[Consensus] Announced %s/%s: term=%d idx=%d", ann.VHost, ann.Queue, ann.Term, ann.Index) + return nil +} diff --git a/internal/recovery/consensus_test.go b/internal/recovery/consensus_test.go new file mode 100644 index 0000000..f67ee6b --- /dev/null +++ b/internal/recovery/consensus_test.go @@ -0,0 +1,188 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package recovery + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sassoftware/phorcys/internal/amqpx/amqptest" +) + +func testAnnouncement() RecoveryAnnouncement { + return RecoveryAnnouncement{ + Hostname: "host1", + VHost: "/", + Queue: "orders", + Timestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + Term: 3, + Index: 17, + } +} + +func TestPublishRecoveryAnnouncement_PublishesJSONToFanoutExchange(t *testing.T) { + ch := amqptest.NewChannel() + conn := amqptest.NewConnection(ch) + dialedURL := useFakeBroker(t, conn, nil) + ann := testAnnouncement() + + err := PublishRecoveryAnnouncement(context.Background(), "amqp://fake/", ann) + + require.NoError(t, err) + assert.Equal(t, "amqp://fake/", *dialedURL) + + exchanges := ch.DeclaredExchanges() + require.Len(t, exchanges, 1) + assert.Equal(t, ConsensusExchange, exchanges[0].Name) + assert.Equal(t, "fanout", exchanges[0].Kind) + assert.True(t, exchanges[0].Durable) + assert.False(t, exchanges[0].AutoDelete) + assert.False(t, exchanges[0].Internal) + assert.False(t, exchanges[0].NoWait) + + pubs := ch.Published() + require.Len(t, pubs, 1) + assert.Equal(t, ConsensusExchange, pubs[0].Exchange) + assert.Empty(t, pubs[0].Key) + assert.False(t, pubs[0].Mandatory) + assert.False(t, pubs[0].Immediate) + assert.Equal(t, "application/json", pubs[0].Msg.ContentType) + assert.True(t, ann.Timestamp.Equal(pubs[0].Msg.Timestamp)) + + var got RecoveryAnnouncement + require.NoError(t, json.Unmarshal(pubs[0].Msg.Body, &got)) + assert.True(t, ann.Timestamp.Equal(got.Timestamp)) + got.Timestamp = ann.Timestamp + assert.Equal(t, ann, got) + + assert.True(t, ch.Closed(), "channel should be closed") + assert.True(t, conn.Closed(), "connection should be closed") +} + +func TestPublishRecoveryAnnouncement_DialError(t *testing.T) { + dialErr := errors.New("dial boom") + useFakeBroker(t, nil, dialErr) + + err := PublishRecoveryAnnouncement(context.Background(), "amqp://fake/", testAnnouncement()) + + require.ErrorIs(t, err, dialErr) + assert.Contains(t, err.Error(), "failed to connect to broker") +} + +func TestPublishRecoveryAnnouncement_ChannelError(t *testing.T) { + chErr := errors.New("channel boom") + conn := &amqptest.Connection{ChannelErr: chErr} + useFakeBroker(t, conn, nil) + + err := PublishRecoveryAnnouncement(context.Background(), "amqp://fake/", testAnnouncement()) + + require.ErrorIs(t, err, chErr) + assert.Contains(t, err.Error(), "failed to open channel") + assert.True(t, conn.Closed(), "connection should be closed") +} + +func TestPublishRecoveryAnnouncement_ExchangeDeclareError(t *testing.T) { + declErr := errors.New("declare boom") + ch := &amqptest.Channel{ExchangeDeclareErr: declErr} + useFakeBroker(t, amqptest.NewConnection(ch), nil) + + err := PublishRecoveryAnnouncement(context.Background(), "amqp://fake/", testAnnouncement()) + + require.ErrorIs(t, err, declErr) + assert.Contains(t, err.Error(), "failed to declare consensus exchange") + assert.Empty(t, ch.Published()) + assert.True(t, ch.Closed()) +} + +func TestPublishRecoveryAnnouncement_MarshalError(t *testing.T) { + ch := amqptest.NewChannel() + useFakeBroker(t, amqptest.NewConnection(ch), nil) + ann := testAnnouncement() + ann.Timestamp = time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) // year outside RFC 3339 range + + err := PublishRecoveryAnnouncement(context.Background(), "amqp://fake/", ann) + + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to marshal announcement") + assert.Empty(t, ch.Published()) +} + +func TestPublishRecoveryAnnouncement_PublishError(t *testing.T) { + pubErr := errors.New("publish boom") + ch := &amqptest.Channel{PublishErrAt: map[int]error{0: pubErr}} + useFakeBroker(t, amqptest.NewConnection(ch), nil) + + err := PublishRecoveryAnnouncement(context.Background(), "amqp://fake/", testAnnouncement()) + + require.ErrorIs(t, err, pubErr) + assert.Contains(t, err.Error(), "failed to publish announcement") +} + +func TestPublishRecoveryAnnouncement_UnreachableBrokerReturnsError(t *testing.T) { + ann := RecoveryAnnouncement{ + Hostname: "host1", + VHost: "/", + Queue: "orders", + Timestamp: time.Now(), + Term: 1, + Index: 2, + } + + err := PublishRecoveryAnnouncement(context.Background(), unreachableAMQPURL, ann) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to connect to broker") +} + +func TestRecoveryAnnouncement_JSONFieldNames(t *testing.T) { + ann := RecoveryAnnouncement{ + Hostname: "host1", + VHost: "/", + Queue: "orders", + Timestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + Term: 7, + Index: 42, + } + + body, err := json.Marshal(ann) + require.NoError(t, err) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(body, &decoded)) + assert.Equal(t, "host1", decoded["hostname"]) + assert.Equal(t, "/", decoded["vhost"]) + assert.Equal(t, "orders", decoded["queue"]) + assert.Equal(t, float64(7), decoded["term"]) + assert.Equal(t, float64(42), decoded["index"]) + assert.Equal(t, "2026-01-02T03:04:05Z", decoded["timestamp"]) +} + +func TestRecoveryAnnouncement_JSONRoundTrip(t *testing.T) { + want := RecoveryAnnouncement{ + Hostname: "rabbit-node-2", + VHost: "production", + Queue: "payments", + Timestamp: time.Date(2026, 6, 15, 12, 30, 0, 0, time.UTC), + Term: 3, + Index: 99, + } + + body, err := json.Marshal(want) + require.NoError(t, err) + + var got RecoveryAnnouncement + require.NoError(t, json.Unmarshal(body, &got)) + assert.True(t, want.Timestamp.Equal(got.Timestamp)) + got.Timestamp = want.Timestamp // compare the rest with a plain Equal + assert.Equal(t, want, got) +} + +func TestConsensusExchange_Name(t *testing.T) { + assert.Equal(t, "phorcys.consensus", ConsensusExchange) +} diff --git a/internal/recovery/pipeline.go b/internal/recovery/pipeline.go index b8bc963..429ea9d 100644 --- a/internal/recovery/pipeline.go +++ b/internal/recovery/pipeline.go @@ -7,29 +7,47 @@ import ( "context" "fmt" "log" + "os" "path/filepath" + "time" "github.com/sassoftware/phorcys/internal/broker" "github.com/sassoftware/phorcys/internal/runtime" ) -// Run executes the full automated recovery for a confirmed unrecoverable queue: +// PreparedRecovery is the output of Prepare: everything Execute needs to +// finish recovering a queue, plus the RaftPosition a consensus process +// compares across a queue's member nodes to pick which one calls Execute. +type PreparedRecovery struct { + Queue string + VHost string + QueueUID string + BackupDir string + WALBackupDir string + Position RaftPosition + PositionFound bool +} + +// Prepare runs the read-only, non-destructive half of recovery for a +// confirmed unrecoverable queue (steps 1-4): // 1. Locate the quorum queue's data directory on disk // 2. Back up segment files to a timestamped archive folder // 3. Back up WAL files that contain records for this queue (only if any exist) -// 4. Log the latest Raft term/index found in the backup (cluster-comparison aid) -// 5. Delete the corrupted queue from the broker -// 6. Carve recoverable payloads from backed-up segment files -// 7. Carve recoverable payloads from backed-up WAL files (queue-UID filtered) -// 8. Republish all payloads — segments first, then WAL — to preserve ordering -func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string) error { - log.Printf("[Recovery] Starting pipeline for queue %s/%s", vhost, queueName) +// 4. Determine the latest Raft term/index found in the backup +// +// It performs no destructive broker calls, so every node hosting a replica of +// the queue can call Prepare independently and in parallel; it does publish a +// RecoveryAnnouncement to ConsensusExchange so the other nodes learn this +// node's RaftPosition, which a consensus process compares — via +// RaftPosition.After — to pick which node calls Execute. +func Prepare(ctx context.Context, cfg runtime.Config, vhost, queueName string) (*PreparedRecovery, error) { + log.Printf("[Recovery] Preparing pipeline for queue %s/%s", vhost, queueName) // Phase 1: Locate the queue's Raft data directory. // The directory basename IS the Ra UID used to identify this queue's records in the WAL. queueDir, err := FindQueueDirectory(cfg.QuorumBasePath, vhost, queueName) if err != nil { - return fmt.Errorf("locate phase: %w", err) + return nil, fmt.Errorf("locate phase: %w", err) } queueUID := filepath.Base(queueDir) log.Printf("[Recovery] Located quorum data at: %s (UID: %s)", queueDir, queueUID) @@ -37,7 +55,7 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, // Phase 2: Backup segment files before any destructive operations. backupDir, err := BackupQueueData(queueDir, cfg.BackupBaseDir, vhost, queueName) if err != nil { - return fmt.Errorf("backup phase: %w", err) + return nil, fmt.Errorf("backup phase: %w", err) } log.Printf("[Recovery] Segment data backed up to: %s", backupDir) @@ -55,36 +73,80 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, log.Printf("[Recovery] No WAL records found for UID %s — skipping WAL backup", queueUID) } - // Phase 4: Log the latest Raft position in this backup so it can be compared - // against other cluster nodes to determine who holds the most recent data. - // TODO: this currently runs inline because backup is still part of this - // pipeline; once backup is split out, this should move with it. - if pos, found, err := LatestQueuePosition(backupDir, walBackupDir, queueUID); err != nil { + // Phase 4: Determine the latest Raft position in this backup so the caller + // can compare it against other cluster nodes to decide who holds the most + // recent data and should therefore run Execute. + pos, found, err := LatestQueuePosition(backupDir, walBackupDir, queueUID) + switch { + case err != nil: log.Printf("[Recovery] WARNING: failed to determine latest Raft position (non-fatal): %v", err) - } else if found { + case found: log.Printf("[Recovery] Latest backed-up Raft position for %s: term=%d idx=%d", queueName, pos.Term, pos.Idx) - } else { + default: log.Printf("[Recovery] No Raft position found in backup for %s", queueName) } + // Announce this node's position so other nodes hosting the queue can run + // their own consensus comparison; a hostname lookup/publish failure must + // not block the local backup that already succeeded. + hostname, hErr := os.Hostname() + if hErr != nil { + log.Printf("[Recovery] WARNING: failed to determine hostname (non-fatal): %v", hErr) + hostname = "unknown" + } + ann := RecoveryAnnouncement{ + Hostname: hostname, + VHost: vhost, + Queue: queueName, + Timestamp: time.Now().UTC(), + Term: pos.Term, + Index: pos.Idx, + } + if err := PublishRecoveryAnnouncement(ctx, cfg.AMQPURL, ann); err != nil { + log.Printf("[Recovery] WARNING: failed to publish recovery announcement (non-fatal): %v", err) + } + + return &PreparedRecovery{ + Queue: queueName, + VHost: vhost, + QueueUID: queueUID, + BackupDir: backupDir, + WALBackupDir: walBackupDir, + Position: pos, + PositionFound: found, + }, nil +} + +// Execute runs the destructive half of recovery (steps 5-8) against the +// backup a prior call to Prepare produced: +// 5. Delete the corrupted queue from the broker +// 6. Carve recoverable payloads from backed-up segment files +// 7. Carve recoverable payloads from backed-up WAL files (queue-UID filtered) +// 8. Republish all payloads — segments first, then WAL — to preserve ordering +// +// Callers must only invoke Execute on the single node a consensus process has +// determined should own recovery for this queue; running it on more than one +// node would delete the queue out from under the others and double-publish +// messages. +func Execute(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, prep *PreparedRecovery) error { // Phase 5: Delete the corrupted queue from the broker. - if err := dm.DeleteOrForceEvict(ctx, vhost, queueName); err != nil { + if err := dm.DeleteOrForceEvict(ctx, prep.VHost, prep.Queue); err != nil { return fmt.Errorf("delete phase: %w", err) } - log.Printf("[Recovery] Queue %s deleted from broker", queueName) + log.Printf("[Recovery] Queue %s deleted from broker", prep.Queue) // Phase 6: Carve messages from backed-up segment files. // CarveMessagesFromDir already skips non-segment/wal extensions; WAL files // in the main backup dir have been separated into the wal/ subdirectory so // this call only processes .segment files here. - segMessages, err := CarveMessagesFromDir(backupDir) + segMessages, err := CarveMessagesFromDir(prep.BackupDir) if err != nil { return fmt.Errorf("segment carve phase: %w", err) } log.Printf("[Recovery] Extracted %d message(s) from segment backup", len(segMessages)) // Phase 7: Carve messages from backed-up WAL files, filtered to this queue's UID. - walMessages, err := CarveWALMessages(walBackupDir, queueUID) + walMessages, err := CarveWALMessages(prep.WALBackupDir, prep.QueueUID) if err != nil { log.Printf("[Recovery] WARNING: WAL carve failed (non-fatal): %v", err) } @@ -94,16 +156,30 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, segCount := len(segMessages) segMessages = append(segMessages, walMessages...) if len(segMessages) == 0 { - log.Printf("[Recovery] No messages found in backup for %s — pipeline complete.", queueName) + log.Printf("[Recovery] No messages found in backup for %s — pipeline complete.", prep.Queue) return nil } // Phase 8: Republish to default exchange; routing key = queue name. - if err := RepublishMessages(ctx, cfg.AMQPURL, queueName, segMessages); err != nil { + if err := RepublishMessages(ctx, cfg.AMQPURL, prep.Queue, segMessages); err != nil { return fmt.Errorf("republish phase: %w", err) } log.Printf("[Recovery] Successfully republished %d message(s) to queue %s (%d from segments, %d from WAL)", - len(segMessages), queueName, segCount, len(walMessages)) + len(segMessages), prep.Queue, segCount, len(walMessages)) return nil } + +// Run performs the full single-node recovery pipeline (Prepare then +// Execute). It is a convenience for callers that don't need multi-node +// consensus — on a cluster where every node runs Phorcys, use Prepare and +// Execute directly so only the node holding the most recent data executes. +func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string) error { + log.Printf("[Recovery] Starting pipeline for queue %s/%s", vhost, queueName) + + prep, err := Prepare(ctx, cfg, vhost, queueName) + if err != nil { + return err + } + return Execute(ctx, cfg, dm, prep) +} diff --git a/internal/recovery/pipeline_test.go b/internal/recovery/pipeline_test.go new file mode 100644 index 0000000..1e33a6b --- /dev/null +++ b/internal/recovery/pipeline_test.go @@ -0,0 +1,189 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package recovery + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/sassoftware/phorcys/internal/broker" + "github.com/sassoftware/phorcys/internal/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// unreachableAMQPURL is deliberately unroutable so amqp.Dial fails fast +// without a live broker, letting tests assert on non-fatal publish/republish +// failures without a network timeout. +const unreachableAMQPURL = "amqp://127.0.0.1:1/" + +// noContentServer returns an httptest server that answers every request with +// 204, satisfying DeleteOrForceEvict's HTTP deletion path. +func noContentServer(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) +} + +// --------------------------------------------------------------------------- +// Prepare +// --------------------------------------------------------------------------- + +func TestPrepare_LocatePhaseFailsReturnsError(t *testing.T) { + cfg := runtime.Config{ + QuorumBasePath: t.TempDir(), + BackupBaseDir: t.TempDir(), + AMQPURL: unreachableAMQPURL, + } + + _, err := Prepare(context.Background(), cfg, "/", "missing-queue") + require.Error(t, err) + assert.Contains(t, err.Error(), "locate phase") +} + +func TestPrepare_SuccessFindsPositionAndSurvivesPublishFailure(t *testing.T) { + quorumBase := t.TempDir() + writeConfigFile(t, quorumBase, "queue-uid-123", "/", "orders") + queueDir := filepath.Join(quorumBase, "queue-uid-123") + writeSegmentBackup(t, queueDir, "0000000000000001.segment", []segmentEntry{ + {idx: 5, term: 2, data: []byte("x")}, + }) + + cfg := runtime.Config{ + QuorumBasePath: quorumBase, + BackupBaseDir: t.TempDir(), + AMQPURL: unreachableAMQPURL, // announce publish must be non-fatal + } + + prep, err := Prepare(context.Background(), cfg, "/", "orders") + require.NoError(t, err, "Prepare should succeed even when the announcement publish fails") + require.NotNil(t, prep) + assert.Equal(t, "queue-uid-123", prep.QueueUID) + assert.DirExists(t, prep.BackupDir) + assert.True(t, prep.PositionFound) + assert.Equal(t, RaftPosition{Idx: 5, Term: 2}, prep.Position) +} + +func TestPrepare_NoBackupDataLeavesPositionNotFound(t *testing.T) { + quorumBase := t.TempDir() + writeConfigFile(t, quorumBase, "queue-uid-456", "/", "empty-queue") + + cfg := runtime.Config{ + QuorumBasePath: quorumBase, + BackupBaseDir: t.TempDir(), + AMQPURL: unreachableAMQPURL, + } + + prep, err := Prepare(context.Background(), cfg, "/", "empty-queue") + require.NoError(t, err) + assert.False(t, prep.PositionFound) + assert.Equal(t, RaftPosition{}, prep.Position) +} + +// --------------------------------------------------------------------------- +// Execute +// --------------------------------------------------------------------------- + +func TestExecute_DeletePhaseFailsReturnsError(t *testing.T) { + // Always-500 server: HTTP delete fails, and rabbitmqctl isn't installed in + // the test environment, so DeleteOrForceEvict is guaranteed to fail too. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + dm := &broker.DiagnosticsManager{APIURL: server.URL, Client: server.Client()} + cfg := runtime.Config{AMQPURL: unreachableAMQPURL} + prep := &PreparedRecovery{Queue: "q", VHost: "/", BackupDir: t.TempDir(), WALBackupDir: t.TempDir()} + + err := Execute(context.Background(), cfg, dm, prep) + require.Error(t, err) + assert.Contains(t, err.Error(), "delete phase") +} + +func TestExecute_SegmentCarveFailsReturnsError(t *testing.T) { + server := noContentServer(t) + defer server.Close() + dm := &broker.DiagnosticsManager{APIURL: server.URL, Client: server.Client()} + cfg := runtime.Config{AMQPURL: unreachableAMQPURL} + prep := &PreparedRecovery{ + Queue: "q", + VHost: "/", + BackupDir: filepath.Join(t.TempDir(), "does-not-exist"), + WALBackupDir: t.TempDir(), + } + + err := Execute(context.Background(), cfg, dm, prep) + require.Error(t, err) + assert.Contains(t, err.Error(), "segment carve phase") +} + +func TestExecute_NoMessagesFoundSkipsRepublishAndSucceeds(t *testing.T) { + server := noContentServer(t) + defer server.Close() + dm := &broker.DiagnosticsManager{APIURL: server.URL, Client: server.Client()} + // AMQPURL is unreachable; if Execute tried to republish it would fail, so + // a nil result here proves the zero-message republish phase was skipped. + cfg := runtime.Config{AMQPURL: unreachableAMQPURL} + prep := &PreparedRecovery{Queue: "q", VHost: "/", BackupDir: t.TempDir(), WALBackupDir: t.TempDir(), QueueUID: "uid"} + + err := Execute(context.Background(), cfg, dm, prep) + assert.NoError(t, err) +} + +func TestExecute_MessagesFoundAttemptsRepublish(t *testing.T) { + server := noContentServer(t) + defer server.Close() + dm := &broker.DiagnosticsManager{APIURL: server.URL, Client: server.Client()} + + backupDir := t.TempDir() + msg, err := encodeETFTerm(makeBasicMessageTerm([]byte("recovered payload"))) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(backupDir, "0000000000000001.segment"), msg, 0600)) + + cfg := runtime.Config{AMQPURL: unreachableAMQPURL} // republish must fail + prep := &PreparedRecovery{Queue: "q", VHost: "/", BackupDir: backupDir, WALBackupDir: t.TempDir(), QueueUID: "uid"} + + err = Execute(context.Background(), cfg, dm, prep) + require.Error(t, err, "carved messages should trigger a republish attempt against the unreachable broker") + assert.Contains(t, err.Error(), "republish phase") +} + +// --------------------------------------------------------------------------- +// Run +// --------------------------------------------------------------------------- + +func TestRun_PropagatesPrepareError(t *testing.T) { + cfg := runtime.Config{ + QuorumBasePath: t.TempDir(), + BackupBaseDir: t.TempDir(), + AMQPURL: unreachableAMQPURL, + } + + err := Run(context.Background(), cfg, &broker.DiagnosticsManager{}, "/", "missing-queue") + require.Error(t, err) + assert.Contains(t, err.Error(), "locate phase") +} + +func TestRun_FullPipelineWithNoBackupDataSucceeds(t *testing.T) { + quorumBase := t.TempDir() + writeConfigFile(t, quorumBase, "queue-uid-789", "/", "orders") + + server := noContentServer(t) + defer server.Close() + dm := &broker.DiagnosticsManager{APIURL: server.URL, Client: server.Client()} + + cfg := runtime.Config{ + QuorumBasePath: quorumBase, + BackupBaseDir: t.TempDir(), + AMQPURL: unreachableAMQPURL, // announce + (skipped) republish must not block success + } + + err := Run(context.Background(), cfg, dm, "/", "orders") + assert.NoError(t, err) +}