Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions internal/recovery/consensus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package recovery

import (
"context"
"encoding/json"
"fmt"
"log"
"time"

amqp "github.com/rabbitmq/amqp091-go"
)

// ConsensusExchange is the fanout exchange every Phorcys instance publishes
// RecoveryAnnouncements to. Fanout ensures every node subscribed to it — i.e.
// every other node hosting a replica of the announced queue — receives every
// announcement regardless of routing key.
const ConsensusExchange = "phorcys.consensus"

// RecoveryAnnouncement is what a node publishes to ConsensusExchange after
// Prepare backs up a queue, so other nodes hosting that queue can compare
// Term/Index (via RaftPosition.After) and agree on which one calls Execute.
type RecoveryAnnouncement struct {
Hostname string `json:"hostname"`
VHost string `json:"vhost"`
Queue string `json:"queue"`
Timestamp time.Time `json:"timestamp"`
Term uint64 `json:"term"`
Index uint64 `json:"index"`
}

// PublishRecoveryAnnouncement declares ConsensusExchange and publishes ann to
// it as JSON.
func PublishRecoveryAnnouncement(ctx context.Context, amqpURL string, ann RecoveryAnnouncement) error {
conn, err := amqp.Dial(amqpURL)
if err != nil {
return fmt.Errorf("failed to connect to broker: %w", err)
}
defer func() { _ = conn.Close() }()

ch, err := conn.Channel()
if err != nil {
return fmt.Errorf("failed to open channel: %w", err)
}
defer func() { _ = ch.Close() }()

if err := ch.ExchangeDeclare(ConsensusExchange, "fanout", true, false, false, false, nil); err != nil {
return fmt.Errorf("failed to declare consensus exchange: %w", err)
}

body, err := json.Marshal(ann)
if err != nil {
return fmt.Errorf("failed to marshal announcement: %w", err)
}

pubCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

err = ch.PublishWithContext(pubCtx,
ConsensusExchange,
"", // fanout ignores routing key
false,
false,
amqp.Publishing{
ContentType: "application/json",
Timestamp: ann.Timestamp,
Body: body,
},
)
if err != nil {
return fmt.Errorf("failed to publish announcement: %w", err)
}

log.Printf("[Consensus] Announced %s/%s: term=%d idx=%d", ann.VHost, ann.Queue, ann.Term, ann.Index)
return nil
}
76 changes: 76 additions & 0 deletions internal/recovery/consensus_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package recovery

import (
"context"
"encoding/json"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestPublishRecoveryAnnouncement_UnreachableBrokerReturnsError(t *testing.T) {
ann := RecoveryAnnouncement{
Hostname: "host1",
VHost: "/",
Queue: "orders",
Timestamp: time.Now(),
Term: 1,
Index: 2,
}

err := PublishRecoveryAnnouncement(context.Background(), unreachableAMQPURL, ann)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to connect to broker")
}

func TestRecoveryAnnouncement_JSONFieldNames(t *testing.T) {
ann := RecoveryAnnouncement{
Hostname: "host1",
VHost: "/",
Queue: "orders",
Timestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
Term: 7,
Index: 42,
}

body, err := json.Marshal(ann)
require.NoError(t, err)

var decoded map[string]any
require.NoError(t, json.Unmarshal(body, &decoded))
assert.Equal(t, "host1", decoded["hostname"])
assert.Equal(t, "/", decoded["vhost"])
assert.Equal(t, "orders", decoded["queue"])
assert.Equal(t, float64(7), decoded["term"])
assert.Equal(t, float64(42), decoded["index"])
assert.Equal(t, "2026-01-02T03:04:05Z", decoded["timestamp"])
}

func TestRecoveryAnnouncement_JSONRoundTrip(t *testing.T) {
want := RecoveryAnnouncement{
Hostname: "rabbit-node-2",
VHost: "production",
Queue: "payments",
Timestamp: time.Date(2026, 6, 15, 12, 30, 0, 0, time.UTC),
Term: 3,
Index: 99,
}

body, err := json.Marshal(want)
require.NoError(t, err)

var got RecoveryAnnouncement
require.NoError(t, json.Unmarshal(body, &got))
assert.True(t, want.Timestamp.Equal(got.Timestamp))
got.Timestamp = want.Timestamp // compare the rest with a plain Equal
assert.Equal(t, want, got)
}

func TestConsensusExchange_Name(t *testing.T) {
assert.Equal(t, "phorcys.consensus", ConsensusExchange)
}
110 changes: 91 additions & 19 deletions internal/recovery/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,37 +7,53 @@ import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"time"

"github.com/sassoftware/phorcys/internal/broker"
"github.com/sassoftware/phorcys/internal/runtime"
)

// Run executes the full automated recovery for a confirmed unrecoverable queue:
// PreparedRecovery is the output of Prepare: everything Execute needs to
// finish recovering a queue, plus the RaftPosition a consensus process
// compares across a queue's member nodes to pick which one calls Execute.
type PreparedRecovery struct {
QueueUID string
BackupDir string
WALBackupDir string
Position RaftPosition
PositionFound bool
}

// Prepare runs the read-only, non-destructive half of recovery for a
// confirmed unrecoverable queue (steps 1-4):
// 1. Locate the quorum queue's data directory on disk
// 2. Back up segment files to a timestamped archive folder
// 3. Back up WAL files that contain records for this queue (only if any exist)
// 4. Log the latest Raft term/index found in the backup (cluster-comparison aid)
// 5. Delete the corrupted queue from the broker
// 6. Carve recoverable payloads from backed-up segment files
// 7. Carve recoverable payloads from backed-up WAL files (queue-UID filtered)
// 8. Republish all payloads — segments first, then WAL — to preserve ordering
func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string) error {
log.Printf("[Recovery] Starting pipeline for queue %s/%s", vhost, queueName)
// 4. Determine the latest Raft term/index found in the backup
//
// It performs no destructive broker calls, so every node hosting a replica of
// the queue can call Prepare independently and in parallel; it does publish a
// RecoveryAnnouncement to ConsensusExchange so the other nodes learn this
// node's RaftPosition, which a consensus process compares — via
// RaftPosition.After — to pick which node calls Execute.
func Prepare(ctx context.Context, cfg runtime.Config, vhost, queueName string) (*PreparedRecovery, error) {
log.Printf("[Recovery] Preparing pipeline for queue %s/%s", vhost, queueName)

// Phase 1: Locate the queue's Raft data directory.
// The directory basename IS the Ra UID used to identify this queue's records in the WAL.
queueDir, err := FindQueueDirectory(cfg.QuorumBasePath, vhost, queueName)
if err != nil {
return fmt.Errorf("locate phase: %w", err)
return nil, fmt.Errorf("locate phase: %w", err)
}
queueUID := filepath.Base(queueDir)
log.Printf("[Recovery] Located quorum data at: %s (UID: %s)", queueDir, queueUID)

// Phase 2: Backup segment files before any destructive operations.
backupDir, err := BackupQueueData(queueDir, cfg.BackupBaseDir, vhost, queueName)
if err != nil {
return fmt.Errorf("backup phase: %w", err)
return nil, fmt.Errorf("backup phase: %w", err)
}
log.Printf("[Recovery] Segment data backed up to: %s", backupDir)

Expand All @@ -55,18 +71,60 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager,
log.Printf("[Recovery] No WAL records found for UID %s — skipping WAL backup", queueUID)
}

// Phase 4: Log the latest Raft position in this backup so it can be compared
// against other cluster nodes to determine who holds the most recent data.
// TODO: this currently runs inline because backup is still part of this
// pipeline; once backup is split out, this should move with it.
if pos, found, err := LatestQueuePosition(backupDir, walBackupDir, queueUID); err != nil {
// Phase 4: Determine the latest Raft position in this backup so the caller
// can compare it against other cluster nodes to decide who holds the most
// recent data and should therefore run Execute.
pos, found, err := LatestQueuePosition(backupDir, walBackupDir, queueUID)
switch {
case err != nil:
log.Printf("[Recovery] WARNING: failed to determine latest Raft position (non-fatal): %v", err)
} else if found {
case found:
log.Printf("[Recovery] Latest backed-up Raft position for %s: term=%d idx=%d", queueName, pos.Term, pos.Idx)
} else {
default:
log.Printf("[Recovery] No Raft position found in backup for %s", queueName)
}

// Announce this node's position so other nodes hosting the queue can run
// their own consensus comparison; a hostname lookup/publish failure must
// not block the local backup that already succeeded.
hostname, hErr := os.Hostname()
if hErr != nil {
log.Printf("[Recovery] WARNING: failed to determine hostname (non-fatal): %v", hErr)
hostname = "unknown"
}
ann := RecoveryAnnouncement{
Hostname: hostname,
VHost: vhost,
Queue: queueName,
Timestamp: time.Now().UTC(),
Term: pos.Term,
Index: pos.Idx,
}
if err := PublishRecoveryAnnouncement(ctx, cfg.AMQPURL, ann); err != nil {
log.Printf("[Recovery] WARNING: failed to publish recovery announcement (non-fatal): %v", err)
}

return &PreparedRecovery{
QueueUID: queueUID,
BackupDir: backupDir,
WALBackupDir: walBackupDir,
Position: pos,
PositionFound: found,
}, nil
}

// Execute runs the destructive half of recovery (steps 5-8) against the
// backup a prior call to Prepare produced:
// 5. Delete the corrupted queue from the broker
// 6. Carve recoverable payloads from backed-up segment files
// 7. Carve recoverable payloads from backed-up WAL files (queue-UID filtered)
// 8. Republish all payloads — segments first, then WAL — to preserve ordering
//
// Callers must only invoke Execute on the single node a consensus process has
// determined should own recovery for this queue; running it on more than one
// node would delete the queue out from under the others and double-publish
// messages.
func Execute(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string, prep *PreparedRecovery) error {
Comment thread
bithckr marked this conversation as resolved.
Resolved
// Phase 5: Delete the corrupted queue from the broker.
if err := dm.DeleteOrForceEvict(ctx, vhost, queueName); err != nil {
return fmt.Errorf("delete phase: %w", err)
Expand All @@ -77,14 +135,14 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager,
// CarveMessagesFromDir already skips non-segment/wal extensions; WAL files
// in the main backup dir have been separated into the wal/ subdirectory so
// this call only processes .segment files here.
segMessages, err := CarveMessagesFromDir(backupDir)
segMessages, err := CarveMessagesFromDir(prep.BackupDir)
if err != nil {
return fmt.Errorf("segment carve phase: %w", err)
}
log.Printf("[Recovery] Extracted %d message(s) from segment backup", len(segMessages))

// Phase 7: Carve messages from backed-up WAL files, filtered to this queue's UID.
walMessages, err := CarveWALMessages(walBackupDir, queueUID)
walMessages, err := CarveWALMessages(prep.WALBackupDir, prep.QueueUID)
if err != nil {
log.Printf("[Recovery] WARNING: WAL carve failed (non-fatal): %v", err)
}
Expand All @@ -107,3 +165,17 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager,

return nil
}

// Run performs the full single-node recovery pipeline (Prepare then
// Execute). It is a convenience for callers that don't need multi-node
// consensus — on a cluster where every node runs Phorcys, use Prepare and
// Execute directly so only the node holding the most recent data executes.
func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string) error {
log.Printf("[Recovery] Starting pipeline for queue %s/%s", vhost, queueName)

prep, err := Prepare(ctx, cfg, vhost, queueName)
if err != nil {
return err
}
return Execute(ctx, cfg, dm, vhost, queueName, prep)
}
Loading
Loading