Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -183,5 +183,5 @@ jobs:
if: always()
id: check-coverage
run: |-
go install github.com/vladopajic/go-test-coverage/v2@latest
go install github.com/vladopajic/go-test-coverage/v2@v2.19.0
go-test-coverage --config=${{ env.LINT_DIR }}/testcoverage.yml
78 changes: 78 additions & 0 deletions internal/recovery/consensus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package recovery

import (
"context"
"encoding/json"
"fmt"
"log"
"time"

amqp "github.com/rabbitmq/amqp091-go"
)

// ConsensusExchange is the fanout exchange every Phorcys instance publishes
// RecoveryAnnouncements to. Fanout ensures every node subscribed to it — i.e.
// every other node hosting a replica of the announced queue — receives every
// announcement regardless of routing key.
const ConsensusExchange = "phorcys.consensus"

// RecoveryAnnouncement is what a node publishes to ConsensusExchange after
// Prepare backs up a queue, so other nodes hosting that queue can compare
// Term/Index (via RaftPosition.After) and agree on which one calls Execute.
type RecoveryAnnouncement struct {
Hostname string `json:"hostname"`
VHost string `json:"vhost"`
Queue string `json:"queue"`
Timestamp time.Time `json:"timestamp"`
Term uint64 `json:"term"`
Index uint64 `json:"index"`
}

// PublishRecoveryAnnouncement declares ConsensusExchange and publishes ann to
// it as JSON.
func PublishRecoveryAnnouncement(ctx context.Context, amqpURL string, ann RecoveryAnnouncement) error {
conn, err := amqp.Dial(amqpURL)
if err != nil {
return fmt.Errorf("failed to connect to broker: %w", err)
}
defer func() { _ = conn.Close() }()

ch, err := conn.Channel()
if err != nil {
return fmt.Errorf("failed to open channel: %w", err)
}
defer func() { _ = ch.Close() }()

if err := ch.ExchangeDeclare(ConsensusExchange, "fanout", true, false, false, false, nil); err != nil {
return fmt.Errorf("failed to declare consensus exchange: %w", err)
}

body, err := json.Marshal(ann)
if err != nil {
return fmt.Errorf("failed to marshal announcement: %w", err)
}

pubCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

err = ch.PublishWithContext(pubCtx,
ConsensusExchange,
"", // fanout ignores routing key
false,
false,
amqp.Publishing{
ContentType: "application/json",
Timestamp: ann.Timestamp,
Body: body,
},
)
if err != nil {
return fmt.Errorf("failed to publish announcement: %w", err)
}

log.Printf("[Consensus] Announced %s/%s: term=%d idx=%d", ann.VHost, ann.Queue, ann.Term, ann.Index)
return nil
}
76 changes: 76 additions & 0 deletions internal/recovery/consensus_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package recovery

import (
"context"
"encoding/json"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestPublishRecoveryAnnouncement_UnreachableBrokerReturnsError(t *testing.T) {
ann := RecoveryAnnouncement{
Hostname: "host1",
VHost: "/",
Queue: "orders",
Timestamp: time.Now(),
Term: 1,
Index: 2,
}

err := PublishRecoveryAnnouncement(context.Background(), unreachableAMQPURL, ann)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to connect to broker")
}

func TestRecoveryAnnouncement_JSONFieldNames(t *testing.T) {
ann := RecoveryAnnouncement{
Hostname: "host1",
VHost: "/",
Queue: "orders",
Timestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
Term: 7,
Index: 42,
}

body, err := json.Marshal(ann)
require.NoError(t, err)

var decoded map[string]any
require.NoError(t, json.Unmarshal(body, &decoded))
assert.Equal(t, "host1", decoded["hostname"])
assert.Equal(t, "/", decoded["vhost"])
assert.Equal(t, "orders", decoded["queue"])
assert.Equal(t, float64(7), decoded["term"])
assert.Equal(t, float64(42), decoded["index"])
assert.Equal(t, "2026-01-02T03:04:05Z", decoded["timestamp"])
}

func TestRecoveryAnnouncement_JSONRoundTrip(t *testing.T) {
want := RecoveryAnnouncement{
Hostname: "rabbit-node-2",
VHost: "production",
Queue: "payments",
Timestamp: time.Date(2026, 6, 15, 12, 30, 0, 0, time.UTC),
Term: 3,
Index: 99,
}

body, err := json.Marshal(want)
require.NoError(t, err)

var got RecoveryAnnouncement
require.NoError(t, json.Unmarshal(body, &got))
assert.True(t, want.Timestamp.Equal(got.Timestamp))
got.Timestamp = want.Timestamp // compare the rest with a plain Equal
assert.Equal(t, want, got)
}

func TestConsensusExchange_Name(t *testing.T) {
assert.Equal(t, "phorcys.consensus", ConsensusExchange)
}
110 changes: 91 additions & 19 deletions internal/recovery/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,37 +7,53 @@ import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"time"

"github.com/sassoftware/phorcys/internal/broker"
"github.com/sassoftware/phorcys/internal/runtime"
)

// Run executes the full automated recovery for a confirmed unrecoverable queue:
// PreparedRecovery is the output of Prepare: everything Execute needs to
// finish recovering a queue, plus the RaftPosition a consensus process
// compares across a queue's member nodes to pick which one calls Execute.
type PreparedRecovery struct {
QueueUID string
BackupDir string
WALBackupDir string
Position RaftPosition
PositionFound bool
}

// Prepare runs the read-only, non-destructive half of recovery for a
// confirmed unrecoverable queue (steps 1-4):
// 1. Locate the quorum queue's data directory on disk
// 2. Back up segment files to a timestamped archive folder
// 3. Back up WAL files that contain records for this queue (only if any exist)
// 4. Log the latest Raft term/index found in the backup (cluster-comparison aid)
// 5. Delete the corrupted queue from the broker
// 6. Carve recoverable payloads from backed-up segment files
// 7. Carve recoverable payloads from backed-up WAL files (queue-UID filtered)
// 8. Republish all payloads — segments first, then WAL — to preserve ordering
func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string) error {
log.Printf("[Recovery] Starting pipeline for queue %s/%s", vhost, queueName)
// 4. Determine the latest Raft term/index found in the backup
//
// It performs no destructive broker calls, so every node hosting a replica of
// the queue can call Prepare independently and in parallel; it does publish a
// RecoveryAnnouncement to ConsensusExchange so the other nodes learn this
// node's RaftPosition, which a consensus process compares — via
// RaftPosition.After — to pick which node calls Execute.
func Prepare(ctx context.Context, cfg runtime.Config, vhost, queueName string) (*PreparedRecovery, error) {
log.Printf("[Recovery] Preparing pipeline for queue %s/%s", vhost, queueName)

// Phase 1: Locate the queue's Raft data directory.
// The directory basename IS the Ra UID used to identify this queue's records in the WAL.
queueDir, err := FindQueueDirectory(cfg.QuorumBasePath, vhost, queueName)
if err != nil {
return fmt.Errorf("locate phase: %w", err)
return nil, fmt.Errorf("locate phase: %w", err)
}
queueUID := filepath.Base(queueDir)
log.Printf("[Recovery] Located quorum data at: %s (UID: %s)", queueDir, queueUID)

// Phase 2: Backup segment files before any destructive operations.
backupDir, err := BackupQueueData(queueDir, cfg.BackupBaseDir, vhost, queueName)
if err != nil {
return fmt.Errorf("backup phase: %w", err)
return nil, fmt.Errorf("backup phase: %w", err)
}
log.Printf("[Recovery] Segment data backed up to: %s", backupDir)

Expand All @@ -55,18 +71,60 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager,
log.Printf("[Recovery] No WAL records found for UID %s — skipping WAL backup", queueUID)
}

// Phase 4: Log the latest Raft position in this backup so it can be compared
// against other cluster nodes to determine who holds the most recent data.
// TODO: this currently runs inline because backup is still part of this
// pipeline; once backup is split out, this should move with it.
if pos, found, err := LatestQueuePosition(backupDir, walBackupDir, queueUID); err != nil {
// Phase 4: Determine the latest Raft position in this backup so the caller
// can compare it against other cluster nodes to decide who holds the most
// recent data and should therefore run Execute.
pos, found, err := LatestQueuePosition(backupDir, walBackupDir, queueUID)
switch {
case err != nil:
log.Printf("[Recovery] WARNING: failed to determine latest Raft position (non-fatal): %v", err)
} else if found {
case found:
log.Printf("[Recovery] Latest backed-up Raft position for %s: term=%d idx=%d", queueName, pos.Term, pos.Idx)
} else {
default:
log.Printf("[Recovery] No Raft position found in backup for %s", queueName)
}

// Announce this node's position so other nodes hosting the queue can run
// their own consensus comparison; a hostname lookup/publish failure must
// not block the local backup that already succeeded.
hostname, hErr := os.Hostname()
if hErr != nil {
log.Printf("[Recovery] WARNING: failed to determine hostname (non-fatal): %v", hErr)
hostname = "unknown"
}
ann := RecoveryAnnouncement{
Hostname: hostname,
VHost: vhost,
Queue: queueName,
Timestamp: time.Now().UTC(),
Term: pos.Term,
Index: pos.Idx,
}
if err := PublishRecoveryAnnouncement(ctx, cfg.AMQPURL, ann); err != nil {
log.Printf("[Recovery] WARNING: failed to publish recovery announcement (non-fatal): %v", err)
}

return &PreparedRecovery{
QueueUID: queueUID,
BackupDir: backupDir,
WALBackupDir: walBackupDir,
Position: pos,
PositionFound: found,
}, nil
}

// Execute runs the destructive half of recovery (steps 5-8) against the
// backup a prior call to Prepare produced:
// 5. Delete the corrupted queue from the broker
// 6. Carve recoverable payloads from backed-up segment files
// 7. Carve recoverable payloads from backed-up WAL files (queue-UID filtered)
// 8. Republish all payloads — segments first, then WAL — to preserve ordering
//
// Callers must only invoke Execute on the single node a consensus process has
// determined should own recovery for this queue; running it on more than one
// node would delete the queue out from under the others and double-publish
// messages.
func Execute(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string, prep *PreparedRecovery) error {
Comment thread
bithckr marked this conversation as resolved.
Resolved
// Phase 5: Delete the corrupted queue from the broker.
if err := dm.DeleteOrForceEvict(ctx, vhost, queueName); err != nil {
return fmt.Errorf("delete phase: %w", err)
Expand All @@ -77,14 +135,14 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager,
// CarveMessagesFromDir already skips non-segment/wal extensions; WAL files
// in the main backup dir have been separated into the wal/ subdirectory so
// this call only processes .segment files here.
segMessages, err := CarveMessagesFromDir(backupDir)
segMessages, err := CarveMessagesFromDir(prep.BackupDir)
if err != nil {
return fmt.Errorf("segment carve phase: %w", err)
}
log.Printf("[Recovery] Extracted %d message(s) from segment backup", len(segMessages))

// Phase 7: Carve messages from backed-up WAL files, filtered to this queue's UID.
walMessages, err := CarveWALMessages(walBackupDir, queueUID)
walMessages, err := CarveWALMessages(prep.WALBackupDir, prep.QueueUID)
if err != nil {
log.Printf("[Recovery] WARNING: WAL carve failed (non-fatal): %v", err)
}
Expand All @@ -107,3 +165,17 @@ func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager,

return nil
}

// Run performs the full single-node recovery pipeline (Prepare then
// Execute). It is a convenience for callers that don't need multi-node
// consensus — on a cluster where every node runs Phorcys, use Prepare and
// Execute directly so only the node holding the most recent data executes.
func Run(ctx context.Context, cfg runtime.Config, dm *broker.DiagnosticsManager, vhost, queueName string) error {
log.Printf("[Recovery] Starting pipeline for queue %s/%s", vhost, queueName)

prep, err := Prepare(ctx, cfg, vhost, queueName)
if err != nil {
return err
}
return Execute(ctx, cfg, dm, vhost, queueName, prep)
}
Loading
Loading