Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,25 @@ export PHORCYS_BACKUP_DIR="/data/phorcys-backups"

Phorcys runs until it receives `SIGINT` or `SIGTERM`.

### Manual recovery

If a quorum queue is corrupted but was not detected by the log monitor, the
recovery pipeline can be run on demand. It uses the same environment variables
as the monitor:

```bash
./phorcys recover -queue orders -vhost /
```

| Flag | Default | Description |
|----------|---------|------------------------------------------------|
| `-queue` | — | Name of the quorum queue to recover (required) |
| `-vhost` | `/` | Virtual host of the queue |
| `-yes` | `false` | Skip the interactive confirmation prompt |

The queue is **deleted** from the broker before its recovered messages are
republished, so you are asked to confirm unless `-yes` is passed.

## Requirements

- RabbitMQ 3.8+ with quorum queues enabled
Expand All @@ -100,6 +119,8 @@ Phorcys runs until it receives `SIGINT` or `SIGTERM`.
phorcys/
├── main.go # Entry point
├── internal/
│ ├── cmd/
│ │ └── recover/ # `phorcys recover` manual recovery subcommand
│ ├── amqpx/
│ │ ├── amqpx.go # AMQP Connection/Channel interfaces
│ │ ├── shim.go # Adapter over amqp091-go (live broker)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/sassoftware/phorcys

go 1.26.6
go 1.26.9

require (
github.com/DeedleFake/etf v0.0.0-20250818194526-023937b5970e
Expand Down
124 changes: 124 additions & 0 deletions internal/cmd/recover/recover.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

// Package recover implements the "recover" CLI subcommand, which manually runs
// the recovery pipeline against a quorum queue that the log monitor did not
// detect as corrupted.
package recover

import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"strings"

"github.com/sassoftware/phorcys/internal/broker"
"github.com/sassoftware/phorcys/internal/recovery"
"github.com/sassoftware/phorcys/internal/runtime"
)

// Overridable for tests.
var (
runPipeline = recovery.Run
loadConfig = runtime.LoadConfig
evaluateQueueHealth = func(ctx context.Context, dm *broker.DiagnosticsManager, vhost, queue string) (broker.QueueHealth, error) {
return dm.EvaluateQueueHealth(ctx, vhost, queue)
}
)

const Command = "recover"

const usageHeader = `Usage: phorcys recover -queue <name> [-vhost <vhost>] [-yes]

Manually runs the recovery pipeline for a corrupted quorum queue: backs up its
Raft segment/WAL data, deletes the queue from the broker, carves recoverable
messages from the backup and republishes them to a new queue of the same name.

Connection settings are read from the same PHORCYS_* environment variables
used by the monitor.

Flags:
`

// RunCommand parses args and executes the recovery pipeline for the requested queue.
// Unless -yes is given, the user is asked to confirm on in before the
// (destructive) pipeline starts. Returns flag.ErrHelp when -h is requested.
func RunCommand(ctx context.Context, args []string, in io.Reader, out io.Writer) error {
// Parse flags and validate required arguments.
fs := flag.NewFlagSet(Command, flag.ContinueOnError)
fs.SetOutput(out)
queue := fs.String("queue", "", "name of the quorum queue to recover (required)")
vhost := fs.String("vhost", "/", "virtual host of the queue")
yes := fs.Bool("yes", false, "skip the interactive confirmation prompt")
fs.Usage = func() {
_, _ = fmt.Fprint(out, usageHeader)
fs.PrintDefaults()
}

if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() > 0 {
fs.Usage()
return fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " "))
}
if strings.TrimSpace(*queue) == "" {
fs.Usage()
return errors.New("-queue is required")
}

// Load configuration and initialize diagnostics manager.
cfg := loadConfig()
dm := broker.NewDiagnosticsManager(cfg.ManagementURL, cfg.ManagementUser, cfg.ManagementPassword)
if cfg.QuorumBasePath == "" {
cfg.QuorumBasePath = dm.ResolveQuorumBasePath(ctx)
}

// Ask for confirmation unless -yes is given.
if !*yes {
qh, err := evaluateQueueHealth(ctx, dm, *vhost, *queue)
if err != nil {
return fmt.Errorf("failed to evaluate queue health: %w", err)
}
ok, err := confirm(in, out, *vhost, *queue, qh, cfg)
if err != nil {
return err
}
if !ok {
_, _ = fmt.Fprintln(out, "Aborted.")
return nil
}
}

// Run the recovery pipeline.
if err := runPipeline(ctx, cfg, dm, *vhost, *queue); err != nil {
return fmt.Errorf("recovery of %s/%s failed: %w", *vhost, *queue, err)
}
_, _ = fmt.Fprintf(out, "Recovery of %s/%s complete.\n", *vhost, *queue)
return nil
}

func confirm(in io.Reader, out io.Writer, vhost, queue string, qh broker.QueueHealth, cfg runtime.Config) (bool, error) {
_, _ = fmt.Fprintf(out,
"About to recover queue %q in vhost %q.\n"+
" Queue health: %s\n"+
" Quorum data path: %s\n"+
" Backup directory: %s\n"+
"The queue will be DELETED from the broker and its recovered messages republished.\n"+
"Continue? [y/N]: ",
queue, vhost, qh, cfg.QuorumBasePath, cfg.BackupBaseDir)

line, err := bufio.NewReader(in).ReadString('\n')
if err != nil && !errors.Is(err, io.EOF) {
return false, fmt.Errorf("reading confirmation: %w", err)
}
switch strings.ToLower(strings.TrimSpace(line)) {
case "y", "yes":
return true, nil
default:
return false, nil
}
}
170 changes: 170 additions & 0 deletions internal/cmd/recover/recover_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package recover

import (
"bytes"
"context"
"errors"
"flag"
"net/http"
"net/http/httptest"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/sassoftware/phorcys/internal/broker"
"github.com/sassoftware/phorcys/internal/runtime"
)

type call struct {
cfg runtime.Config
vhost, queue string
}

func stub(t *testing.T, cfg runtime.Config, pipelineErr error) *[]call {
t.Helper()
calls := &[]call{}
origRun, origLoad, origEvaluateQueueHealth := runPipeline, loadConfig, evaluateQueueHealth
t.Cleanup(func() {
runPipeline, loadConfig, evaluateQueueHealth = origRun, origLoad, origEvaluateQueueHealth
})
loadConfig = func() runtime.Config { return cfg }
evaluateQueueHealth = func(_ context.Context, _ *broker.DiagnosticsManager, _, _ string) (broker.QueueHealth, error) {
return broker.HealthGreen, nil
}
runPipeline = func(_ context.Context, c runtime.Config, dm *broker.DiagnosticsManager, vhost, queue string) error {
require.NotNil(t, dm)
*calls = append(*calls, call{c, vhost, queue})
return pipelineErr
}
return calls
}

func baseCfg() runtime.Config {
return runtime.Config{QuorumBasePath: "/q", BackupBaseDir: "/b"}
}

func TestRun_WithYesRunsPipeline(t *testing.T) {
calls := stub(t, baseCfg(), nil)
var out bytes.Buffer

err := RunCommand(context.Background(), []string{"-queue", "orders", "-vhost", "prod", "-yes"}, strings.NewReader(""), &out)

require.NoError(t, err)
require.Len(t, *calls, 1)
assert.Equal(t, "prod", (*calls)[0].vhost)
assert.Equal(t, "orders", (*calls)[0].queue)
assert.Equal(t, "/q", (*calls)[0].cfg.QuorumBasePath)
assert.Contains(t, out.String(), "complete")
}

func TestRun_DefaultVhost(t *testing.T) {
calls := stub(t, baseCfg(), nil)

err := RunCommand(context.Background(), []string{"-queue", "orders", "-yes"}, nil, &bytes.Buffer{})

require.NoError(t, err)
require.Len(t, *calls, 1)
assert.Equal(t, "/", (*calls)[0].vhost)
}

func TestRun_ConfirmYes(t *testing.T) {
for _, answer := range []string{"y\n", "YES\n", " yes"} {
calls := stub(t, baseCfg(), nil)
var out bytes.Buffer

err := RunCommand(context.Background(), []string{"-queue", "orders"}, strings.NewReader(answer), &out)

require.NoError(t, err)
assert.Len(t, *calls, 1, "answer %q", answer)
assert.Contains(t, out.String(), "Continue? [y/N]")
assert.Contains(t, out.String(), "/q")
assert.Contains(t, out.String(), "/b")
}
}

func TestRun_ConfirmDeclined(t *testing.T) {
for _, answer := range []string{"n\n", "\n", ""} {
calls := stub(t, baseCfg(), nil)
var out bytes.Buffer

err := RunCommand(context.Background(), []string{"-queue", "orders"}, strings.NewReader(answer), &out)

require.NoError(t, err)
assert.Empty(t, *calls, "answer %q", answer)
assert.Contains(t, out.String(), "Aborted.")
}
}

type errReader struct{}

func (errReader) Read([]byte) (int, error) { return 0, errors.New("boom") }

func TestRun_ConfirmReadError(t *testing.T) {
calls := stub(t, baseCfg(), nil)

err := RunCommand(context.Background(), []string{"-queue", "orders"}, errReader{}, &bytes.Buffer{})

require.ErrorContains(t, err, "reading confirmation")
assert.Empty(t, *calls)
}

func TestRun_MissingQueue(t *testing.T) {
calls := stub(t, baseCfg(), nil)
var out bytes.Buffer

err := RunCommand(context.Background(), []string{"-yes"}, nil, &out)

require.ErrorContains(t, err, "-queue is required")
assert.Empty(t, *calls)
assert.Contains(t, out.String(), "Usage: phorcys recover")
}

func TestRun_UnexpectedArgs(t *testing.T) {
calls := stub(t, baseCfg(), nil)

err := RunCommand(context.Background(), []string{"-queue", "orders", "extra"}, nil, &bytes.Buffer{})

require.ErrorContains(t, err, "unexpected arguments: extra")
assert.Empty(t, *calls)
}

func TestRun_Help(t *testing.T) {
stub(t, baseCfg(), nil)
var out bytes.Buffer

err := RunCommand(context.Background(), []string{"-h"}, nil, &out)

require.ErrorIs(t, err, flag.ErrHelp)
assert.Contains(t, out.String(), "-queue")
}

func TestRun_PipelineError(t *testing.T) {
stub(t, baseCfg(), errors.New("locate failed"))

err := RunCommand(context.Background(), []string{"-queue", "orders", "-yes"}, nil, &bytes.Buffer{})

require.ErrorContains(t, err, "recovery of //orders failed: locate failed")
}

func TestRun_ResolvesQuorumPathWhenUnset(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`[{"name":"rabbit@node1","running":true}]`))
}))
defer srv.Close()

cfg := baseCfg()
cfg.QuorumBasePath = ""
cfg.ManagementURL = srv.URL
calls := stub(t, cfg, nil)

err := RunCommand(context.Background(), []string{"-queue", "orders", "-yes"}, nil, &bytes.Buffer{})

require.NoError(t, err)
require.Len(t, *calls, 1)
assert.Contains(t, (*calls)[0].cfg.QuorumBasePath, "rabbit@node1")
}
22 changes: 22 additions & 0 deletions internal/runtime/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@ const (
EnvVarMgmtPass = "PHORCYS_MGMT_PASS" //nolint:gosec
EnvVarBackupDir = "PHORCYS_BACKUP_DIR"
EnvVarQuorumPath = "PHORCYS_QUORUM_PATH"

amqpURLDefault = "amqp://guest:guest@localhost:5672/" // #nosec G101 -- local-development default
managementURLDefault = "http://localhost:15672"
managementUserDefault = "guest"
managementPassDefault = "guest" // #nosec G101 -- local-development default
backupBaseDirDefault = "/var/lib/rabbitmq/phorcys-backups"
workerCountDefault = 3
)

// Config holds all runtime configuration for Phorcys.
Expand All @@ -31,6 +38,21 @@ type Config struct {
WorkerCount int
}

// LoadConfig builds a Config from environment variables, falling back to
// local-development defaults. QuorumBasePath is left empty when
// PHORCYS_QUORUM_PATH is unset so callers can resolve it from the broker.
func LoadConfig() Config {
return Config{
AMQPURL: GetEnv(EnvVarAMQPURL, amqpURLDefault),
ManagementURL: GetEnv(EnvVarMgmtURL, managementURLDefault),
ManagementUser: GetEnv(EnvVarMgmtUser, managementUserDefault),
ManagementPassword: GetEnv(EnvVarMgmtPass, managementPassDefault),
BackupBaseDir: GetEnv(EnvVarBackupDir, backupBaseDirDefault),
QuorumBasePath: os.Getenv(EnvVarQuorumPath),
WorkerCount: workerCountDefault,
}
}

// GetEnv returns the value of the environment variable named by key, or
// fallback if the variable is unset or empty.
func GetEnv(key, fallback string) string {
Expand Down
Loading
Loading