diff --git a/README.md b/README.md index cffbfc7..2e31611 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,25 @@ export PHORCYS_BACKUP_DIR="/data/phorcys-backups" Phorcys runs until it receives `SIGINT` or `SIGTERM`. +### Manual recovery + +If a quorum queue is corrupted but was not detected by the log monitor, the +recovery pipeline can be run on demand. It uses the same environment variables +as the monitor: + +```bash +./phorcys recover -queue orders -vhost / +``` + +| Flag | Default | Description | +|----------|---------|------------------------------------------------| +| `-queue` | — | Name of the quorum queue to recover (required) | +| `-vhost` | `/` | Virtual host of the queue | +| `-yes` | `false` | Skip the interactive confirmation prompt | + +The queue is **deleted** from the broker before its recovered messages are +republished, so you are asked to confirm unless `-yes` is passed. + ## Requirements - RabbitMQ 3.8+ with quorum queues enabled @@ -100,6 +119,8 @@ Phorcys runs until it receives `SIGINT` or `SIGTERM`. phorcys/ ├── main.go # Entry point ├── internal/ +│ ├── cmd/ +│ │ └── recover/ # `phorcys recover` manual recovery subcommand │ ├── amqpx/ │ │ ├── amqpx.go # AMQP Connection/Channel interfaces │ │ ├── shim.go # Adapter over amqp091-go (live broker) diff --git a/go.mod b/go.mod index b25ae8d..5636beb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/sassoftware/phorcys -go 1.26.6 +go 1.26.9 require ( github.com/DeedleFake/etf v0.0.0-20250818194526-023937b5970e diff --git a/internal/cmd/recover/recover.go b/internal/cmd/recover/recover.go new file mode 100644 index 0000000..ae633e9 --- /dev/null +++ b/internal/cmd/recover/recover.go @@ -0,0 +1,124 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package recover implements the "recover" CLI subcommand, which manually runs +// the recovery pipeline against a quorum queue that the log monitor did not +// detect as corrupted. +package recover + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "strings" + + "github.com/sassoftware/phorcys/internal/broker" + "github.com/sassoftware/phorcys/internal/recovery" + "github.com/sassoftware/phorcys/internal/runtime" +) + +// Overridable for tests. +var ( + runPipeline = recovery.Run + loadConfig = runtime.LoadConfig + evaluateQueueHealth = func(ctx context.Context, dm *broker.DiagnosticsManager, vhost, queue string) (broker.QueueHealth, error) { + return dm.EvaluateQueueHealth(ctx, vhost, queue) + } +) + +const Command = "recover" + +const usageHeader = `Usage: phorcys recover -queue [-vhost ] [-yes] + +Manually runs the recovery pipeline for a corrupted quorum queue: backs up its +Raft segment/WAL data, deletes the queue from the broker, carves recoverable +messages from the backup and republishes them to a new queue of the same name. + +Connection settings are read from the same PHORCYS_* environment variables +used by the monitor. + +Flags: +` + +// RunCommand parses args and executes the recovery pipeline for the requested queue. +// Unless -yes is given, the user is asked to confirm on in before the +// (destructive) pipeline starts. Returns flag.ErrHelp when -h is requested. +func RunCommand(ctx context.Context, args []string, in io.Reader, out io.Writer) error { + // Parse flags and validate required arguments. + fs := flag.NewFlagSet(Command, flag.ContinueOnError) + fs.SetOutput(out) + queue := fs.String("queue", "", "name of the quorum queue to recover (required)") + vhost := fs.String("vhost", "/", "virtual host of the queue") + yes := fs.Bool("yes", false, "skip the interactive confirmation prompt") + fs.Usage = func() { + _, _ = fmt.Fprint(out, usageHeader) + fs.PrintDefaults() + } + + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() > 0 { + fs.Usage() + return fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " ")) + } + if strings.TrimSpace(*queue) == "" { + fs.Usage() + return errors.New("-queue is required") + } + + // Load configuration and initialize diagnostics manager. + cfg := loadConfig() + dm := broker.NewDiagnosticsManager(cfg.ManagementURL, cfg.ManagementUser, cfg.ManagementPassword) + if cfg.QuorumBasePath == "" { + cfg.QuorumBasePath = dm.ResolveQuorumBasePath(ctx) + } + + // Ask for confirmation unless -yes is given. + if !*yes { + qh, err := evaluateQueueHealth(ctx, dm, *vhost, *queue) + if err != nil { + return fmt.Errorf("failed to evaluate queue health: %w", err) + } + ok, err := confirm(in, out, *vhost, *queue, qh, cfg) + if err != nil { + return err + } + if !ok { + _, _ = fmt.Fprintln(out, "Aborted.") + return nil + } + } + + // Run the recovery pipeline. + if err := runPipeline(ctx, cfg, dm, *vhost, *queue); err != nil { + return fmt.Errorf("recovery of %s/%s failed: %w", *vhost, *queue, err) + } + _, _ = fmt.Fprintf(out, "Recovery of %s/%s complete.\n", *vhost, *queue) + return nil +} + +func confirm(in io.Reader, out io.Writer, vhost, queue string, qh broker.QueueHealth, cfg runtime.Config) (bool, error) { + _, _ = fmt.Fprintf(out, + "About to recover queue %q in vhost %q.\n"+ + " Queue health: %s\n"+ + " Quorum data path: %s\n"+ + " Backup directory: %s\n"+ + "The queue will be DELETED from the broker and its recovered messages republished.\n"+ + "Continue? [y/N]: ", + queue, vhost, qh, cfg.QuorumBasePath, cfg.BackupBaseDir) + + line, err := bufio.NewReader(in).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return false, fmt.Errorf("reading confirmation: %w", err) + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + return true, nil + default: + return false, nil + } +} diff --git a/internal/cmd/recover/recover_test.go b/internal/cmd/recover/recover_test.go new file mode 100644 index 0000000..0e9b7ad --- /dev/null +++ b/internal/cmd/recover/recover_test.go @@ -0,0 +1,170 @@ +// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package recover + +import ( + "bytes" + "context" + "errors" + "flag" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sassoftware/phorcys/internal/broker" + "github.com/sassoftware/phorcys/internal/runtime" +) + +type call struct { + cfg runtime.Config + vhost, queue string +} + +func stub(t *testing.T, cfg runtime.Config, pipelineErr error) *[]call { + t.Helper() + calls := &[]call{} + origRun, origLoad, origEvaluateQueueHealth := runPipeline, loadConfig, evaluateQueueHealth + t.Cleanup(func() { + runPipeline, loadConfig, evaluateQueueHealth = origRun, origLoad, origEvaluateQueueHealth + }) + loadConfig = func() runtime.Config { return cfg } + evaluateQueueHealth = func(_ context.Context, _ *broker.DiagnosticsManager, _, _ string) (broker.QueueHealth, error) { + return broker.HealthGreen, nil + } + runPipeline = func(_ context.Context, c runtime.Config, dm *broker.DiagnosticsManager, vhost, queue string) error { + require.NotNil(t, dm) + *calls = append(*calls, call{c, vhost, queue}) + return pipelineErr + } + return calls +} + +func baseCfg() runtime.Config { + return runtime.Config{QuorumBasePath: "/q", BackupBaseDir: "/b"} +} + +func TestRun_WithYesRunsPipeline(t *testing.T) { + calls := stub(t, baseCfg(), nil) + var out bytes.Buffer + + err := RunCommand(context.Background(), []string{"-queue", "orders", "-vhost", "prod", "-yes"}, strings.NewReader(""), &out) + + require.NoError(t, err) + require.Len(t, *calls, 1) + assert.Equal(t, "prod", (*calls)[0].vhost) + assert.Equal(t, "orders", (*calls)[0].queue) + assert.Equal(t, "/q", (*calls)[0].cfg.QuorumBasePath) + assert.Contains(t, out.String(), "complete") +} + +func TestRun_DefaultVhost(t *testing.T) { + calls := stub(t, baseCfg(), nil) + + err := RunCommand(context.Background(), []string{"-queue", "orders", "-yes"}, nil, &bytes.Buffer{}) + + require.NoError(t, err) + require.Len(t, *calls, 1) + assert.Equal(t, "/", (*calls)[0].vhost) +} + +func TestRun_ConfirmYes(t *testing.T) { + for _, answer := range []string{"y\n", "YES\n", " yes"} { + calls := stub(t, baseCfg(), nil) + var out bytes.Buffer + + err := RunCommand(context.Background(), []string{"-queue", "orders"}, strings.NewReader(answer), &out) + + require.NoError(t, err) + assert.Len(t, *calls, 1, "answer %q", answer) + assert.Contains(t, out.String(), "Continue? [y/N]") + assert.Contains(t, out.String(), "/q") + assert.Contains(t, out.String(), "/b") + } +} + +func TestRun_ConfirmDeclined(t *testing.T) { + for _, answer := range []string{"n\n", "\n", ""} { + calls := stub(t, baseCfg(), nil) + var out bytes.Buffer + + err := RunCommand(context.Background(), []string{"-queue", "orders"}, strings.NewReader(answer), &out) + + require.NoError(t, err) + assert.Empty(t, *calls, "answer %q", answer) + assert.Contains(t, out.String(), "Aborted.") + } +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, errors.New("boom") } + +func TestRun_ConfirmReadError(t *testing.T) { + calls := stub(t, baseCfg(), nil) + + err := RunCommand(context.Background(), []string{"-queue", "orders"}, errReader{}, &bytes.Buffer{}) + + require.ErrorContains(t, err, "reading confirmation") + assert.Empty(t, *calls) +} + +func TestRun_MissingQueue(t *testing.T) { + calls := stub(t, baseCfg(), nil) + var out bytes.Buffer + + err := RunCommand(context.Background(), []string{"-yes"}, nil, &out) + + require.ErrorContains(t, err, "-queue is required") + assert.Empty(t, *calls) + assert.Contains(t, out.String(), "Usage: phorcys recover") +} + +func TestRun_UnexpectedArgs(t *testing.T) { + calls := stub(t, baseCfg(), nil) + + err := RunCommand(context.Background(), []string{"-queue", "orders", "extra"}, nil, &bytes.Buffer{}) + + require.ErrorContains(t, err, "unexpected arguments: extra") + assert.Empty(t, *calls) +} + +func TestRun_Help(t *testing.T) { + stub(t, baseCfg(), nil) + var out bytes.Buffer + + err := RunCommand(context.Background(), []string{"-h"}, nil, &out) + + require.ErrorIs(t, err, flag.ErrHelp) + assert.Contains(t, out.String(), "-queue") +} + +func TestRun_PipelineError(t *testing.T) { + stub(t, baseCfg(), errors.New("locate failed")) + + err := RunCommand(context.Background(), []string{"-queue", "orders", "-yes"}, nil, &bytes.Buffer{}) + + require.ErrorContains(t, err, "recovery of //orders failed: locate failed") +} + +func TestRun_ResolvesQuorumPathWhenUnset(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[{"name":"rabbit@node1","running":true}]`)) + })) + defer srv.Close() + + cfg := baseCfg() + cfg.QuorumBasePath = "" + cfg.ManagementURL = srv.URL + calls := stub(t, cfg, nil) + + err := RunCommand(context.Background(), []string{"-queue", "orders", "-yes"}, nil, &bytes.Buffer{}) + + require.NoError(t, err) + require.Len(t, *calls, 1) + assert.Contains(t, (*calls)[0].cfg.QuorumBasePath, "rabbit@node1") +} diff --git a/internal/runtime/config.go b/internal/runtime/config.go index 9d83098..171ecfc 100644 --- a/internal/runtime/config.go +++ b/internal/runtime/config.go @@ -12,6 +12,13 @@ const ( EnvVarMgmtPass = "PHORCYS_MGMT_PASS" //nolint:gosec EnvVarBackupDir = "PHORCYS_BACKUP_DIR" EnvVarQuorumPath = "PHORCYS_QUORUM_PATH" + + amqpURLDefault = "amqp://guest:guest@localhost:5672/" // #nosec G101 -- local-development default + managementURLDefault = "http://localhost:15672" + managementUserDefault = "guest" + managementPassDefault = "guest" // #nosec G101 -- local-development default + backupBaseDirDefault = "/var/lib/rabbitmq/phorcys-backups" + workerCountDefault = 3 ) // Config holds all runtime configuration for Phorcys. @@ -31,6 +38,21 @@ type Config struct { WorkerCount int } +// LoadConfig builds a Config from environment variables, falling back to +// local-development defaults. QuorumBasePath is left empty when +// PHORCYS_QUORUM_PATH is unset so callers can resolve it from the broker. +func LoadConfig() Config { + return Config{ + AMQPURL: GetEnv(EnvVarAMQPURL, amqpURLDefault), + ManagementURL: GetEnv(EnvVarMgmtURL, managementURLDefault), + ManagementUser: GetEnv(EnvVarMgmtUser, managementUserDefault), + ManagementPassword: GetEnv(EnvVarMgmtPass, managementPassDefault), + BackupBaseDir: GetEnv(EnvVarBackupDir, backupBaseDirDefault), + QuorumBasePath: os.Getenv(EnvVarQuorumPath), + WorkerCount: workerCountDefault, + } +} + // GetEnv returns the value of the environment variable named by key, or // fallback if the variable is unset or empty. func GetEnv(key, fallback string) string { diff --git a/internal/runtime/config_test.go b/internal/runtime/config_test.go index e8a5281..b1ea5ff 100644 --- a/internal/runtime/config_test.go +++ b/internal/runtime/config_test.go @@ -36,3 +36,35 @@ func TestGetEnv_EmptyEnvValueUsesFallback(t *testing.T) { assert.Equal(t, "fallback", got, "expected fallback for empty env var, got %q", got) } } + +// --------------------------------------------------------------------------- +// LoadConfig +// --------------------------------------------------------------------------- + +func TestLoadConfig_ReadsEnv(t *testing.T) { + t.Setenv(EnvVarAMQPURL, "amqp://u:p@h:5672/") + t.Setenv(EnvVarMgmtURL, "http://h:15672") + t.Setenv(EnvVarMgmtUser, "u") + t.Setenv(EnvVarMgmtPass, "p") + t.Setenv(EnvVarBackupDir, "/backup") + t.Setenv(EnvVarQuorumPath, "/quorum") + + cfg := LoadConfig() + + assert.Equal(t, "amqp://u:p@h:5672/", cfg.AMQPURL) + assert.Equal(t, "http://h:15672", cfg.ManagementURL) + assert.Equal(t, "u", cfg.ManagementUser) + assert.Equal(t, "p", cfg.ManagementPassword) + assert.Equal(t, "/backup", cfg.BackupBaseDir) + assert.Equal(t, "/quorum", cfg.QuorumBasePath) + assert.Equal(t, 3, cfg.WorkerCount) +} + +func TestLoadConfig_QuorumPathEmptyWhenUnset(t *testing.T) { + t.Setenv(EnvVarQuorumPath, "") + + cfg := LoadConfig() + + assert.Empty(t, cfg.QuorumBasePath) + assert.Equal(t, managementURLDefault, cfg.ManagementURL) +} diff --git a/main.go b/main.go index 8be180e..51da16e 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ package main import ( "context" "errors" + "flag" "log" "os" "os/signal" @@ -13,25 +14,24 @@ import ( "github.com/sassoftware/phorcys/internal/amqpx" "github.com/sassoftware/phorcys/internal/broker" + "github.com/sassoftware/phorcys/internal/cmd/recover" "github.com/sassoftware/phorcys/internal/monitor" "github.com/sassoftware/phorcys/internal/runtime" ) func main() { - cfg := runtime.Config{ - AMQPURL: runtime.GetEnv(runtime.EnvVarAMQPURL, "amqp://guest:guest@localhost:5672/"), - ManagementURL: runtime.GetEnv(runtime.EnvVarMgmtURL, "http://localhost:15672"), - ManagementUser: runtime.GetEnv(runtime.EnvVarMgmtUser, "guest"), - ManagementPassword: runtime.GetEnv(runtime.EnvVarMgmtPass, "guest"), - BackupBaseDir: runtime.GetEnv(runtime.EnvVarBackupDir, "/var/lib/rabbitmq/phorcys-backups"), - WorkerCount: 3, + if len(os.Args) > 1 && os.Args[1] == recover.Command { + if err := runRecover(os.Args[2:]); err != nil && !errors.Is(err, flag.ErrHelp) { + log.Fatalf("FATAL: %v", err) + } + return } + cfg := runtime.LoadConfig() + // If the quorum path is not explicitly set, derive it from the node name reported // by the management API so the default works for any node name (e.g. rabbit@rabbitmq). - if path := os.Getenv(runtime.EnvVarQuorumPath); path != "" { - cfg.QuorumBasePath = path - } else { + if cfg.QuorumBasePath == "" { dm0 := broker.NewDiagnosticsManager(cfg.ManagementURL, cfg.ManagementUser, cfg.ManagementPassword) cfg.QuorumBasePath = dm0.ResolveQuorumBasePath(context.Background()) } @@ -55,3 +55,10 @@ func main() { } log.Println("Phorcys shutdown complete.") } + +// runRecover executes the "recover" subcommand, cancelling on SIGINT/SIGTERM. +func runRecover(args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + return recover.RunCommand(ctx, args, os.Stdin, os.Stdout) +}