Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ on:
- main
- release-*
env:
GO_VERSION: 1.26.6
GO_VERSION: 1.26.9
ARTIFACT_DIR: ./artifacts
LINT_DIR: ./.github/linters
COV_DIR: ./artifacts/cov
Expand Down Expand Up @@ -183,5 +183,5 @@ jobs:
if: always()
id: check-coverage
run: |-
go install github.com/vladopajic/go-test-coverage/v2@latest
go install github.com/vladopajic/go-test-coverage/v2@v2.19.0
go-test-coverage --config=${{ env.LINT_DIR }}/testcoverage.yml
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ go build -o phorcys .

## Running

Ensure the following options are configured within RabbitMQ:

```bash
log.exchange = true
log.exchange.level = warning
```

```bash
export PHORCYS_AMQP_URL="amqp://admin:secret@rabbitmq:5672/"
export PHORCYS_MGMT_URL="http://rabbitmq:15672"
Expand All @@ -81,6 +88,10 @@ Phorcys runs until it receives `SIGINT` or `SIGTERM`.
- RabbitMQ 3.8+ with quorum queues enabled
- The process must run on the **same host** as the RabbitMQ node (for
filesystem access to Raft data and `rabbitmqctl` fallback eviction)
- The user configured by PHORCYS_MGMT_USER and the user embedded in
PHORCYS_AMQP_URL must have full(configure, read, write) access
to all Virtual Host within RabbitMQ.
- The RabbitMQ System log exchange must be enabled.
- Go 1.21+ to build from source

## Project layout
Expand Down
36 changes: 32 additions & 4 deletions internal/amqpx/amqptest/amqptest.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,17 @@ type Binding struct {
Args amqp.Table
}

// DeclaredExchange records an ExchangeDeclare call.
type DeclaredExchange struct {
Name string
Kind string
Durable bool
AutoDelete bool
Internal bool
NoWait bool
Args amqp.Table
}

// Consumer records a Consume call.
type Consumer struct {
Queue string
Expand Down Expand Up @@ -68,10 +79,11 @@ type Publication struct {
// overridden per zero-based publish index via NackAt, NoConfirmAt, or
// CloseConfirmsAt.
type Channel struct {
ConfirmErr error
DeclareErr error
BindErr error
ConsumeErr error
ConfirmErr error
ExchangeDeclareErr error
DeclareErr error
BindErr error
ConsumeErr error
// PublishErrAt fails the publish at the given index; the message is not recorded.
PublishErrAt map[int]error
// NackAt negatively acknowledges the publish at the given index.
Expand All @@ -89,6 +101,7 @@ type Channel struct {
confirmMode bool
confirmNoWait bool
confirms chan amqp.Confirmation
exchanges []DeclaredExchange
declared []DeclaredQueue
bindings []Binding
consumers []Consumer
Expand Down Expand Up @@ -121,6 +134,14 @@ func (c *Channel) NotifyPublish(confirm chan amqp.Confirmation) chan amqp.Confir
return confirm
}

// ExchangeDeclare implements amqpx.Channel.
func (c *Channel) ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp.Table) error {
c.mu.Lock()
defer c.mu.Unlock()
c.exchanges = append(c.exchanges, DeclaredExchange{name, kind, durable, autoDelete, internal, noWait, args})
return c.ExchangeDeclareErr
}

// QueueDeclare implements amqpx.Channel.
func (c *Channel) QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) (amqp.Queue, error) {
c.mu.Lock()
Expand Down Expand Up @@ -207,6 +228,13 @@ func (c *Channel) ConfirmNoWait() bool {
return c.confirmNoWait
}

// DeclaredExchanges returns all recorded ExchangeDeclare calls.
func (c *Channel) DeclaredExchanges() []DeclaredExchange {
c.mu.Lock()
defer c.mu.Unlock()
return append([]DeclaredExchange(nil), c.exchanges...)
}

// Declared returns all recorded QueueDeclare calls.
func (c *Channel) Declared() []DeclaredQueue {
c.mu.Lock()
Expand Down
1 change: 1 addition & 0 deletions internal/amqpx/amqpx.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
// Channel is the subset of *amqp.Channel used by Phorcys.
type Channel interface {
Confirm(noWait bool) error
ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp.Table) error
NotifyPublish(confirm chan amqp.Confirmation) chan amqp.Confirmation
QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) (amqp.Queue, error)
QueueBind(name, key, exchange string, noWait bool, args amqp.Table) error
Expand Down
123 changes: 123 additions & 0 deletions internal/broker/diagnostics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,129 @@ func TestFetchSpecificQueue_NonOKNon500ReturnsError(t *testing.T) {
assert.False(t, is500, "is500 must be false for 404")
}

// ---------------------------------------------------------------------------
// ResolveQuorumBasePath
// ---------------------------------------------------------------------------

func TestResolveQuorumBasePath_UsesFirstNodeName(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, json.NewEncoder(w).Encode([]RabbitNode{{Name: "rabbit@n1"}, {Name: "rabbit@n2"}}))
}))
defer server.Close()

got := newTestManager(server).ResolveQuorumBasePath(context.Background())

assert.Equal(t, "/var/lib/rabbitmq/mnesia/rabbit@n1/quorum/rabbit@n1", got)
}

func TestResolveQuorumBasePath_FallsBackOnAPIError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()

got := newTestManager(server).ResolveQuorumBasePath(context.Background())

assert.Equal(t, "/var/lib/rabbitmq/mnesia/rabbit@localhost/quorum/rabbit@localhost", got)
}

func TestResolveQuorumBasePath_FallsBackOnEmptyNodeList(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("[]"))
}))
defer server.Close()

got := newTestManager(server).ResolveQuorumBasePath(context.Background())

assert.Equal(t, "/var/lib/rabbitmq/mnesia/rabbit@localhost/quorum/rabbit@localhost", got)
}

// ---------------------------------------------------------------------------
// probeQueueGet
// ---------------------------------------------------------------------------

func TestProbeQueueGet(t *testing.T) {
tests := []struct {
name string
status int
body string
want bool
}{
{"healthy 200", http.StatusOK, "[]", false},
{"500 with noproc", http.StatusInternalServerError, `{"reason":"noproc"}`, true},
{"400 with noproc", http.StatusBadRequest, `{"reason":"noproc"}`, true},
{"500 without noproc", http.StatusInternalServerError, "boom", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "/api/queues/%2F/q1/get", r.URL.EscapedPath())
w.WriteHeader(tt.status)
_, _ = w.Write([]byte(tt.body))
}))
defer server.Close()

assert.Equal(t, tt.want, newTestManager(server).probeQueueGet(context.Background(), "/", "q1"))
})
}
}

func TestProbeQueueGet_TransportAndRequestErrorsReturnFalse(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
dm := newTestManager(server)
server.Close() // connection refused

assert.False(t, dm.probeQueueGet(context.Background(), "/", "q1"))

dm.APIURL = "http://bad host/" // invalid URL -> NewRequest error
assert.False(t, dm.probeQueueGet(context.Background(), "/", "q1"))
}

// ---------------------------------------------------------------------------
// request / transport / decode error paths
// ---------------------------------------------------------------------------

func TestFetchers_InvalidJSONReturnsError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("not json"))
}))
defer server.Close()
dm := newTestManager(server)

_, err := dm.fetchNodes(context.Background())
require.Error(t, err)
_, err = dm.FetchAllQueues(context.Background())
require.Error(t, err)
_, _, err = dm.fetchSpecificQueue(context.Background(), "/", "q")
require.Error(t, err)
}

func TestFetchers_TransportErrorReturnsError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
dm := newTestManager(server)
server.Close()

_, err := dm.fetchNodes(context.Background())
require.Error(t, err)
_, err = dm.FetchAllQueues(context.Background())
require.Error(t, err)
_, is500, err := dm.fetchSpecificQueue(context.Background(), "/", "q")
require.Error(t, err)
assert.False(t, is500)
}

func TestFetchers_InvalidURLReturnsError(t *testing.T) {
dm := &DiagnosticsManager{APIURL: "http://bad host/", Client: http.DefaultClient}

_, err := dm.fetchNodes(context.Background())
require.Error(t, err)
_, err = dm.FetchAllQueues(context.Background())
require.Error(t, err)
_, _, err = dm.fetchSpecificQueue(context.Background(), "/", "q")
require.Error(t, err)
}

func newTestManager(server *httptest.Server) *DiagnosticsManager {
return &DiagnosticsManager{
APIURL: server.URL,
Expand Down
78 changes: 78 additions & 0 deletions internal/recovery/consensus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright © 2026, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package recovery

import (
"context"
"encoding/json"
"fmt"
"log"
"time"

amqp "github.com/rabbitmq/amqp091-go"
)

// ConsensusExchange is the fanout exchange every Phorcys instance publishes
// RecoveryAnnouncements to. Fanout ensures every node subscribed to it — i.e.
// every other node hosting a replica of the announced queue — receives every
// announcement regardless of routing key.
const ConsensusExchange = "phorcys.consensus"

// RecoveryAnnouncement is what a node publishes to ConsensusExchange after
// Prepare backs up a queue, so other nodes hosting that queue can compare
// Term/Index (via RaftPosition.After) and agree on which one calls Execute.
type RecoveryAnnouncement struct {
Hostname string `json:"hostname"`
VHost string `json:"vhost"`
Queue string `json:"queue"`
Timestamp time.Time `json:"timestamp"`
Term uint64 `json:"term"`
Index uint64 `json:"index"`
}

// PublishRecoveryAnnouncement declares ConsensusExchange and publishes ann to
// it as JSON.
func PublishRecoveryAnnouncement(ctx context.Context, amqpURL string, ann RecoveryAnnouncement) error {
conn, err := dialAMQP(amqpURL)
if err != nil {
return fmt.Errorf("failed to connect to broker: %w", err)
}
defer func() { _ = conn.Close() }()

ch, err := conn.Channel()
if err != nil {
return fmt.Errorf("failed to open channel: %w", err)
}
defer func() { _ = ch.Close() }()

if err := ch.ExchangeDeclare(ConsensusExchange, "fanout", true, false, false, false, nil); err != nil {
return fmt.Errorf("failed to declare consensus exchange: %w", err)
}

body, err := json.Marshal(ann)
if err != nil {
return fmt.Errorf("failed to marshal announcement: %w", err)
}

pubCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

err = ch.PublishWithContext(pubCtx,
ConsensusExchange,
"", // fanout ignores routing key
false,
false,
amqp.Publishing{
ContentType: "application/json",
Timestamp: ann.Timestamp,
Body: body,
},
)
if err != nil {
return fmt.Errorf("failed to publish announcement: %w", err)
}

log.Printf("[Consensus] Announced %s/%s: term=%d idx=%d", ann.VHost, ann.Queue, ann.Term, ann.Index)
return nil
}
Loading
Loading