From 890af38cb3a76f995c7af1f0de2ba2bda15f4762 Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 18:18:48 +0200 Subject: [PATCH 1/9] feat(relay): add per-day egress budget (Denial-of-Wallet cap) Add FSEND_RELAY_MAX_BYTES_PER_DAY: a server-wide cap on relay bytes per UTC day. Default 0 = unlimited. Once spent, the relay refuses new transfers and drops in-flight ones until 00:00 UTC. Bounds total egress cost, which per-IP and per-session limits cannot: a distributed abuser sidesteps per-IP caps, and a per-session cap only limits one transfer. Enforcement has two paths. New transfers fail fast at allocate time (deterministic, both peers get the reason). A budget that trips mid-transfer evicts via the same path as the per-session cap. Both surface as E037 with a descriptive message on sender and receiver. Every rate-limit error (E017/E023/E037) now ends with a self-hosting link so users know they can run their own server. Rename two env vars for consistency (breaking, pre-1.0): - FSEND_PAIRING_ADDR -> FSEND_SERVER_ADDR - FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN -> ..._PER_MINUTE Metrics gain budget_max_bytes_per_day, budget_bytes_today, and transfers_budget_capped_total. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/fsend/internet.go | 5 ++ cmd/fsend/internet_test.go | 6 ++ cmd/fsend/sendpair.go | 10 ++++ cmd/fsend/server.go | 37 ++++++++---- cmd/fsend/server_lifecycle_unix_test.go | 2 +- cmd/fsend/server_test.go | 54 ++++++++++------- docs/development.md | 6 +- docs/self-hosting.md | 33 +++++----- internal/fserrors/fserrors.go | 33 +++++++--- internal/fserrors/fserrors_test.go | 1 + internal/relay/budget.go | 65 ++++++++++++++++++++ internal/relay/budget_test.go | 58 ++++++++++++++++++ internal/relay/relay_test.go | 57 ++++++++++++++++++ internal/relay/server.go | 80 ++++++++++++++++++------- internal/server/metrics_test.go | 4 +- internal/server/server.go | 8 +++ internal/server/server_test.go | 41 ++++++++++++- internal/server/types.go | 5 ++ test/e2e/harness_test.go | 4 +- test/e2e/smoke_test.go | 6 +- 20 files changed, 425 insertions(+), 90 deletions(-) create mode 100644 internal/relay/budget.go create mode 100644 internal/relay/budget_test.go diff --git a/cmd/fsend/internet.go b/cmd/fsend/internet.go index 3767887..36e0416 100644 --- a/cmd/fsend/internet.go +++ b/cmd/fsend/internet.go @@ -124,6 +124,9 @@ func runReceiveOverInternet(ctx context.Context, f *flags, c string, cfg *config if allocErr != nil { return fmt.Errorf("%w: %v", fserrors.ErrConnectFailed, allocErr) } + if alloc.BudgetExhausted { + return fserrors.ErrRelayBudgetExhausted + } if alloc.ForwardingDisabled { return fmt.Errorf("%w: direct connection failed and this server has relay forwarding disabled", fserrors.ErrConnectFailed) } @@ -159,6 +162,8 @@ func classifyRelayDrop(ctx context.Context, client *signaling.Client, sessionID, fserrors.ErrRelayCapHit, uxlog.HumanBytes(int64(status.LimitBytes))) } return fserrors.ErrRelayCapHit + case relay.ReasonBudgetHit: + return fserrors.ErrRelayBudgetExhausted case relay.ReasonIdle: return fserrors.ErrRelayIdleTimeout } diff --git a/cmd/fsend/internet_test.go b/cmd/fsend/internet_test.go index 0e94f01..706871f 100644 --- a/cmd/fsend/internet_test.go +++ b/cmd/fsend/internet_test.go @@ -175,6 +175,12 @@ func TestClassifyRelayDrop_MapsReasons(t *testing.T) { runErr: errors.New("idle timeout"), wantErr: fserrors.ErrRelayIdleTimeout, }, + { + name: "daily_budget_is_bare_sentinel", + body: `{"state":"evicted","reason":"daily_budget"}`, + runErr: errors.New("connection reset"), + wantErr: fserrors.ErrRelayBudgetExhausted, + }, { name: "active_keeps_run_err", body: `{"state":"active"}`, diff --git a/cmd/fsend/sendpair.go b/cmd/fsend/sendpair.go index 0ff3cf8..b83d31f 100644 --- a/cmd/fsend/sendpair.go +++ b/cmd/fsend/sendpair.go @@ -190,6 +190,13 @@ func pairOverInternet(ctx context.Context, f *flags, code string, cfg *config.Co // Establish the underlying data path: ICE-direct first, relay fallback. pc, pathInfo, err := establishInternetDataPath(ctx, f, client, created, waitResp) if err != nil { + // A spent daily budget is terminal and self-explanatory. Surface it + // as-is (not the generic errPairedGone), and leave the session for the + // receiver to reach the relay and learn the same reason — it's the + // budget, not the pairing, that failed. The session expires via TTL. + if errors.Is(err, fserrors.ErrRelayBudgetExhausted) { + return nil, err + } deleteSession() if ctx.Err() == nil { err = fmt.Errorf("%w: %v", errPairedGone, err) @@ -357,6 +364,9 @@ func allocAndDialRelay(ctx context.Context, client *signaling.Client, sessionID, if err != nil { return nil, connpath.Info{}, fmt.Errorf("%w: %v", fserrors.ErrConnectFailed, err) } + if alloc.BudgetExhausted { + return nil, connpath.Info{}, fserrors.ErrRelayBudgetExhausted + } if alloc.ForwardingDisabled { return nil, connpath.Info{}, fmt.Errorf("%w: this server has relay forwarding disabled", fserrors.ErrConnectFailed) } diff --git a/cmd/fsend/server.go b/cmd/fsend/server.go index c70f576..645346c 100644 --- a/cmd/fsend/server.go +++ b/cmd/fsend/server.go @@ -100,6 +100,7 @@ func runServer() error { defer func() { _ = udpListener.Close() }() relaySrv := relay.NewServer(udpListener, relay.ServerConfig{ MaxBytesPerSession: cfg.maxBytesPerSession, + MaxBytesPerDay: cfg.maxBytesPerDay, Logger: logger, DisableForwarding: !cfg.enableRelay, }) @@ -202,22 +203,24 @@ func drainRelay(ctx context.Context, r *relay.Server, logger *slog.Logger) { // loader and the startup summary so the two can't drift on a rename or a // changed default. const ( - envPairingAddr = "FSEND_PAIRING_ADDR" + envServerAddr = "FSEND_SERVER_ADDR" envRelayAddr = "FSEND_RELAY_ADDR" envMaxSessionsPerIP = "FSEND_SERVER_MAX_SESSIONS_PER_IP" - envMaxNewSessionsPerMin = "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN" + envMaxNewSessionsPerMin = "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE" envMaxBytesPerSession = "FSEND_RELAY_MAX_BYTES_PER_SESSION" + envMaxBytesPerDay = "FSEND_RELAY_MAX_BYTES_PER_DAY" envRelayEnabled = "FSEND_RELAY_ENABLED" envLogLevel = "FSEND_LOG_LEVEL" - defaultPairingAddr = ":8080" - defaultRelayAddr = ":443" - // The three caps default to 0 = unlimited: an unconfigured server - // imposes no per-IP session limits and no relay byte cap. Operators - // opt into protection by setting a positive value. + defaultServerAddr = ":8080" + defaultRelayAddr = ":443" + // The caps default to 0 = unlimited: an unconfigured server imposes no + // per-IP session limits and no relay byte caps. Operators opt into + // protection by setting a positive value. defaultMaxSessionsPerIP = 0 defaultMaxNewSessionsPerMin = 0 defaultMaxBytesPerSession = 0 + defaultMaxBytesPerDay = 0 defaultRelayEnabled = true ) @@ -229,13 +232,14 @@ type serverRuntimeConfig struct { maxSessionsPerIP int maxNewSessionsPerMin int maxBytesPerSession uint64 + maxBytesPerDay uint64 serverPassword string enableRelay bool } func loadServerConfig() (serverRuntimeConfig, error) { cfg := serverRuntimeConfig{ - httpAddr: envOr(envPairingAddr, defaultPairingAddr), + httpAddr: envOr(envServerAddr, defaultServerAddr), udpAddr: envOr(envRelayAddr, defaultRelayAddr), serverPassword: os.Getenv("FSEND_SERVER_PASSWORD"), } @@ -249,6 +253,9 @@ func loadServerConfig() (serverRuntimeConfig, error) { if cfg.maxBytesPerSession, err = envBytes(envMaxBytesPerSession, defaultMaxBytesPerSession); err != nil { return cfg, err } + if cfg.maxBytesPerDay, err = envBytes(envMaxBytesPerDay, defaultMaxBytesPerDay); err != nil { + return cfg, err + } if cfg.enableRelay, err = envBool(envRelayEnabled, defaultRelayEnabled); err != nil { return cfg, err } @@ -287,10 +294,11 @@ func formatServerConfig(cfg serverRuntimeConfig) string { {"FSEND_SERVER_PASSWORD", password, cfg.serverPassword != ""}, {envMaxSessionsPerIP, capCount(cfg.maxSessionsPerIP), cfg.maxSessionsPerIP != defaultMaxSessionsPerIP}, {envMaxNewSessionsPerMin, capCount(cfg.maxNewSessionsPerMin), cfg.maxNewSessionsPerMin != defaultMaxNewSessionsPerMin}, - {envPairingAddr, cfg.httpAddr, cfg.httpAddr != defaultPairingAddr}, + {envServerAddr, cfg.httpAddr, cfg.httpAddr != defaultServerAddr}, {envRelayEnabled, strconv.FormatBool(cfg.enableRelay), !cfg.enableRelay}, {envRelayAddr, cfg.udpAddr, cfg.udpAddr != defaultRelayAddr}, {envMaxBytesPerSession, capBytes(cfg.maxBytesPerSession), cfg.maxBytesPerSession != defaultMaxBytesPerSession}, + {envMaxBytesPerDay, capBytes(cfg.maxBytesPerDay), cfg.maxBytesPerDay != defaultMaxBytesPerDay}, } width, modified := 0, 0 @@ -439,7 +447,7 @@ func envBytes(name string, def uint64) (uint64, error) { // HTTP address. Exits 0 on healthy, 1 on anything else. Designed for // Docker HEALTHCHECK. func healthCheck() error { - addr := envOr(envPairingAddr, defaultPairingAddr) + addr := envOr(envServerAddr, defaultServerAddr) if strings.HasPrefix(addr, ":") { addr = "127.0.0.1" + addr } @@ -486,13 +494,13 @@ CONFIGURATION (environment variables — all optional) alive at once (concurrency cap; gates relay access too). Default 0 = unlimited; set a positive value to cap. - FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN + FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE How many new sessions one IP may create per minute (rate cap). Default 0 = unlimited; set a positive value to cap. Pairing (TCP signaling/control plane): - FSEND_PAIRING_ADDR Default :8080 (TCP). + FSEND_SERVER_ADDR Default :8080 (TCP). Relay (UDP data plane — also answers STUN): FSEND_RELAY_ENABLED Default true. false = pairing + STUN @@ -505,6 +513,11 @@ CONFIGURATION (environment variables — all optional) compression. Accepts B, KB, MB, GB, TB, or a plain byte count. Default 0 = unlimited; set a value to cap. + FSEND_RELAY_MAX_BYTES_PER_DAY Server-wide wire bytes forwarded per + UTC day — the egress/cost ceiling. + Same units. Default 0 = unlimited; once + hit, the relay stops forwarding until + 00:00 UTC. LEARN MORE https://github.com/polius/fsend diff --git a/cmd/fsend/server_lifecycle_unix_test.go b/cmd/fsend/server_lifecycle_unix_test.go index 6e53301..f6ca661 100644 --- a/cmd/fsend/server_lifecycle_unix_test.go +++ b/cmd/fsend/server_lifecycle_unix_test.go @@ -34,7 +34,7 @@ func TestServerLifecycle(t *testing.T) { udpPort := srvFreePortUDP(t) httpAddr := fmt.Sprintf("127.0.0.1:%d", httpPort) udpAddr := fmt.Sprintf("127.0.0.1:%d", udpPort) - t.Setenv("FSEND_PAIRING_ADDR", httpAddr) + t.Setenv("FSEND_SERVER_ADDR", httpAddr) t.Setenv("FSEND_RELAY_ADDR", udpAddr) t.Setenv("FSEND_LOG_LEVEL", "error") diff --git a/cmd/fsend/server_test.go b/cmd/fsend/server_test.go index 58949d0..7063d2c 100644 --- a/cmd/fsend/server_test.go +++ b/cmd/fsend/server_test.go @@ -148,12 +148,13 @@ func TestServerEnvBytes(t *testing.T) { func clearServerEnv(t *testing.T) { t.Helper() for _, k := range []string{ - "FSEND_PAIRING_ADDR", + "FSEND_SERVER_ADDR", "FSEND_RELAY_ADDR", "FSEND_LOG_LEVEL", "FSEND_SERVER_MAX_SESSIONS_PER_IP", - "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN", + "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE", "FSEND_RELAY_MAX_BYTES_PER_SESSION", + "FSEND_RELAY_MAX_BYTES_PER_DAY", } { os.Unsetenv(k) } @@ -181,6 +182,9 @@ func TestServerLoadConfigDefaults(t *testing.T) { if cfg.maxBytesPerSession != 0 { t.Errorf("maxBytesPerSession: got %d, want 0 (unlimited)", cfg.maxBytesPerSession) } + if cfg.maxBytesPerDay != 0 { + t.Errorf("maxBytesPerDay: got %d, want 0 (unlimited)", cfg.maxBytesPerDay) + } if cfg.logLevel != slog.LevelInfo { t.Errorf("logLevel: got %v, want info", cfg.logLevel) } @@ -188,11 +192,12 @@ func TestServerLoadConfigDefaults(t *testing.T) { func TestServerLoadConfigOverrides(t *testing.T) { clearServerEnv(t) - t.Setenv("FSEND_PAIRING_ADDR", ":19999") + t.Setenv("FSEND_SERVER_ADDR", ":19999") t.Setenv("FSEND_RELAY_ADDR", ":29999") t.Setenv("FSEND_SERVER_MAX_SESSIONS_PER_IP", "12") - t.Setenv("FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN", "120") - t.Setenv("FSEND_RELAY_MAX_BYTES_PER_SESSION", "250MB") + t.Setenv("FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE", "120") + t.Setenv("FSEND_RELAY_MAX_BYTES_PER_SESSION", "2MB") + t.Setenv("FSEND_RELAY_MAX_BYTES_PER_DAY", "10MB") cfg, err := loadServerConfig() if err != nil { t.Fatalf("loadServerConfig: %v", err) @@ -209,17 +214,20 @@ func TestServerLoadConfigOverrides(t *testing.T) { if cfg.maxNewSessionsPerMin != 120 { t.Errorf("maxNewSessionsPerMin: got %d", cfg.maxNewSessionsPerMin) } - if cfg.maxBytesPerSession != 250*srvMB { + if cfg.maxBytesPerSession != 2*srvMB { t.Errorf("maxBytesPerSession: got %d", cfg.maxBytesPerSession) } + if cfg.maxBytesPerDay != 10*srvMB { + t.Errorf("maxBytesPerDay: got %d", cfg.maxBytesPerDay) + } } func TestServerLoadConfigRejectsBadValues(t *testing.T) { // A typo'd limit must stop the server, not silently run the default. cases := map[string]string{ - "FSEND_RELAY_MAX_BYTES_PER_SESSION": "1GiB", - "FSEND_SERVER_MAX_SESSIONS_PER_IP": "many", - "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN": "-1", + "FSEND_RELAY_MAX_BYTES_PER_SESSION": "1GiB", + "FSEND_SERVER_MAX_SESSIONS_PER_IP": "many", + "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE": "-1", } for k, v := range cases { t.Run(k+"="+v, func(t *testing.T) { @@ -262,7 +270,7 @@ func TestServerLoadConfigLogLevels(t *testing.T) { func defaultServerConfig() serverRuntimeConfig { return serverRuntimeConfig{ - httpAddr: defaultPairingAddr, + httpAddr: defaultServerAddr, udpAddr: defaultRelayAddr, logLevel: slog.LevelInfo, maxSessionsPerIP: defaultMaxSessionsPerIP, @@ -277,15 +285,15 @@ func TestFormatServerConfig_AllDefaults(t *testing.T) { if strings.Contains(out, "* FSEND") { t.Errorf("all-default config must mark nothing customized:\n%s", out) } - if !strings.Contains(out, "0 of 8 customized") { - t.Errorf("want '0 of 8 customized':\n%s", out) + if !strings.Contains(out, "0 of 9 customized") { + t.Errorf("want '0 of 9 customized':\n%s", out) } if !strings.Contains(out, "(not set)") { t.Errorf("password must read '(not set)' when empty:\n%s", out) } - // The three caps default to 0 and must render as "0 (unlimited)". - if n := strings.Count(out, "0 (unlimited)"); n != 3 { - t.Errorf("want 3 '0 (unlimited)' lines (the default caps), got %d:\n%s", n, out) + // The four caps default to 0 and must render as "0 (unlimited)". + if n := strings.Count(out, "0 (unlimited)"); n != 4 { + t.Errorf("want 4 '0 (unlimited)' lines (the default caps), got %d:\n%s", n, out) } } @@ -311,8 +319,8 @@ func TestFormatServerConfig_Overrides(t *testing.T) { if !strings.Contains(out, "0 (unlimited)") { t.Errorf("default caps must still render as '0 (unlimited)':\n%s", out) } - if !strings.Contains(out, "3 of 8 customized") { - t.Errorf("want '3 of 8 customized':\n%s", out) + if !strings.Contains(out, "3 of 9 customized") { + t.Errorf("want '3 of 9 customized':\n%s", out) } for _, name := range []string{"FSEND_SERVER_PASSWORD", envMaxSessionsPerIP, envRelayEnabled} { if !strings.Contains(out, "* "+name) { @@ -320,15 +328,15 @@ func TestFormatServerConfig_Overrides(t *testing.T) { } } // An unchanged setting must stay unmarked. - if strings.Contains(out, "* "+envPairingAddr) { - t.Errorf("unchanged %s must not be marked:\n%s", envPairingAddr, out) + if strings.Contains(out, "* "+envServerAddr) { + t.Errorf("unchanged %s must not be marked:\n%s", envServerAddr, out) } } func TestServerHealthCheckUnreachable(t *testing.T) { // Bind+close to take a known-free port, then probe it (nothing listens). port := srvFreePortTCP(t) - t.Setenv("FSEND_PAIRING_ADDR", fmt.Sprintf("127.0.0.1:%d", port)) + t.Setenv("FSEND_SERVER_ADDR", fmt.Sprintf("127.0.0.1:%d", port)) if err := healthCheck(); err == nil { t.Fatal("expected error against closed port") } @@ -336,7 +344,7 @@ func TestServerHealthCheckUnreachable(t *testing.T) { func TestServerHealthCheckBadStatus(t *testing.T) { addr := startStubHealth(t, http.StatusInternalServerError, nil) - t.Setenv("FSEND_PAIRING_ADDR", addr) + t.Setenv("FSEND_SERVER_ADDR", addr) err := healthCheck() if err == nil { t.Fatal("expected error for 500 response") @@ -345,7 +353,7 @@ func TestServerHealthCheckBadStatus(t *testing.T) { func TestServerHealthCheckOK(t *testing.T) { addr := startStubHealth(t, http.StatusOK, []byte(`{"ok":true}`)) - t.Setenv("FSEND_PAIRING_ADDR", addr) + t.Setenv("FSEND_SERVER_ADDR", addr) if err := healthCheck(); err != nil { t.Errorf("healthCheck: %v", err) } @@ -353,7 +361,7 @@ func TestServerHealthCheckOK(t *testing.T) { // startStubHealth boots a minimal HTTP server that answers /v1/health with // the given status + body, and shuts it down when the test ends. Returns -// the host:port the test should set as FSEND_PAIRING_ADDR. +// the host:port the test should set as FSEND_SERVER_ADDR. func startStubHealth(t *testing.T, status int, body []byte) string { t.Helper() port := srvFreePortTCP(t) diff --git a/docs/development.md b/docs/development.md index e6249b8..d9e11e6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -68,7 +68,7 @@ For internet transfers (one peer behind NAT), the client needs a pairing server. To exercise the full client↔server↔client path: ```sh -FSEND_PAIRING_ADDR=":18080" \ +FSEND_SERVER_ADDR=":18080" \ FSEND_RELAY_ADDR=":18443" \ FSEND_LOG_LEVEL=debug \ /tmp/fsend server @@ -93,7 +93,7 @@ Reset to the public default: Health check (the env var tells the probe where to look): ```sh -FSEND_PAIRING_ADDR=":18080" /tmp/fsend server --health-check +FSEND_SERVER_ADDR=":18080" /tmp/fsend server --health-check ``` ### Common server env vars @@ -101,7 +101,7 @@ FSEND_PAIRING_ADDR=":18080" /tmp/fsend server --health-check | Var | Default | Notes | |---|---|---| | `FSEND_LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error` | -| `FSEND_PAIRING_ADDR` | `:8080` | Signaling listener (TCP) | +| `FSEND_SERVER_ADDR` | `:8080` | Signaling listener (TCP) | | `FSEND_RELAY_ADDR` | `:443` | Relay listener (UDP); also the STUN endpoint | | `FSEND_RELAY_MAX_BYTES_PER_SESSION` | `1GB` | Per-session relay cap (wire bytes, post-compression). | diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 1f13cc5..49cc3fb 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -187,16 +187,19 @@ so anyone can verify the server holds nothing sensitive. | `sessions_active` | Pairing sessions live right now — current load. | | `sessions_created_total` | Sessions a sender opened since boot — total usage. | | `sessions_paired_total` | Sessions a receiver successfully joined. Compared to `sessions_created_total`, the gap is senders who never found a receiver (abandoned codes). | -| `sessions_rejected_total.rate_limit` | Sessions refused by the per-IP rate cap (`FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN`) — bursts or brute-force. | +| `sessions_rejected_total.rate_limit` | Sessions refused by the per-IP rate cap (`FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE`) — bursts or brute-force. | | `sessions_rejected_total.concurrency_limit` | Sessions refused by the per-IP concurrency cap (`FSEND_SERVER_MAX_SESSIONS_PER_IP`). | | `sessions_rejected_total.unauthorized` | Wrong/missing `FSEND_SERVER_PASSWORD` — brute-force or misconfigured clients (always `0` on an open server). | | `relay.forwarding` | Whether the relay carries data (`false` in pairing + STUN-only mode). | | `relay.healthy` | Relay read loop alive — mirrors `/v1/health`'s 503 signal. | | `relay.transfers_active` | Relay transfers moving bytes right now. | | `relay.transfers_total` | Relay transfers that actually forwarded data since boot. Divide by `sessions_paired_total` for your relay-fallback rate; the rest connected directly (P2P). A climbing ratio means hole-punching is failing more often. | -| `relay.transfers_capped_total` | Transfers cut off for hitting the byte cap — raise the cap if this climbs. | -| `relay.bytes_forwarded_total` | Cumulative bytes relayed — what the relay is costing you. | +| `relay.transfers_capped_total` | Transfers cut off for hitting the per-session byte cap — raise the cap if this climbs. | +| `relay.transfers_budget_capped_total` | Transfers cut off because the daily budget was spent — the circuit breaker tripped. Climbing here means legit transfers are being denied; raise `FSEND_RELAY_MAX_BYTES_PER_DAY`. | +| `relay.bytes_forwarded_total` | Cumulative bytes relayed since boot — what the relay is costing you. | | `relay.peak_transfer_bytes` | Largest single transfer so far; compare to your byte cap to see headroom. | +| `relay.budget_max_bytes_per_day` | The configured daily budget (`0` = unlimited). | +| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day — watch it approach the budget to alert before the breaker trips. | The `relay` block is omitted when the server runs without a relay. @@ -245,15 +248,16 @@ your overrides were picked up (the password value is never printed, only whether one is set): ``` -fsend server configuration (2 of 8 customized; * = changed from default): - * FSEND_SERVER_PASSWORD (set) - * FSEND_SERVER_MAX_SESSIONS_PER_IP 10 - FSEND_LOG_LEVEL info - FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN 0 (unlimited) - FSEND_PAIRING_ADDR :8080 - FSEND_RELAY_ENABLED true - FSEND_RELAY_ADDR :443 - FSEND_RELAY_MAX_BYTES_PER_SESSION 0 (unlimited) +fsend server configuration (2 of 9 customized; * = changed from default): + * FSEND_SERVER_PASSWORD (set) + * FSEND_SERVER_MAX_SESSIONS_PER_IP 10 + FSEND_LOG_LEVEL info + FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE 0 (unlimited) + FSEND_SERVER_ADDR :8080 + FSEND_RELAY_ENABLED true + FSEND_RELAY_ADDR :443 + FSEND_RELAY_MAX_BYTES_PER_SESSION 0 (unlimited) + FSEND_RELAY_MAX_BYTES_PER_DAY 0 (unlimited) ``` Variables are grouped into **server-wide** controls (apply to the whole @@ -266,11 +270,12 @@ STUN). | `FSEND_LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error`. | | `FSEND_SERVER_PASSWORD` | _(unset)_ | Shared secret restricting all endpoints except `/v1/health` — see [Require a password](#require-a-password-optional). | | `FSEND_SERVER_MAX_SESSIONS_PER_IP` | `0` (unlimited) | **Concurrency cap** — how many sessions one source IP may have **alive at once**. A session gates relay allocation too, so this caps relay access as well. Defaults to unlimited; **set a positive value to enable this DoS protection** (recommended on a public server). | -| `FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN` | `0` (unlimited) | **Rate cap** — how many **new** sessions one source IP may **create per minute**. Distinct from the concurrency cap above: this limits inflow over time, that limits standing count. Defaults to unlimited; **set a positive value to enable this DoS protection** (recommended on a public server). | -| `FSEND_PAIRING_ADDR` | `:8080` | TCP signaling listener. | +| `FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE` | `0` (unlimited) | **Rate cap** — how many **new** sessions one source IP may **create per minute**. Distinct from the concurrency cap above: this limits inflow over time, that limits standing count. Defaults to unlimited; **set a positive value to enable this DoS protection** (recommended on a public server). | +| `FSEND_SERVER_ADDR` | `:8080` | TCP signaling listener. | | `FSEND_RELAY_ENABLED` | `true` | Set `false` for **pairing + STUN only**: the server still helps peers hole-punch a direct path, but carries no file data. See [Pairing-only mode](#pairing-only-mode-no-relay). | | `FSEND_RELAY_ADDR` | `:443` | UDP relay listener — also the STUN endpoint, so it stays in use even when forwarding is off. The server tells clients to dial `:`, so no separate public-address knob is needed. | | `FSEND_RELAY_MAX_BYTES_PER_SESSION` | `0` (unlimited) | Per-session relay cap — wire bytes after compression. Defaults to unlimited; set a value to bound per-transfer bandwidth. Accepts `B`, `KB`, `MB`, `GB`, `TB` suffixes (decimal, e.g. `500MB`, `1GB`) or a plain byte count (`1000000000`). | +| `FSEND_RELAY_MAX_BYTES_PER_DAY` | `0` (unlimited) | **Egress budget** — wire bytes the relay forwards **per UTC day** across all sessions. The Denial-of-Wallet ceiling: once spent, the relay stops forwarding and refuses new transfers until 00:00 UTC. Bounds a distributed abuser that the per-IP caps can't. Same units as above. Defaults to unlimited; **set a value to bound your bandwidth bill**. | ### Pairing-only mode (no relay) diff --git a/internal/fserrors/fserrors.go b/internal/fserrors/fserrors.go index 40b3be6..814f181 100644 --- a/internal/fserrors/fserrors.go +++ b/internal/fserrors/fserrors.go @@ -72,6 +72,10 @@ var ( // was torn down. Tells the user *why* the transfer stopped, instead // of "connection interrupted, retrying" forever. ErrRelayCapHit = errors.New("relay byte cap reached") + // E037 — the relay's server-wide daily byte budget is spent, so it + // stopped forwarding until the next UTC day. Distinct from E023: the + // limit is global, not per-session, and time-based rather than size. + ErrRelayBudgetExhausted = errors.New("relay daily budget exhausted") // E024 — any CLI usage error (bad flag, bad arg shape, conflicting // modes). Routine; never asks the user to file a bug. ErrUsage = errors.New("usage error") @@ -184,6 +188,13 @@ func (e Entry) Render() string { // // Placeholders like {addr}, {code}, {path} are filled in by the caller via // fmt.Sprintf when known; we use plain text so unknown contexts still render. + +// selfHostHint closes every server-limit error: the limits are the operator's +// choice, so the surest fix is running your own server. Kept identical across +// the rate-limit entries so the guidance never drifts. +const selfHostHint = "\n Or run your own fsend server, where you set the limits:\n" + + " https://github.com/polius/fsend/blob/main/docs/self-hosting.md" + var catalog = map[error]Entry{ ErrServerUnreachable: { Code: "E001", Exit: 1, @@ -280,9 +291,8 @@ var catalog = map[error]Entry{ Code: "E017", Exit: 17, Message: "Too many attempts from your network — rate limit hit on the server.", Action: "Wait a minute and try again, or switch servers:\n" + - " fsend --connect \n" + - " Or self-host your own server (`fsend server`) and raise\n" + - " FSEND_SERVER_MAX_SESSIONS_PER_IP / FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN.", + " fsend --connect " + + selfHostHint, }, ErrServerRetired: { Code: "E018", Exit: 18, @@ -330,10 +340,17 @@ var catalog = map[error]Entry{ Message: "The server's transfer-size limit was reached. Transfer aborted.", Action: "Only fallback transfers routed through the server count against this\n" + " limit; same-network and direct internet transfers are uncapped.\n" + - " Workarounds:\n" + - " - Run again from a different network so fsend can connect you directly.\n" + - " - Self-host your own server (`fsend server`) and raise\n" + - " FSEND_RELAY_MAX_BYTES_PER_SESSION.", + " Run again from a different network so fsend can connect you directly." + + selfHostHint, + }, + ErrRelayBudgetExhausted: { + Code: "E037", Exit: 37, + Message: "The server hit its daily transfer budget. Transfer aborted.", + Action: "Only fallback transfers routed through the server count against this\n" + + " budget; same-network and direct internet transfers don't.\n" + + " Run again from a different network so fsend can connect you directly,\n" + + " or try again after 00:00 UTC, when the budget resets." + + selfHostHint, }, ErrUsage: { Code: "E024", Exit: 24, @@ -370,7 +387,7 @@ var catalog = map[error]Entry{ Code: "E030", Exit: 30, Message: "The server could not start.", Action: "Fix the setting named above, or check that the ports are free and you\n" + - " have permission to bind them (FSEND_PAIRING_ADDR, FSEND_RELAY_ADDR).\n" + + " have permission to bind them (FSEND_SERVER_ADDR, FSEND_RELAY_ADDR).\n" + " Binding :443 may need elevated privileges.", }, ErrPasswordRequired: { diff --git a/internal/fserrors/fserrors_test.go b/internal/fserrors/fserrors_test.go index e6fcdc3..bc695b2 100644 --- a/internal/fserrors/fserrors_test.go +++ b/internal/fserrors/fserrors_test.go @@ -117,6 +117,7 @@ func TestNewSentinels_LookupAndExit(t *testing.T) { {ErrSourceNotFound, "E025", 25}, {ErrUserCancelled, "E026", 130}, {ErrRelayIdleTimeout, "E029", 29}, + {ErrRelayBudgetExhausted, "E037", 37}, } for _, c := range cases { entry, ok := Lookup(c.err) diff --git a/internal/relay/budget.go b/internal/relay/budget.go new file mode 100644 index 0000000..f3ad5a9 --- /dev/null +++ b/internal/relay/budget.go @@ -0,0 +1,65 @@ +package relay + +import ( + "sync" + "time" +) + +// dayBudget caps the bytes the relay forwards per UTC day — the +// Denial-of-Wallet backstop that bounds egress spend no matter how the +// per-IP and per-session limits are set. A limit of 0 means unlimited. +// +// The window rolls at UTC midnight. charge() and exhausted() both roll, so +// the counter resets on a new day even when no datagrams are flowing. +type dayBudget struct { + limit uint64 // bytes per day; 0 = unlimited + + mu sync.Mutex + day int64 // UTC-midnight unix seconds of the current window + used uint64 // bytes forwarded so far this window +} + +// roll resets the window when now lands on a different UTC day. Caller holds mu. +func (b *dayBudget) roll(now time.Time) { + if d := now.UTC().Truncate(24 * time.Hour).Unix(); d != b.day { + b.day = d + b.used = 0 + } +} + +// charge records n forwarded bytes and returns true once the day's budget +// is exceeded, meaning the caller should drop the datagram rather than +// forward it. Mirrors the per-session cap: the datagram that crosses the +// limit is counted but not forwarded. +func (b *dayBudget) charge(now time.Time, n uint64) bool { + if b.limit == 0 { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + b.roll(now) + b.used += n + return b.used > b.limit +} + +// exhausted reports whether the day's budget is spent, without charging. +// Allocate uses it to refuse a new transfer up front — deterministically, +// before the lossy relay pairing — so both peers fail fast with a clear +// reason instead of racing the in-flight breaker. +func (b *dayBudget) exhausted(now time.Time) bool { + if b.limit == 0 { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + b.roll(now) + return b.used >= b.limit +} + +// usedToday returns bytes forwarded in the current window, for /v1/metrics. +func (b *dayBudget) usedToday(now time.Time) uint64 { + b.mu.Lock() + defer b.mu.Unlock() + b.roll(now) + return b.used +} diff --git a/internal/relay/budget_test.go b/internal/relay/budget_test.go new file mode 100644 index 0000000..10f9815 --- /dev/null +++ b/internal/relay/budget_test.go @@ -0,0 +1,58 @@ +package relay + +import ( + "testing" + "time" +) + +// day1 and day2 are noon on two consecutive UTC days, so Truncate(24h) +// maps them to different windows. +var ( + day1 = time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC) + day2 = time.Date(2026, 7, 2, 12, 0, 0, 0, time.UTC) +) + +func TestDayBudget_ZeroMeansUnlimited(t *testing.T) { + b := &dayBudget{limit: 0} + for i := 0; i < 100; i++ { + if b.charge(day1, 1<<30) { + t.Fatalf("charge dropped under a 0 (unlimited) budget") + } + } +} + +func TestDayBudget_ChargeUpToLimit(t *testing.T) { + b := &dayBudget{limit: 100} + // The datagram that reaches the limit is counted and allowed; the next + // one is dropped. + if b.charge(day1, 100) { + t.Fatal("charge dropped the datagram that reaches the limit") + } + if !b.charge(day1, 1) { + t.Fatal("charge allowed a datagram past the spent budget") + } +} + +func TestDayBudget_RollsAtUTCMidnight(t *testing.T) { + b := &dayBudget{limit: 100} + b.charge(day1, 100) + if !b.charge(day1, 1) { + t.Fatal("budget should be spent on day1") + } + // A new UTC day resets the window even with no charge in between. + if got := b.usedToday(day2); got != 0 { + t.Fatalf("usedToday(day2) = %d, want 0 after rollover", got) + } + if b.charge(day2, 50) { + t.Fatal("fresh day should accept a charge under the limit") + } +} + +func TestDayBudget_UsedToday(t *testing.T) { + b := &dayBudget{limit: 1000} + b.charge(day1, 300) + b.charge(day1, 200) + if got := b.usedToday(day1); got != 500 { + t.Fatalf("usedToday = %d, want 500", got) + } +} diff --git a/internal/relay/relay_test.go b/internal/relay/relay_test.go index 2b3916f..410e7f4 100644 --- a/internal/relay/relay_test.go +++ b/internal/relay/relay_test.go @@ -3,6 +3,7 @@ package relay import ( "bytes" "context" + "errors" "net" "sync" "testing" @@ -426,6 +427,62 @@ func TestMetricsCapHit(t *testing.T) { <-srvErr } +// TestMetricsBudgetHit checks the daily budget trips the circuit breaker: +// once the day's bytes are spent the transfer is evicted with the +// daily_budget reason and transfers_budget_capped_total counts it. +func TestMetricsBudgetHit(t *testing.T) { + serverConn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + if err != nil { + t.Fatal(err) + } + relayAddr := serverConn.LocalAddr().(*net.UDPAddr) + srv := NewServer(serverConn, ServerConfig{MaxBytesPerDay: 1}) // any forwarded datagram exceeds it + tok, _ := srv.Allocate() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + srvErr := make(chan error, 1) + go func() { srvErr <- srv.Run(ctx) }() + + clientA, _ := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + defer clientA.Close() + clientB, _ := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + defer clientB.Close() + connA := NewClient(clientA, relayAddr, tok) + connB := NewClient(clientB, relayAddr, tok) + + _, _ = connA.WriteTo([]byte("x"), nil) // registers A as peerA + _, _ = connB.WriteTo([]byte("y"), nil) // first forwarded datagram trips the 1-byte budget + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if srv.Metrics().TransfersBudgetCappedTotal > 0 { + break + } + time.Sleep(20 * time.Millisecond) + } + if got := srv.Metrics().TransfersBudgetCappedTotal; got != 1 { + t.Errorf("transfers_budget_capped_total = %d, want 1", got) + } + if got := srv.Status(tok); got != ReasonBudgetHit { + t.Errorf("Status = %q, want %q", got, ReasonBudgetHit) + } + + cancel() + _ = serverConn.Close() + <-srvErr +} + +// TestAllocate_BudgetExhausted checks a new allocation is refused with the +// sentinel once the day's budget is spent, so the signaling layer can flag +// the response and clients fail fast rather than dialing a dead slot. +func TestAllocate_BudgetExhausted(t *testing.T) { + srv := NewServer(nil, ServerConfig{MaxBytesPerDay: 10}) + srv.budget.charge(time.Now(), 10) // spend it + if _, err := srv.Allocate(); !errors.Is(err, ErrBudgetExhausted) { + t.Fatalf("Allocate err = %v, want ErrBudgetExhausted", err) + } +} + // TestMaxBytes_ZeroMeansUnlimited locks in the operator escape hatch: a // 0 byte cap forwards without ever tripping cap_hit. The hot path guards // with `MaxBytesPerSession > 0`, so this guards against Default() (or a diff --git a/internal/relay/server.go b/internal/relay/server.go index 3f4fc37..02f55dd 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -49,34 +49,45 @@ type Server struct { // can converge while in-flight sessions finish. draining atomic.Bool + // budget is the global per-UTC-day byte ceiling (Denial-of-Wallet + // circuit breaker). Zero limit = unlimited. + budget dayBudget + // Aggregate counters for /v1/metrics (cumulative since boot). - bytesForwarded atomic.Uint64 - peakTransferBytes atomic.Uint64 // most any single transfer has forwarded - transfersCapped atomic.Uint64 // transfers cut off for exceeding the byte cap - transfersTotal atomic.Uint64 // transfers that forwarded ≥1 byte; vs paired sessions = relay-fallback rate + bytesForwarded atomic.Uint64 + peakTransferBytes atomic.Uint64 // most any single transfer has forwarded + transfersCapped atomic.Uint64 // transfers cut off for exceeding the per-session byte cap + transfersBudgetCapped atomic.Uint64 // transfers cut off because the daily budget was spent + transfersTotal atomic.Uint64 // transfers that forwarded ≥1 byte; vs paired sessions = relay-fallback rate } // Metrics is an aggregate snapshot of relay activity for /v1/metrics. type Metrics struct { - Forwarding bool `json:"forwarding"` - Healthy bool `json:"healthy"` - TransfersActive int `json:"transfers_active"` - TransfersTotal uint64 `json:"transfers_total"` - TransfersCappedTotal uint64 `json:"transfers_capped_total"` - BytesForwardedTotal uint64 `json:"bytes_forwarded_total"` - PeakTransferBytes uint64 `json:"peak_transfer_bytes"` + Forwarding bool `json:"forwarding"` + Healthy bool `json:"healthy"` + TransfersActive int `json:"transfers_active"` + TransfersTotal uint64 `json:"transfers_total"` + TransfersCappedTotal uint64 `json:"transfers_capped_total"` + TransfersBudgetCappedTotal uint64 `json:"transfers_budget_capped_total"` + BytesForwardedTotal uint64 `json:"bytes_forwarded_total"` + PeakTransferBytes uint64 `json:"peak_transfer_bytes"` + BudgetMaxBytesPerDay uint64 `json:"budget_max_bytes_per_day"` // 0 = unlimited + BudgetBytesToday uint64 `json:"budget_bytes_today"` } // Metrics returns the current aggregate snapshot. func (s *Server) Metrics() Metrics { return Metrics{ - Forwarding: !s.cfg.DisableForwarding, - Healthy: s.healthy.Load(), - TransfersActive: s.ActiveAllocations(), - TransfersTotal: s.transfersTotal.Load(), - TransfersCappedTotal: s.transfersCapped.Load(), - BytesForwardedTotal: s.bytesForwarded.Load(), - PeakTransferBytes: s.peakTransferBytes.Load(), + Forwarding: !s.cfg.DisableForwarding, + Healthy: s.healthy.Load(), + TransfersActive: s.ActiveAllocations(), + TransfersTotal: s.transfersTotal.Load(), + TransfersCappedTotal: s.transfersCapped.Load(), + TransfersBudgetCappedTotal: s.transfersBudgetCapped.Load(), + BytesForwardedTotal: s.bytesForwarded.Load(), + PeakTransferBytes: s.peakTransferBytes.Load(), + BudgetMaxBytesPerDay: s.cfg.MaxBytesPerDay, + BudgetBytesToday: s.budget.usedToday(time.Now()), } } @@ -113,7 +124,10 @@ type ServerConfig struct { // 0 means unlimited. The unset default (1 GB) is applied by the env // layer in cmd/fsend, so a bare ServerConfig{} here is uncapped. MaxBytesPerSession uint64 - Logger *slog.Logger + // MaxBytesPerDay caps wire bytes the relay forwards per UTC day across + // all sessions — the Denial-of-Wallet ceiling. 0 means unlimited. + MaxBytesPerDay uint64 + Logger *slog.Logger // DisableForwarding makes the relay answer STUN but never carry data // (pairing + hole-punching only). Negative so the zero value forwards. DisableForwarding bool @@ -164,6 +178,7 @@ func NewServer(conn net.PacketConn, cfg ServerConfig) *Server { allocs: make(map[Token]*allocation), tombstone: make(map[Token]tombstoneEntry), } + s.budget.limit = cfg.MaxBytesPerDay s.healthy.Store(true) // assume up until Run proves otherwise return s } @@ -229,6 +244,12 @@ func (s *Server) Allocate() (Token, error) { if s.draining.Load() { return Token{}, errors.New("relay: server is draining") } + // Refuse up front when the day's budget is already spent, so both peers + // fail fast with a clear reason. (A budget that trips mid-transfer is + // handled in handle, via the same eviction path as the per-session cap.) + if s.budget.exhausted(time.Now()) { + return Token{}, ErrBudgetExhausted + } var t Token if _, err := rand.Read(t[:]); err != nil { return Token{}, fmt.Errorf("relay: token rand: %w", err) @@ -319,7 +340,8 @@ func (s *Server) handle(datagram []byte, src *net.UDPAddr) { return // unknown token = silently drop (likely stale) } - a.lastActivity.Store(time.Now().UnixNano()) + now := time.Now() + a.lastActivity.Store(now.UnixNano()) // Determine forward direction. var dst *net.UDPAddr @@ -356,6 +378,14 @@ func (s *Server) handle(datagram []byte, src *net.UDPAddr) { return } + // Daily budget (Denial-of-Wallet circuit breaker): once the day's bytes + // are spent, evict and drop until the UTC-midnight rollover. + if s.budget.charge(now, wireSize) { + s.transfersBudgetCapped.Add(1) + s.evict(token, ReasonBudgetHit) + return + } + s.bytesForwarded.Add(wireSize) if a.dataCounted.CompareAndSwap(false, true) { s.transfersTotal.Add(1) // first byte forwarded = this allocation is a real transfer @@ -475,10 +505,16 @@ func (s *Server) sweep(now time.Time) { // Eviction reasons surfaced through Status(). const ( - ReasonCapHit = "cap_hit" - ReasonIdle = "idle" + ReasonCapHit = "cap_hit" + ReasonBudgetHit = "daily_budget" + ReasonIdle = "idle" ) +// ErrBudgetExhausted is returned by Allocate when the daily byte budget is +// already spent, so the signaling layer can flag the response and clients +// can fail fast with a specific reason. +var ErrBudgetExhausted = errors.New("relay: daily byte budget exhausted") + // udpEqual reports whether two UDPAddrs refer to the same endpoint. func udpEqual(a, b *net.UDPAddr) bool { return a != nil && b != nil && a.Port == b.Port && a.IP.Equal(b.IP) diff --git a/internal/server/metrics_test.go b/internal/server/metrics_test.go index 8efca1c..30b1fed 100644 --- a/internal/server/metrics_test.go +++ b/internal/server/metrics_test.go @@ -64,7 +64,9 @@ func TestMetrics_Shape(t *testing.T) { "rate_limit", "concurrency_limit", "unauthorized") assertKeys(t, raw["relay"], "relay", "forwarding", "healthy", "transfers_active", "transfers_total", - "transfers_capped_total", "bytes_forwarded_total", "peak_transfer_bytes") + "transfers_capped_total", "transfers_budget_capped_total", + "bytes_forwarded_total", "peak_transfer_bytes", + "budget_max_bytes_per_day", "budget_bytes_today") } func assertKeys(t *testing.T, v any, name string, want ...string) { diff --git a/internal/server/server.go b/internal/server/server.go index 6ef265f..040b5b4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -6,6 +6,7 @@ import ( "crypto/subtle" "encoding/base32" "encoding/json" + "errors" "fmt" "log/slog" "net" @@ -336,6 +337,13 @@ func (s *Server) allocateRelay(w http.ResponseWriter, r *http.Request) { t, err := s.relayAllocator.Allocate() if err != nil { s.mu.Unlock() + // Budget spent: answer 200 with a flag (like ForwardingDisabled) + // so the client fails fast with a specific reason rather than + // treating an opaque 5xx as a generic connectivity failure. + if errors.Is(err, relay.ErrBudgetExhausted) { + writeJSON(w, http.StatusOK, RelayAllocateResponse{BudgetExhausted: true}) + return + } writeJSONError(w, http.StatusInternalServerError, "alloc failed") return } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 089e3da..414482e 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -533,9 +533,10 @@ type fakeRelay struct { maxBytes uint64 dead bool noForward bool + allocErr error } -func (f *fakeRelay) Allocate() (relay.Token, error) { return f.tok, nil } +func (f *fakeRelay) Allocate() (relay.Token, error) { return f.tok, f.allocErr } func (f *fakeRelay) Status(t relay.Token) string { return f.reason } func (f *fakeRelay) MaxBytesPerSession() uint64 { return f.maxBytes } func (f *fakeRelay) Healthy() bool { return !f.dead } @@ -641,6 +642,44 @@ func TestRelayStatus_IncludesConfiguredLimits(t *testing.T) { } } +// TestRelayAllocate_BudgetExhausted checks a spent daily budget surfaces as +// a 200 with the budget_exhausted flag (mirroring forwarding_disabled), so +// the client fails fast with a specific reason instead of reading an opaque +// 5xx as a generic connectivity failure. +func TestRelayAllocate_BudgetExhausted(t *testing.T) { + s := New(Config{ + ServerVersion: "0.0.0-test", + UnpairedTTL: 2 * time.Second, + PairedTTL: 5 * time.Second, + MaxSessionsPerIP: 10, + MaxNewSessionsPerMin: 100, + }) + s.WithRelay(&fakeRelay{tok: relay.Token{1, 2, 3, 4}, allocErr: relay.ErrBudgetExhausted}, 9999) + ts := httptest.NewServer(s.Handler()) + defer ts.Close() + + cr := createSession(t, ts.URL, testSlot) + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/v1/relay/allocate", + bytes.NewReader(mustJSON(t, RelayAllocateRequest{SessionID: cr.SessionID}))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+cr.RoleToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var body RelayAllocateResponse + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if !body.BudgetExhausted { + t.Errorf("budget_exhausted = false, want true") + } +} + // TestRelayAllocate_ForwardingDisabledFlag checks the allocate response // mirrors the relay's forwarding mode: the flag is set only in stun-only // mode, and RelayAddr is always populated (it's the STUN server). diff --git a/internal/server/types.go b/internal/server/types.go index 43a1479..0f4acb8 100644 --- a/internal/server/types.go +++ b/internal/server/types.go @@ -97,11 +97,16 @@ type RelayAllocateRequest struct { // won't carry data. RelayAddr is still returned (it's the STUN server); // the client uses it for srflx gathering but skips the relay fallback. // omitempty so older servers round-trip as false (forwarding capable). +// +// BudgetExhausted is set when the relay's daily byte budget is already +// spent. Like ForwardingDisabled it lets the client fail fast with a +// specific reason instead of dialing a slot that can't forward. type RelayAllocateResponse struct { RelayAddr string `json:"relay_addr"` SessionToken string `json:"session_token"` // Crockford-base32 encoded 16 bytes TTLSeconds int `json:"ttl_seconds"` ForwardingDisabled bool `json:"forwarding_disabled,omitempty"` + BudgetExhausted bool `json:"budget_exhausted,omitempty"` } // HealthResponse is the body of GET /v1/health. diff --git a/test/e2e/harness_test.go b/test/e2e/harness_test.go index 1d5f8d6..e9fbf5d 100644 --- a/test/e2e/harness_test.go +++ b/test/e2e/harness_test.go @@ -106,7 +106,7 @@ func startHarness() (*harness, error) { hh.serverCmd = exec.Command(hh.fsendBin, "server") hh.serverCmd.Env = append(os.Environ(), - "FSEND_PAIRING_ADDR=:"+strconv.Itoa(hh.httpPort), + "FSEND_SERVER_ADDR=:"+strconv.Itoa(hh.httpPort), "FSEND_RELAY_ADDR=:"+strconv.Itoa(hh.udpPort), "FSEND_LOG_LEVEL=warn", // Both peers in this harness come from 127.0.0.1, so a single @@ -115,7 +115,7 @@ func startHarness() (*harness, error) { // the production default of 30/min would throttle the harness // itself. Production sender/receiver come from distinct IPs // and each get their own bucket. - "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN=10000", + "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE=10000", "FSEND_SERVER_MAX_SESSIONS_PER_IP=1000", ) hh.serverCmd.Stdout = &hh.serverOutput diff --git a/test/e2e/smoke_test.go b/test/e2e/smoke_test.go index aa58982..b6044ce 100644 --- a/test/e2e/smoke_test.go +++ b/test/e2e/smoke_test.go @@ -85,7 +85,7 @@ func TestServer_Help(t *testing.T) { func TestServer_HealthCheck(t *testing.T) { requireE2E(t) cmd := exec.Command(h.fsendBin, "server", "--health-check") - cmd.Env = append(os.Environ(), "FSEND_PAIRING_ADDR=:"+strconv.Itoa(h.httpPort)) + cmd.Env = append(os.Environ(), "FSEND_SERVER_ADDR=:"+strconv.Itoa(h.httpPort)) if err := cmd.Run(); err != nil { t.Fatalf("--health-check exit: %v", err) } @@ -113,11 +113,11 @@ func TestServer_PasswordGate(t *testing.T) { cmd := exec.Command(h.fsendBin, "server") cmd.Env = append(os.Environ(), - "FSEND_PAIRING_ADDR=:"+strconv.Itoa(httpPort), + "FSEND_SERVER_ADDR=:"+strconv.Itoa(httpPort), "FSEND_RELAY_ADDR=:"+strconv.Itoa(udpPort), "FSEND_LOG_LEVEL=warn", "FSEND_SERVER_PASSWORD=swordfish", - "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MIN=10000", + "FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE=10000", "FSEND_SERVER_MAX_SESSIONS_PER_IP=1000", ) if err := cmd.Start(); err != nil { From 1188e4d9b05743fb37d0395d042fd73fc555898a Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:11:34 +0200 Subject: [PATCH 2/9] refactor(relay): drop transfers_budget_capped_total metric The two budget gauges already report the actionable state: budget_bytes_today >= budget_max_bytes_per_day means the breaker tripped. The counter only tallied mid-flight evictions (not up-front refusals), so it was a partial, easily-misread signal on top of gauges that answer the question exactly. Keep it simple. The per-session transfers_capped_total stays: that cap has no gauge, so its counter is the only signal. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/self-hosting.md | 3 +-- internal/relay/relay_test.go | 11 +++----- internal/relay/server.go | 48 +++++++++++++++------------------ internal/server/metrics_test.go | 3 +-- 4 files changed, 28 insertions(+), 37 deletions(-) diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 49cc3fb..471d40a 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -195,11 +195,10 @@ so anyone can verify the server holds nothing sensitive. | `relay.transfers_active` | Relay transfers moving bytes right now. | | `relay.transfers_total` | Relay transfers that actually forwarded data since boot. Divide by `sessions_paired_total` for your relay-fallback rate; the rest connected directly (P2P). A climbing ratio means hole-punching is failing more often. | | `relay.transfers_capped_total` | Transfers cut off for hitting the per-session byte cap — raise the cap if this climbs. | -| `relay.transfers_budget_capped_total` | Transfers cut off because the daily budget was spent — the circuit breaker tripped. Climbing here means legit transfers are being denied; raise `FSEND_RELAY_MAX_BYTES_PER_DAY`. | | `relay.bytes_forwarded_total` | Cumulative bytes relayed since boot — what the relay is costing you. | | `relay.peak_transfer_bytes` | Largest single transfer so far; compare to your byte cap to see headroom. | | `relay.budget_max_bytes_per_day` | The configured daily budget (`0` = unlimited). | -| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day — watch it approach the budget to alert before the breaker trips. | +| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day. Approaching `budget_max_bytes_per_day` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | The `relay` block is omitted when the server runs without a relay. diff --git a/internal/relay/relay_test.go b/internal/relay/relay_test.go index 410e7f4..30bda63 100644 --- a/internal/relay/relay_test.go +++ b/internal/relay/relay_test.go @@ -427,10 +427,10 @@ func TestMetricsCapHit(t *testing.T) { <-srvErr } -// TestMetricsBudgetHit checks the daily budget trips the circuit breaker: +// TestBudgetHitEvicts checks the daily budget trips the circuit breaker: // once the day's bytes are spent the transfer is evicted with the -// daily_budget reason and transfers_budget_capped_total counts it. -func TestMetricsBudgetHit(t *testing.T) { +// daily_budget reason. +func TestBudgetHitEvicts(t *testing.T) { serverConn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) if err != nil { t.Fatal(err) @@ -455,14 +455,11 @@ func TestMetricsBudgetHit(t *testing.T) { deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { - if srv.Metrics().TransfersBudgetCappedTotal > 0 { + if srv.Status(tok) == ReasonBudgetHit { break } time.Sleep(20 * time.Millisecond) } - if got := srv.Metrics().TransfersBudgetCappedTotal; got != 1 { - t.Errorf("transfers_budget_capped_total = %d, want 1", got) - } if got := srv.Status(tok); got != ReasonBudgetHit { t.Errorf("Status = %q, want %q", got, ReasonBudgetHit) } diff --git a/internal/relay/server.go b/internal/relay/server.go index 02f55dd..23d4294 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -54,40 +54,37 @@ type Server struct { budget dayBudget // Aggregate counters for /v1/metrics (cumulative since boot). - bytesForwarded atomic.Uint64 - peakTransferBytes atomic.Uint64 // most any single transfer has forwarded - transfersCapped atomic.Uint64 // transfers cut off for exceeding the per-session byte cap - transfersBudgetCapped atomic.Uint64 // transfers cut off because the daily budget was spent - transfersTotal atomic.Uint64 // transfers that forwarded ≥1 byte; vs paired sessions = relay-fallback rate + bytesForwarded atomic.Uint64 + peakTransferBytes atomic.Uint64 // most any single transfer has forwarded + transfersCapped atomic.Uint64 // transfers cut off for exceeding the per-session byte cap + transfersTotal atomic.Uint64 // transfers that forwarded ≥1 byte; vs paired sessions = relay-fallback rate } // Metrics is an aggregate snapshot of relay activity for /v1/metrics. type Metrics struct { - Forwarding bool `json:"forwarding"` - Healthy bool `json:"healthy"` - TransfersActive int `json:"transfers_active"` - TransfersTotal uint64 `json:"transfers_total"` - TransfersCappedTotal uint64 `json:"transfers_capped_total"` - TransfersBudgetCappedTotal uint64 `json:"transfers_budget_capped_total"` - BytesForwardedTotal uint64 `json:"bytes_forwarded_total"` - PeakTransferBytes uint64 `json:"peak_transfer_bytes"` - BudgetMaxBytesPerDay uint64 `json:"budget_max_bytes_per_day"` // 0 = unlimited - BudgetBytesToday uint64 `json:"budget_bytes_today"` + Forwarding bool `json:"forwarding"` + Healthy bool `json:"healthy"` + TransfersActive int `json:"transfers_active"` + TransfersTotal uint64 `json:"transfers_total"` + TransfersCappedTotal uint64 `json:"transfers_capped_total"` + BytesForwardedTotal uint64 `json:"bytes_forwarded_total"` + PeakTransferBytes uint64 `json:"peak_transfer_bytes"` + BudgetMaxBytesPerDay uint64 `json:"budget_max_bytes_per_day"` // 0 = unlimited + BudgetBytesToday uint64 `json:"budget_bytes_today"` } // Metrics returns the current aggregate snapshot. func (s *Server) Metrics() Metrics { return Metrics{ - Forwarding: !s.cfg.DisableForwarding, - Healthy: s.healthy.Load(), - TransfersActive: s.ActiveAllocations(), - TransfersTotal: s.transfersTotal.Load(), - TransfersCappedTotal: s.transfersCapped.Load(), - TransfersBudgetCappedTotal: s.transfersBudgetCapped.Load(), - BytesForwardedTotal: s.bytesForwarded.Load(), - PeakTransferBytes: s.peakTransferBytes.Load(), - BudgetMaxBytesPerDay: s.cfg.MaxBytesPerDay, - BudgetBytesToday: s.budget.usedToday(time.Now()), + Forwarding: !s.cfg.DisableForwarding, + Healthy: s.healthy.Load(), + TransfersActive: s.ActiveAllocations(), + TransfersTotal: s.transfersTotal.Load(), + TransfersCappedTotal: s.transfersCapped.Load(), + BytesForwardedTotal: s.bytesForwarded.Load(), + PeakTransferBytes: s.peakTransferBytes.Load(), + BudgetMaxBytesPerDay: s.cfg.MaxBytesPerDay, + BudgetBytesToday: s.budget.usedToday(time.Now()), } } @@ -381,7 +378,6 @@ func (s *Server) handle(datagram []byte, src *net.UDPAddr) { // Daily budget (Denial-of-Wallet circuit breaker): once the day's bytes // are spent, evict and drop until the UTC-midnight rollover. if s.budget.charge(now, wireSize) { - s.transfersBudgetCapped.Add(1) s.evict(token, ReasonBudgetHit) return } diff --git a/internal/server/metrics_test.go b/internal/server/metrics_test.go index 30b1fed..cca48e2 100644 --- a/internal/server/metrics_test.go +++ b/internal/server/metrics_test.go @@ -64,8 +64,7 @@ func TestMetrics_Shape(t *testing.T) { "rate_limit", "concurrency_limit", "unauthorized") assertKeys(t, raw["relay"], "relay", "forwarding", "healthy", "transfers_active", "transfers_total", - "transfers_capped_total", "transfers_budget_capped_total", - "bytes_forwarded_total", "peak_transfer_bytes", + "transfers_capped_total", "bytes_forwarded_total", "peak_transfer_bytes", "budget_max_bytes_per_day", "budget_bytes_today") } From f6eb9b3f49fe6c68a6cfad8c0dfcbe2aa81192b5 Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:16:51 +0200 Subject: [PATCH 3/9] refactor(relay): don't expose the daily budget limit in metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop budget_max_bytes_per_day from /v1/metrics. It's the operator's own config, it's the only limit value the metrics exposed (per-IP and per-session caps expose only counts), and on an open server it handed a Denial-of-Wallet abuser the exact target. Keep budget_bytes_today — live usage, the operator's early warning. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/self-hosting.md | 3 +-- internal/relay/server.go | 5 ++--- internal/server/metrics_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 471d40a..fbfde6f 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -197,8 +197,7 @@ so anyone can verify the server holds nothing sensitive. | `relay.transfers_capped_total` | Transfers cut off for hitting the per-session byte cap — raise the cap if this climbs. | | `relay.bytes_forwarded_total` | Cumulative bytes relayed since boot — what the relay is costing you. | | `relay.peak_transfer_bytes` | Largest single transfer so far; compare to your byte cap to see headroom. | -| `relay.budget_max_bytes_per_day` | The configured daily budget (`0` = unlimited). | -| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day. Approaching `budget_max_bytes_per_day` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | +| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day. Approaching your configured `FSEND_RELAY_MAX_BYTES_PER_DAY` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | The `relay` block is omitted when the server runs without a relay. diff --git a/internal/relay/server.go b/internal/relay/server.go index 23d4294..48838b4 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -69,8 +69,8 @@ type Metrics struct { TransfersCappedTotal uint64 `json:"transfers_capped_total"` BytesForwardedTotal uint64 `json:"bytes_forwarded_total"` PeakTransferBytes uint64 `json:"peak_transfer_bytes"` - BudgetMaxBytesPerDay uint64 `json:"budget_max_bytes_per_day"` // 0 = unlimited - BudgetBytesToday uint64 `json:"budget_bytes_today"` + // Live usage only; the configured limit is not exposed on purpose. + BudgetBytesToday uint64 `json:"budget_bytes_today"` } // Metrics returns the current aggregate snapshot. @@ -83,7 +83,6 @@ func (s *Server) Metrics() Metrics { TransfersCappedTotal: s.transfersCapped.Load(), BytesForwardedTotal: s.bytesForwarded.Load(), PeakTransferBytes: s.peakTransferBytes.Load(), - BudgetMaxBytesPerDay: s.cfg.MaxBytesPerDay, BudgetBytesToday: s.budget.usedToday(time.Now()), } } diff --git a/internal/server/metrics_test.go b/internal/server/metrics_test.go index cca48e2..16ccfe2 100644 --- a/internal/server/metrics_test.go +++ b/internal/server/metrics_test.go @@ -65,7 +65,7 @@ func TestMetrics_Shape(t *testing.T) { assertKeys(t, raw["relay"], "relay", "forwarding", "healthy", "transfers_active", "transfers_total", "transfers_capped_total", "bytes_forwarded_total", "peak_transfer_bytes", - "budget_max_bytes_per_day", "budget_bytes_today") + "budget_bytes_today") } func assertKeys(t *testing.T, v any, name string, want ...string) { From 62ab2f5e13ea105e4bfc1703af3ed058de5027db Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:27:45 +0200 Subject: [PATCH 4/9] docs: reorder config by plane (server-wide / pairing / relay) Group the config table, startup summary, and CLI help the same way: server-wide (log, auth), then the pairing control plane (its listener + session limits), then the relay data plane (its listener + byte limits). Previously FSEND_SERVER_ADDR sat after the session limits, so the table didn't read plane-by-plane. Also fix the startup-example alignment to match the actual output. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/fsend/server.go | 11 +++++++---- docs/self-hosting.md | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/cmd/fsend/server.go b/cmd/fsend/server.go index 645346c..8b884b4 100644 --- a/cmd/fsend/server.go +++ b/cmd/fsend/server.go @@ -288,13 +288,16 @@ func formatServerConfig(cfg serverRuntimeConfig) string { password = "(set)" } settings := []setting{ + // Grouped: server-wide (log, password), then the pairing control + // plane (its listener + session limits), then the relay data plane + // (its listener + byte limits). Matches the docs table order. {envLogLevel, logLevelName(cfg.logLevel), cfg.logLevel != slog.LevelInfo}, // Env-var name inlined (not a shared const) so gosec G101 doesn't // mistake a password-named identifier for a hardcoded credential. {"FSEND_SERVER_PASSWORD", password, cfg.serverPassword != ""}, + {envServerAddr, cfg.httpAddr, cfg.httpAddr != defaultServerAddr}, {envMaxSessionsPerIP, capCount(cfg.maxSessionsPerIP), cfg.maxSessionsPerIP != defaultMaxSessionsPerIP}, {envMaxNewSessionsPerMin, capCount(cfg.maxNewSessionsPerMin), cfg.maxNewSessionsPerMin != defaultMaxNewSessionsPerMin}, - {envServerAddr, cfg.httpAddr, cfg.httpAddr != defaultServerAddr}, {envRelayEnabled, strconv.FormatBool(cfg.enableRelay), !cfg.enableRelay}, {envRelayAddr, cfg.udpAddr, cfg.udpAddr != defaultRelayAddr}, {envMaxBytesPerSession, capBytes(cfg.maxBytesPerSession), cfg.maxBytesPerSession != defaultMaxBytesPerSession}, @@ -490,6 +493,9 @@ CONFIGURATION (environment variables — all optional) endpoints except /v1/health require the X-Fsend-Auth header. Connect with fsend --connect ,. + + Pairing (TCP signaling/control plane): + FSEND_SERVER_ADDR Default :8080 (TCP). FSEND_SERVER_MAX_SESSIONS_PER_IP How many sessions one IP may have alive at once (concurrency cap; gates relay access too). Default 0 = @@ -499,9 +505,6 @@ CONFIGURATION (environment variables — all optional) per minute (rate cap). Default 0 = unlimited; set a positive value to cap. - Pairing (TCP signaling/control plane): - FSEND_SERVER_ADDR Default :8080 (TCP). - Relay (UDP data plane — also answers STUN): FSEND_RELAY_ENABLED Default true. false = pairing + STUN only: peers still hole-punch, but the diff --git a/docs/self-hosting.md b/docs/self-hosting.md index fbfde6f..f33bae7 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -247,29 +247,29 @@ whether one is set): ``` fsend server configuration (2 of 9 customized; * = changed from default): - * FSEND_SERVER_PASSWORD (set) - * FSEND_SERVER_MAX_SESSIONS_PER_IP 10 - FSEND_LOG_LEVEL info - FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE 0 (unlimited) - FSEND_SERVER_ADDR :8080 - FSEND_RELAY_ENABLED true - FSEND_RELAY_ADDR :443 - FSEND_RELAY_MAX_BYTES_PER_SESSION 0 (unlimited) - FSEND_RELAY_MAX_BYTES_PER_DAY 0 (unlimited) + FSEND_LOG_LEVEL info + * FSEND_SERVER_PASSWORD (set) + FSEND_SERVER_ADDR :8080 + * FSEND_SERVER_MAX_SESSIONS_PER_IP 10 + FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE 0 (unlimited) + FSEND_RELAY_ENABLED true + FSEND_RELAY_ADDR :443 + FSEND_RELAY_MAX_BYTES_PER_SESSION 0 (unlimited) + FSEND_RELAY_MAX_BYTES_PER_DAY 0 (unlimited) ``` -Variables are grouped into **server-wide** controls (apply to the whole -server, relay included), the **pairing** listener (TCP signaling/control -plane), and **relay** settings (the UDP data plane, which also answers -STUN). +Variables fall into three groups: **server-wide** controls (logging and +auth, which apply to every endpoint), the **pairing / control plane** (its +TCP listener and per-IP session limits), and the **relay / data plane** +(the UDP listener — which also answers STUN — and its byte limits). | Variable | Default | Notes | |---|---|---| | `FSEND_LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error`. | | `FSEND_SERVER_PASSWORD` | _(unset)_ | Shared secret restricting all endpoints except `/v1/health` — see [Require a password](#require-a-password-optional). | +| `FSEND_SERVER_ADDR` | `:8080` | TCP signaling listener. | | `FSEND_SERVER_MAX_SESSIONS_PER_IP` | `0` (unlimited) | **Concurrency cap** — how many sessions one source IP may have **alive at once**. A session gates relay allocation too, so this caps relay access as well. Defaults to unlimited; **set a positive value to enable this DoS protection** (recommended on a public server). | | `FSEND_SERVER_MAX_SESSIONS_PER_IP_PER_MINUTE` | `0` (unlimited) | **Rate cap** — how many **new** sessions one source IP may **create per minute**. Distinct from the concurrency cap above: this limits inflow over time, that limits standing count. Defaults to unlimited; **set a positive value to enable this DoS protection** (recommended on a public server). | -| `FSEND_SERVER_ADDR` | `:8080` | TCP signaling listener. | | `FSEND_RELAY_ENABLED` | `true` | Set `false` for **pairing + STUN only**: the server still helps peers hole-punch a direct path, but carries no file data. See [Pairing-only mode](#pairing-only-mode-no-relay). | | `FSEND_RELAY_ADDR` | `:443` | UDP relay listener — also the STUN endpoint, so it stays in use even when forwarding is off. The server tells clients to dial `:`, so no separate public-address knob is needed. | | `FSEND_RELAY_MAX_BYTES_PER_SESSION` | `0` (unlimited) | Per-session relay cap — wire bytes after compression. Defaults to unlimited; set a value to bound per-transfer bandwidth. Accepts `B`, `KB`, `MB`, `GB`, `TB` suffixes (decimal, e.g. `500MB`, `1GB`) or a plain byte count (`1000000000`). | From d81d332dbf6097da198e82b2b2fdce3558479d45 Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:43:19 +0200 Subject: [PATCH 5/9] fix(scripts): rename stale FSEND_PAIRING_ADDR in demo.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The env-var rename missed scripts/preview/demo.sh, which set a custom port via FSEND_PAIRING_ADDR — now ignored, so the server bound :8080 while the client dialed :8799 and the demo broke. Point it at FSEND_SERVER_ADDR. Also add comments: the both-peers allocate asymmetry (only the first caller sees the budget refusal; the second converges via the in-flight breaker) and the per-IP-count retention when a budget-exhausted session is left to PairedTTL. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/fsend/sendpair.go | 4 +++- internal/server/server.go | 5 ++++- scripts/preview/demo.sh | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/fsend/sendpair.go b/cmd/fsend/sendpair.go index b83d31f..894c6aa 100644 --- a/cmd/fsend/sendpair.go +++ b/cmd/fsend/sendpair.go @@ -193,7 +193,9 @@ func pairOverInternet(ctx context.Context, f *flags, code string, cfg *config.Co // A spent daily budget is terminal and self-explanatory. Surface it // as-is (not the generic errPairedGone), and leave the session for the // receiver to reach the relay and learn the same reason — it's the - // budget, not the pairing, that failed. The session expires via TTL. + // budget, not the pairing, that failed. The session (and its per-IP + // count) is reclaimed at PairedTTL rather than now; fine since the + // budget is already the degraded state and the message says to wait. if errors.Is(err, fserrors.ErrRelayBudgetExhausted) { return nil, err } diff --git a/internal/server/server.go b/internal/server/server.go index 040b5b4..9b30b2a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -332,7 +332,10 @@ func (s *Server) allocateRelay(w http.ResponseWriter, r *http.Request) { } // One token per session: both peers must end up with the same value // so the relay's source-addr de-mux pairs them. Allocate lazily on - // first call; subsequent calls reuse. + // first call; subsequent calls reuse. So only the first caller sees a + // budget-exhausted refusal here; if the budget is spent between the two + // calls, the second peer gets the cached token and instead converges on + // the same reason via the in-flight breaker (handle → status probe). if !sess.relayTokenSet { t, err := s.relayAllocator.Allocate() if err != nil { diff --git a/scripts/preview/demo.sh b/scripts/preview/demo.sh index 6be2635..400f6f7 100755 --- a/scripts/preview/demo.sh +++ b/scripts/preview/demo.sh @@ -69,7 +69,7 @@ do_setup() { ensure_payload ln -f "$PAYLOAD" "$HOME_DIR/$FILE" - FSEND_PAIRING_ADDR="$HTTP_ADDR" FSEND_RELAY_ADDR="$UDP_ADDR" FSEND_LOG_LEVEL=error \ + FSEND_SERVER_ADDR="$HTTP_ADDR" FSEND_RELAY_ADDR="$UDP_ADDR" FSEND_LOG_LEVEL=error \ "$bin" server >"$ROOT/server.log" 2>&1 & echo $! > "$ROOT/server.pid" disown From 3403ee2a8bbe862155d0e5b10820cf1a097f26d3 Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:48:37 +0200 Subject: [PATCH 6/9] test(server): cover both-peers relay-token reuse Prove the lazy-once contract: the relay token is minted on the first peer's allocate and reused for the second, so the second peer never re-invokes Allocate. The fake relay is rigged to report the budget spent on any second Allocate; the test asserts it never fires (allocCalls==1, no budget_exhausted, shared token). This is the asymmetry that lets a budget spent mid-pairing converge via the in-flight breaker. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/server/server_test.go | 89 +++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 414482e..128c1f7 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -528,15 +528,23 @@ func TestServerPassword_OpenByDefault(t *testing.T) { // fakeRelay implements RelayAllocator with fixed responses, so the // relay-status handler can be exercised without standing up a UDP relay. type fakeRelay struct { - tok relay.Token - reason string - maxBytes uint64 - dead bool - noForward bool - allocErr error + tok relay.Token + reason string + maxBytes uint64 + dead bool + noForward bool + allocErr error + allocCalls int // number of Allocate() invocations + budgetAfter int // if >0, Allocate reports the budget spent once allocCalls exceeds it } -func (f *fakeRelay) Allocate() (relay.Token, error) { return f.tok, f.allocErr } +func (f *fakeRelay) Allocate() (relay.Token, error) { + f.allocCalls++ + if f.budgetAfter > 0 && f.allocCalls > f.budgetAfter { + return relay.Token{}, relay.ErrBudgetExhausted + } + return f.tok, f.allocErr +} func (f *fakeRelay) Status(t relay.Token) string { return f.reason } func (f *fakeRelay) MaxBytesPerSession() uint64 { return f.maxBytes } func (f *fakeRelay) Healthy() bool { return !f.dead } @@ -680,6 +688,73 @@ func TestRelayAllocate_BudgetExhausted(t *testing.T) { } } +// TestRelayAllocate_BothPeersReuseToken locks in the lazy-once contract: +// the relay token is minted on the first peer's allocate and reused for the +// second, so the second peer never re-invokes Allocate. That's why a budget +// spent between the two calls can't refuse the second peer — it gets the +// cached token and converges via the in-flight breaker instead. The fake is +// rigged to report the budget spent on any *second* Allocate; that must +// never fire, proving the reuse. +func TestRelayAllocate_BothPeersReuseToken(t *testing.T) { + s := New(Config{ + ServerVersion: "0.0.0-test", + UnpairedTTL: 2 * time.Second, + PairedTTL: 5 * time.Second, + MaxSessionsPerIP: 10, + MaxNewSessionsPerMin: 100, + }) + fr := &fakeRelay{tok: relay.Token{9, 9, 9}, budgetAfter: 1} + s.WithRelay(fr, 9999) + ts := httptest.NewServer(s.Handler()) + defer ts.Close() + + cr := createSession(t, ts.URL, testSlot) + jresp := postJSON(t, ts.URL+"/v1/session/"+testSlot+"/join", JoinSessionRequest{}) + var jr JoinSessionResponse + if err := json.NewDecoder(jresp.Body).Decode(&jr); err != nil { + t.Fatal(err) + } + jresp.Body.Close() + + sStatus, sBody := postRelayAllocate(t, ts.URL, cr.SessionID, cr.RoleToken) + rStatus, rBody := postRelayAllocate(t, ts.URL, jr.SessionID, jr.RoleToken) + + if sStatus != 200 || rStatus != 200 { + t.Fatalf("allocate status: sender=%d receiver=%d, want 200/200", sStatus, rStatus) + } + if sBody.SessionToken == "" { + t.Fatal("sender got an empty relay token") + } + if rBody.SessionToken != sBody.SessionToken { + t.Errorf("token mismatch: sender=%q receiver=%q (both peers must share one token)", + sBody.SessionToken, rBody.SessionToken) + } + if rBody.BudgetExhausted { + t.Error("second peer got budget_exhausted; it must reuse the cached token, not re-allocate") + } + if fr.allocCalls != 1 { + t.Errorf("Allocate called %d times, want 1 (second peer must reuse the cached token)", fr.allocCalls) + } +} + +// postRelayAllocate POSTs /v1/relay/allocate as one session peer and returns +// the status and decoded body. +func postRelayAllocate(t *testing.T, baseURL, sessionID, roleToken string) (int, RelayAllocateResponse) { + t.Helper() + req, _ := http.NewRequest(http.MethodPost, baseURL+"/v1/relay/allocate", + bytes.NewReader(mustJSON(t, RelayAllocateRequest{SessionID: sessionID}))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+roleToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var body RelayAllocateResponse + _ = json.NewDecoder(resp.Body).Decode(&body) + return resp.StatusCode, body +} + // TestRelayAllocate_ForwardingDisabledFlag checks the allocate response // mirrors the relay's forwarding mode: the flag is set only in stun-only // mode, and RelayAddr is always populated (it's the STUN server). From fe0d1736c03de85c40df85ca9a88c41835c6f663 Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:49:17 +0200 Subject: [PATCH 7/9] style: gofmt fakeRelay method alignment Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/server/server_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 128c1f7..fd611a7 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -545,10 +545,10 @@ func (f *fakeRelay) Allocate() (relay.Token, error) { } return f.tok, f.allocErr } -func (f *fakeRelay) Status(t relay.Token) string { return f.reason } -func (f *fakeRelay) MaxBytesPerSession() uint64 { return f.maxBytes } -func (f *fakeRelay) Healthy() bool { return !f.dead } -func (f *fakeRelay) Forwarding() bool { return !f.noForward } +func (f *fakeRelay) Status(t relay.Token) string { return f.reason } +func (f *fakeRelay) MaxBytesPerSession() uint64 { return f.maxBytes } +func (f *fakeRelay) Healthy() bool { return !f.dead } +func (f *fakeRelay) Forwarding() bool { return !f.noForward } func (f *fakeRelay) Metrics() relay.Metrics { return relay.Metrics{Forwarding: !f.noForward, Healthy: !f.dead} } From 2a19ff3290b8bec713d7f1540089816ace187800 Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:55:08 +0200 Subject: [PATCH 8/9] docs: clarify the daily budget counts egress in both directions Spell out what FSEND_RELAY_MAX_BYTES_PER_DAY meters: every forwarded datagram, once, in both directions (payload one way, ACKs/control the other) = outbound bytes, matching how egress is billed. Note STUN and direct/LAN transfers aren't counted, and that a host billing inbound too sees ~2x. Applied to the config table, the budget_bytes_today metric row, and the CLI help. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/fsend/server.go | 2 ++ docs/self-hosting.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/fsend/server.go b/cmd/fsend/server.go index 8b884b4..5e9f7af 100644 --- a/cmd/fsend/server.go +++ b/cmd/fsend/server.go @@ -518,6 +518,8 @@ CONFIGURATION (environment variables — all optional) unlimited; set a value to cap. FSEND_RELAY_MAX_BYTES_PER_DAY Server-wide wire bytes forwarded per UTC day — the egress/cost ceiling. + Counts every forwarded datagram once, + both directions (= outbound bytes). Same units. Default 0 = unlimited; once hit, the relay stops forwarding until 00:00 UTC. diff --git a/docs/self-hosting.md b/docs/self-hosting.md index f33bae7..45666c7 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -197,7 +197,7 @@ so anyone can verify the server holds nothing sensitive. | `relay.transfers_capped_total` | Transfers cut off for hitting the per-session byte cap — raise the cap if this climbs. | | `relay.bytes_forwarded_total` | Cumulative bytes relayed since boot — what the relay is costing you. | | `relay.peak_transfer_bytes` | Largest single transfer so far; compare to your byte cap to see headroom. | -| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day. Approaching your configured `FSEND_RELAY_MAX_BYTES_PER_DAY` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | +| `relay.budget_bytes_today` | Wire bytes forwarded so far in the current UTC day (both directions, same count as the budget). Approaching your configured `FSEND_RELAY_MAX_BYTES_PER_DAY` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | The `relay` block is omitted when the server runs without a relay. @@ -273,7 +273,7 @@ TCP listener and per-IP session limits), and the **relay / data plane** | `FSEND_RELAY_ENABLED` | `true` | Set `false` for **pairing + STUN only**: the server still helps peers hole-punch a direct path, but carries no file data. See [Pairing-only mode](#pairing-only-mode-no-relay). | | `FSEND_RELAY_ADDR` | `:443` | UDP relay listener — also the STUN endpoint, so it stays in use even when forwarding is off. The server tells clients to dial `:`, so no separate public-address knob is needed. | | `FSEND_RELAY_MAX_BYTES_PER_SESSION` | `0` (unlimited) | Per-session relay cap — wire bytes after compression. Defaults to unlimited; set a value to bound per-transfer bandwidth. Accepts `B`, `KB`, `MB`, `GB`, `TB` suffixes (decimal, e.g. `500MB`, `1GB`) or a plain byte count (`1000000000`). | -| `FSEND_RELAY_MAX_BYTES_PER_DAY` | `0` (unlimited) | **Egress budget** — wire bytes the relay forwards **per UTC day** across all sessions. The Denial-of-Wallet ceiling: once spent, the relay stops forwarding and refuses new transfers until 00:00 UTC. Bounds a distributed abuser that the per-IP caps can't. Same units as above. Defaults to unlimited; **set a value to bound your bandwidth bill**. | +| `FSEND_RELAY_MAX_BYTES_PER_DAY` | `0` (unlimited) | **Egress budget** — wire bytes the relay forwards **per UTC day** across all sessions. The Denial-of-Wallet ceiling: once spent, the relay stops forwarding and refuses new transfers until 00:00 UTC. Bounds a distributed abuser that the per-IP caps can't. Same units as above. Defaults to unlimited; **set a value to bound your bandwidth bill**. **What's counted:** every datagram the relay forwards, once, in **both directions** (payload one way, ACKs/control the other) — i.e. outbound bytes, matching how egress is billed. Not counted: STUN (separate path), and same-network/direct transfers (never touch the relay). Note the relay physically receives *and* sends each byte, so if your host also bills **inbound** traffic, real cost ≈ 2× the budget. | ### Pairing-only mode (no relay) From 092a84ea1d20b0be5c029b9b656c9d898f15231e Mon Sep 17 00:00:00 2001 From: Pol Alzina Date: Wed, 1 Jul 2026 20:58:04 +0200 Subject: [PATCH 9/9] docs: simplify daily-budget wording, fix the counting example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous wording over-explained "both directions" and a 2x inbound caveat, which read as if the budget double-counts. It doesn't: each byte is counted once when forwarded, so a 1 MB relayed file ≈ 1 MB of budget. State that plainly and drop the confusing bits. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/fsend/server.go | 13 ++++++------- docs/self-hosting.md | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/fsend/server.go b/cmd/fsend/server.go index 5e9f7af..5156058 100644 --- a/cmd/fsend/server.go +++ b/cmd/fsend/server.go @@ -516,13 +516,12 @@ CONFIGURATION (environment variables — all optional) compression. Accepts B, KB, MB, GB, TB, or a plain byte count. Default 0 = unlimited; set a value to cap. - FSEND_RELAY_MAX_BYTES_PER_DAY Server-wide wire bytes forwarded per - UTC day — the egress/cost ceiling. - Counts every forwarded datagram once, - both directions (= outbound bytes). - Same units. Default 0 = unlimited; once - hit, the relay stops forwarding until - 00:00 UTC. + FSEND_RELAY_MAX_BYTES_PER_DAY Server-wide outbound bytes forwarded + per UTC day — the egress/cost ceiling. + Each byte counts once (a 1 MB relayed + file ≈ 1 MB). Same units. Default 0 = + unlimited; once hit, the relay stops + forwarding until 00:00 UTC. LEARN MORE https://github.com/polius/fsend diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 45666c7..4cd9aa3 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -197,7 +197,7 @@ so anyone can verify the server holds nothing sensitive. | `relay.transfers_capped_total` | Transfers cut off for hitting the per-session byte cap — raise the cap if this climbs. | | `relay.bytes_forwarded_total` | Cumulative bytes relayed since boot — what the relay is costing you. | | `relay.peak_transfer_bytes` | Largest single transfer so far; compare to your byte cap to see headroom. | -| `relay.budget_bytes_today` | Wire bytes forwarded so far in the current UTC day (both directions, same count as the budget). Approaching your configured `FSEND_RELAY_MAX_BYTES_PER_DAY` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | +| `relay.budget_bytes_today` | Bytes forwarded so far in the current UTC day (same count as the budget). Approaching your configured `FSEND_RELAY_MAX_BYTES_PER_DAY` is your early warning; reaching it means the breaker has tripped and transfers are being refused until 00:00 UTC. | The `relay` block is omitted when the server runs without a relay. @@ -273,7 +273,7 @@ TCP listener and per-IP session limits), and the **relay / data plane** | `FSEND_RELAY_ENABLED` | `true` | Set `false` for **pairing + STUN only**: the server still helps peers hole-punch a direct path, but carries no file data. See [Pairing-only mode](#pairing-only-mode-no-relay). | | `FSEND_RELAY_ADDR` | `:443` | UDP relay listener — also the STUN endpoint, so it stays in use even when forwarding is off. The server tells clients to dial `:`, so no separate public-address knob is needed. | | `FSEND_RELAY_MAX_BYTES_PER_SESSION` | `0` (unlimited) | Per-session relay cap — wire bytes after compression. Defaults to unlimited; set a value to bound per-transfer bandwidth. Accepts `B`, `KB`, `MB`, `GB`, `TB` suffixes (decimal, e.g. `500MB`, `1GB`) or a plain byte count (`1000000000`). | -| `FSEND_RELAY_MAX_BYTES_PER_DAY` | `0` (unlimited) | **Egress budget** — wire bytes the relay forwards **per UTC day** across all sessions. The Denial-of-Wallet ceiling: once spent, the relay stops forwarding and refuses new transfers until 00:00 UTC. Bounds a distributed abuser that the per-IP caps can't. Same units as above. Defaults to unlimited; **set a value to bound your bandwidth bill**. **What's counted:** every datagram the relay forwards, once, in **both directions** (payload one way, ACKs/control the other) — i.e. outbound bytes, matching how egress is billed. Not counted: STUN (separate path), and same-network/direct transfers (never touch the relay). Note the relay physically receives *and* sends each byte, so if your host also bills **inbound** traffic, real cost ≈ 2× the budget. | +| `FSEND_RELAY_MAX_BYTES_PER_DAY` | `0` (unlimited) | **Egress budget** — outbound bytes the relay forwards **per UTC day** across all sessions. The Denial-of-Wallet ceiling: once spent, the relay stops forwarding and refuses new transfers until 00:00 UTC. Bounds a distributed abuser that the per-IP caps can't. Each byte is counted once, so a 1 MB file sent over the relay uses about 1 MB of budget. Only relay-fallback transfers count — same-network and direct transfers don't, and STUN doesn't. Same units as above; default unlimited — **set a value to bound your bandwidth bill**. | ### Pairing-only mode (no relay)