From 9720e45a02888542970210055529f0be7c9e7f26 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Thu, 11 Jun 2026 22:26:18 -0400 Subject: [PATCH 01/19] feat(ingest): fula-ingest - verified byte ingress node (Phase 2) PUT /v0/block?cid=: verifies the blake3 raw-leaf CID over the body BEFORE storing (single flipped bit => 422, nothing written), then kubo block/put?cid-codec=raw&mhtype=blake3 and cross-checks kubo's key. S1 quota gate mirrors the gateway (GET /api/v1/storage with the client Bearer; INGEST_QUOTA_MODE=open matches gateway fail-open, strict denies when unverifiable); per-token 30s cache so a thousand-chunk upload makes one quota call per window. Size cap + bounded concurrency + /health + graceful shutdown. 10 Go tests (containerized run: all pass). Part of functionland/fula-ota#74 (cross-repo: fula-api#31) Co-Authored-By: Claude Fable 5 --- docker/fula-ingest/Dockerfile | 14 ++ docker/fula-ingest/go.mod | 22 ++ docker/fula-ingest/go.sum | 28 +++ docker/fula-ingest/main.go | 376 ++++++++++++++++++++++++++++++++ docker/fula-ingest/main_test.go | 262 ++++++++++++++++++++++ 5 files changed, 702 insertions(+) create mode 100644 docker/fula-ingest/Dockerfile create mode 100644 docker/fula-ingest/go.mod create mode 100644 docker/fula-ingest/go.sum create mode 100644 docker/fula-ingest/main.go create mode 100644 docker/fula-ingest/main_test.go diff --git a/docker/fula-ingest/Dockerfile b/docker/fula-ingest/Dockerfile new file mode 100644 index 00000000..0b543d4f --- /dev/null +++ b/docker/fula-ingest/Dockerfile @@ -0,0 +1,14 @@ +# fula-ingest — verified byte ingress (Phase 2). +# Build (repo root): docker build -f docker/fula-ingest/Dockerfile -t functionland/fula-ingest:latest docker/fula-ingest +FROM golang:1.22-alpine AS build +WORKDIR /src +COPY go.mod go.sum* ./ +RUN go mod download 2>/dev/null || true +COPY . . +RUN go mod tidy && CGO_ENABLED=0 go build -o /fula-ingest . + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates +COPY --from=build /fula-ingest /usr/local/bin/fula-ingest +EXPOSE 3601 +ENTRYPOINT ["fula-ingest"] diff --git a/docker/fula-ingest/go.mod b/docker/fula-ingest/go.mod new file mode 100644 index 00000000..b099c3e4 --- /dev/null +++ b/docker/fula-ingest/go.mod @@ -0,0 +1,22 @@ +module github.com/functionland/fula-ota/docker/fula-ingest + +go 1.22 + +require ( + github.com/ipfs/go-cid v0.4.1 + github.com/multiformats/go-multihash v0.2.3 +) + +require ( + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.0.3 // indirect + github.com/multiformats/go-base36 v0.1.0 // indirect + github.com/multiformats/go-multibase v0.0.3 // indirect + github.com/multiformats/go-varint v0.0.6 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect + lukechampine.com/blake3 v1.1.6 // indirect +) diff --git a/docker/fula-ingest/go.sum b/docker/fula-ingest/go.sum new file mode 100644 index 00000000..2be0bae6 --- /dev/null +++ b/docker/fula-ingest/go.sum @@ -0,0 +1,28 @@ +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= +lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/docker/fula-ingest/main.go b/docker/fula-ingest/main.go new file mode 100644 index 00000000..0b6c4437 --- /dev/null +++ b/docker/fula-ingest/main.go @@ -0,0 +1,376 @@ +// fula-ingest — verified byte ingress for the Fula network (Phase 2). +// +// Accepts client-side-encrypted chunk BYTES with a client-DECLARED blake3 +// raw-leaf CID, verifies the CID over the body BEFORE storing (tampered +// byte => 422, nothing stored), then writes the block to the local kubo +// (block/put?cid-codec=raw&mhtype=blake3) and double-checks kubo's returned +// key. This is the server-side half of upload tamper-evidence; the client +// half (per-chunk pre-compute + ETag self-verify) ships in fula-client +// (walkable-v8). +// +// S1 quota gate (safeguards invariant: never unmetered ingestion): mirrors +// the gateway's check exactly — GET {STORAGE_API_URL}/api/v1/storage with +// the client's Bearer JWT => {canUpload}. INGEST_QUOTA_MODE=open matches the +// gateway's fail-open semantics; =strict denies when the check cannot pass. +// +// Endpoints: +// PUT /v0/block?cid= body = chunk bytes +// GET /health +// +// Env: INGEST_PORT (3601), INGEST_BIND (0.0.0.0), KUBO_API +// (http://127.0.0.1:5001), STORAGE_API_URL (empty disables the quota call — +// pair with INGEST_NO_AUTH only in tests), INGEST_QUOTA_MODE (open|strict), +// INGEST_MAX_BLOCK_BYTES (4194304), INGEST_MAX_CONCURRENT (32), +// INGEST_NO_AUTH (0|1, tests only). +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "mime/multipart" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + _ "github.com/multiformats/go-multihash/register/blake3" +) + +type config struct { + port string + bind string + kuboAPI string + storageAPIURL string + quotaStrict bool + maxBlockBytes int64 + maxConcurrent int + noAuth bool +} + +func envOr(k, d string) string { + if v := os.Getenv(k); v != "" { + return v + } + return d +} + +func loadConfig() config { + maxBytes, _ := strconv.ParseInt(envOr("INGEST_MAX_BLOCK_BYTES", "4194304"), 10, 64) + if maxBytes <= 0 { + maxBytes = 4194304 + } + maxConc, _ := strconv.Atoi(envOr("INGEST_MAX_CONCURRENT", "32")) + if maxConc <= 0 { + maxConc = 32 + } + return config{ + port: envOr("INGEST_PORT", "3601"), + bind: envOr("INGEST_BIND", "0.0.0.0"), + kuboAPI: strings.TrimRight(envOr("KUBO_API", "http://127.0.0.1:5001"), "/"), + storageAPIURL: strings.TrimRight(os.Getenv("STORAGE_API_URL"), "/"), + quotaStrict: envOr("INGEST_QUOTA_MODE", "open") == "strict", + maxBlockBytes: maxBytes, + maxConcurrent: maxConc, + noAuth: envOr("INGEST_NO_AUTH", "0") == "1", + } +} + +// computeBlake3RawCID returns the CIDv1(raw, blake3-256) for body — the same +// construction as fula-client's local_blake3_raw_cid and kubo's +// block/put?cid-codec=raw&mhtype=blake3 (codec 0x55, multihash 0x1e). +func computeBlake3RawCID(body []byte) (cid.Cid, error) { + mh, err := multihash.Sum(body, multihash.BLAKE3, 32) + if err != nil { + return cid.Undef, err + } + return cid.NewCidV1(cid.Raw, mh), nil +} + +// quotaCache: short per-token cache so a multi-thousand-chunk upload performs +// one storage-API call per TTL window, not one per chunk. +type quotaCache struct { + mu sync.Mutex + m map[string]quotaEntry + ttl time.Duration +} + +type quotaEntry struct { + allowed bool + exp time.Time +} + +func newQuotaCache(ttl time.Duration) *quotaCache { + return "aCache{m: make(map[string]quotaEntry), ttl: ttl} +} + +func (q *quotaCache) get(tok string) (bool, bool) { + q.mu.Lock() + defer q.mu.Unlock() + e, ok := q.m[tok] + if !ok || time.Now().After(e.exp) { + return false, false + } + return e.allowed, true +} + +func (q *quotaCache) put(tok string, allowed bool) { + q.mu.Lock() + defer q.mu.Unlock() + q.m[tok] = quotaEntry{allowed: allowed, exp: time.Now().Add(q.ttl)} + // Opportunistic bound: drop expired entries when the map grows. + if len(q.m) > 4096 { + now := time.Now() + for k, e := range q.m { + if now.After(e.exp) { + delete(q.m, k) + } + } + } +} + +type server struct { + cfg config + http *http.Client + quota *quotaCache + sem chan struct{} +} + +func newServer(cfg config) *server { + return &server{ + cfg: cfg, + http: &http.Client{Timeout: 30 * time.Second}, + quota: newQuotaCache(30 * time.Second), + sem: make(chan struct{}, cfg.maxConcurrent), + } +} + +// checkQuota mirrors fula-cli's check_can_upload: GET /api/v1/storage with the +// user's Bearer token => {canUpload}. mode=open fails open (gateway parity); +// mode=strict denies whenever a positive canUpload cannot be obtained. +func (s *server) checkQuota(token string) (bool, string) { + if s.cfg.storageAPIURL == "" { + if s.cfg.quotaStrict { + return false, "quota strict mode but no STORAGE_API_URL configured" + } + return true, "" + } + if allowed, ok := s.quota.get(token); ok { + if !allowed { + return false, "quota exceeded (cached)" + } + return true, "" + } + req, _ := http.NewRequest(http.MethodGet, s.cfg.storageAPIURL+"/api/v1/storage", nil) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := s.http.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + if resp != nil { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + if s.cfg.quotaStrict { + return false, "quota check unavailable (strict mode denies)" + } + return true, "" // fail-open: gateway parity + } + defer resp.Body.Close() + var st struct { + CanUpload bool `json:"canUpload"` + } + if err := json.NewDecoder(resp.Body).Decode(&st); err != nil { + if s.cfg.quotaStrict { + return false, "quota response unparseable (strict mode denies)" + } + return true, "" + } + s.quota.put(token, st.CanUpload) + if !st.CanUpload { + return false, "insufficient credits or quota exceeded" + } + return true, "" +} + +// kuboBlockPut stores body via kubo block/put with blake3 raw addressing and +// returns the key kubo computed. +func (s *server) kuboBlockPut(ctx context.Context, body []byte) (string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + part, err := w.CreateFormFile("data", "blob") + if err != nil { + return "", err + } + if _, err := part.Write(body); err != nil { + return "", err + } + w.Close() + + url := s.cfg.kuboAPI + "/api/v0/block/put?cid-codec=raw&mhtype=blake3&mhlen=32&pin=false" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &buf) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", w.FormDataContentType()) + resp, err := s.http.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + rb, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("kubo block/put status %d: %s", resp.StatusCode, strings.TrimSpace(string(rb))) + } + var out struct { + Key string `json:"Key"` + } + if err := json.Unmarshal(rb, &out); err != nil { + return "", fmt.Errorf("kubo block/put bad response: %w", err) + } + return out.Key, nil +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + json.NewEncoder(w).Encode(v) +} + +func (s *server) handleBlockPut(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut && r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "use PUT"}) + return + } + + // Bounded concurrency — a thundering herd of chunks queues here instead of + // exhausting kubo or memory. + s.sem <- struct{}{} + defer func() { <-s.sem }() + + declared := r.URL.Query().Get("cid") + if declared == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "missing ?cid= (declared blake3 raw CID)"}) + return + } + declaredCid, err := cid.Decode(declared) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid cid: " + err.Error()}) + return + } + + // Auth + S1 quota gate BEFORE reading the full body where possible. + token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if !s.cfg.noAuth { + if token == "" || token == r.Header.Get("Authorization") { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "missing Bearer token"}) + return + } + if ok, reason := s.checkQuota(token); !ok { + writeJSON(w, http.StatusPaymentRequired, map[string]string{"error": reason}) + return + } + } + + body, err := io.ReadAll(io.LimitReader(r.Body, s.cfg.maxBlockBytes+1)) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "read body: " + err.Error()}) + return + } + if int64(len(body)) > s.cfg.maxBlockBytes { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": fmt.Sprintf("block exceeds %d bytes", s.cfg.maxBlockBytes)}) + return + } + if len(body) == 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "empty body"}) + return + } + + // TAMPER-EVIDENCE: verify the declared CID over the received bytes BEFORE + // anything touches the blockstore. A single flipped bit => 422, not stored. + actual, err := computeBlake3RawCID(body) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "hash: " + err.Error()}) + return + } + if !actual.Equals(declaredCid) { + writeJSON(w, http.StatusUnprocessableEntity, map[string]string{ + "error": "cid mismatch: body does not hash to the declared cid (tampered or corrupted)", + "declared": declaredCid.String(), + "actual": actual.String(), + }) + return + } + + key, err := s.kuboBlockPut(r.Context(), body) + if err != nil { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": "kubo: " + err.Error()}) + return + } + // Belt and suspenders: kubo's independently computed key must agree. + if kc, err := cid.Decode(key); err != nil || !kc.Equals(declaredCid) { + writeJSON(w, http.StatusInternalServerError, map[string]string{ + "error": "kubo key disagrees with declared cid", "kubo": key, "declared": declaredCid.String(), + }) + return + } + + writeJSON(w, http.StatusOK, map[string]any{"cid": declaredCid.String(), "size": len(body)}) +} + +func (s *server) handleHealth(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.kuboAPI+"/api/v0/id", nil) + resp, err := s.http.Do(req) + kubo := err == nil && resp.StatusCode == http.StatusOK + if resp != nil { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + code := http.StatusOK + if !kubo { + code = http.StatusServiceUnavailable + } + writeJSON(w, code, map[string]any{"status": "ok", "kubo": kubo}) +} + +func main() { + cfg := loadConfig() + s := newServer(cfg) + + mux := http.NewServeMux() + mux.HandleFunc("/v0/block", s.handleBlockPut) + mux.HandleFunc("/health", s.handleHealth) + + srv := &http.Server{ + Addr: cfg.bind + ":" + cfg.port, + Handler: mux, + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, + } + + go func() { + log.Printf("[fula-ingest] listening on %s (kubo=%s, quota=%s, max-block=%d, no-auth=%v)", + srv.Addr, cfg.kuboAPI, map[bool]string{true: "strict", false: "open"}[cfg.quotaStrict], cfg.maxBlockBytes, cfg.noAuth) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("[fula-ingest] server error: %v", err) + } + }() + + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _ = srv.Shutdown(ctx) + log.Printf("[fula-ingest] stopped") +} diff --git a/docker/fula-ingest/main_test.go b/docker/fula-ingest/main_test.go new file mode 100644 index 00000000..cd5e47aa --- /dev/null +++ b/docker/fula-ingest/main_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +// mockKubo returns an httptest server that computes the real blake3 raw CID +// for received block/put bodies (exactly what kubo does with +// cid-codec=raw&mhtype=blake3) and counts calls. +func mockKubo(t *testing.T, calls *int64) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/api/v0/block/put"): + atomic.AddInt64(calls, 1) + if err := r.ParseMultipartForm(8 << 20); err != nil { + http.Error(w, err.Error(), 400) + return + } + f, _, err := r.FormFile("data") + if err != nil { + http.Error(w, err.Error(), 400) + return + } + var buf bytes.Buffer + buf.ReadFrom(f) + c, err := computeBlake3RawCID(buf.Bytes()) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + json.NewEncoder(w).Encode(map[string]any{"Key": c.String(), "Size": buf.Len()}) + case strings.HasPrefix(r.URL.Path, "/api/v0/id"): + json.NewEncoder(w).Encode(map[string]string{"ID": "12D3KooWMockKubo"}) + default: + http.NotFound(w, r) + } + })) +} + +// mockStorage returns a storage-API mock answering /api/v1/storage. +func mockStorage(canUpload bool, status int) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/storage" { + http.NotFound(w, r) + return + } + if status != http.StatusOK { + w.WriteHeader(status) + return + } + json.NewEncoder(w).Encode(map[string]any{"canUpload": canUpload}) + })) +} + +func newTestServer(kubo, storage string, strict, noAuth bool) *server { + cfg := config{ + kuboAPI: kubo, + storageAPIURL: storage, + quotaStrict: strict, + maxBlockBytes: 1 << 20, + maxConcurrent: 4, + noAuth: noAuth, + } + return newServer(cfg) +} + +func doPut(s *server, cid string, body []byte, token string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPut, "/v0/block?cid="+cid, bytes.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rr := httptest.NewRecorder() + s.handleBlockPut(rr, req) + return rr +} + +func TestValidBlockAcceptedAndStored(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + s := newTestServer(k.URL, "", false, true) + + body := []byte("fula-ingest-valid-chunk") + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "") + if rr.Code != 200 { + t.Fatalf("want 200, got %d: %s", rr.Code, rr.Body.String()) + } + if atomic.LoadInt64(&kuboCalls) != 1 { + t.Fatalf("kubo should be called exactly once, got %d", kuboCalls) + } + var resp map[string]any + json.Unmarshal(rr.Body.Bytes(), &resp) + if resp["cid"] != c.String() { + t.Fatalf("response cid mismatch: %v", resp) + } +} + +func TestTamperedByteRejectedNotStored(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + s := newTestServer(k.URL, "", false, true) + + body := []byte("fula-ingest-original-bytes") + c, _ := computeBlake3RawCID(body) + tampered := append([]byte{}, body...) + tampered[3] ^= 0x01 // single flipped bit + rr := doPut(s, c.String(), tampered, "") + if rr.Code != 422 { + t.Fatalf("want 422 for tampered body, got %d: %s", rr.Code, rr.Body.String()) + } + if atomic.LoadInt64(&kuboCalls) != 0 { + t.Fatalf("tampered block must NOT reach the blockstore (kubo calls=%d)", kuboCalls) + } +} + +func TestMissingOrInvalidCid(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + s := newTestServer(k.URL, "", false, true) + + rr := doPut(s, "", []byte("x"), "") + if rr.Code != 400 { + t.Fatalf("missing cid: want 400, got %d", rr.Code) + } + rr = doPut(s, "not-a-cid", []byte("x"), "") + if rr.Code != 400 { + t.Fatalf("invalid cid: want 400, got %d", rr.Code) + } +} + +func TestOversizeRejected(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + s := newTestServer(k.URL, "", false, true) + s.cfg.maxBlockBytes = 64 + + body := bytes.Repeat([]byte("A"), 65) + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "") + if rr.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("want 413, got %d", rr.Code) + } +} + +func TestAuthRequiredWhenEnabled(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + s := newTestServer(k.URL, "", false, false) // auth ON + + body := []byte("needs-auth") + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "") + if rr.Code != http.StatusUnauthorized { + t.Fatalf("want 401 without token, got %d", rr.Code) + } +} + +func TestQuotaDeniedBlocksIngestion(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + st := mockStorage(false, http.StatusOK) // canUpload=false + defer st.Close() + s := newTestServer(k.URL, st.URL, false, false) + + body := []byte("over-quota-user") + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "user-jwt") + if rr.Code != http.StatusPaymentRequired { + t.Fatalf("want 402 for quota-denied, got %d: %s", rr.Code, rr.Body.String()) + } + if atomic.LoadInt64(&kuboCalls) != 0 { + t.Fatalf("quota-denied upload must not store anything") + } +} + +func TestQuotaFailOpenMatchesGateway(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + st := mockStorage(false, http.StatusInternalServerError) // storage API down + defer st.Close() + s := newTestServer(k.URL, st.URL, false, false) // mode=open + + body := []byte("storage-api-down-open-mode") + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "user-jwt") + if rr.Code != 200 { + t.Fatalf("open mode must fail open like the gateway: want 200, got %d", rr.Code) + } +} + +func TestQuotaStrictDeniesWhenUnavailable(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + st := mockStorage(false, http.StatusInternalServerError) + defer st.Close() + s := newTestServer(k.URL, st.URL, true, false) // mode=strict + + body := []byte("storage-api-down-strict-mode") + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "user-jwt") + if rr.Code != http.StatusPaymentRequired { + t.Fatalf("strict mode must deny when quota unverifiable: want 402, got %d", rr.Code) + } + if atomic.LoadInt64(&kuboCalls) != 0 { + t.Fatalf("strict-denied upload must not store anything") + } +} + +func TestQuotaCacheOneCallPerWindow(t *testing.T) { + var kuboCalls int64 + var storageCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + st := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&storageCalls, 1) + json.NewEncoder(w).Encode(map[string]any{"canUpload": true}) + })) + defer st.Close() + s := newTestServer(k.URL, st.URL, false, false) + + for i := 0; i < 5; i++ { + body := []byte(fmt.Sprintf("chunk-%d", i)) + c, _ := computeBlake3RawCID(body) + rr := doPut(s, c.String(), body, "same-jwt") + if rr.Code != 200 { + t.Fatalf("chunk %d: want 200, got %d", i, rr.Code) + } + } + if atomic.LoadInt64(&storageCalls) != 1 { + t.Fatalf("quota cache: want exactly 1 storage-API call for 5 chunks, got %d", storageCalls) + } +} + +func TestHealth(t *testing.T) { + var kuboCalls int64 + k := mockKubo(t, &kuboCalls) + defer k.Close() + s := newTestServer(k.URL, "", false, true) + + req := httptest.NewRequest(http.MethodGet, "/health", nil) + rr := httptest.NewRecorder() + s.handleHealth(rr, req) + if rr.Code != 200 { + t.Fatalf("health: want 200, got %d", rr.Code) + } +} From 81a5cff2a8b3cf7e3cc270bcd33b77adc24fd80b Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Thu, 11 Jun 2026 23:08:08 -0400 Subject: [PATCH 02/19] test(e2e): Phase 2 ingest drills - tamper, quota-strict, gateway-down, live mapping PUT I1-I10 on the test master: image build+run (strict quota vs the live webui storage API via a seeded legacy api key); valid block stored; tampered body 422 + never stored; suspended user 402 + never stored; missing token 401; bytes still ingested with the gateway container STOPPED; gateway rebuilt from the phase-2 branch flips /fula/capabilities false->true; live empty-body remote-cid mapping PUT returns ETag=cid and GET round-trips the exact ingested bytes; absent cid rejected (client-fallback contract). Part of #74 Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 150 ++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 tests/e2e/phase-2/40-ingest-drills.sh diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh new file mode 100644 index 00000000..3f83e467 --- /dev/null +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# +# Phase 2 e2e drills — verified byte ingress (TEST SERVER ONLY). +# Requires: Stage-A master stack up (join-as-master), fxe2e writer kubo on +# :5001, /root/fula-ota on phase-2-ingest, /root/fula-api on +# phase-2-client-ingest. Run as root from anywhere. +# +# I1 build + run fula-ingest (strict quota, auth ON) +# I2 health +# I3 valid block accepted + stored in kubo +# I4 tampered body -> 422 AND not stored +# I5 suspended user -> 402 AND not stored (S1, strict) +# I6 missing token -> 401 +# I7 gateway container STOPPED -> ingest still ingests bytes +# I8 rebuild gateway from phase-2 branch; /fula/capabilities flips false->true +# I9 live remote-cid mapping PUT: empty body + header -> 200, ETag=cid, +# GET returns the ingested bytes end-to-end +# I10 mapping PUT for an ABSENT cid -> 409 (client-fallback contract) +# +set -uo pipefail +PASS=0; FAIL=0 +ok() { echo "ok - $1"; PASS=$((PASS+1)); } +bad() { echo "FAIL - $1"; FAIL=$((FAIL+1)); } +. /opt/fula-master/.env +psqlc() { docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user}" -d "${POSTGRES_DB:-pinning_service}" -tA -c "$1"; } + +U="e2e-drill-user"; EMAIL="e2e-drill@fxe2e.local"; APIKEY="fxe2e-ingest-drill-key-001" +ING=http://127.0.0.1:3601 + +# blake3 raw-leaf CID of a file, computed by kubo itself (no store). +cid_of() { docker exec -i ipfs_host ipfs add --only-hash --raw-leaves --hash=blake3 -q < "$1"; } + +echo "== I1 build + run fula-ingest ==" +cd /root/fula-ota && git fetch origin phase-2-ingest -q && git checkout -q phase-2-ingest && git pull -q +docker build -q -f docker/fula-ingest/Dockerfile -t functionland/fula-ingest:e2e docker/fula-ingest >/dev/null \ + && ok "I1 image built" || { bad "I1 image build failed"; exit 1; } +docker stop fula-ingest-e2e >/dev/null 2>&1; docker rm fula-ingest-e2e >/dev/null 2>&1 +docker run -d --name fula-ingest-e2e --network host \ + -e INGEST_BIND=127.0.0.1 -e INGEST_PORT=3601 \ + -e KUBO_API=http://127.0.0.1:5001 \ + -e STORAGE_API_URL=http://127.0.0.1:3001 \ + -e INGEST_QUOTA_MODE=strict \ + functionland/fula-ingest:e2e >/dev/null +sleep 3 + +echo "== seed drill identity (webui_users + legacy api_key + healthy credits) ==" +psqlc "INSERT INTO webui_users (email) VALUES ('$EMAIL') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true +psqlc "INSERT INTO api_keys (key_id, user_email, user_id) VALUES ('$APIKEY', '$EMAIL', '$U') ON CONFLICT (key_id) DO UPDATE SET is_deleted=0, user_id='$U'" >/dev/null \ + && ok "seeded api key" || bad "could not seed api key" +psqlc "UPDATE user_credits SET is_suspended=0, balance_fula=50 WHERE user_id='$U'" >/dev/null + +echo "== I2 health ==" +curl -s -m 5 "$ING/health" | grep -q '"kubo":true' && ok "I2 ingest healthy (kubo reachable)" || bad "I2 health failed" + +echo "== I3 valid block accepted + stored ==" +F1=/tmp/p2-valid.bin; head -c 2048 /dev/urandom > "$F1" +C1="$(cid_of "$F1")" +code=$(curl -s -m 15 -o /tmp/p2-r1.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C1" -H "Authorization: Bearer $APIKEY" --data-binary @"$F1") +[ "$code" = 200 ] && ok "I3 valid block -> 200" || bad "I3 got $code: $(cat /tmp/p2-r1.json)" +docker exec ipfs_host ipfs block stat "$C1" >/dev/null 2>&1 && ok "I3 block present in kubo" || bad "I3 block missing from kubo" + +echo "== I4 tampered body -> 422, not stored ==" +F2=/tmp/p2-orig.bin; head -c 2048 /dev/urandom > "$F2" +C2="$(cid_of "$F2")" +F2T=/tmp/p2-tampered.bin; cp "$F2" "$F2T"; printf 'X' | dd of="$F2T" bs=1 seek=10 count=1 conv=notrunc 2>/dev/null +C2T="$(cid_of "$F2T")" # true cid of the tampered bytes — must NOT appear in kubo +code=$(curl -s -m 15 -o /tmp/p2-r2.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C2" -H "Authorization: Bearer $APIKEY" --data-binary @"$F2T") +[ "$code" = 422 ] && ok "I4 tampered -> 422" || bad "I4 got $code: $(cat /tmp/p2-r2.json)" +docker exec ipfs_host ipfs block stat --offline "$C2T" >/dev/null 2>&1 && bad "I4 tampered bytes WERE stored" || ok "I4 tampered bytes not stored" + +echo "== I5 suspended user -> 402, not stored (strict S1) ==" +psqlc "UPDATE user_credits SET is_suspended=1 WHERE user_id='$U'" >/dev/null +sleep 31 # outlive the ingest quota cache TTL (30s) +F3=/tmp/p2-quota.bin; head -c 2048 /dev/urandom > "$F3" +C3="$(cid_of "$F3")" +code=$(curl -s -m 15 -o /tmp/p2-r3.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C3" -H "Authorization: Bearer $APIKEY" --data-binary @"$F3") +[ "$code" = 402 ] && ok "I5 suspended -> 402" || bad "I5 got $code: $(cat /tmp/p2-r3.json)" +docker exec ipfs_host ipfs block stat --offline "$C3" >/dev/null 2>&1 && bad "I5 quota-denied bytes WERE stored" || ok "I5 quota-denied bytes not stored" +psqlc "UPDATE user_credits SET is_suspended=0 WHERE user_id='$U'" >/dev/null + +echo "== I6 missing token -> 401 ==" +code=$(curl -s -m 5 -o /tmp/p2-r4.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C3" --data-binary @"$F3") +[ "$code" = 401 ] && ok "I6 no token -> 401" || bad "I6 got $code" + +echo "== I7 gateway DOWN -> ingest still ingests ==" +docker stop fula-gateway-1 >/dev/null 2>&1 +sleep 31 # fresh quota window so the check runs while gateway is down (webui still up — quota is webui's) +F4=/tmp/p2-gwdown.bin; head -c 2048 /dev/urandom > "$F4" +C4="$(cid_of "$F4")" +code=$(curl -s -m 15 -o /tmp/p2-r5.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C4" -H "Authorization: Bearer $APIKEY" --data-binary @"$F4") +[ "$code" = 200 ] && ok "I7 bytes ingested with the gateway STOPPED" || bad "I7 got $code: $(cat /tmp/p2-r5.json)" +docker start fula-gateway-1 >/dev/null 2>&1 + +echo "== I8 rebuild gateway from phase-2 branch; capabilities flips ==" +code=$(curl -s -m 5 -o /tmp/p2-cap0.json -w "%{http_code}" http://127.0.0.1:9000/fula/capabilities) +grep -q '"remoteCidPut":true' /tmp/p2-cap0.json 2>/dev/null && bad "I8 pre-rebuild gateway already advertises (unexpected)" || ok "I8 old gateway does not advertise remoteCidPut (code=$code)" +cd /root/fula-api && git fetch origin phase-2-client-ingest -q && git checkout -q phase-2-client-ingest && git pull -q +docker build -q -f docker/Dockerfile.gateway -t fula-gateway:latest . >/dev/null && ok "I8 gateway image rebuilt from phase-2 branch" || { bad "I8 gateway rebuild failed"; exit 1; } +cat > /opt/fula-master/fula-gateway.env </dev/null 2>&1 +for i in $(seq 1 20); do curl -s -m 3 http://127.0.0.1:9000/fula/capabilities | grep -q remoteCidPut && break; sleep 3; done +curl -s -m 5 http://127.0.0.1:9000/fula/capabilities | grep -q '"remoteCidPut":true' \ + && ok "I8 new gateway advertises remoteCidPut:true" || bad "I8 capabilities did not flip" + +echo "== I9 live remote-cid mapping PUT (empty body) -> 200 + GET round-trip ==" +JWT=$(python3 - "$JWT_SECRET" <<'PYEOF' +import sys, hmac, hashlib, base64, json, time +def b64u(b): return base64.urlsafe_b64encode(b).rstrip(b"=") +secret = sys.argv[1].encode() +h = b64u(json.dumps({"alg":"HS256","typ":"JWT"}).encode()) +p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","iat":int(time.time()),"exp":int(time.time())+3600}).encode()) +sig = b64u(hmac.new(secret, h+b"."+p, hashlib.sha256).digest()) +print((h+b"."+p+b"."+sig).decode()) +PYEOF +) +[ -n "$JWT" ] && ok "I9 minted gateway JWT" || bad "I9 JWT mint failed" +code=$(curl -s -m 20 -o /tmp/p2-map.txt -D /tmp/p2-map-h.txt -w "%{http_code}" -X PUT \ + "http://127.0.0.1:9000/p2-drill-bucket/chunk-0001" \ + -H "Authorization: Bearer $JWT" \ + -H "x-amz-meta-fula-remote-cid: $C1" \ + -H "x-amz-meta-fula-remote-size: 2048" \ + -H "Content-Length: 0") +[ "$code" = 200 ] && ok "I9 mapping PUT -> 200" || bad "I9 mapping PUT got $code: $(head -c200 /tmp/p2-map.txt)" +grep -qi "etag.*$C1" /tmp/p2-map-h.txt && ok "I9 ETag echoes the declared cid" || bad "I9 ETag mismatch: $(grep -i etag /tmp/p2-map-h.txt)" +body=$(curl -s -m 20 -H "Authorization: Bearer $JWT" "http://127.0.0.1:9000/p2-drill-bucket/chunk-0001" -o /tmp/p2-get.bin -w "%{http_code}") +if [ "$body" = 200 ] && cmp -s /tmp/p2-get.bin "$F1"; then ok "I9 GET returns the exact ingested bytes (end-to-end)"; else bad "I9 GET round-trip failed (code=$body)"; fi + +echo "== I10 mapping PUT for ABSENT cid -> 409 ==" +FA=/tmp/p2-absent.bin; head -c 1024 /dev/urandom > "$FA" +CA="$(cid_of "$FA")" # never uploaded anywhere +code=$(curl -s -m 30 -o /tmp/p2-abs.txt -w "%{http_code}" -X PUT \ + "http://127.0.0.1:9000/p2-drill-bucket/chunk-absent" \ + -H "Authorization: Bearer $JWT" \ + -H "x-amz-meta-fula-remote-cid: $CA" \ + -H "x-amz-meta-fula-remote-size: 1024" \ + -H "Content-Length: 0") +case "$code" in 409|4*) ok "I10 absent cid rejected (code=$code — client falls back to full bytes)";; *) bad "I10 got $code: $(head -c200 /tmp/p2-abs.txt)";; esac + +echo +echo "RESULT: pass=$PASS fail=$FAIL" +[ "$FAIL" = 0 ] || exit 1 From 64bd3a3ee935daa82737238a8299c784da8fab2b Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Thu, 11 Jun 2026 23:12:24 -0400 Subject: [PATCH 03/19] test(e2e): Phase 2 fidelity runner - live ingest round-trips, v8 on/off, FxFiles flow, 1 GiB Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/60-fidelity.sh | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/e2e/phase-2/60-fidelity.sh diff --git a/tests/e2e/phase-2/60-fidelity.sh b/tests/e2e/phase-2/60-fidelity.sh new file mode 100644 index 00000000..ad7b25de --- /dev/null +++ b/tests/e2e/phase-2/60-fidelity.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Phase 2 fidelity suite (TEST SERVER ONLY). Runs AFTER 40-ingest-drills.sh +# (gateway rebuilt with FULA_REMOTE_CID_PUT=true; fula-ingest-e2e running). +# +# F1 live ingest round-trip (client CID ON, bytes via ingest) + server-side +# proof: the ingest container's kubo-stored block count INCREASES +# F2 v8-OFF legacy round-trip (client CID off matrix leg) +# F3 FxFiles-faithful suite: offline_e2e single + chunked upload/download +# (the byte-for-byte FxFiles flow, legacy path) +# F4 ≥1 GiB chunked via ingest (FULA_BIG=1; scale invariant) +# +set -uo pipefail +PASS=0; FAIL=0 +ok() { echo "ok - $1"; PASS=$((PASS+1)); } +bad() { echo "FAIL - $1"; FAIL=$((FAIL+1)); } +. /opt/fula-master/.env + +JWT=$(python3 - "$JWT_SECRET" <<'PYEOF' +import sys, hmac, hashlib, base64, json, time +def b64u(b): return base64.urlsafe_b64encode(b).rstrip(b"=") +secret = sys.argv[1].encode() +h = b64u(json.dumps({"alg":"HS256","typ":"JWT"}).encode()) +p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","iat":int(time.time()),"exp":int(time.time())+7200}).encode()) +sig = b64u(hmac.new(secret, h+b"."+p, hashlib.sha256).digest()) +print((h+b"."+p+b"."+sig).decode()) +PYEOF +) +[ -n "$JWT" ] && ok "minted 2h gateway JWT" || { bad "JWT mint failed"; exit 1; } + +cd /root/fula-api && git pull -q + +run_tests() { # $1=extra-env $2=test-filter + docker run --rm --network host -v /root/fula-api:/src \ + -v fula-cargo-cache:/usr/local/cargo/registry -v fula-cargo-cache-target:/src/target \ + -w /src -e CARGO_TERM_COLOR=never \ + -e FULA_S3=http://127.0.0.1:9000 -e FULA_JWT="$JWT" $1 \ + rust:1-bookworm bash -c "set -o pipefail; cargo test -p fula-client --release --test $2 -- --ignored --nocapture 2>&1 | tail -8" +} + +echo "== F1 live ingest round-trip + server-side block-count proof ==" +BLOCKS_BEFORE=$(docker exec ipfs_host ipfs repo stat 2>/dev/null | awk '/NumObjects/{print $2}') +if run_tests "-e FULA_INGEST=http://127.0.0.1:3601" "live_ingest_e2e live_chunked_via_ingest_round_trip"; then + ok "F1 client round-trip via ingest" +else + bad "F1 client round-trip failed" +fi +BLOCKS_AFTER=$(docker exec ipfs_host ipfs repo stat 2>/dev/null | awk '/NumObjects/{print $2}') +[ "${BLOCKS_AFTER:-0}" -gt "${BLOCKS_BEFORE:-0}" ] \ + && ok "F1 kubo block count grew ($BLOCKS_BEFORE -> $BLOCKS_AFTER)" \ + || bad "F1 no new blocks landed" + +echo "== F2 v8-off legacy round-trip (CID-off matrix leg) ==" +if run_tests "" "live_ingest_e2e live_chunked_v8_off_legacy_round_trip"; then + ok "F2 legacy round-trip (v8 off)" +else + bad "F2 legacy round-trip failed" +fi + +echo "== F3 FxFiles-faithful offline_e2e (single + chunked) ==" +if run_tests "" "offline_e2e offline_upload_download_single_object_e2e"; then + ok "F3 FxFiles single-object flow" +else + bad "F3 single-object flow failed" +fi +if run_tests "" "offline_e2e offline_upload_download_chunked_e2e"; then + ok "F3 FxFiles chunked flow" +else + bad "F3 chunked flow failed" +fi + +echo "== F4 1 GiB chunked via ingest (scale invariant) ==" +if run_tests "-e FULA_INGEST=http://127.0.0.1:3601 -e FULA_BIG=1" "live_ingest_e2e live_1gib_chunked_via_ingest"; then + ok "F4 1 GiB via ingest round-trip" +else + bad "F4 1 GiB failed" +fi + +echo +echo "RESULT: pass=$PASS fail=$FAIL" +[ "$FAIL" = 0 ] || exit 1 From 4be7ea1717d9ae82056c78b5dd57f140d4abe99d Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Thu, 11 Jun 2026 23:14:41 -0400 Subject: [PATCH 04/19] fix(e2e): seed webui_users.user_id - api_keys FK requires it Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index 3f83e467..8c2923b2 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -44,7 +44,8 @@ docker run -d --name fula-ingest-e2e --network host \ sleep 3 echo "== seed drill identity (webui_users + legacy api_key + healthy credits) ==" -psqlc "INSERT INTO webui_users (email) VALUES ('$EMAIL') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true +# api_keys.user_id has an FK to webui_users.user_id — seed both columns. +psqlc "INSERT INTO webui_users (email, user_id) VALUES ('$EMAIL', '$U') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true psqlc "INSERT INTO api_keys (key_id, user_email, user_id) VALUES ('$APIKEY', '$EMAIL', '$U') ON CONFLICT (key_id) DO UPDATE SET is_deleted=0, user_id='$U'" >/dev/null \ && ok "seeded api key" || bad "could not seed api key" psqlc "UPDATE user_credits SET is_suspended=0, balance_fula=50 WHERE user_id='$U'" >/dev/null From 9a67f854d17cf730c6c62f0c92a24c8d6e312ec3 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Thu, 11 Jun 2026 23:42:43 -0400 Subject: [PATCH 05/19] fix(e2e): minted JWTs need scope=storage:* (gateway can_write gate) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 14 +++++++------- tests/e2e/phase-2/60-fidelity.sh | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index 8c2923b2..a562e291 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash # -# Phase 2 e2e drills — verified byte ingress (TEST SERVER ONLY). +# Phase 2 e2e drills — verified byte ingress (TEST SERVER ONLY). # Requires: Stage-A master stack up (join-as-master), fxe2e writer kubo on # :5001, /root/fula-ota on phase-2-ingest, /root/fula-api on # phase-2-client-ingest. Run as root from anywhere. @@ -44,7 +44,7 @@ docker run -d --name fula-ingest-e2e --network host \ sleep 3 echo "== seed drill identity (webui_users + legacy api_key + healthy credits) ==" -# api_keys.user_id has an FK to webui_users.user_id — seed both columns. +# api_keys.user_id has an FK to webui_users.user_id — seed both columns. psqlc "INSERT INTO webui_users (email, user_id) VALUES ('$EMAIL', '$U') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true psqlc "INSERT INTO api_keys (key_id, user_email, user_id) VALUES ('$APIKEY', '$EMAIL', '$U') ON CONFLICT (key_id) DO UPDATE SET is_deleted=0, user_id='$U'" >/dev/null \ && ok "seeded api key" || bad "could not seed api key" @@ -64,7 +64,7 @@ echo "== I4 tampered body -> 422, not stored ==" F2=/tmp/p2-orig.bin; head -c 2048 /dev/urandom > "$F2" C2="$(cid_of "$F2")" F2T=/tmp/p2-tampered.bin; cp "$F2" "$F2T"; printf 'X' | dd of="$F2T" bs=1 seek=10 count=1 conv=notrunc 2>/dev/null -C2T="$(cid_of "$F2T")" # true cid of the tampered bytes — must NOT appear in kubo +C2T="$(cid_of "$F2T")" # true cid of the tampered bytes — must NOT appear in kubo code=$(curl -s -m 15 -o /tmp/p2-r2.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C2" -H "Authorization: Bearer $APIKEY" --data-binary @"$F2T") [ "$code" = 422 ] && ok "I4 tampered -> 422" || bad "I4 got $code: $(cat /tmp/p2-r2.json)" docker exec ipfs_host ipfs block stat --offline "$C2T" >/dev/null 2>&1 && bad "I4 tampered bytes WERE stored" || ok "I4 tampered bytes not stored" @@ -85,7 +85,7 @@ code=$(curl -s -m 5 -o /tmp/p2-r4.json -w "%{http_code}" -X PUT "$ING/v0/block?c echo "== I7 gateway DOWN -> ingest still ingests ==" docker stop fula-gateway-1 >/dev/null 2>&1 -sleep 31 # fresh quota window so the check runs while gateway is down (webui still up — quota is webui's) +sleep 31 # fresh quota window so the check runs while gateway is down (webui still up — quota is webui's) F4=/tmp/p2-gwdown.bin; head -c 2048 /dev/urandom > "$F4" C4="$(cid_of "$F4")" code=$(curl -s -m 15 -o /tmp/p2-r5.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C4" -H "Authorization: Bearer $APIKEY" --data-binary @"$F4") @@ -118,7 +118,7 @@ import sys, hmac, hashlib, base64, json, time def b64u(b): return base64.urlsafe_b64encode(b).rstrip(b"=") secret = sys.argv[1].encode() h = b64u(json.dumps({"alg":"HS256","typ":"JWT"}).encode()) -p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","iat":int(time.time()),"exp":int(time.time())+3600}).encode()) +p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","scope":"storage:*","iat":int(time.time()),"exp":int(time.time())+3600}).encode()) sig = b64u(hmac.new(secret, h+b"."+p, hashlib.sha256).digest()) print((h+b"."+p+b"."+sig).decode()) PYEOF @@ -144,7 +144,7 @@ code=$(curl -s -m 30 -o /tmp/p2-abs.txt -w "%{http_code}" -X PUT \ -H "x-amz-meta-fula-remote-cid: $CA" \ -H "x-amz-meta-fula-remote-size: 1024" \ -H "Content-Length: 0") -case "$code" in 409|4*) ok "I10 absent cid rejected (code=$code — client falls back to full bytes)";; *) bad "I10 got $code: $(head -c200 /tmp/p2-abs.txt)";; esac +case "$code" in 409|4*) ok "I10 absent cid rejected (code=$code — client falls back to full bytes)";; *) bad "I10 got $code: $(head -c200 /tmp/p2-abs.txt)";; esac echo echo "RESULT: pass=$PASS fail=$FAIL" diff --git a/tests/e2e/phase-2/60-fidelity.sh b/tests/e2e/phase-2/60-fidelity.sh index ad7b25de..ecbdde80 100644 --- a/tests/e2e/phase-2/60-fidelity.sh +++ b/tests/e2e/phase-2/60-fidelity.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash # # Phase 2 fidelity suite (TEST SERVER ONLY). Runs AFTER 40-ingest-drills.sh # (gateway rebuilt with FULA_REMOTE_CID_PUT=true; fula-ingest-e2e running). @@ -8,7 +8,7 @@ # F2 v8-OFF legacy round-trip (client CID off matrix leg) # F3 FxFiles-faithful suite: offline_e2e single + chunked upload/download # (the byte-for-byte FxFiles flow, legacy path) -# F4 ≥1 GiB chunked via ingest (FULA_BIG=1; scale invariant) +# F4 ≥1 GiB chunked via ingest (FULA_BIG=1; scale invariant) # set -uo pipefail PASS=0; FAIL=0 @@ -21,7 +21,7 @@ import sys, hmac, hashlib, base64, json, time def b64u(b): return base64.urlsafe_b64encode(b).rstrip(b"=") secret = sys.argv[1].encode() h = b64u(json.dumps({"alg":"HS256","typ":"JWT"}).encode()) -p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","iat":int(time.time()),"exp":int(time.time())+7200}).encode()) +p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","scope":"storage:*","iat":int(time.time()),"exp":int(time.time())+7200}).encode()) sig = b64u(hmac.new(secret, h+b"."+p, hashlib.sha256).digest()) print((h+b"."+p+b"."+sig).decode()) PYEOF From 787c68a2a7923b3a9d42f004ce604a47e4785095 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Thu, 11 Jun 2026 23:52:51 -0400 Subject: [PATCH 06/19] fix(e2e): restore clean encoding for phase-2 scripts (scope fix reapplied) The previous commit re-encoded both files with a BOM + mojibake comments (PowerShell 5.1 round-trip). Restored from the clean parent and reapplied the scope=storage:* JWT claim via a targeted edit. Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 12 ++++++------ tests/e2e/phase-2/60-fidelity.sh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index a562e291..836e9cea 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash # -# Phase 2 e2e drills — verified byte ingress (TEST SERVER ONLY). +# Phase 2 e2e drills — verified byte ingress (TEST SERVER ONLY). # Requires: Stage-A master stack up (join-as-master), fxe2e writer kubo on # :5001, /root/fula-ota on phase-2-ingest, /root/fula-api on # phase-2-client-ingest. Run as root from anywhere. @@ -44,7 +44,7 @@ docker run -d --name fula-ingest-e2e --network host \ sleep 3 echo "== seed drill identity (webui_users + legacy api_key + healthy credits) ==" -# api_keys.user_id has an FK to webui_users.user_id — seed both columns. +# api_keys.user_id has an FK to webui_users.user_id — seed both columns. psqlc "INSERT INTO webui_users (email, user_id) VALUES ('$EMAIL', '$U') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true psqlc "INSERT INTO api_keys (key_id, user_email, user_id) VALUES ('$APIKEY', '$EMAIL', '$U') ON CONFLICT (key_id) DO UPDATE SET is_deleted=0, user_id='$U'" >/dev/null \ && ok "seeded api key" || bad "could not seed api key" @@ -64,7 +64,7 @@ echo "== I4 tampered body -> 422, not stored ==" F2=/tmp/p2-orig.bin; head -c 2048 /dev/urandom > "$F2" C2="$(cid_of "$F2")" F2T=/tmp/p2-tampered.bin; cp "$F2" "$F2T"; printf 'X' | dd of="$F2T" bs=1 seek=10 count=1 conv=notrunc 2>/dev/null -C2T="$(cid_of "$F2T")" # true cid of the tampered bytes — must NOT appear in kubo +C2T="$(cid_of "$F2T")" # true cid of the tampered bytes — must NOT appear in kubo code=$(curl -s -m 15 -o /tmp/p2-r2.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C2" -H "Authorization: Bearer $APIKEY" --data-binary @"$F2T") [ "$code" = 422 ] && ok "I4 tampered -> 422" || bad "I4 got $code: $(cat /tmp/p2-r2.json)" docker exec ipfs_host ipfs block stat --offline "$C2T" >/dev/null 2>&1 && bad "I4 tampered bytes WERE stored" || ok "I4 tampered bytes not stored" @@ -85,7 +85,7 @@ code=$(curl -s -m 5 -o /tmp/p2-r4.json -w "%{http_code}" -X PUT "$ING/v0/block?c echo "== I7 gateway DOWN -> ingest still ingests ==" docker stop fula-gateway-1 >/dev/null 2>&1 -sleep 31 # fresh quota window so the check runs while gateway is down (webui still up — quota is webui's) +sleep 31 # fresh quota window so the check runs while gateway is down (webui still up — quota is webui's) F4=/tmp/p2-gwdown.bin; head -c 2048 /dev/urandom > "$F4" C4="$(cid_of "$F4")" code=$(curl -s -m 15 -o /tmp/p2-r5.json -w "%{http_code}" -X PUT "$ING/v0/block?cid=$C4" -H "Authorization: Bearer $APIKEY" --data-binary @"$F4") @@ -144,7 +144,7 @@ code=$(curl -s -m 30 -o /tmp/p2-abs.txt -w "%{http_code}" -X PUT \ -H "x-amz-meta-fula-remote-cid: $CA" \ -H "x-amz-meta-fula-remote-size: 1024" \ -H "Content-Length: 0") -case "$code" in 409|4*) ok "I10 absent cid rejected (code=$code — client falls back to full bytes)";; *) bad "I10 got $code: $(head -c200 /tmp/p2-abs.txt)";; esac +case "$code" in 409|4*) ok "I10 absent cid rejected (code=$code — client falls back to full bytes)";; *) bad "I10 got $code: $(head -c200 /tmp/p2-abs.txt)";; esac echo echo "RESULT: pass=$PASS fail=$FAIL" diff --git a/tests/e2e/phase-2/60-fidelity.sh b/tests/e2e/phase-2/60-fidelity.sh index ecbdde80..3560deb7 100644 --- a/tests/e2e/phase-2/60-fidelity.sh +++ b/tests/e2e/phase-2/60-fidelity.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash # # Phase 2 fidelity suite (TEST SERVER ONLY). Runs AFTER 40-ingest-drills.sh # (gateway rebuilt with FULA_REMOTE_CID_PUT=true; fula-ingest-e2e running). @@ -8,7 +8,7 @@ # F2 v8-OFF legacy round-trip (client CID off matrix leg) # F3 FxFiles-faithful suite: offline_e2e single + chunked upload/download # (the byte-for-byte FxFiles flow, legacy path) -# F4 ≥1 GiB chunked via ingest (FULA_BIG=1; scale invariant) +# F4 ≥1 GiB chunked via ingest (FULA_BIG=1; scale invariant) # set -uo pipefail PASS=0; FAIL=0 From f03cc77295e0bc1cab5ae33ba5d44e536cf0e732 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 00:03:05 -0400 Subject: [PATCH 07/19] fix(e2e): create the drill bucket before the mapping PUT (S3 semantics) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index 836e9cea..24c1af58 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -124,6 +124,10 @@ print((h+b"."+p+b"."+sig).decode()) PYEOF ) [ -n "$JWT" ] && ok "I9 minted gateway JWT" || bad "I9 JWT mint failed" +# S3 semantics: the bucket must exist before object PUTs. +code=$(curl -s -m 20 -o /tmp/p2-mkbkt.txt -w "%{http_code}" -X PUT \ + "http://127.0.0.1:9000/p2-drill-bucket" -H "Authorization: Bearer $JWT") +case "$code" in 200|409) ok "I9 bucket ready (code=$code)";; *) bad "I9 create bucket got $code: $(head -c200 /tmp/p2-mkbkt.txt)";; esac code=$(curl -s -m 20 -o /tmp/p2-map.txt -D /tmp/p2-map-h.txt -w "%{http_code}" -X PUT \ "http://127.0.0.1:9000/p2-drill-bucket/chunk-0001" \ -H "Authorization: Bearer $JWT" \ From a4a688a506d3d5396adc8f8f3d09068478e8a2e6 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 00:22:19 -0400 Subject: [PATCH 08/19] fix(e2e): gateway env needs PINNING_SERVICE_ENDPOINT (registry persist pins via it) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index 24c1af58..d5d62c16 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -102,6 +102,7 @@ JWT_SECRET=$JWT_SECRET STORAGE_API_URL=http://127.0.0.1:3001 CLUSTER_API_URL=http://127.0.0.1:9094 IPFS_API_URL=http://127.0.0.1:5001 +PINNING_SERVICE_ENDPOINT=http://127.0.0.1:6000 FULA_HOST=127.0.0.1 FULA_PORT=9000 FULA_REMOTE_CID_PUT=true From fad22f0196c10e8a557640f72a612786d74f73b6 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 00:50:59 -0400 Subject: [PATCH 09/19] fix(e2e): register the minted JWT as a pinning-service session (prod parity) The registry pin forwards the user JWT to the pinning API, which validates sessions IT issued (/auth/google) - a minted drill token must be seeded into sessions (hashed per migration 009) for the pin to authenticate, exactly as prod tokens are. Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index d5d62c16..978d6163 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -125,6 +125,15 @@ print((h+b"."+p+b"."+sig).decode()) PYEOF ) [ -n "$JWT" ] && ok "I9 minted gateway JWT" || bad "I9 JWT mint failed" +# Prod parity: gateway JWTs are ISSUED by the pinning service (/auth/google), +# so they exist in its sessions table. The gateway forwards the user's JWT for +# the registry pin — a minted token must be registered as a session too +# (hashed storage per migration 009; legacy-plaintext fallback also tried). +JWT_HASH=$(printf '%s' "$JWT" | sha256sum | cut -d' ' -f1) +psqlc "INSERT INTO users (username, password_hash) VALUES ('$U','x') ON CONFLICT (username) DO NOTHING" >/dev/null 2>&1 || true +psqlc "INSERT INTO sessions (username, session_token) VALUES ('$U', '$JWT_HASH') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ + || psqlc "INSERT INTO sessions (username, session_token, expires_at) VALUES ('$U', '$JWT_HASH', NOW() + interval '2 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ + || true # S3 semantics: the bucket must exist before object PUTs. code=$(curl -s -m 20 -o /tmp/p2-mkbkt.txt -w "%{http_code}" -X PUT \ "http://127.0.0.1:9000/p2-drill-bucket" -H "Authorization: Bearer $JWT") From eb583f020b6c0a3bb1a922cfa2f1411b1c7fcd85 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 01:10:11 -0400 Subject: [PATCH 10/19] fix(e2e): seed sessions WITH expires_at - NULL expiry reads as expired Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index 978d6163..d9fb4103 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -131,9 +131,11 @@ PYEOF # (hashed storage per migration 009; legacy-plaintext fallback also tried). JWT_HASH=$(printf '%s' "$JWT" | sha256sum | cut -d' ' -f1) psqlc "INSERT INTO users (username, password_hash) VALUES ('$U','x') ON CONFLICT (username) DO NOTHING" >/dev/null 2>&1 || true -psqlc "INSERT INTO sessions (username, session_token) VALUES ('$U', '$JWT_HASH') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ - || psqlc "INSERT INTO sessions (username, session_token, expires_at) VALUES ('$U', '$JWT_HASH', NOW() + interval '2 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ +# expires_at MUST be set — a NULL expiry reads as "expired" to the service. +psqlc "INSERT INTO sessions (username, session_token, expires_at) VALUES ('$U', '$JWT_HASH', NOW() + interval '2 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ + || psqlc "INSERT INTO sessions (username, session_token) VALUES ('$U', '$JWT_HASH') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ || true +psqlc "UPDATE sessions SET expires_at = NOW() + interval '2 hours' WHERE session_token = '$JWT_HASH'" >/dev/null 2>&1 || true # S3 semantics: the bucket must exist before object PUTs. code=$(curl -s -m 20 -o /tmp/p2-mkbkt.txt -w "%{http_code}" -X PUT \ "http://127.0.0.1:9000/p2-drill-bucket" -H "Authorization: Bearer $JWT") From 6268a02813171d4235d0d21f277e051c477e708c Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 01:29:09 -0400 Subject: [PATCH 11/19] fix(e2e): session seed must populate token_hash (the column the service matches) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/40-ingest-drills.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/phase-2/40-ingest-drills.sh b/tests/e2e/phase-2/40-ingest-drills.sh index d9fb4103..1d3e1b70 100644 --- a/tests/e2e/phase-2/40-ingest-drills.sh +++ b/tests/e2e/phase-2/40-ingest-drills.sh @@ -131,11 +131,11 @@ PYEOF # (hashed storage per migration 009; legacy-plaintext fallback also tried). JWT_HASH=$(printf '%s' "$JWT" | sha256sum | cut -d' ' -f1) psqlc "INSERT INTO users (username, password_hash) VALUES ('$U','x') ON CONFLICT (username) DO NOTHING" >/dev/null 2>&1 || true -# expires_at MUST be set — a NULL expiry reads as "expired" to the service. -psqlc "INSERT INTO sessions (username, session_token, expires_at) VALUES ('$U', '$JWT_HASH', NOW() + interval '2 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ - || psqlc "INSERT INTO sessions (username, session_token) VALUES ('$U', '$JWT_HASH') ON CONFLICT DO NOTHING" >/dev/null 2>&1 \ - || true -psqlc "UPDATE sessions SET expires_at = NOW() + interval '2 hours' WHERE session_token = '$JWT_HASH'" >/dev/null 2>&1 || true +# The service validates `token_hash = sha256(presented)` (postgres_service.go:804; +# NULL expiry is allowed). session_token is NOT NULL UNIQUE — fill it with the +# hash too (it's never matched for hashed sessions). +psqlc "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('$U', '$JWT_HASH', '$JWT_HASH', NOW() + interval '2 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true +psqlc "UPDATE sessions SET token_hash = '$JWT_HASH', expires_at = NOW() + interval '2 hours' WHERE session_token = '$JWT_HASH'" >/dev/null 2>&1 || true # S3 semantics: the bucket must exist before object PUTs. code=$(curl -s -m 20 -o /tmp/p2-mkbkt.txt -w "%{http_code}" -X PUT \ "http://127.0.0.1:9000/p2-drill-bucket" -H "Authorization: Bearer $JWT") From 67d37675e2a2e4c19fc8d8ed3d6ac6decbb76bb9 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 08:11:13 -0400 Subject: [PATCH 12/19] fix(e2e): fidelity runner seeds the JWT session (token_hash) like the drills Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/60-fidelity.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/e2e/phase-2/60-fidelity.sh b/tests/e2e/phase-2/60-fidelity.sh index 3560deb7..2c66cae6 100644 --- a/tests/e2e/phase-2/60-fidelity.sh +++ b/tests/e2e/phase-2/60-fidelity.sh @@ -28,6 +28,16 @@ PYEOF ) [ -n "$JWT" ] && ok "minted 2h gateway JWT" || { bad "JWT mint failed"; exit 1; } +# Register the minted JWT as a pinning-service session (token_hash = +# sha256(token), postgres_service.go:804) so per-PUT registry pins authenticate +# — exactly how production tokens work (the service issues + stores them). +psqlc() { docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user}" -d "${POSTGRES_DB:-pinning_service}" -tA -c "$1"; } +U="e2e-drill-user" +JWT_HASH=$(printf '%s' "$JWT" | sha256sum | cut -d' ' -f1) +psqlc "INSERT INTO users (username, password_hash) VALUES ('$U','x') ON CONFLICT (username) DO NOTHING" >/dev/null 2>&1 || true +psqlc "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('$U', '$JWT_HASH', '$JWT_HASH', NOW() + interval '4 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true +psqlc "UPDATE user_credits SET is_suspended=0, balance_fula=50 WHERE user_id='$U'" >/dev/null 2>&1 || true + cd /root/fula-api && git pull -q run_tests() { # $1=extra-env $2=test-filter From 2fa1a2dddfb6558a635dab4ca97b06cc75272cad Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 08:20:24 -0400 Subject: [PATCH 13/19] fix(e2e): pre-create the suite buckets (fresh stack has none; prod does) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/60-fidelity.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/phase-2/60-fidelity.sh b/tests/e2e/phase-2/60-fidelity.sh index 2c66cae6..fd5ba316 100644 --- a/tests/e2e/phase-2/60-fidelity.sh +++ b/tests/e2e/phase-2/60-fidelity.sh @@ -38,6 +38,14 @@ psqlc "INSERT INTO users (username, password_hash) VALUES ('$U','x') ON CONFLICT psqlc "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('$U', '$JWT_HASH', '$JWT_HASH', NOW() + interval '4 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true psqlc "UPDATE user_credits SET is_suspended=0, balance_fula=50 WHERE user_id='$U'" >/dev/null 2>&1 || true +# Pre-create every bucket the suites touch (prod has them; a fresh stack +# doesn't — offline_e2e's default bucket is "other"). +for b in other p2-live-ingest p2-live-legacy p2-live-big; do + code=$(curl -s -m 15 -o /dev/null -w "%{http_code}" -X PUT "http://127.0.0.1:9000/$b" -H "Authorization: Bearer $JWT") + case "$code" in 200|409) :;; *) bad "could not create bucket $b (code=$code)";; esac +done +ok "buckets ready (other, p2-live-*)" + cd /root/fula-api && git pull -q run_tests() { # $1=extra-env $2=test-filter From 6777a9521347b6b1235eef9c86ca741834023d1d Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 10:35:33 -0400 Subject: [PATCH 14/19] fix(e2e): 12h token for the 1 GiB leg + targeted F4 rerun script The 1 GiB upload alone runs ~2h on the contended 4-core box; the 2h fixed-expiry test token died at chunk 4019/4096 (real clients refresh). Measured insight recorded: chunked-upload throughput is metadata-commit bound (per-chunk bucket lock + tree flush) - pre-existing gateway property, unchanged by Phase 2. Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/60-fidelity.sh | 6 ++++-- tests/e2e/phase-2/61-f4-only.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/phase-2/61-f4-only.sh diff --git a/tests/e2e/phase-2/60-fidelity.sh b/tests/e2e/phase-2/60-fidelity.sh index fd5ba316..3b678bd0 100644 --- a/tests/e2e/phase-2/60-fidelity.sh +++ b/tests/e2e/phase-2/60-fidelity.sh @@ -21,7 +21,7 @@ import sys, hmac, hashlib, base64, json, time def b64u(b): return base64.urlsafe_b64encode(b).rstrip(b"=") secret = sys.argv[1].encode() h = b64u(json.dumps({"alg":"HS256","typ":"JWT"}).encode()) -p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","scope":"storage:*","iat":int(time.time()),"exp":int(time.time())+7200}).encode()) +p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","scope":"storage:*","iat":int(time.time()),"exp":int(time.time())+43200}).encode()) sig = b64u(hmac.new(secret, h+b"."+p, hashlib.sha256).digest()) print((h+b"."+p+b"."+sig).decode()) PYEOF @@ -35,7 +35,9 @@ psqlc() { docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user U="e2e-drill-user" JWT_HASH=$(printf '%s' "$JWT" | sha256sum | cut -d' ' -f1) psqlc "INSERT INTO users (username, password_hash) VALUES ('$U','x') ON CONFLICT (username) DO NOTHING" >/dev/null 2>&1 || true -psqlc "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('$U', '$JWT_HASH', '$JWT_HASH', NOW() + interval '4 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true +# 12h: the 1 GiB leg alone runs ~2h on the contended test box — a fixed-expiry +# token must outlive it (real clients refresh; e2e tokens cannot). +psqlc "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('$U', '$JWT_HASH', '$JWT_HASH', NOW() + interval '12 hours') ON CONFLICT DO NOTHING" >/dev/null 2>&1 || true psqlc "UPDATE user_credits SET is_suspended=0, balance_fula=50 WHERE user_id='$U'" >/dev/null 2>&1 || true # Pre-create every bucket the suites touch (prod has them; a fresh stack diff --git a/tests/e2e/phase-2/61-f4-only.sh b/tests/e2e/phase-2/61-f4-only.sh new file mode 100644 index 00000000..21f28429 --- /dev/null +++ b/tests/e2e/phase-2/61-f4-only.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Targeted F4 rerun (1 GiB via ingest) with a 12h token — F1-F3 already green. +set -uo pipefail +. /opt/fula-master/.env +JWT=$(python3 - "$JWT_SECRET" <<'PYEOF' +import sys, hmac, hashlib, base64, json, time +def b64u(b): return base64.urlsafe_b64encode(b).rstrip(b"=") +secret = sys.argv[1].encode() +h = b64u(json.dumps({"alg":"HS256","typ":"JWT"}).encode()) +p = b64u(json.dumps({"sub":"e2e-drill@fxe2e.local","scope":"storage:*","iat":int(time.time()),"exp":int(time.time())+43200}).encode()) +sig = b64u(hmac.new(secret, h+b"."+p, hashlib.sha256).digest()) +print((h+b"."+p+b"."+sig).decode()) +PYEOF +) +JWT_HASH=$(printf '%s' "$JWT" | sha256sum | cut -d' ' -f1) +docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user}" -d "${POSTGRES_DB:-pinning_service}" -tA -c \ + "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('e2e-drill-user', '$JWT_HASH', '$JWT_HASH', NOW() + interval '12 hours') ON CONFLICT DO NOTHING" >/dev/null +curl -s -m 15 -o /dev/null -X PUT "http://127.0.0.1:9000/p2-live-big" -H "Authorization: Bearer $JWT" + +cd /root/fula-api && git pull -q +docker run --rm --network host -v /root/fula-api:/src \ + -v fula-cargo-cache:/usr/local/cargo/registry -v fula-cargo-cache-target:/src/target \ + -w /src -e CARGO_TERM_COLOR=never \ + -e FULA_S3=http://127.0.0.1:9000 -e FULA_JWT="$JWT" \ + -e FULA_INGEST=http://127.0.0.1:3601 -e FULA_BIG=1 \ + rust:1-bookworm bash -c "cargo test -p fula-client --release --test live_ingest_e2e live_1gib_chunked_via_ingest -- --ignored --nocapture 2>&1 | tail -20" +rc=$? +[ $rc -eq 0 ] && echo "RESULT: pass=1 fail=0" || echo "RESULT: pass=0 fail=1" From 4de8b90037d10e31b26ab203f32003e3d748247e Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 14:01:20 -0400 Subject: [PATCH 15/19] fix(e2e): pipefail in the F4 runner - the RESULT line must not mask a failed test Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/61-f4-only.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/phase-2/61-f4-only.sh b/tests/e2e/phase-2/61-f4-only.sh index 21f28429..7e27e3fd 100644 --- a/tests/e2e/phase-2/61-f4-only.sh +++ b/tests/e2e/phase-2/61-f4-only.sh @@ -23,6 +23,6 @@ docker run --rm --network host -v /root/fula-api:/src \ -w /src -e CARGO_TERM_COLOR=never \ -e FULA_S3=http://127.0.0.1:9000 -e FULA_JWT="$JWT" \ -e FULA_INGEST=http://127.0.0.1:3601 -e FULA_BIG=1 \ - rust:1-bookworm bash -c "cargo test -p fula-client --release --test live_ingest_e2e live_1gib_chunked_via_ingest -- --ignored --nocapture 2>&1 | tail -20" + rust:1-bookworm bash -c "set -o pipefail; cargo test -p fula-client --release --test live_ingest_e2e live_1gib_chunked_via_ingest -- --ignored --nocapture 2>&1 | tail -20" rc=$? [ $rc -eq 0 ] && echo "RESULT: pass=1 fail=0" || echo "RESULT: pass=0 fail=1" From b7a1f97fc8abc0de88fd5647645f84e3b9660201 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Fri, 12 Jun 2026 16:22:16 -0400 Subject: [PATCH 16/19] fix(e2e): F4 runner pins phase-2-client-ingest - a bare pull tested whatever branch the previous job left checked out (run #3 exercised the 2.5 branch without the timeout fix, repeating run #2 failure verbatim) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/61-f4-only.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/phase-2/61-f4-only.sh b/tests/e2e/phase-2/61-f4-only.sh index 7e27e3fd..0ced41e6 100644 --- a/tests/e2e/phase-2/61-f4-only.sh +++ b/tests/e2e/phase-2/61-f4-only.sh @@ -17,7 +17,10 @@ docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user}" -d "${P "INSERT INTO sessions (username, session_token, token_hash, expires_at) VALUES ('e2e-drill-user', '$JWT_HASH', '$JWT_HASH', NOW() + interval '12 hours') ON CONFLICT DO NOTHING" >/dev/null curl -s -m 15 -o /dev/null -X PUT "http://127.0.0.1:9000/p2-live-big" -H "Authorization: Bearer $JWT" -cd /root/fula-api && git pull -q +# Pin the branch — other jobs (e.g. the Phase-2.5 gate) may have left the +# checkout elsewhere; a bare pull would silently test the wrong code. +cd /root/fula-api && git fetch origin phase-2-client-ingest -q && git checkout -q phase-2-client-ingest && git pull -q +git log --oneline -1 docker run --rm --network host -v /root/fula-api:/src \ -v fula-cargo-cache:/usr/local/cargo/registry -v fula-cargo-cache-target:/src/target \ -w /src -e CARGO_TERM_COLOR=never \ From f408bea9c4b9830fa1dc1b5d145c83922d3bc2ed Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Mon, 15 Jun 2026 11:23:25 -0400 Subject: [PATCH 17/19] fix(e2e): F4 runner targets the renamed large-file test (512MB default) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2/61-f4-only.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/phase-2/61-f4-only.sh b/tests/e2e/phase-2/61-f4-only.sh index 0ced41e6..ff058bed 100644 --- a/tests/e2e/phase-2/61-f4-only.sh +++ b/tests/e2e/phase-2/61-f4-only.sh @@ -26,6 +26,6 @@ docker run --rm --network host -v /root/fula-api:/src \ -w /src -e CARGO_TERM_COLOR=never \ -e FULA_S3=http://127.0.0.1:9000 -e FULA_JWT="$JWT" \ -e FULA_INGEST=http://127.0.0.1:3601 -e FULA_BIG=1 \ - rust:1-bookworm bash -c "set -o pipefail; cargo test -p fula-client --release --test live_ingest_e2e live_1gib_chunked_via_ingest -- --ignored --nocapture 2>&1 | tail -20" + rust:1-bookworm bash -c "set -o pipefail; cargo test -p fula-client --release --test live_ingest_e2e live_large_file_chunked_via_ingest -- --ignored --nocapture 2>&1 | tail -20" rc=$? [ $rc -eq 0 ] && echo "RESULT: pass=1 fail=0" || echo "RESULT: pass=0 fail=1" From f504c3a8779be29d3f9d36525ad6d9bf70d05948 Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Mon, 15 Jun 2026 18:36:29 -0400 Subject: [PATCH 18/19] test(e2e): Phase 2.5 FM-1 PgRootStore integration runner (real stack Postgres) Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2.5/80-fm1-pg-it.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/e2e/phase-2.5/80-fm1-pg-it.sh diff --git a/tests/e2e/phase-2.5/80-fm1-pg-it.sh b/tests/e2e/phase-2.5/80-fm1-pg-it.sh new file mode 100644 index 00000000..35c4a155 --- /dev/null +++ b/tests/e2e/phase-2.5/80-fm1-pg-it.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Phase 2.5 FM-1 acceptance: PgRootStore CAS against the real stack Postgres. +# Migration 020 (bucket_roots) must be applied first (idempotent here). +set -uo pipefail +. /opt/fula-master/.env + +echo "== ensure migration 020 (bucket_roots) ==" +docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user}" -d "${POSTGRES_DB:-pinning_service}" -v ON_ERROR_STOP=1 \ + -c "CREATE TABLE IF NOT EXISTS bucket_roots (owner_id TEXT NOT NULL, bucket TEXT NOT NULL, root_cid TEXT NOT NULL, version BIGINT NOT NULL DEFAULT 1, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (owner_id, bucket));" \ + && echo " bucket_roots ready" + +cd /root/fula-api +git fetch origin phase-2.5-multimaster -q && git checkout -q phase-2.5-multimaster && git pull -q +git log --oneline -1 + +docker run --rm --network host -v /root/fula-api:/src \ + -v fula-cargo-cache:/usr/local/cargo/registry -v fula-cargo-cache-target:/src/target \ + -w /src -e CARGO_TERM_COLOR=never \ + -e POSTGRES_HOST=127.0.0.1 -e POSTGRES_PORT=5432 \ + -e POSTGRES_DB="${POSTGRES_DB:-pinning_service}" -e POSTGRES_USER="${POSTGRES_USER:-pinning_user}" \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + rust:1-bookworm bash -c "set -o pipefail; cargo test -p fula-cli --test root_store_pg_it -- --ignored --nocapture 2>&1 | tail -16" +rc=$? +[ $rc -eq 0 ] && echo "RESULT: pass fail=0" || echo "RESULT: fail" From b7af6952a9730aa2fa856c67b4b97d6eb7e4f34b Mon Sep 17 00:00:00 2001 From: ehsan shariati Date: Mon, 15 Jun 2026 18:39:22 -0400 Subject: [PATCH 19/19] test(e2e): Phase 2.5 combined gate - FM-1 unit + FM-4 unit + FM-1 PG integration Co-Authored-By: Claude Fable 5 --- tests/e2e/phase-2.5/81-units-and-it.sh | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/e2e/phase-2.5/81-units-and-it.sh diff --git a/tests/e2e/phase-2.5/81-units-and-it.sh b/tests/e2e/phase-2.5/81-units-and-it.sh new file mode 100644 index 00000000..0d150e34 --- /dev/null +++ b/tests/e2e/phase-2.5/81-units-and-it.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Phase 2.5 combined gate on the latest commit: FM-1 unit (root_pointer) + +# FM-4 unit (auth_eip712) + FM-1 PG integration (real bucket_roots). +set -uo pipefail +. /opt/fula-master/.env + +docker exec -i postgres-pinning psql -U "${POSTGRES_USER:-pinning_user}" -d "${POSTGRES_DB:-pinning_service}" \ + -c "CREATE TABLE IF NOT EXISTS bucket_roots (owner_id TEXT NOT NULL, bucket TEXT NOT NULL, root_cid TEXT NOT NULL, version BIGINT NOT NULL DEFAULT 1, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (owner_id, bucket));" >/dev/null && echo "bucket_roots ready" + +cd /root/fula-api +git fetch origin phase-2.5-multimaster -q && git checkout -q phase-2.5-multimaster && git pull -q +git log --oneline -1 + +docker run --rm --network host -v /root/fula-api:/src \ + -v fula-cargo-cache:/usr/local/cargo/registry -v fula-cargo-cache-target:/src/target \ + -w /src -e CARGO_TERM_COLOR=never \ + -e POSTGRES_HOST=127.0.0.1 -e POSTGRES_PORT=5432 \ + -e POSTGRES_DB="${POSTGRES_DB:-pinning_service}" -e POSTGRES_USER="${POSTGRES_USER:-pinning_user}" \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + rust:1-bookworm bash -c ' + set -o pipefail + echo "===== FM-1 unit (root_pointer) =====" + cargo test -p fula-core root_pointer 2>&1 | tail -10 + echo "===== FM-4 unit (auth_eip712) =====" + cargo test -p fula-cli auth_eip712 2>&1 | tail -12 + echo "===== FM-1 integration (PgRootStore vs real Postgres) =====" + cargo test -p fula-cli --test root_store_pg_it -- --ignored --nocapture 2>&1 | tail -14 + ' +echo "P25-ALL-RC=$?"