fix(cli): make the CI gate fail on an unreadable findings response

countFindings treated any response it could not parse as zero findings,
so `nullify ci gate` printed "Gate passed: no findings above threshold".

Parse the real {findings, numItems, nextToken} envelope, return an error
for anything else, and exit ExitNetworkError rather than counting an
unreadable response as clean. Drop the dead items/total shapes and the
duplicate totalFindingsCount. The gate requests limit=1 and the API has
no grand total, so the printed counts were the number of failing
scanner/severity pairs; the messages now say that instead.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: tim-thacker-nullify

cmd/cli/cmd/ci.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"sync"
@@ -80,8 +81,9 @@ Exit codes:
 			}
 		}
 
-		var totalFindings int64
+		var findingsFound int64
 		var apiErrors int64
+		var unreadable int64
 		var mu sync.Mutex
 		g, gctx := errgroup.WithContext(ctx)
 
@@ -104,14 +106,19 @@ Exit codes:
 						return nil
 					}
 
-					// limit=1 keeps the payload small; the accurate count
-					// comes from the response's "total" field, not the
-					// truncated items array.
-					count := totalFindingsCount(body)
+					count, err := countFindings(body)
+					if err != nil {
+						mu.Lock()
+						fmt.Fprintf(os.Stderr, "Error: unreadable response from %s (%s): %v\n", ep.name, sev, err)
+						mu.Unlock()
+						atomic.AddInt64(&unreadable, 1)
+						return nil
+					}
+
 					if count > 0 {
-						atomic.AddInt64(&totalFindings, int64(count))
+						atomic.AddInt64(&findingsFound, 1)
 						mu.Lock()
-						fmt.Printf("FAIL: %s has %d %s findings\n", ep.name, count, sev)
+						fmt.Printf("FAIL: %s has open %s findings\n", ep.name, sev)
 						mu.Unlock()
 					}
 					return nil
@@ -126,10 +133,15 @@ Exit codes:
 			return withExitCode(ExitNetworkError, err)
 		}
 
-		if totalFindings > 0 {
-			fmt.Printf("\nGate failed: %d findings at or above %s severity\n", totalFindings, severityThreshold)
+		if unreadable > 0 {
+			err := fmt.Errorf("%d scanner response(s) could not be read; failing the gate (cannot confirm a clean result)", unreadable)
+			return withExitCode(ExitNetworkError, err)
+		}
+
+		if findingsFound > 0 {
+			fmt.Printf("\nGate failed: open findings at or above %s severity\n", severityThreshold)
 			cmd.SilenceErrors = true
-			return withExitCode(ExitFindings, fmt.Errorf("gate failed: %d findings at or above %s severity", totalFindings, severityThreshold))
+			return withExitCode(ExitFindings, fmt.Errorf("gate failed: open findings at or above %s severity", severityThreshold))
 		}
 
 		fmt.Println("Gate passed: no findings above threshold")
@@ -202,12 +214,20 @@ format emits a SARIF v2.1.0 document for upload to code-scanning tools.`,
 						mu.Unlock()
 						return nil
 					}
+					count, err := countFindings(body)
+					if err != nil {
+						atomic.AddInt64(&apiErrors, 1)
+						mu.Lock()
+						fmt.Fprintf(os.Stderr, "Warning: unreadable response from %s (%s): %v\n", ep.name, sev, err)
+						mu.Unlock()
+						return nil
+					}
 					atomic.AddInt64(&successCount, 1)
 
 					rows[i*len(severities)+j] = reportRow{
 						scanner:  ep.name,
 						severity: sev,
-						count:    totalFindingsCount(body),
+						count:    count,
 						findings: extractFindings(body),
 					}
 					return nil
@@ -278,55 +298,37 @@ func severitiesAboveThreshold(threshold string) []string {
 	return []string{"critical", "high"}
 }
 
-// countFindings extracts a count from API response JSON. When used with limit=1,
-// it returns 0 or 1 to indicate whether findings exist at a given severity.
-func countFindings(body string) int {
-	var result any
-	if err := json.Unmarshal([]byte(body), &result); err != nil {
-		return 0
+// errUnreadableFindings reports a findings response that does not carry the
+// scanner envelope. It is never reported as a count of zero: a gate that cannot
+// read the response must block the build rather than declare the repo clean.
+var errUnreadableFindings = errors.New("unrecognised findings response envelope")
+
+// countFindings returns the number of findings in a scanner API response. Every
+// endpoint in allScannerEndpoints returns {"findings":[...],"numItems":N,
+// "nextToken":"..."}, except /dast/bughunt/findings which omits numItems. The
+// count is page-scoped and therefore bounded by the request's limit; the API
+// exposes no grand total.
+func countFindings(body string) (int, error) {
+	var envelope struct {
+		Findings json.RawMessage `json:"findings"`
+		NumItems *int            `json:"numItems"`
 	}
-
-	switch v := result.(type) {
-	case []any:
-		return len(v)
-	case map[string]any:
-		if items, ok := v["items"].([]any); ok {
-			return len(items)
-		}
-		if total, ok := v["total"].(float64); ok {
-			return int(total)
-		}
+	if err := json.Unmarshal([]byte(body), &envelope); err != nil {
+		return 0, fmt.Errorf("%w: %w", errUnreadableFindings, err)
 	}
-
-	return 0
-}
-
-// totalFindingsCount extracts an accurate finding count from an API response.
-// It prefers the response's "total" field (which reflects the full result set
-// regardless of the request's limit) over the length of the truncated items
-// array. Falls back to array/items length when no total is present.
-func totalFindingsCount(body string) int {
-	var result any
-	if err := json.Unmarshal([]byte(body), &result); err != nil {
-		return 0
+	if len(envelope.Findings) == 0 {
+		return 0, fmt.Errorf("%w: no \"findings\" field", errUnreadableFindings)
 	}
 
-	switch v := result.(type) {
-	case []any:
-		return len(v)
-	case map[string]any:
-		if total, ok := v["total"].(float64); ok {
-			return int(total)
-		}
-		if items, ok := v["items"].([]any); ok {
-			return len(items)
-		}
-		if findings, ok := v["findings"].([]any); ok {
-			return len(findings)
-		}
+	if envelope.NumItems != nil {
+		return *envelope.NumItems, nil
 	}
 
-	return 0
+	var findings []json.RawMessage
+	if err := json.Unmarshal(envelope.Findings, &findings); err != nil {
+		return 0, fmt.Errorf("%w: \"findings\" is not an array: %w", errUnreadableFindings, err)
+	}
+	return len(findings), nil
 }
 
 // extractFindings pulls the finding objects out of an API response body so they

cmd/cli/cmd/ci_test.go

@@ -33,41 +33,75 @@ func TestCountFindings(t *testing.T) {
 		expected int
 	}{
 		{
-			name:     "empty array",
-			body:     `[]`,
-			expected: 0,
+			name:     "sast envelope",
+			body:     `{"version":"1","findings":[{"id":"f1"}],"numItems":1,"nextToken":""}`,
+			expected: 1,
 		},
 		{
-			name:     "array with items",
-			body:     `[{"id": 1}, {"id": 2}, {"id": 3}]`,
-			expected: 3,
+			name:     "sca dependencies envelope",
+			body:     `{"version":"1","findings":[{"id":"f1"},{"id":"f2"}],"numItems":2,"nextToken":"abc"}`,
+			expected: 2,
 		},
 		{
-			name:     "object with items array",
-			body:     `{"items": [{"id": 1}]}`,
+			name:     "sca containers envelope",
+			body:     `{"findings":[{"id":"f1"}],"numItems":1,"nextToken":""}`,
 			expected: 1,
 		},
 		{
-			name:     "object with total field",
-			body:     `{"total": 42}`,
-			expected: 42,
+			name:     "secrets envelope",
+			body:     `{"findings":[],"numItems":0,"nextToken":""}`,
+			expected: 0,
 		},
 		{
-			name:     "invalid json",
-			body:     `not json`,
-			expected: 0,
+			name:     "pentest envelope",
+			body:     `{"findings":[{"id":"f1"},{"id":"f2"},{"id":"f3"}],"numItems":3,"nextToken":""}`,
+			expected: 3,
+		},
+		{
+			name:     "bughunt envelope omits numItems",
+			body:     `{"findings":[{"id":"f1"},{"id":"f2"}]}`,
+			expected: 2,
+		},
+		{
+			name:     "cspm envelope omits empty nextToken",
+			body:     `{"findings":[{"id":"f1"}],"numItems":1,"version":"1"}`,
+			expected: 1,
 		},
 		{
-			name:     "empty object",
-			body:     `{}`,
+			name:     "nil findings slice serialises as null",
+			body:     `{"findings":null,"numItems":0,"nextToken":""}`,
 			expected: 0,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := countFindings(tt.body)
+			result, err := countFindings(tt.body)
+			require.NoError(t, err)
 			require.Equal(t, tt.expected, result)
 		})
 	}
 }
+
+func TestCountFindingsRejectsUnknownEnvelopes(t *testing.T) {
+	tests := []struct {
+		name string
+		body string
+	}{
+		{name: "invalid json", body: `not json`},
+		{name: "empty object", body: `{}`},
+		{name: "bare array", body: `[{"id":"f1"}]`},
+		{name: "items envelope", body: `{"items":[{"id":"f1"}]}`},
+		{name: "total envelope", body: `{"total":42}`},
+		{name: "findings is not an array", body: `{"findings":{"id":"f1"}}`},
+		{name: "empty body", body: ``},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			count, err := countFindings(tt.body)
+			require.ErrorIs(t, err, errUnreadableFindings)
+			require.Zero(t, count)
+		})
+	}
+}

cmd/cli/cmd/status.go

@@ -113,33 +113,14 @@ func init() {
 
 // summarizeFindingsResponse extracts a human-readable summary from a findings API response.
 func summarizeFindingsResponse(body string) string {
-	var result any
-	if err := json.Unmarshal([]byte(body), &result); err != nil {
-		return "data available"
+	count, err := countFindings(body)
+	if err != nil {
+		return "unreadable response"
 	}
-
-	switch v := result.(type) {
-	case []any:
-		if len(v) == 1 {
-			return "1 finding returned"
-		}
-		return fmt.Sprintf("%d findings returned", len(v))
-	case map[string]any:
-		if items, ok := v["items"].([]any); ok {
-			if len(items) == 1 {
-				return "1 finding returned"
-			}
-			return fmt.Sprintf("%d findings returned", len(items))
-		}
-		if total, ok := v["total"].(float64); ok {
-			if total == 1 {
-				return "1 total finding"
-			}
-			return fmt.Sprintf("%.0f total findings", total)
-		}
+	if count == 1 {
+		return "1 finding returned"
 	}
-
-	return "data available"
+	return fmt.Sprintf("%d findings returned", count)
 }
 
 type securityStatusOutput struct {

cmd/cli/cmd/status_test.go

@@ -14,29 +14,34 @@ func TestSummarizeFindingsResponse(t *testing.T) {
 		expected string
 	}{
 		{
-			name:     "array response",
-			body:     `[{"id":1},{"id":2}]`,
+			name:     "scanner envelope with findings",
+			body:     `{"findings":[{"id":"f1"},{"id":"f2"}],"numItems":2,"nextToken":""}`,
 			expected: "2 findings returned",
 		},
 		{
-			name:     "object with items",
-			body:     `{"items":[{"id":1}]}`,
+			name:     "scanner envelope with one finding",
+			body:     `{"findings":[{"id":"f1"}],"numItems":1,"nextToken":""}`,
 			expected: "1 finding returned",
 		},
 		{
-			name:     "object with total",
-			body:     `{"total":99}`,
-			expected: "99 total findings",
+			name:     "bughunt envelope omits numItems",
+			body:     `{"findings":[{"id":"f1"},{"id":"f2"},{"id":"f3"}]}`,
+			expected: "3 findings returned",
+		},
+		{
+			name:     "empty scanner envelope",
+			body:     `{"findings":[],"numItems":0,"nextToken":""}`,
+			expected: "0 findings returned",
 		},
 		{
 			name:     "empty object",
 			body:     `{}`,
-			expected: "data available",
+			expected: "unreadable response",
 		},
 		{
 			name:     "invalid json",
 			body:     `nope`,
-			expected: "data available",
+			expected: "unreadable response",
 		},
 	}