Fix: JSON based scrapers fail with Chrome CDP

pkg/scraper/url.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/chromedp/cdproto/cdp"
@@ -253,6 +254,21 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO
 	var res string
 	headers := cdpHeaders(driverOptions)
 
+	// Track the main document response so that, if it turns out to be JSON,
+	// we can pull the raw response body via CDP's Network domain instead of
+	// reading the rendered DOM. Chrome's built-in JSON viewer wraps raw JSON
+	// responses in an HTML pretty-printer when navigated to directly, so
+	// OuterHTML on such a page returns HTML containing the JSON rather than
+	// the JSON itself.
+	var jsonDoc jsonDocumentTracker
+	chromedp.ListenTarget(ctx, func(ev interface{}) {
+		if ev, ok := ev.(*network.EventResponseReceived); ok {
+			if ev.Type == network.ResourceTypeDocument {
+				jsonDoc.markDocument(ev.RequestID, ev.Response.MimeType)
+			}
+		}
+	})
+
 	if proxyUsesAuth(globalConfig.GetProxy()) {
 		_, user, pass := splitProxyAuth(globalConfig.GetProxy())
 
@@ -294,7 +310,27 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO
 		chromedp.Navigate(urlCDP),
 		chromedp.Sleep(sleepDuration),
 		setCDPClicks(driverOptions),
-		chromedp.OuterHTML("html", &res, chromedp.ByQuery),
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			requestID, isJSON := jsonDoc.mainDocument()
+
+			if isJSON {
+				body, err := network.GetResponseBody(requestID).Do(ctx)
+				if err != nil {
+					// Fall back to OuterHTML if the response body is no
+					// longer available (e.g. evicted from Chrome's cache).
+					// The scraper will most likely still fail downstream on
+					// this fallback, since OuterHTML returns Chrome's
+					// HTML-wrapped view of the JSON rather than the JSON
+					// itself - but it's a better failure mode than losing
+					// the response entirely.
+					logger.Warnf("[scraper] could not get raw response body for JSON document, falling back to OuterHTML (the scrape will likely still fail as a result): %v", err)
+					return chromedp.OuterHTML("html", &res, chromedp.ByQuery).Do(ctx)
+				}
+				res = string(body)
+				return nil
+			}
+			return chromedp.OuterHTML("html", &res, chromedp.ByQuery).Do(ctx)
+		}),
 		printCDPCookies(driverOptions, "Cookies set"),
 	)
 
@@ -361,6 +397,72 @@ func getRemoteCDPWSAddress(ctx context.Context, url string) (string, error) {
 	return remote, err
 }
 
+// isJSONMimeType returns true if the given response MIME type indicates
+// JSON content (e.g. "application/json", "application/ld+json",
+// "text/json; charset=utf-8"), for deciding whether a CDP-fetched document
+// should be read via its raw network response body rather than the
+// rendered DOM.
+func isJSONMimeType(mimeType string) bool {
+	mimeType, _, _ = strings.Cut(mimeType, ";")
+	mimeType = strings.TrimSpace(strings.ToLower(mimeType))
+
+	return mimeType == "application/json" ||
+		strings.HasSuffix(mimeType, "+json") ||
+		mimeType == "text/json"
+}
+
+// jsonDocumentTracker records whether the CDP-loaded page's main document
+// turned out to be JSON, and if so, which network request to fetch its raw
+// body from.
+//
+// It only ever considers the *first* Document-type response it sees, via
+// markDocument. Network.responseReceived fires with resourceType Document
+// for iframe navigations too, not just the top-level one, so a naive
+// "first Document that happens to be JSON" rule could latch onto an
+// iframe's JSON response instead of the main page's HTML - misidentifying
+// an HTML scrape as a JSON one. Redirects don't complicate this: a
+// pre-redirect response never gets its own responseReceived event (that
+// history arrives via requestWillBeSent's redirectResponse on the same
+// request instead), so the first Document response reliably corresponds to
+// the top-level navigation. A click-triggered navigation (via
+// setCDPClicks, which runs after the initial Navigate) is not treated as a
+// new "first" document either, so if a click leads to a JSON page, this
+// still reports the original navigation's result.
+//
+// It's written from chromedp's event-processing goroutine (via
+// markDocument, called from a ListenTarget callback) and read from the
+// action sequence passed to chromedp.Run (via mainDocument), so access is
+// synchronized with a mutex.
+type jsonDocumentTracker struct {
+	mutex     sync.Mutex
+	requestID network.RequestID
+	isJSON    bool
+	recorded  bool
+}
+
+// markDocument records requestID as the main document if no document has
+// been recorded yet. Subsequent calls after the first are no-ops.
+func (t *jsonDocumentTracker) markDocument(requestID network.RequestID, mimeType string) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	if t.recorded {
+		return
+	}
+	t.recorded = true
+	t.requestID = requestID
+	t.isJSON = isJSONMimeType(mimeType)
+}
+
+// mainDocument returns the main document's request ID and whether it was
+// JSON.
+func (t *jsonDocumentTracker) mainDocument() (network.RequestID, bool) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	return t.requestID, t.isJSON
+}
+
 func cdpHeaders(driverOptions scraperDriverOptions) map[string]interface{} {
 	headers := map[string]interface{}{}
 	if driverOptions.Headers != nil {

pkg/scraper/url_test.go

@@ -0,0 +1,108 @@
+package scraper
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/chromedp/cdproto/network"
+)
+
+func TestIsJSONMimeType(t *testing.T) {
+	tests := []struct {
+		name     string
+		mimeType string
+		want     bool
+	}{
+		{"plain json", "application/json", true},
+		{"json with charset", "application/json; charset=utf-8", true},
+		{"json with charset and spacing", "application/json;  charset=UTF-8", true},
+		{"structured syntax suffix", "application/ld+json", true},
+		{"uppercase", "APPLICATION/JSON", true},
+		{"text/json", "text/json", true},
+		{"html", "text/html", false},
+		{"html with charset", "text/html; charset=utf-8", false},
+		{"plain text", "text/plain", false},
+		{"empty", "", false},
+		{"contains json as substring but isn't json", "application/jsonp", false},
+		{"contains json as substring but isn't json 2", "multipart/json-form-data", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := isJSONMimeType(tt.mimeType)
+			if got != tt.want {
+				t.Errorf("isJSONMimeType(%q) = %v, want %v", tt.mimeType, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestJSONDocumentTrackerOnlyRecordsFirstDocument(t *testing.T) {
+	t.Run("first document is JSON, later ones don't override it", func(t *testing.T) {
+		var tracker jsonDocumentTracker
+
+		tracker.markDocument("req-1", "application/json")
+		tracker.markDocument("req-2", "application/json")
+		tracker.markDocument("req-3", "text/html")
+
+		if requestID, isJSON := tracker.mainDocument(); !isJSON || requestID != "req-1" {
+			t.Fatalf("mainDocument() = (%q, %v), want (%q, true)", requestID, isJSON, "req-1")
+		}
+	})
+
+	t.Run("first document is HTML, a later JSON response doesn't override it", func(t *testing.T) {
+		// This is the main-frame-vs-iframe scenario: an HTML scrape whose
+		// page contains an iframe that loads JSON (an embed widget, an ad
+		// frame) also fires a Document-type responseReceived for that
+		// iframe. Since the top-level HTML document is always the first
+		// Document response chromedp sees for a given navigation, the
+		// tracker must not let this later JSON response override it -
+		// otherwise an HTML scraper would incorrectly receive the iframe's
+		// JSON body instead of the page's HTML.
+		var tracker jsonDocumentTracker
+
+		tracker.markDocument("req-main-page", "text/html")
+		tracker.markDocument("req-iframe", "application/json")
+
+		if requestID, isJSON := tracker.mainDocument(); isJSON || requestID != "req-main-page" {
+			t.Fatalf("mainDocument() = (%q, %v), want (%q, false)", requestID, isJSON, "req-main-page")
+		}
+	})
+}
+
+// TestJSONDocumentTrackerConcurrentAccess exercises jsonDocumentTracker the
+// way urlFromCDP actually uses it: one goroutine (standing in for chromedp's
+// event-processing goroutine) calls markDocument while another goroutine
+// (standing in for the action sequence passed to chromedp.Run) calls
+// mainDocument, concurrently and repeatedly. Before requestID/isJSON/recorded
+// were guarded by a mutex, `go test -race` reliably flagged this exact
+// access pattern as a data race.
+func TestJSONDocumentTrackerConcurrentAccess(t *testing.T) {
+	var tracker jsonDocumentTracker
+
+	const n = 200
+	var wg sync.WaitGroup
+	wg.Add(2 * n)
+
+	for i := range n {
+		go func() {
+			defer wg.Done()
+			tracker.markDocument(network.RequestID(fmt.Sprintf("req-%d", i)), "application/json")
+		}()
+		go func() {
+			defer wg.Done()
+			tracker.mainDocument()
+		}()
+	}
+
+	wg.Wait()
+
+	requestID, isJSON := tracker.mainDocument()
+	if !isJSON {
+		t.Fatal("expected a JSON document to have been recorded")
+	}
+	if requestID == "" {
+		t.Fatal("expected a non-empty request ID to have been recorded")
+	}
+}