diff --git a/pkg/scraper/url.go b/pkg/scraper/url.go index d95f4563f1..8441bc4f83 100644 --- a/pkg/scraper/url.go +++ b/pkg/scraper/url.go @@ -11,6 +11,7 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/chromedp/cdproto/cdp" @@ -253,6 +254,21 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO var res string headers := cdpHeaders(driverOptions) + // Track the main document response so that, if it turns out to be JSON, + // we can pull the raw response body via CDP's Network domain instead of + // reading the rendered DOM. Chrome's built-in JSON viewer wraps raw JSON + // responses in an HTML pretty-printer when navigated to directly, so + // OuterHTML on such a page returns HTML containing the JSON rather than + // the JSON itself. + var jsonDoc jsonDocumentTracker + chromedp.ListenTarget(ctx, func(ev interface{}) { + if ev, ok := ev.(*network.EventResponseReceived); ok { + if ev.Type == network.ResourceTypeDocument { + jsonDoc.markDocument(ev.RequestID, ev.Response.MimeType) + } + } + }) + if proxyUsesAuth(globalConfig.GetProxy()) { _, user, pass := splitProxyAuth(globalConfig.GetProxy()) @@ -294,7 +310,27 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO chromedp.Navigate(urlCDP), chromedp.Sleep(sleepDuration), setCDPClicks(driverOptions), - chromedp.OuterHTML("html", &res, chromedp.ByQuery), + chromedp.ActionFunc(func(ctx context.Context) error { + requestID, isJSON := jsonDoc.mainDocument() + + if isJSON { + body, err := network.GetResponseBody(requestID).Do(ctx) + if err != nil { + // Fall back to OuterHTML if the response body is no + // longer available (e.g. evicted from Chrome's cache). + // The scraper will most likely still fail downstream on + // this fallback, since OuterHTML returns Chrome's + // HTML-wrapped view of the JSON rather than the JSON + // itself - but it's a better failure mode than losing + // the response entirely. + logger.Warnf("[scraper] could not get raw response body for JSON document, falling back to OuterHTML (the scrape will likely still fail as a result): %v", err) + return chromedp.OuterHTML("html", &res, chromedp.ByQuery).Do(ctx) + } + res = string(body) + return nil + } + return chromedp.OuterHTML("html", &res, chromedp.ByQuery).Do(ctx) + }), printCDPCookies(driverOptions, "Cookies set"), ) @@ -361,6 +397,72 @@ func getRemoteCDPWSAddress(ctx context.Context, url string) (string, error) { return remote, err } +// isJSONMimeType returns true if the given response MIME type indicates +// JSON content (e.g. "application/json", "application/ld+json", +// "text/json; charset=utf-8"), for deciding whether a CDP-fetched document +// should be read via its raw network response body rather than the +// rendered DOM. +func isJSONMimeType(mimeType string) bool { + mimeType, _, _ = strings.Cut(mimeType, ";") + mimeType = strings.TrimSpace(strings.ToLower(mimeType)) + + return mimeType == "application/json" || + strings.HasSuffix(mimeType, "+json") || + mimeType == "text/json" +} + +// jsonDocumentTracker records whether the CDP-loaded page's main document +// turned out to be JSON, and if so, which network request to fetch its raw +// body from. +// +// It only ever considers the *first* Document-type response it sees, via +// markDocument. Network.responseReceived fires with resourceType Document +// for iframe navigations too, not just the top-level one, so a naive +// "first Document that happens to be JSON" rule could latch onto an +// iframe's JSON response instead of the main page's HTML - misidentifying +// an HTML scrape as a JSON one. Redirects don't complicate this: a +// pre-redirect response never gets its own responseReceived event (that +// history arrives via requestWillBeSent's redirectResponse on the same +// request instead), so the first Document response reliably corresponds to +// the top-level navigation. A click-triggered navigation (via +// setCDPClicks, which runs after the initial Navigate) is not treated as a +// new "first" document either, so if a click leads to a JSON page, this +// still reports the original navigation's result. +// +// It's written from chromedp's event-processing goroutine (via +// markDocument, called from a ListenTarget callback) and read from the +// action sequence passed to chromedp.Run (via mainDocument), so access is +// synchronized with a mutex. +type jsonDocumentTracker struct { + mutex sync.Mutex + requestID network.RequestID + isJSON bool + recorded bool +} + +// markDocument records requestID as the main document if no document has +// been recorded yet. Subsequent calls after the first are no-ops. +func (t *jsonDocumentTracker) markDocument(requestID network.RequestID, mimeType string) { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.recorded { + return + } + t.recorded = true + t.requestID = requestID + t.isJSON = isJSONMimeType(mimeType) +} + +// mainDocument returns the main document's request ID and whether it was +// JSON. +func (t *jsonDocumentTracker) mainDocument() (network.RequestID, bool) { + t.mutex.Lock() + defer t.mutex.Unlock() + + return t.requestID, t.isJSON +} + func cdpHeaders(driverOptions scraperDriverOptions) map[string]interface{} { headers := map[string]interface{}{} if driverOptions.Headers != nil { diff --git a/pkg/scraper/url_test.go b/pkg/scraper/url_test.go new file mode 100644 index 0000000000..9f84e24b0f --- /dev/null +++ b/pkg/scraper/url_test.go @@ -0,0 +1,108 @@ +package scraper + +import ( + "fmt" + "sync" + "testing" + + "github.com/chromedp/cdproto/network" +) + +func TestIsJSONMimeType(t *testing.T) { + tests := []struct { + name string + mimeType string + want bool + }{ + {"plain json", "application/json", true}, + {"json with charset", "application/json; charset=utf-8", true}, + {"json with charset and spacing", "application/json; charset=UTF-8", true}, + {"structured syntax suffix", "application/ld+json", true}, + {"uppercase", "APPLICATION/JSON", true}, + {"text/json", "text/json", true}, + {"html", "text/html", false}, + {"html with charset", "text/html; charset=utf-8", false}, + {"plain text", "text/plain", false}, + {"empty", "", false}, + {"contains json as substring but isn't json", "application/jsonp", false}, + {"contains json as substring but isn't json 2", "multipart/json-form-data", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isJSONMimeType(tt.mimeType) + if got != tt.want { + t.Errorf("isJSONMimeType(%q) = %v, want %v", tt.mimeType, got, tt.want) + } + }) + } +} + +func TestJSONDocumentTrackerOnlyRecordsFirstDocument(t *testing.T) { + t.Run("first document is JSON, later ones don't override it", func(t *testing.T) { + var tracker jsonDocumentTracker + + tracker.markDocument("req-1", "application/json") + tracker.markDocument("req-2", "application/json") + tracker.markDocument("req-3", "text/html") + + if requestID, isJSON := tracker.mainDocument(); !isJSON || requestID != "req-1" { + t.Fatalf("mainDocument() = (%q, %v), want (%q, true)", requestID, isJSON, "req-1") + } + }) + + t.Run("first document is HTML, a later JSON response doesn't override it", func(t *testing.T) { + // This is the main-frame-vs-iframe scenario: an HTML scrape whose + // page contains an iframe that loads JSON (an embed widget, an ad + // frame) also fires a Document-type responseReceived for that + // iframe. Since the top-level HTML document is always the first + // Document response chromedp sees for a given navigation, the + // tracker must not let this later JSON response override it - + // otherwise an HTML scraper would incorrectly receive the iframe's + // JSON body instead of the page's HTML. + var tracker jsonDocumentTracker + + tracker.markDocument("req-main-page", "text/html") + tracker.markDocument("req-iframe", "application/json") + + if requestID, isJSON := tracker.mainDocument(); isJSON || requestID != "req-main-page" { + t.Fatalf("mainDocument() = (%q, %v), want (%q, false)", requestID, isJSON, "req-main-page") + } + }) +} + +// TestJSONDocumentTrackerConcurrentAccess exercises jsonDocumentTracker the +// way urlFromCDP actually uses it: one goroutine (standing in for chromedp's +// event-processing goroutine) calls markDocument while another goroutine +// (standing in for the action sequence passed to chromedp.Run) calls +// mainDocument, concurrently and repeatedly. Before requestID/isJSON/recorded +// were guarded by a mutex, `go test -race` reliably flagged this exact +// access pattern as a data race. +func TestJSONDocumentTrackerConcurrentAccess(t *testing.T) { + var tracker jsonDocumentTracker + + const n = 200 + var wg sync.WaitGroup + wg.Add(2 * n) + + for i := range n { + go func() { + defer wg.Done() + tracker.markDocument(network.RequestID(fmt.Sprintf("req-%d", i)), "application/json") + }() + go func() { + defer wg.Done() + tracker.mainDocument() + }() + } + + wg.Wait() + + requestID, isJSON := tracker.mainDocument() + if !isJSON { + t.Fatal("expected a JSON document to have been recorded") + } + if requestID == "" { + t.Fatal("expected a non-empty request ID to have been recorded") + } +}