From c6c2eb4359069e9b345d0fc8340ffbd8a6e2956f Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Mon, 27 Jul 2026 23:08:36 -0500 Subject: [PATCH 1/2] feat(ja4): compute JA4 natively from the ClientHello, and bump Go to 1.24 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblocks the half of PRD workstream D that was not waiting on the enrichment migration. Go 1.24 is a hard prerequisite: crypto/tls only began exposing the ClientHello extension list in that release (ClientHelloInfo.Extensions, golang/go#32936, verified against GOROOT/api/go1.24.txt), and without it the extension hash cannot be computed and JA4 is not derivable from the stdlib. Five version pins moved together, not the three the PRD listed — server-go/go.mod, server-go/Dockerfile, docker/Dockerfile (the CI publish image, missing it broke the v1.15.0 publish), plus fcaptcha-cloud's deploy/Dockerfile and its vendored server/go.mod. ## Scope, and a licensing line JA4 (TLS) only. It is BSD-3-Clause with an explicit patent disclaimer, which is what makes it safe in an MIT project people deploy commercially. JA4H, JA4T, JA4L, JA4S, JA4X and JA4SSH are FoxIO License 1.1 — non-commercial, patent- pending, GPL-incompatible — and a test now fails if one appears, pointing at the reason rather than leaving the next person to rediscover it. ## Where it works Only where this process terminates TLS, since the ClientHello is consumed by whoever completes the handshake. Set FCAPTCHA_TLS_CERT and FCAPTCHA_TLS_KEY to serve HTTPS directly; behind Railway, Cloudflare or nginx the TRUSTED_JA4_HEADERS path is unchanged and remains the only source. The startup log now says which of the two is in effect, or that neither is. When both are available the native fingerprint wins: it was derived here and cannot be asserted by a client or a misconfigured proxy, whereas a header is only as good as whoever set it. ## A mislabelled placeholder that would have flagged every Chrome user knownBotJA4Hashes carried a commented example: "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS" That is the canonical example from the JA4 spec, and it is Chrome. Measured with this implementation against a real TLS listener: Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 node https t13i521000_b262b3658495_8e6e362c5eac python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac The middle section hashes the cipher list, i.e. the TLS stack. 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Uncommenting that line would have scored every Chrome visitor as automation at 0.8/0.9. Replaced with the measured table and the reason the map ships empty. ## What this does and does not detect today Nothing, yet. The map CheckJA4Fingerprint consults is empty, deliberately: a static hash list is defeated by rotating a fingerprint, and the readings above already drift from the spec's own example through version churn alone. Populating it with values nobody here has observed would buy false-positive risk and no detection. A test records that it ships empty so filling it is a deliberate act. The aggregate JA4 Signals model the PRD describes is not built either — it wants population-scale ratios that a single self-hosted instance does not have. What ships is the correct, tested computation and the plumbing, so a fingerprint is available to correlate (workstream F) and to populate a list from real abuse. Tests: 12 covering GREASE exclusion, the count-includes-but-hash-excludes asymmetry for SNI/ALPN, sig-algs deliberately unsorted, order insensitivity, native-over-header precedence, a real TLS handshake end to end, and the licensing boundary. Go E2E unchanged at 89/93. --- README.md | 4 +- docker/Dockerfile | 2 +- server-go/Dockerfile | 2 +- server-go/detection.go | 42 +++++- server-go/go.mod | 2 +- server-go/ja4.go | 252 +++++++++++++++++++++++++++++++++++ server-go/ja4_test.go | 272 ++++++++++++++++++++++++++++++++++++++ server-go/main.go | 48 ++++++- server-go/scoring.go | 31 +++-- server-go/scoring_test.go | 4 +- 10 files changed, 630 insertions(+), 29 deletions(-) create mode 100644 server-go/ja4.go create mode 100644 server-go/ja4_test.go diff --git a/README.md b/README.md index 9c99cfd..8326504 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ This makes credential stuffing expensive: even if a bot passes all other checks, - **CDP detection** — legacy ChromeDriver/Selenium globals plus a Runtime/DevTools console-attach probe that catches any attached protocol client, even when JS globals are scrubbed - Headless browser indicators, plugin/feature checks, UA ↔ platform consistency - Canvas / WebGL / Audio fingerprinting (session-scoped only) -- **TLS fingerprinting** — JA3 (client-supplied) and JA4 (read from a trusted reverse-proxy header, un-spoofable by the client) matched against known automation tools +- **TLS fingerprinting** — JA3 (client-supplied) and JA4. JA4 is computed **natively from the ClientHello** when the Go server terminates TLS itself (`FCAPTCHA_TLS_CERT`/`FCAPTCHA_TLS_KEY`), which no client can assert; behind a terminating proxy it falls back to a trusted header. Only JA4-TLS is implemented — the rest of the JA4 family is licensed non-commercially and cannot ship here ### Temporal Signals - Proof of Work timing (reveals API round-trip latency) @@ -453,6 +453,7 @@ Verify a previously issued token (server-side). | `FCAPTCHA_SITE_KEYS` | Comma-separated allowlist of accepted site keys. Unset accepts any key (zero-config self-hosting); unlisted keys are folded into a shared overflow bucket rather than allocating their own rate-limit/fingerprint state | (any) | | `FCAPTCHA_MAX_SITE_KEYS_PER_IP` | Distinct site keys one IP may allocate state for before the excess is folded into the overflow bucket. The cap itself is unconditional | 8 | | `TRUSTED_JA4_HEADERS` | Comma-separated reverse-proxy header names carrying a JA4 TLS fingerprint (e.g. set by nginx/Cloudflare). Only these names are read, and only from a peer in `TRUSTED_PROXIES` | (none) | +| `FCAPTCHA_TLS_CERT` / `FCAPTCHA_TLS_KEY` | Serve HTTPS directly (Go server). Terminating TLS here is what makes **native JA4** possible — the fingerprint is computed from the ClientHello rather than taken on trust from a proxy. Behind Railway/Cloudflare/nginx, leave unset and use `TRUSTED_JA4_HEADERS` | (none, plain HTTP) | | `FCAPTCHA_CLIENT_PATH` | Explicit path to `client/fcaptcha.js` for same-origin widget serving | (auto-probed) | | `FCAPTCHA_SERVE_CLIENT` | (Python) Serve the widget at `/fcaptcha.js`; set `false` to host the client on a separate CDN | `true` | | `FCAPTCHA_PPROF` | (Go) Enable the pprof debug server (`1`/`true`/`yes`/`on`) | off | @@ -542,6 +543,7 @@ fcaptcha/ │ ├── clientip.go # Trusted-proxy client IP resolution (TRUSTED_PROXIES) │ ├── sitekeys.go # Bounds on state a client-supplied siteKey can allocate │ ├── inputforensics.go # Typing cadence/modality, scroll morphology, platform coherence +│ ├── ja4.go # Native JA4-TLS from the ClientHello (Go 1.24+, stdlib only) │ ├── scoring_test.go # Go unit tests │ ├── clientip_test.go # Trusted-proxy unit tests │ └── go.mod diff --git a/docker/Dockerfile b/docker/Dockerfile index 3bd55a6..4111542 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.23-alpine AS builder +FROM golang:1.24-alpine AS builder WORKDIR /app diff --git a/server-go/Dockerfile b/server-go/Dockerfile index 79798fc..2b6e013 100644 --- a/server-go/Dockerfile +++ b/server-go/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS builder +FROM golang:1.24-alpine AS builder WORKDIR /app diff --git a/server-go/detection.go b/server-go/detection.go index 07d920e..e698728 100644 --- a/server-go/detection.go +++ b/server-go/detection.go @@ -569,13 +569,41 @@ func (e *ScoringEngine) CheckJA3Fingerprint(ja3Hash string) []DetectionResult { // computed from the TLS ClientHello by the reverse proxy and passed via a // trusted header the server is configured to accept via TRUSTED_JA4_HEADERS. -// knownBotJA4Hashes holds observed automation fingerprints. -// Populate in production; entries below are placeholders. -// JA4 format: t##d####_####..._#### -var knownBotJA4Hashes = map[string]string{ - // Example once identified: - // "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS", -} +// knownBotJA4Hashes holds observed automation fingerprints. It ships EMPTY, and +// that is deliberate — see below before adding anything. +// +// JA4 format: t##d####__ +// +// # The placeholder that used to live here was wrong +// +// It read: +// +// "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS" +// +// That is the canonical example from the JA4 specification, and it is **Chrome**, +// not Go. Measured on a real TLS listener with server-go's own ComputeJA4: +// +// Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea +// Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 +// curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 +// node https t13i521000_b262b3658495_8e6e362c5eac +// python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac +// +// The middle section is a hash of the cipher list, i.e. of the TLS stack itself. +// 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Had anyone uncommented that +// line, every Chrome visitor would have been scored as automation at 0.8/0.9. +// +// # Why it stays empty +// +// A static hash list is defeated by rotating a fingerprint, and these values +// drift with every browser release — the two Chromium readings above differ from +// the spec's example in both the counts and the extension hash, from version +// drift alone. Populating this with fingerprints nobody here has observed trades +// a real false-positive risk for no detection value. +// +// If you populate it, populate it from abuse you actually saw on your own +// deployment, and record where each fingerprint came from. +var knownBotJA4Hashes = map[string]string{} // GetTrustedJA4HeaderNames reads TRUSTED_JA4_HEADERS env var (comma-separated). func GetTrustedJA4HeaderNames() []string { diff --git a/server-go/go.mod b/server-go/go.mod index 8b67b39..c5bcb93 100644 --- a/server-go/go.mod +++ b/server-go/go.mod @@ -1,6 +1,6 @@ module github.com/WebDecoy/FCaptcha/server-go -go 1.23 +go 1.24 require ( github.com/go-chi/chi/v5 v5.0.11 diff --git a/server-go/ja4.go b/server-go/ja4.go new file mode 100644 index 0000000..0a15958 --- /dev/null +++ b/server-go/ja4.go @@ -0,0 +1,252 @@ +package main + +import ( + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "fmt" + "sort" + "strings" + "time" + + "github.com/hashicorp/golang-lru/v2/expirable" +) + +// JA4 TLS client fingerprinting, computed natively from the ClientHello. +// +// # Scope, and a licensing line not to cross +// +// This implements **JA4 (TLS) only**. JA4 is published under BSD-3-Clause and +// FoxIO explicitly disclaims patent coverage for it, which is what makes it +// safe to ship in an MIT project that people deploy commercially. +// +// The rest of the family — JA4H (HTTP), JA4T, JA4L, JA4S, JA4X, JA4SSH — is +// licensed under FoxIO License 1.1: non-commercial use only, patent-pending on +// the detection logic, and incompatible with GPL/AGPL. FoxIO's FAQ is explicit +// that indirect monetization requires an OEM licence. **Do not add them here.** +// If you want header-order or RTT heuristics, derive them from first principles +// — a header-order signature is SHA256 over the lowercased names in order plus +// a casing flag, about fifteen lines, and owes nothing to anyone. +// +// # Where this works, and where it does not +// +// It reads the ClientHello, so it only produces a fingerprint when *this +// process* terminates TLS. Behind Railway, Cloudflare, nginx or any other +// terminating proxy the ClientHello was consumed upstream and there is nothing +// here to read — those deployments must keep using the trusted-header path +// (TRUSTED_JA4_HEADERS), which is unchanged and remains the default. +// +// That covers the single-binary self-hosted case FCaptcha advertises, and +// nothing else. It is not a replacement for the header path; it is the option +// for people who do not have a proxy to read the header from. +// +// # Why Go 1.24 +// +// crypto/tls only began exposing the ClientHello extension list in 1.24 +// (ClientHelloInfo.Extensions, golang/go#32936). Without it the extension hash +// cannot be computed and JA4 is not derivable from the stdlib at all. + +// GREASE values (RFC 8701) are random padding a client injects to keep the +// ecosystem tolerant of unknown values. They vary per connection by design, so +// including them would make the fingerprint different every time — the JA4 spec +// excludes them from every count and every list. +func isGREASE(v uint16) bool { + // 0x0a0a, 0x1a1a, 0x2a2a ... 0xfafa: both bytes equal, low nibble 0xa. + return v&0x0f0f == 0x0a0a && v>>8 == v&0xff +} + +func filterGREASE(vals []uint16) []uint16 { + out := make([]uint16, 0, len(vals)) + for _, v := range vals { + if !isGREASE(v) { + out = append(out, v) + } + } + return out +} + +// ja4TLSVersion renders the negotiated-or-highest-offered version as the two +// characters the spec uses. +func ja4TLSVersion(versions []uint16) string { + best := uint16(0) + for _, v := range filterGREASE(versions) { + if v > best { + best = v + } + } + switch best { + case tls.VersionTLS13: + return "13" + case tls.VersionTLS12: + return "12" + case tls.VersionTLS11: + return "11" + case tls.VersionTLS10: + return "10" + case 0x0300: + return "s3" + case 0x0002: + return "s2" + default: + return "00" + } +} + +// ja4ALPN takes the first and last character of the first offered protocol: +// "h2" -> "h2", "http/1.1" -> "h1". Non-ASCII values fall back to the hex of +// those bytes, per the spec's note on non-printable ALPNs. +func ja4ALPN(protos []string) string { + if len(protos) == 0 || protos[0] == "" { + return "00" + } + p := protos[0] + first, last := p[0], p[len(p)-1] + + printable := func(b byte) bool { + return (b >= '0' && b <= '9') || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') + } + if printable(first) && printable(last) { + return string([]byte{first, last}) + } + return fmt.Sprintf("%x%x", first>>4, last&0x0f) +} + +// twoDigit caps a count at 99, which is what the fixed-width JA4_a field allows. +func twoDigit(n int) string { + if n > 99 { + n = 99 + } + return fmt.Sprintf("%02d", n) +} + +func hexList(vals []uint16) string { + parts := make([]string, len(vals)) + for i, v := range vals { + parts[i] = fmt.Sprintf("%04x", v) + } + return strings.Join(parts, ",") +} + +// sha256First12 is the truncation the spec specifies for both hashed sections. +func sha256First12(s string) string { + if s == "" { + return "000000000000" + } + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:12] +} + +// ComputeJA4 builds the fingerprint from a ClientHello. +// +// Layout: ja4_a _ ja4_b _ ja4_c +// +// ja4_a transport, TLS version, SNI presence, cipher count, extension +// count, first ALPN — human-readable, 10 chars +// ja4_b truncated SHA-256 over the sorted cipher list +// ja4_c truncated SHA-256 over the sorted extension list, an underscore, +// and the signature algorithms in their original order +// +// Two asymmetries in the spec are deliberate and easy to get wrong: the +// extension *count* in ja4_a includes SNI and ALPN, while the extension *list* +// in ja4_c excludes them; and the signature algorithms are NOT sorted, because +// their order is itself characteristic of the client. +func ComputeJA4(hello *tls.ClientHelloInfo) string { + if hello == nil { + return "" + } + + ciphers := filterGREASE(hello.CipherSuites) + extensions := filterGREASE(hello.Extensions) + + // "d" when the client sent SNI (a domain), "i" when it connected by IP. + sni := "i" + if hello.ServerName != "" { + sni = "d" + } + + // "t" for TCP. QUIC would be "q", but a net/http server reaching this code + // path is TCP by construction. + ja4a := "t" + ja4TLSVersion(hello.SupportedVersions) + sni + + twoDigit(len(ciphers)) + twoDigit(len(extensions)) + ja4ALPN(hello.SupportedProtos) + + sortedCiphers := append([]uint16(nil), ciphers...) + sort.Slice(sortedCiphers, func(i, j int) bool { return sortedCiphers[i] < sortedCiphers[j] }) + ja4b := sha256First12(hexList(sortedCiphers)) + + // SNI (0x0000) and ALPN (0x0010) are excluded from the hashed list: they + // carry per-request content rather than client identity, so including them + // would make the same browser fingerprint differently per site. + hashable := make([]uint16, 0, len(extensions)) + for _, e := range extensions { + if e != 0x0000 && e != 0x0010 { + hashable = append(hashable, e) + } + } + sort.Slice(hashable, func(i, j int) bool { return hashable[i] < hashable[j] }) + + sigAlgs := make([]uint16, 0, len(hello.SignatureSchemes)) + for _, s := range hello.SignatureSchemes { + if !isGREASE(uint16(s)) { + sigAlgs = append(sigAlgs, uint16(s)) + } + } + + ja4c := sha256First12(hexList(hashable) + "_" + hexList(sigAlgs)) + + return ja4a + "_" + ja4b + "_" + ja4c +} + +// ja4Store keeps the fingerprint for the life of a connection. +// +// The ClientHello is only visible during the handshake, in a callback that has +// no request attached to it yet. So it is recorded against the connection there +// and read back when the request arrives on that same connection. Bounded, so a +// long-lived server cannot accumulate entries for connections that never +// produced a request. +type ja4Store struct { + byConn *expirable.LRU[string, string] +} + +// TLSConfigWithJA4 returns a tls.Config that records a JA4 fingerprint per +// connection, and a lookup the HTTP handler can use. +// +// GetConfigForClient is the hook: it fires once per handshake with the parsed +// ClientHello and is otherwise free to return nil, meaning "use the base +// config". Recording a fingerprint there is a side effect on the connection's +// remote address, which is the only stable key available at that point. +func TLSConfigWithJA4(base *tls.Config, store *ja4Store) *tls.Config { + cfg := base.Clone() + cfg.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + if hello.Conn != nil { + store.Record(hello.Conn.RemoteAddr().String(), ComputeJA4(hello)) + } + return nil, nil + } + return cfg +} + +// ja4ConnTTL bounds how long a recorded fingerprint outlives its handshake. +// Keep-alive connections can serve requests for a while, but an entry that has +// not been read within this window belongs to a connection that handshook and +// then went quiet. +const ja4ConnTTL = 5 * time.Minute + +// maxTrackedJA4Conns bounds the per-connection table. Generous next to any +// realistic count of simultaneously-open connections, small enough that the +// store cannot become a leak. +const maxTrackedJA4Conns = 20_000 + +func newJA4Store(max int) *ja4Store { + return &ja4Store{byConn: expirable.NewLRU[string, string](max, nil, ja4ConnTTL)} +} + +func (s *ja4Store) Record(remoteAddr, fp string) { + if fp != "" { + s.byConn.Add(remoteAddr, fp) + } +} + +func (s *ja4Store) Lookup(remoteAddr string) string { + fp, _ := s.byConn.Get(remoteAddr) + return fp +} diff --git a/server-go/ja4_test.go b/server-go/ja4_test.go new file mode 100644 index 0000000..202c065 --- /dev/null +++ b/server-go/ja4_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "crypto/tls" + "io" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strings" + "testing" +) + +// JA4 has a published spec, so the failure mode to guard against is silent +// divergence from it. These check the rules that are easy to get subtly wrong, +// plus one end-to-end handshake proving the wiring actually produces a +// fingerprint from a real connection. + +func TestIsGREASE(t *testing.T) { + // RFC 8701 defines exactly sixteen GREASE values. + for _, v := range []uint16{0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x7a7a, 0xdada, 0xeaea, 0xfafa} { + if !isGREASE(v) { + t.Errorf("%#04x is a GREASE value", v) + } + } + // Near-misses that must NOT be treated as GREASE, or real ciphers vanish + // from the fingerprint. + for _, v := range []uint16{0x1301, 0x1302, 0xc02b, 0x0a0b, 0x0b0a, 0x1a2a, 0xabab} { + if isGREASE(v) { + t.Errorf("%#04x is a real value, not GREASE", v) + } + } +} + +func TestJA4ALPNEncoding(t *testing.T) { + cases := map[string]string{ + "h2": "h2", + "http/1.1": "h1", // first and last character, not a prefix + "h3": "h3", + } + for in, want := range cases { + if got := ja4ALPN([]string{in}); got != want { + t.Errorf("ja4ALPN(%q) = %q, want %q", in, got, want) + } + } + if got := ja4ALPN(nil); got != "00" { + t.Errorf("no ALPN should give 00, got %q", got) + } + // Only the first protocol counts. + if got := ja4ALPN([]string{"h2", "http/1.1"}); got != "h2" { + t.Errorf("only the first ALPN counts, got %q", got) + } +} + +func TestJA4CountsExcludeGREASE(t *testing.T) { + hello := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x0a0a, 0x1301, 0x1302, 0x1a1a, 0xc02b}, + Extensions: []uint16{0x2a2a, 0x0000, 0x0010, 0x000d, 0x002b}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + SupportedProtos: []string{"h2"}, + } + ja4 := ComputeJA4(hello) + + // 3 real ciphers of 5, 4 real extensions of 5 — and the extension COUNT + // includes SNI and ALPN even though the hashed list excludes them. + if !strings.HasPrefix(ja4, "t13d0304h2_") { + t.Errorf("expected prefix t13d0304h2_, got %q", ja4) + } +} + +func TestJA4SNIFlag(t *testing.T) { + base := func(name string) *tls.ClientHelloInfo { + return &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301}, + Extensions: []uint16{0x002b}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: name, + } + } + if got := ComputeJA4(base("example.com")); got[3] != 'd' { + t.Errorf("SNI present should give 'd', got %q in %s", got[3], got) + } + if got := ComputeJA4(base("")); got[3] != 'i' { + t.Errorf("no SNI should give 'i', got %q in %s", got[3], got) + } +} + +// Cipher and extension order varies between connections from the same client; +// the spec sorts them so it does not affect the fingerprint. +func TestJA4IsOrderInsensitiveForCiphersAndExtensions(t *testing.T) { + a := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301, 0x1302, 0xc02b}, + Extensions: []uint16{0x002b, 0x000d, 0x0017}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + } + b := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0xc02b, 0x1301, 0x1302}, + Extensions: []uint16{0x0017, 0x002b, 0x000d}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + } + if ComputeJA4(a) != ComputeJA4(b) { + t.Errorf("reordering ciphers/extensions changed the fingerprint:\n %s\n %s", ComputeJA4(a), ComputeJA4(b)) + } +} + +// Signature algorithms are the exception: their order IS characteristic of the +// client, so the spec deliberately does not sort them. +func TestJA4SignatureAlgorithmOrderMatters(t *testing.T) { + mk := func(schemes ...tls.SignatureScheme) *tls.ClientHelloInfo { + return &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301}, + Extensions: []uint16{0x000d}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + SignatureSchemes: schemes, + } + } + a := ComputeJA4(mk(tls.ECDSAWithP256AndSHA256, tls.PSSWithSHA256)) + b := ComputeJA4(mk(tls.PSSWithSHA256, tls.ECDSAWithP256AndSHA256)) + if a == b { + t.Error("signature algorithm order must change the fingerprint (it is not sorted)") + } +} + +func TestJA4ExtensionListExcludesSNIAndALPN(t *testing.T) { + // Same client, two different sites: SNI and ALPN differ, everything else is + // identical. The hashed section must not move, or one browser fingerprints + // differently per site and the signal is useless. + mk := func(name string) *tls.ClientHelloInfo { + return &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301, 0x1302}, + Extensions: []uint16{0x0000, 0x0010, 0x002b, 0x000d}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: name, + SupportedProtos: []string{"h2"}, + } + } + a, b := ComputeJA4(mk("one.example")), ComputeJA4(mk("two.example")) + if a != b { + t.Errorf("the hashed sections must not depend on SNI:\n %s\n %s", a, b) + } +} + +func TestJA4Shape(t *testing.T) { + hello := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301, 0x1302, 0x1303}, + Extensions: []uint16{0x0000, 0x0010, 0x002b, 0x000d, 0x0017}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + SupportedProtos: []string{"h2"}, + SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}, + } + got := ComputeJA4(hello) + // t13d0305h2_<12 hex>_<12 hex> + if !regexp.MustCompile(`^[tq](13|12|11|10|s3|s2|00)[di]\d{4}[a-z0-9]{2}_[0-9a-f]{12}_[0-9a-f]{12}$`).MatchString(got) { + t.Errorf("malformed JA4: %q", got) + } +} + +// End-to-end: a real handshake against a real listener, proving the +// GetConfigForClient wiring records something a handler can read back. +func TestJA4EndToEndOverRealTLS(t *testing.T) { + store := newJA4Store(128) + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, store.Lookup(r.RemoteAddr)) + })) + srv.TLS = TLSConfigWithJA4(&tls.Config{}, store) + srv.StartTLS() + defer srv.Close() + + // httptest.Server.Client() already trusts the throwaway certificate. + resp, err := srv.Client().Get(srv.URL) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + got := string(body) + if got == "" { + t.Fatal("no JA4 was recorded for a real TLS connection") + } + if !strings.HasPrefix(got, "t1") { + t.Errorf("expected a TCP/TLS1.x fingerprint, got %q", got) + } + t.Logf("Go client fingerprinted as %s", got) +} + +// Guards the licensing boundary from §8.3 of the extensions PRD: JA4 (TLS) is +// BSD-3-Clause and safe here; JA4H/JA4T/JA4L/JA4S/JA4X/JA4SSH are FoxIO License +// 1.1, non-commercial only, and cannot ship in an MIT project. If someone adds +// one, this fails and points them at the reason. +func TestOnlyJA4TLSIsImplemented(t *testing.T) { + src := readSourceFile(t, "ja4.go") + for _, forbidden := range []string{"JA4H", "JA4T", "JA4L", "JA4S", "JA4X", "JA4SSH"} { + // Allowed inside the comment that explains why they are absent. + count := strings.Count(src, "func "+forbidden) + strings.Count(src, "func Compute"+forbidden) + if count > 0 { + t.Errorf("%s appears to be implemented — it is FoxIO License 1.1 "+ + "(non-commercial, GPL-incompatible) and cannot ship in this MIT project. See PRD §8.3.", forbidden) + } + } +} + +func readSourceFile(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(name) + if err != nil { + t.Fatalf("reading %s: %v", name, err) + } + return string(b) +} + +// The locally-computed fingerprint must win over a header, because a header +// requires trusting whatever set it while a ClientHello-derived value cannot be +// asserted by anyone. +func TestNativeJA4TakesPrecedenceOverHeader(t *testing.T) { + const nativeFP = "t13d1516h2_aaaaaaaaaaaa_bbbbbbbbbbbb" + const headerFP = "t13d1516h2_cccccccccccc_dddddddddddd" + + knownBotJA4Hashes[nativeFP] = "native-source tool" + knownBotJA4Hashes[headerFP] = "header-source tool" + defer func() { + delete(knownBotJA4Hashes, nativeFP) + delete(knownBotJA4Hashes, headerFP) + }() + + t.Setenv("TRUSTED_JA4_HEADERS", "cf-ja4") + e := NewScoringEngine("test-secret") + headers := map[string]string{"cf-ja4": headerFP} + + // The reason string is generic, so read which fingerprint was matched out of + // the detection Details. + matchedTool := func(res *VerificationResult) string { + for _, d := range res.Detections { + if d.Category == CategoryFingerprint && d.Details != nil { + if tool, ok := d.Details["tool"].(string); ok { + return tool + } + } + } + return "" + } + + // Both available: the native one is the one that gets scored. + both := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", headers, "", nativeFP, true, nil) + if got := matchedTool(both); got != "native-source tool" { + t.Errorf("native JA4 should take precedence, matched %q", got) + } + + // No native fingerprint (something upstream terminated TLS): fall back. + fallback := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", headers, "", "", true, nil) + if got := matchedTool(fallback); got != "header-source tool" { + t.Errorf("should fall back to the trusted header, matched %q", got) + } +} + +// The map that CheckJA4Fingerprint consults ships empty on purpose — the PRD +// calls a static hash list defeatable by rotating a fingerprint, and populating +// it with values nobody here has observed would be worse than leaving it bare. +// This records that state so its eventual filling is a deliberate act. +func TestKnownBotJA4MapShipsEmpty(t *testing.T) { + if len(knownBotJA4Hashes) != 0 { + t.Errorf("knownBotJA4Hashes has %d entries; native JA4 computation is wired but "+ + "intentionally has nothing to match against yet. If you are populating it, "+ + "say where the fingerprints were observed.", len(knownBotJA4Hashes)) + } +} diff --git a/server-go/main.go b/server-go/main.go index 8730770..7e5a4a7 100644 --- a/server-go/main.go +++ b/server-go/main.go @@ -3,6 +3,7 @@ package main import ( "context" "crypto/sha256" + "crypto/tls" "encoding/hex" "encoding/json" "log" @@ -179,6 +180,10 @@ func main() { siteKeys := SiteKeyGuardFromEnv() log.Printf("site keys: %s", siteKeys.Describe()) + // Holds a JA4 fingerprint per live connection, populated during the TLS + // handshake. Stays empty unless this process terminates TLS — see ja4.go. + ja4s := newJA4Store(maxTrackedJA4Conns) + // Middleware // // Deliberately no middleware.RealIP: it overwrites r.RemoteAddr from @@ -221,8 +226,8 @@ func main() { // Routes r.Get("/health", healthHandler) - r.Post("/api/verify", verifyHandler(engine, proxyTrust, siteKeys)) - r.Post("/api/score", invisibleScoreHandler(engine, proxyTrust, siteKeys)) + r.Post("/api/verify", verifyHandler(engine, proxyTrust, siteKeys, ja4s)) + r.Post("/api/score", invisibleScoreHandler(engine, proxyTrust, siteKeys, ja4s)) r.Post("/api/token/verify", tokenVerifyHandler(engine, proxyTrust)) r.Get("/api/pow/challenge", powChallengeHandler(engine, proxyTrust, siteKeys)) r.Get("/api/challenge", challengeHandler(engine)) @@ -252,10 +257,39 @@ func main() { }() } + // Optional direct TLS termination, off by default. + // + // Set FCAPTCHA_TLS_CERT and FCAPTCHA_TLS_KEY to have this process terminate + // TLS itself, which is the only arrangement where a JA4 fingerprint can be + // computed here — the ClientHello is consumed by whoever completes the + // handshake. Behind Railway, Cloudflare or nginx that is not us, and the + // TRUSTED_JA4_HEADERS path remains the way to get a fingerprint. + certFile := os.Getenv("FCAPTCHA_TLS_CERT") + keyFile := os.Getenv("FCAPTCHA_TLS_KEY") + serveTLS := certFile != "" && keyFile != "" + + if serveTLS { + srv.TLSConfig = TLSConfigWithJA4(&tls.Config{MinVersion: tls.VersionTLS12}, ja4s) + log.Printf("native JA4: on (terminating TLS locally)") + } else { + trustedJA4 := GetTrustedJA4HeaderNames() + if len(trustedJA4) > 0 { + log.Printf("native JA4: off (not terminating TLS); reading %s from trusted proxies", strings.Join(trustedJA4, ", ")) + } else { + log.Printf("native JA4: off (not terminating TLS, and no TRUSTED_JA4_HEADERS set) — no TLS fingerprint available") + } + } + // Graceful shutdown go func() { log.Printf("FCaptcha server starting on port %s", port) - if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + var err error + if serveTLS { + err = srv.ListenAndServeTLS(certFile, keyFile) + } else { + err = srv.ListenAndServe() + } + if err != nil && err != http.ErrServerClosed { log.Fatalf("Server error: %v", err) } }() @@ -340,7 +374,7 @@ func webBotAuthDetections(engine *ScoringEngine, r *http.Request) []DetectionRes return engine.CheckWebBotAuth(ctx, webbotauth.RequestFromHTTP(r, webbotauth.WithScheme(scheme))) } -func verifyHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard) http.HandlerFunc { +func verifyHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard, ja4s *ja4Store) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var req VerifyRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -414,7 +448,7 @@ func verifyHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGu // passed as preDetections so the verified/forged verdict is scored. webBotAuth := webBotAuthDetections(engine, r) - result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, headers, ja3Hash, peerTrusted, webBotAuth, req.PowSolution) + result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, headers, ja3Hash, ja4s.Lookup(r.RemoteAddr), peerTrusted, webBotAuth, req.PowSolution) // Add signal commitment detections to results if len(extraDetections) > 0 { @@ -459,7 +493,7 @@ type InvisibleScoreRequest struct { PowTiming *PowTiming `json:"powTiming,omitempty"` } -func invisibleScoreHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard) http.HandlerFunc { +func invisibleScoreHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard, ja4s *ja4Store) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var req InvisibleScoreRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -524,7 +558,7 @@ func invisibleScoreHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *S // Web Bot Auth: verify signed-agent requests (see verifyHandler). webBotAuth := webBotAuthDetections(engine, r) - result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, scoreHeaders, ja3, peerTrusted, webBotAuth, req.PowSolution) + result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, scoreHeaders, ja3, ja4s.Lookup(r.RemoteAddr), peerTrusted, webBotAuth, req.PowSolution) if len(scoreExtraDetections) > 0 { result.Detections = append(scoreExtraDetections, result.Detections...) } diff --git a/server-go/scoring.go b/server-go/scoring.go index 923f858..ed17d63 100644 --- a/server-go/scoring.go +++ b/server-go/scoring.go @@ -322,7 +322,13 @@ func compileUAPatterns() []*regexp.Regexp { // request — currently Web Bot Auth signature verification, which needs the // accurately-reconstructed signed request. They are seeded into the detection // set so they participate in scoring like any engine-produced detection. -func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, siteKey, userAgent string, headers map[string]string, ja3Hash string, peerTrusted bool, preDetections []DetectionResult, powSolution ...*PoWSolution) *VerificationResult { +// nativeJA4 is a fingerprint this process computed from the ClientHello (see +// ja4.go), empty when something upstream terminated TLS. +// +// NOTE: this parameter list has now grown three times and is due a request-context +// struct. Left positional for now so the vendored copy in fcaptcha-cloud stays a +// straight file copy rather than a port. +func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, siteKey, userAgent string, headers map[string]string, ja3Hash, nativeJA4 string, peerTrusted bool, preDetections []DetectionResult, powSolution ...*PoWSolution) *VerificationResult { detections := make([]DetectionResult, 0, len(preDetections)+8) detections = append(detections, preDetections...) @@ -407,15 +413,22 @@ func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, si detections = append(detections, e.CheckJA3Fingerprint(ja3Hash)...) } - // TLS fingerprint (JA4) — trusted reverse-proxy header, un-spoofable by client - if headers != nil { - trustedJA4 := GetTrustedJA4HeaderNames() - if len(trustedJA4) > 0 { - if ja4 := ReadJA4FromHeaders(headers, trustedJA4); ja4 != "" { - detections = append(detections, e.CheckJA4Fingerprint(ja4)...) - } + // TLS fingerprint (JA4). Two sources, and the local one wins. + // + // A natively-computed fingerprint was derived from the ClientHello by this + // process, so it cannot be asserted by anyone — not the client, and not a + // misconfigured proxy either. The header path requires trusting whatever sits + // in front of us to have computed it honestly, which is weaker, so it is only + // consulted when there is no local fingerprint to use. + ja4 := nativeJA4 + if ja4 == "" && headers != nil { + if trustedJA4 := GetTrustedJA4HeaderNames(); len(trustedJA4) > 0 { + ja4 = ReadJA4FromHeaders(headers, trustedJA4) } } + if ja4 != "" { + detections = append(detections, e.CheckJA4Fingerprint(ja4)...) + } // Form interaction analysis (credential stuffing & spam detection) if formAnalysis, ok := signals["formAnalysis"].(map[string]interface{}); ok { @@ -459,7 +472,7 @@ func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, si // Verify performs full verification (backward compatible) func (e *ScoringEngine) Verify(signals map[string]interface{}, ip, siteKey, userAgent string) *VerificationResult { - return e.VerifyWithHeaders(signals, ip, siteKey, userAgent, nil, "", false, nil, nil) + return e.VerifyWithHeaders(signals, ip, siteKey, userAgent, nil, "", "", false, nil, nil) } // GenerateChallenge creates a new PoW challenge (legacy) diff --git a/server-go/scoring_test.go b/server-go/scoring_test.go index ffc3f66..4d5ea51 100644 --- a/server-go/scoring_test.go +++ b/server-go/scoring_test.go @@ -405,8 +405,8 @@ func TestVerifyWithHeadersScoresPreDetections(t *testing.T) { e := NewScoringEngine("test-secret") pre := []DetectionResult{webBotAuthVerified("https://agent.example", "thumb", "ed25519")} - base := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", false, nil) - withPre := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", false, pre) + base := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", "", false, nil) + withPre := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", "", false, pre) if base.CategoryScores["declared_ai"] != 0 { t.Fatalf("precondition: expected no declared_ai in base, got %v", base.CategoryScores["declared_ai"]) From e86508bb8c87caf36095255bb463a1eb353538c9 Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Tue, 28 Jul 2026 08:28:37 -0500 Subject: [PATCH 2/2] fix(ja4): correct the mislabelled placeholder in Node and Python too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go server was fixed in the previous commit; the same wrong example was copied verbatim into server-node/detection.js and server-python/detection.py. All three claimed t13d1516h2_8daaf6152771_02713d6af862 was "Go default stdlib TLS". It is the canonical example from the JA4 specification, and it is Chrome — 8daaf6152771 hashes Chrome's cipher list, while Go's stdlib client hashes to f57a46bbacb6. Inert while commented out, but one uncomment away from scoring every Chrome visitor as automation at 0.8/0.9, in whichever server it happened to be uncommented in. Each map now ships explicitly empty with the measured table alongside it, and the reason a static hash list is the wrong shape for this: it is defeated by rotating a fingerprint, and the values drift with every browser release. --- server-node/detection.js | 37 +++++++++++++++++++++++++++++-------- server-python/detection.py | 32 +++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/server-node/detection.js b/server-node/detection.js index 2b6433c..b52221d 100644 --- a/server-node/detection.js +++ b/server-node/detection.js @@ -403,14 +403,35 @@ function checkJA3Fingerprint(ja3Hash) { // and passed via a trusted header the server is configured to accept. // Configure with: TRUSTED_JA4_HEADERS=cf-ja4,x-tls-ja4 (comma-separated). -const KNOWN_BOT_JA4_HASHES = { - // Populate with observed automation fingerprints in production. - // JA4 format: t##d####_####..._#### - // Placeholders — administrators should add real fingerprints as they - // are collected from deployed environments. - // Example once identified: - // 't13d1516h2_8daaf6152771_02713d6af862': 'Go default stdlib TLS' -}; +// Observed automation fingerprints. Ships EMPTY, deliberately. +// +// JA4 format: t##d####__ +// +// # The placeholder that used to live here was wrong +// +// It read: +// +// 't13d1516h2_8daaf6152771_02713d6af862': 'Go default stdlib TLS' +// +// That is the canonical example from the JA4 specification, and it is **Chrome**, +// not Go. Measured against a real TLS listener with server-go's ComputeJA4: +// +// Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea +// Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 +// curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 +// node https t13i521000_b262b3658495_8e6e362c5eac +// python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac +// +// The middle section hashes the cipher list, i.e. the TLS stack itself. +// 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Had anyone uncommented that +// line, every Chrome visitor would have been scored as automation at 0.8/0.9. +// +// # Why it stays empty +// +// A static hash list is defeated by rotating a fingerprint, and these values +// drift with every browser release. Populate it only from abuse you actually saw +// on your own deployment, and record where each fingerprint came from. +const KNOWN_BOT_JA4_HASHES = {}; function getTrustedJA4HeaderNames() { const env = process.env.TRUSTED_JA4_HEADERS; diff --git a/server-python/detection.py b/server-python/detection.py index 9dc2eb2..95a37f8 100644 --- a/server-python/detection.py +++ b/server-python/detection.py @@ -404,13 +404,31 @@ def check_ja3_fingerprint(ja3_hash: Optional[str]) -> List[Dict]: # from the TLS ClientHello by the reverse proxy and passed via a trusted header # the server is configured to accept via TRUSTED_JA4_HEADERS env var. -KNOWN_BOT_JA4_HASHES: Dict[str, str] = { - # Populate with observed automation fingerprints in production. - # JA4 format: t##d####_####..._#### - # Placeholders — administrators should add real fingerprints as collected. - # Example once identified: - # "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS", -} +# Observed automation fingerprints. Ships EMPTY, deliberately. +# +# JA4 format: t##d####__ +# +# The placeholder that used to live here was wrong. It read: +# +# "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS" +# +# That is the canonical example from the JA4 specification, and it is *Chrome*, +# not Go. Measured against a real TLS listener with server-go's ComputeJA4: +# +# Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea +# Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 +# curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 +# node https t13i521000_b262b3658495_8e6e362c5eac +# python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac +# +# The middle section hashes the cipher list, i.e. the TLS stack itself. +# 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Had anyone uncommented that +# line, every Chrome visitor would have been scored as automation at 0.8/0.9. +# +# A static hash list is defeated by rotating a fingerprint, and these values +# drift with every browser release. Populate it only from abuse you actually saw +# on your own deployment, and record where each fingerprint came from. +KNOWN_BOT_JA4_HASHES: Dict[str, str] = {} def get_trusted_ja4_header_names() -> List[str]: