diff --git a/README.md b/README.md index 9c99cfd..8326504 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ This makes credential stuffing expensive: even if a bot passes all other checks, - **CDP detection** — legacy ChromeDriver/Selenium globals plus a Runtime/DevTools console-attach probe that catches any attached protocol client, even when JS globals are scrubbed - Headless browser indicators, plugin/feature checks, UA ↔ platform consistency - Canvas / WebGL / Audio fingerprinting (session-scoped only) -- **TLS fingerprinting** — JA3 (client-supplied) and JA4 (read from a trusted reverse-proxy header, un-spoofable by the client) matched against known automation tools +- **TLS fingerprinting** — JA3 (client-supplied) and JA4. JA4 is computed **natively from the ClientHello** when the Go server terminates TLS itself (`FCAPTCHA_TLS_CERT`/`FCAPTCHA_TLS_KEY`), which no client can assert; behind a terminating proxy it falls back to a trusted header. Only JA4-TLS is implemented — the rest of the JA4 family is licensed non-commercially and cannot ship here ### Temporal Signals - Proof of Work timing (reveals API round-trip latency) @@ -453,6 +453,7 @@ Verify a previously issued token (server-side). | `FCAPTCHA_SITE_KEYS` | Comma-separated allowlist of accepted site keys. Unset accepts any key (zero-config self-hosting); unlisted keys are folded into a shared overflow bucket rather than allocating their own rate-limit/fingerprint state | (any) | | `FCAPTCHA_MAX_SITE_KEYS_PER_IP` | Distinct site keys one IP may allocate state for before the excess is folded into the overflow bucket. The cap itself is unconditional | 8 | | `TRUSTED_JA4_HEADERS` | Comma-separated reverse-proxy header names carrying a JA4 TLS fingerprint (e.g. set by nginx/Cloudflare). Only these names are read, and only from a peer in `TRUSTED_PROXIES` | (none) | +| `FCAPTCHA_TLS_CERT` / `FCAPTCHA_TLS_KEY` | Serve HTTPS directly (Go server). Terminating TLS here is what makes **native JA4** possible — the fingerprint is computed from the ClientHello rather than taken on trust from a proxy. Behind Railway/Cloudflare/nginx, leave unset and use `TRUSTED_JA4_HEADERS` | (none, plain HTTP) | | `FCAPTCHA_CLIENT_PATH` | Explicit path to `client/fcaptcha.js` for same-origin widget serving | (auto-probed) | | `FCAPTCHA_SERVE_CLIENT` | (Python) Serve the widget at `/fcaptcha.js`; set `false` to host the client on a separate CDN | `true` | | `FCAPTCHA_PPROF` | (Go) Enable the pprof debug server (`1`/`true`/`yes`/`on`) | off | @@ -542,6 +543,7 @@ fcaptcha/ │ ├── clientip.go # Trusted-proxy client IP resolution (TRUSTED_PROXIES) │ ├── sitekeys.go # Bounds on state a client-supplied siteKey can allocate │ ├── inputforensics.go # Typing cadence/modality, scroll morphology, platform coherence +│ ├── ja4.go # Native JA4-TLS from the ClientHello (Go 1.24+, stdlib only) │ ├── scoring_test.go # Go unit tests │ ├── clientip_test.go # Trusted-proxy unit tests │ └── go.mod diff --git a/docker/Dockerfile b/docker/Dockerfile index 3bd55a6..4111542 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.23-alpine AS builder +FROM golang:1.24-alpine AS builder WORKDIR /app diff --git a/server-go/Dockerfile b/server-go/Dockerfile index 79798fc..2b6e013 100644 --- a/server-go/Dockerfile +++ b/server-go/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS builder +FROM golang:1.24-alpine AS builder WORKDIR /app diff --git a/server-go/detection.go b/server-go/detection.go index 07d920e..e698728 100644 --- a/server-go/detection.go +++ b/server-go/detection.go @@ -569,13 +569,41 @@ func (e *ScoringEngine) CheckJA3Fingerprint(ja3Hash string) []DetectionResult { // computed from the TLS ClientHello by the reverse proxy and passed via a // trusted header the server is configured to accept via TRUSTED_JA4_HEADERS. -// knownBotJA4Hashes holds observed automation fingerprints. -// Populate in production; entries below are placeholders. -// JA4 format: t##d####_####..._#### -var knownBotJA4Hashes = map[string]string{ - // Example once identified: - // "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS", -} +// knownBotJA4Hashes holds observed automation fingerprints. It ships EMPTY, and +// that is deliberate — see below before adding anything. +// +// JA4 format: t##d####__ +// +// # The placeholder that used to live here was wrong +// +// It read: +// +// "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS" +// +// That is the canonical example from the JA4 specification, and it is **Chrome**, +// not Go. Measured on a real TLS listener with server-go's own ComputeJA4: +// +// Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea +// Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 +// curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 +// node https t13i521000_b262b3658495_8e6e362c5eac +// python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac +// +// The middle section is a hash of the cipher list, i.e. of the TLS stack itself. +// 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Had anyone uncommented that +// line, every Chrome visitor would have been scored as automation at 0.8/0.9. +// +// # Why it stays empty +// +// A static hash list is defeated by rotating a fingerprint, and these values +// drift with every browser release — the two Chromium readings above differ from +// the spec's example in both the counts and the extension hash, from version +// drift alone. Populating this with fingerprints nobody here has observed trades +// a real false-positive risk for no detection value. +// +// If you populate it, populate it from abuse you actually saw on your own +// deployment, and record where each fingerprint came from. +var knownBotJA4Hashes = map[string]string{} // GetTrustedJA4HeaderNames reads TRUSTED_JA4_HEADERS env var (comma-separated). func GetTrustedJA4HeaderNames() []string { diff --git a/server-go/go.mod b/server-go/go.mod index 8b67b39..c5bcb93 100644 --- a/server-go/go.mod +++ b/server-go/go.mod @@ -1,6 +1,6 @@ module github.com/WebDecoy/FCaptcha/server-go -go 1.23 +go 1.24 require ( github.com/go-chi/chi/v5 v5.0.11 diff --git a/server-go/ja4.go b/server-go/ja4.go new file mode 100644 index 0000000..0a15958 --- /dev/null +++ b/server-go/ja4.go @@ -0,0 +1,252 @@ +package main + +import ( + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "fmt" + "sort" + "strings" + "time" + + "github.com/hashicorp/golang-lru/v2/expirable" +) + +// JA4 TLS client fingerprinting, computed natively from the ClientHello. +// +// # Scope, and a licensing line not to cross +// +// This implements **JA4 (TLS) only**. JA4 is published under BSD-3-Clause and +// FoxIO explicitly disclaims patent coverage for it, which is what makes it +// safe to ship in an MIT project that people deploy commercially. +// +// The rest of the family — JA4H (HTTP), JA4T, JA4L, JA4S, JA4X, JA4SSH — is +// licensed under FoxIO License 1.1: non-commercial use only, patent-pending on +// the detection logic, and incompatible with GPL/AGPL. FoxIO's FAQ is explicit +// that indirect monetization requires an OEM licence. **Do not add them here.** +// If you want header-order or RTT heuristics, derive them from first principles +// — a header-order signature is SHA256 over the lowercased names in order plus +// a casing flag, about fifteen lines, and owes nothing to anyone. +// +// # Where this works, and where it does not +// +// It reads the ClientHello, so it only produces a fingerprint when *this +// process* terminates TLS. Behind Railway, Cloudflare, nginx or any other +// terminating proxy the ClientHello was consumed upstream and there is nothing +// here to read — those deployments must keep using the trusted-header path +// (TRUSTED_JA4_HEADERS), which is unchanged and remains the default. +// +// That covers the single-binary self-hosted case FCaptcha advertises, and +// nothing else. It is not a replacement for the header path; it is the option +// for people who do not have a proxy to read the header from. +// +// # Why Go 1.24 +// +// crypto/tls only began exposing the ClientHello extension list in 1.24 +// (ClientHelloInfo.Extensions, golang/go#32936). Without it the extension hash +// cannot be computed and JA4 is not derivable from the stdlib at all. + +// GREASE values (RFC 8701) are random padding a client injects to keep the +// ecosystem tolerant of unknown values. They vary per connection by design, so +// including them would make the fingerprint different every time — the JA4 spec +// excludes them from every count and every list. +func isGREASE(v uint16) bool { + // 0x0a0a, 0x1a1a, 0x2a2a ... 0xfafa: both bytes equal, low nibble 0xa. + return v&0x0f0f == 0x0a0a && v>>8 == v&0xff +} + +func filterGREASE(vals []uint16) []uint16 { + out := make([]uint16, 0, len(vals)) + for _, v := range vals { + if !isGREASE(v) { + out = append(out, v) + } + } + return out +} + +// ja4TLSVersion renders the negotiated-or-highest-offered version as the two +// characters the spec uses. +func ja4TLSVersion(versions []uint16) string { + best := uint16(0) + for _, v := range filterGREASE(versions) { + if v > best { + best = v + } + } + switch best { + case tls.VersionTLS13: + return "13" + case tls.VersionTLS12: + return "12" + case tls.VersionTLS11: + return "11" + case tls.VersionTLS10: + return "10" + case 0x0300: + return "s3" + case 0x0002: + return "s2" + default: + return "00" + } +} + +// ja4ALPN takes the first and last character of the first offered protocol: +// "h2" -> "h2", "http/1.1" -> "h1". Non-ASCII values fall back to the hex of +// those bytes, per the spec's note on non-printable ALPNs. +func ja4ALPN(protos []string) string { + if len(protos) == 0 || protos[0] == "" { + return "00" + } + p := protos[0] + first, last := p[0], p[len(p)-1] + + printable := func(b byte) bool { + return (b >= '0' && b <= '9') || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') + } + if printable(first) && printable(last) { + return string([]byte{first, last}) + } + return fmt.Sprintf("%x%x", first>>4, last&0x0f) +} + +// twoDigit caps a count at 99, which is what the fixed-width JA4_a field allows. +func twoDigit(n int) string { + if n > 99 { + n = 99 + } + return fmt.Sprintf("%02d", n) +} + +func hexList(vals []uint16) string { + parts := make([]string, len(vals)) + for i, v := range vals { + parts[i] = fmt.Sprintf("%04x", v) + } + return strings.Join(parts, ",") +} + +// sha256First12 is the truncation the spec specifies for both hashed sections. +func sha256First12(s string) string { + if s == "" { + return "000000000000" + } + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:12] +} + +// ComputeJA4 builds the fingerprint from a ClientHello. +// +// Layout: ja4_a _ ja4_b _ ja4_c +// +// ja4_a transport, TLS version, SNI presence, cipher count, extension +// count, first ALPN — human-readable, 10 chars +// ja4_b truncated SHA-256 over the sorted cipher list +// ja4_c truncated SHA-256 over the sorted extension list, an underscore, +// and the signature algorithms in their original order +// +// Two asymmetries in the spec are deliberate and easy to get wrong: the +// extension *count* in ja4_a includes SNI and ALPN, while the extension *list* +// in ja4_c excludes them; and the signature algorithms are NOT sorted, because +// their order is itself characteristic of the client. +func ComputeJA4(hello *tls.ClientHelloInfo) string { + if hello == nil { + return "" + } + + ciphers := filterGREASE(hello.CipherSuites) + extensions := filterGREASE(hello.Extensions) + + // "d" when the client sent SNI (a domain), "i" when it connected by IP. + sni := "i" + if hello.ServerName != "" { + sni = "d" + } + + // "t" for TCP. QUIC would be "q", but a net/http server reaching this code + // path is TCP by construction. + ja4a := "t" + ja4TLSVersion(hello.SupportedVersions) + sni + + twoDigit(len(ciphers)) + twoDigit(len(extensions)) + ja4ALPN(hello.SupportedProtos) + + sortedCiphers := append([]uint16(nil), ciphers...) + sort.Slice(sortedCiphers, func(i, j int) bool { return sortedCiphers[i] < sortedCiphers[j] }) + ja4b := sha256First12(hexList(sortedCiphers)) + + // SNI (0x0000) and ALPN (0x0010) are excluded from the hashed list: they + // carry per-request content rather than client identity, so including them + // would make the same browser fingerprint differently per site. + hashable := make([]uint16, 0, len(extensions)) + for _, e := range extensions { + if e != 0x0000 && e != 0x0010 { + hashable = append(hashable, e) + } + } + sort.Slice(hashable, func(i, j int) bool { return hashable[i] < hashable[j] }) + + sigAlgs := make([]uint16, 0, len(hello.SignatureSchemes)) + for _, s := range hello.SignatureSchemes { + if !isGREASE(uint16(s)) { + sigAlgs = append(sigAlgs, uint16(s)) + } + } + + ja4c := sha256First12(hexList(hashable) + "_" + hexList(sigAlgs)) + + return ja4a + "_" + ja4b + "_" + ja4c +} + +// ja4Store keeps the fingerprint for the life of a connection. +// +// The ClientHello is only visible during the handshake, in a callback that has +// no request attached to it yet. So it is recorded against the connection there +// and read back when the request arrives on that same connection. Bounded, so a +// long-lived server cannot accumulate entries for connections that never +// produced a request. +type ja4Store struct { + byConn *expirable.LRU[string, string] +} + +// TLSConfigWithJA4 returns a tls.Config that records a JA4 fingerprint per +// connection, and a lookup the HTTP handler can use. +// +// GetConfigForClient is the hook: it fires once per handshake with the parsed +// ClientHello and is otherwise free to return nil, meaning "use the base +// config". Recording a fingerprint there is a side effect on the connection's +// remote address, which is the only stable key available at that point. +func TLSConfigWithJA4(base *tls.Config, store *ja4Store) *tls.Config { + cfg := base.Clone() + cfg.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + if hello.Conn != nil { + store.Record(hello.Conn.RemoteAddr().String(), ComputeJA4(hello)) + } + return nil, nil + } + return cfg +} + +// ja4ConnTTL bounds how long a recorded fingerprint outlives its handshake. +// Keep-alive connections can serve requests for a while, but an entry that has +// not been read within this window belongs to a connection that handshook and +// then went quiet. +const ja4ConnTTL = 5 * time.Minute + +// maxTrackedJA4Conns bounds the per-connection table. Generous next to any +// realistic count of simultaneously-open connections, small enough that the +// store cannot become a leak. +const maxTrackedJA4Conns = 20_000 + +func newJA4Store(max int) *ja4Store { + return &ja4Store{byConn: expirable.NewLRU[string, string](max, nil, ja4ConnTTL)} +} + +func (s *ja4Store) Record(remoteAddr, fp string) { + if fp != "" { + s.byConn.Add(remoteAddr, fp) + } +} + +func (s *ja4Store) Lookup(remoteAddr string) string { + fp, _ := s.byConn.Get(remoteAddr) + return fp +} diff --git a/server-go/ja4_test.go b/server-go/ja4_test.go new file mode 100644 index 0000000..202c065 --- /dev/null +++ b/server-go/ja4_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "crypto/tls" + "io" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strings" + "testing" +) + +// JA4 has a published spec, so the failure mode to guard against is silent +// divergence from it. These check the rules that are easy to get subtly wrong, +// plus one end-to-end handshake proving the wiring actually produces a +// fingerprint from a real connection. + +func TestIsGREASE(t *testing.T) { + // RFC 8701 defines exactly sixteen GREASE values. + for _, v := range []uint16{0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x7a7a, 0xdada, 0xeaea, 0xfafa} { + if !isGREASE(v) { + t.Errorf("%#04x is a GREASE value", v) + } + } + // Near-misses that must NOT be treated as GREASE, or real ciphers vanish + // from the fingerprint. + for _, v := range []uint16{0x1301, 0x1302, 0xc02b, 0x0a0b, 0x0b0a, 0x1a2a, 0xabab} { + if isGREASE(v) { + t.Errorf("%#04x is a real value, not GREASE", v) + } + } +} + +func TestJA4ALPNEncoding(t *testing.T) { + cases := map[string]string{ + "h2": "h2", + "http/1.1": "h1", // first and last character, not a prefix + "h3": "h3", + } + for in, want := range cases { + if got := ja4ALPN([]string{in}); got != want { + t.Errorf("ja4ALPN(%q) = %q, want %q", in, got, want) + } + } + if got := ja4ALPN(nil); got != "00" { + t.Errorf("no ALPN should give 00, got %q", got) + } + // Only the first protocol counts. + if got := ja4ALPN([]string{"h2", "http/1.1"}); got != "h2" { + t.Errorf("only the first ALPN counts, got %q", got) + } +} + +func TestJA4CountsExcludeGREASE(t *testing.T) { + hello := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x0a0a, 0x1301, 0x1302, 0x1a1a, 0xc02b}, + Extensions: []uint16{0x2a2a, 0x0000, 0x0010, 0x000d, 0x002b}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + SupportedProtos: []string{"h2"}, + } + ja4 := ComputeJA4(hello) + + // 3 real ciphers of 5, 4 real extensions of 5 — and the extension COUNT + // includes SNI and ALPN even though the hashed list excludes them. + if !strings.HasPrefix(ja4, "t13d0304h2_") { + t.Errorf("expected prefix t13d0304h2_, got %q", ja4) + } +} + +func TestJA4SNIFlag(t *testing.T) { + base := func(name string) *tls.ClientHelloInfo { + return &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301}, + Extensions: []uint16{0x002b}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: name, + } + } + if got := ComputeJA4(base("example.com")); got[3] != 'd' { + t.Errorf("SNI present should give 'd', got %q in %s", got[3], got) + } + if got := ComputeJA4(base("")); got[3] != 'i' { + t.Errorf("no SNI should give 'i', got %q in %s", got[3], got) + } +} + +// Cipher and extension order varies between connections from the same client; +// the spec sorts them so it does not affect the fingerprint. +func TestJA4IsOrderInsensitiveForCiphersAndExtensions(t *testing.T) { + a := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301, 0x1302, 0xc02b}, + Extensions: []uint16{0x002b, 0x000d, 0x0017}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + } + b := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0xc02b, 0x1301, 0x1302}, + Extensions: []uint16{0x0017, 0x002b, 0x000d}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + } + if ComputeJA4(a) != ComputeJA4(b) { + t.Errorf("reordering ciphers/extensions changed the fingerprint:\n %s\n %s", ComputeJA4(a), ComputeJA4(b)) + } +} + +// Signature algorithms are the exception: their order IS characteristic of the +// client, so the spec deliberately does not sort them. +func TestJA4SignatureAlgorithmOrderMatters(t *testing.T) { + mk := func(schemes ...tls.SignatureScheme) *tls.ClientHelloInfo { + return &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301}, + Extensions: []uint16{0x000d}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + SignatureSchemes: schemes, + } + } + a := ComputeJA4(mk(tls.ECDSAWithP256AndSHA256, tls.PSSWithSHA256)) + b := ComputeJA4(mk(tls.PSSWithSHA256, tls.ECDSAWithP256AndSHA256)) + if a == b { + t.Error("signature algorithm order must change the fingerprint (it is not sorted)") + } +} + +func TestJA4ExtensionListExcludesSNIAndALPN(t *testing.T) { + // Same client, two different sites: SNI and ALPN differ, everything else is + // identical. The hashed section must not move, or one browser fingerprints + // differently per site and the signal is useless. + mk := func(name string) *tls.ClientHelloInfo { + return &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301, 0x1302}, + Extensions: []uint16{0x0000, 0x0010, 0x002b, 0x000d}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: name, + SupportedProtos: []string{"h2"}, + } + } + a, b := ComputeJA4(mk("one.example")), ComputeJA4(mk("two.example")) + if a != b { + t.Errorf("the hashed sections must not depend on SNI:\n %s\n %s", a, b) + } +} + +func TestJA4Shape(t *testing.T) { + hello := &tls.ClientHelloInfo{ + CipherSuites: []uint16{0x1301, 0x1302, 0x1303}, + Extensions: []uint16{0x0000, 0x0010, 0x002b, 0x000d, 0x0017}, + SupportedVersions: []uint16{tls.VersionTLS13}, + ServerName: "example.com", + SupportedProtos: []string{"h2"}, + SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256}, + } + got := ComputeJA4(hello) + // t13d0305h2_<12 hex>_<12 hex> + if !regexp.MustCompile(`^[tq](13|12|11|10|s3|s2|00)[di]\d{4}[a-z0-9]{2}_[0-9a-f]{12}_[0-9a-f]{12}$`).MatchString(got) { + t.Errorf("malformed JA4: %q", got) + } +} + +// End-to-end: a real handshake against a real listener, proving the +// GetConfigForClient wiring records something a handler can read back. +func TestJA4EndToEndOverRealTLS(t *testing.T) { + store := newJA4Store(128) + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, store.Lookup(r.RemoteAddr)) + })) + srv.TLS = TLSConfigWithJA4(&tls.Config{}, store) + srv.StartTLS() + defer srv.Close() + + // httptest.Server.Client() already trusts the throwaway certificate. + resp, err := srv.Client().Get(srv.URL) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + got := string(body) + if got == "" { + t.Fatal("no JA4 was recorded for a real TLS connection") + } + if !strings.HasPrefix(got, "t1") { + t.Errorf("expected a TCP/TLS1.x fingerprint, got %q", got) + } + t.Logf("Go client fingerprinted as %s", got) +} + +// Guards the licensing boundary from §8.3 of the extensions PRD: JA4 (TLS) is +// BSD-3-Clause and safe here; JA4H/JA4T/JA4L/JA4S/JA4X/JA4SSH are FoxIO License +// 1.1, non-commercial only, and cannot ship in an MIT project. If someone adds +// one, this fails and points them at the reason. +func TestOnlyJA4TLSIsImplemented(t *testing.T) { + src := readSourceFile(t, "ja4.go") + for _, forbidden := range []string{"JA4H", "JA4T", "JA4L", "JA4S", "JA4X", "JA4SSH"} { + // Allowed inside the comment that explains why they are absent. + count := strings.Count(src, "func "+forbidden) + strings.Count(src, "func Compute"+forbidden) + if count > 0 { + t.Errorf("%s appears to be implemented — it is FoxIO License 1.1 "+ + "(non-commercial, GPL-incompatible) and cannot ship in this MIT project. See PRD §8.3.", forbidden) + } + } +} + +func readSourceFile(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(name) + if err != nil { + t.Fatalf("reading %s: %v", name, err) + } + return string(b) +} + +// The locally-computed fingerprint must win over a header, because a header +// requires trusting whatever set it while a ClientHello-derived value cannot be +// asserted by anyone. +func TestNativeJA4TakesPrecedenceOverHeader(t *testing.T) { + const nativeFP = "t13d1516h2_aaaaaaaaaaaa_bbbbbbbbbbbb" + const headerFP = "t13d1516h2_cccccccccccc_dddddddddddd" + + knownBotJA4Hashes[nativeFP] = "native-source tool" + knownBotJA4Hashes[headerFP] = "header-source tool" + defer func() { + delete(knownBotJA4Hashes, nativeFP) + delete(knownBotJA4Hashes, headerFP) + }() + + t.Setenv("TRUSTED_JA4_HEADERS", "cf-ja4") + e := NewScoringEngine("test-secret") + headers := map[string]string{"cf-ja4": headerFP} + + // The reason string is generic, so read which fingerprint was matched out of + // the detection Details. + matchedTool := func(res *VerificationResult) string { + for _, d := range res.Detections { + if d.Category == CategoryFingerprint && d.Details != nil { + if tool, ok := d.Details["tool"].(string); ok { + return tool + } + } + } + return "" + } + + // Both available: the native one is the one that gets scored. + both := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", headers, "", nativeFP, true, nil) + if got := matchedTool(both); got != "native-source tool" { + t.Errorf("native JA4 should take precedence, matched %q", got) + } + + // No native fingerprint (something upstream terminated TLS): fall back. + fallback := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", headers, "", "", true, nil) + if got := matchedTool(fallback); got != "header-source tool" { + t.Errorf("should fall back to the trusted header, matched %q", got) + } +} + +// The map that CheckJA4Fingerprint consults ships empty on purpose — the PRD +// calls a static hash list defeatable by rotating a fingerprint, and populating +// it with values nobody here has observed would be worse than leaving it bare. +// This records that state so its eventual filling is a deliberate act. +func TestKnownBotJA4MapShipsEmpty(t *testing.T) { + if len(knownBotJA4Hashes) != 0 { + t.Errorf("knownBotJA4Hashes has %d entries; native JA4 computation is wired but "+ + "intentionally has nothing to match against yet. If you are populating it, "+ + "say where the fingerprints were observed.", len(knownBotJA4Hashes)) + } +} diff --git a/server-go/main.go b/server-go/main.go index 8730770..7e5a4a7 100644 --- a/server-go/main.go +++ b/server-go/main.go @@ -3,6 +3,7 @@ package main import ( "context" "crypto/sha256" + "crypto/tls" "encoding/hex" "encoding/json" "log" @@ -179,6 +180,10 @@ func main() { siteKeys := SiteKeyGuardFromEnv() log.Printf("site keys: %s", siteKeys.Describe()) + // Holds a JA4 fingerprint per live connection, populated during the TLS + // handshake. Stays empty unless this process terminates TLS — see ja4.go. + ja4s := newJA4Store(maxTrackedJA4Conns) + // Middleware // // Deliberately no middleware.RealIP: it overwrites r.RemoteAddr from @@ -221,8 +226,8 @@ func main() { // Routes r.Get("/health", healthHandler) - r.Post("/api/verify", verifyHandler(engine, proxyTrust, siteKeys)) - r.Post("/api/score", invisibleScoreHandler(engine, proxyTrust, siteKeys)) + r.Post("/api/verify", verifyHandler(engine, proxyTrust, siteKeys, ja4s)) + r.Post("/api/score", invisibleScoreHandler(engine, proxyTrust, siteKeys, ja4s)) r.Post("/api/token/verify", tokenVerifyHandler(engine, proxyTrust)) r.Get("/api/pow/challenge", powChallengeHandler(engine, proxyTrust, siteKeys)) r.Get("/api/challenge", challengeHandler(engine)) @@ -252,10 +257,39 @@ func main() { }() } + // Optional direct TLS termination, off by default. + // + // Set FCAPTCHA_TLS_CERT and FCAPTCHA_TLS_KEY to have this process terminate + // TLS itself, which is the only arrangement where a JA4 fingerprint can be + // computed here — the ClientHello is consumed by whoever completes the + // handshake. Behind Railway, Cloudflare or nginx that is not us, and the + // TRUSTED_JA4_HEADERS path remains the way to get a fingerprint. + certFile := os.Getenv("FCAPTCHA_TLS_CERT") + keyFile := os.Getenv("FCAPTCHA_TLS_KEY") + serveTLS := certFile != "" && keyFile != "" + + if serveTLS { + srv.TLSConfig = TLSConfigWithJA4(&tls.Config{MinVersion: tls.VersionTLS12}, ja4s) + log.Printf("native JA4: on (terminating TLS locally)") + } else { + trustedJA4 := GetTrustedJA4HeaderNames() + if len(trustedJA4) > 0 { + log.Printf("native JA4: off (not terminating TLS); reading %s from trusted proxies", strings.Join(trustedJA4, ", ")) + } else { + log.Printf("native JA4: off (not terminating TLS, and no TRUSTED_JA4_HEADERS set) — no TLS fingerprint available") + } + } + // Graceful shutdown go func() { log.Printf("FCaptcha server starting on port %s", port) - if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + var err error + if serveTLS { + err = srv.ListenAndServeTLS(certFile, keyFile) + } else { + err = srv.ListenAndServe() + } + if err != nil && err != http.ErrServerClosed { log.Fatalf("Server error: %v", err) } }() @@ -340,7 +374,7 @@ func webBotAuthDetections(engine *ScoringEngine, r *http.Request) []DetectionRes return engine.CheckWebBotAuth(ctx, webbotauth.RequestFromHTTP(r, webbotauth.WithScheme(scheme))) } -func verifyHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard) http.HandlerFunc { +func verifyHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard, ja4s *ja4Store) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var req VerifyRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -414,7 +448,7 @@ func verifyHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGu // passed as preDetections so the verified/forged verdict is scored. webBotAuth := webBotAuthDetections(engine, r) - result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, headers, ja3Hash, peerTrusted, webBotAuth, req.PowSolution) + result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, headers, ja3Hash, ja4s.Lookup(r.RemoteAddr), peerTrusted, webBotAuth, req.PowSolution) // Add signal commitment detections to results if len(extraDetections) > 0 { @@ -459,7 +493,7 @@ type InvisibleScoreRequest struct { PowTiming *PowTiming `json:"powTiming,omitempty"` } -func invisibleScoreHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard) http.HandlerFunc { +func invisibleScoreHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *SiteKeyGuard, ja4s *ja4Store) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var req InvisibleScoreRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -524,7 +558,7 @@ func invisibleScoreHandler(engine *ScoringEngine, trust *ProxyTrust, siteKeys *S // Web Bot Auth: verify signed-agent requests (see verifyHandler). webBotAuth := webBotAuthDetections(engine, r) - result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, scoreHeaders, ja3, peerTrusted, webBotAuth, req.PowSolution) + result := engine.VerifyWithHeaders(signals, ip, req.SiteKey, userAgent, scoreHeaders, ja3, ja4s.Lookup(r.RemoteAddr), peerTrusted, webBotAuth, req.PowSolution) if len(scoreExtraDetections) > 0 { result.Detections = append(scoreExtraDetections, result.Detections...) } diff --git a/server-go/scoring.go b/server-go/scoring.go index 923f858..ed17d63 100644 --- a/server-go/scoring.go +++ b/server-go/scoring.go @@ -322,7 +322,13 @@ func compileUAPatterns() []*regexp.Regexp { // request — currently Web Bot Auth signature verification, which needs the // accurately-reconstructed signed request. They are seeded into the detection // set so they participate in scoring like any engine-produced detection. -func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, siteKey, userAgent string, headers map[string]string, ja3Hash string, peerTrusted bool, preDetections []DetectionResult, powSolution ...*PoWSolution) *VerificationResult { +// nativeJA4 is a fingerprint this process computed from the ClientHello (see +// ja4.go), empty when something upstream terminated TLS. +// +// NOTE: this parameter list has now grown three times and is due a request-context +// struct. Left positional for now so the vendored copy in fcaptcha-cloud stays a +// straight file copy rather than a port. +func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, siteKey, userAgent string, headers map[string]string, ja3Hash, nativeJA4 string, peerTrusted bool, preDetections []DetectionResult, powSolution ...*PoWSolution) *VerificationResult { detections := make([]DetectionResult, 0, len(preDetections)+8) detections = append(detections, preDetections...) @@ -407,15 +413,22 @@ func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, si detections = append(detections, e.CheckJA3Fingerprint(ja3Hash)...) } - // TLS fingerprint (JA4) — trusted reverse-proxy header, un-spoofable by client - if headers != nil { - trustedJA4 := GetTrustedJA4HeaderNames() - if len(trustedJA4) > 0 { - if ja4 := ReadJA4FromHeaders(headers, trustedJA4); ja4 != "" { - detections = append(detections, e.CheckJA4Fingerprint(ja4)...) - } + // TLS fingerprint (JA4). Two sources, and the local one wins. + // + // A natively-computed fingerprint was derived from the ClientHello by this + // process, so it cannot be asserted by anyone — not the client, and not a + // misconfigured proxy either. The header path requires trusting whatever sits + // in front of us to have computed it honestly, which is weaker, so it is only + // consulted when there is no local fingerprint to use. + ja4 := nativeJA4 + if ja4 == "" && headers != nil { + if trustedJA4 := GetTrustedJA4HeaderNames(); len(trustedJA4) > 0 { + ja4 = ReadJA4FromHeaders(headers, trustedJA4) } } + if ja4 != "" { + detections = append(detections, e.CheckJA4Fingerprint(ja4)...) + } // Form interaction analysis (credential stuffing & spam detection) if formAnalysis, ok := signals["formAnalysis"].(map[string]interface{}); ok { @@ -459,7 +472,7 @@ func (e *ScoringEngine) VerifyWithHeaders(signals map[string]interface{}, ip, si // Verify performs full verification (backward compatible) func (e *ScoringEngine) Verify(signals map[string]interface{}, ip, siteKey, userAgent string) *VerificationResult { - return e.VerifyWithHeaders(signals, ip, siteKey, userAgent, nil, "", false, nil, nil) + return e.VerifyWithHeaders(signals, ip, siteKey, userAgent, nil, "", "", false, nil, nil) } // GenerateChallenge creates a new PoW challenge (legacy) diff --git a/server-go/scoring_test.go b/server-go/scoring_test.go index ffc3f66..4d5ea51 100644 --- a/server-go/scoring_test.go +++ b/server-go/scoring_test.go @@ -405,8 +405,8 @@ func TestVerifyWithHeadersScoresPreDetections(t *testing.T) { e := NewScoringEngine("test-secret") pre := []DetectionResult{webBotAuthVerified("https://agent.example", "thumb", "ed25519")} - base := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", false, nil) - withPre := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", false, pre) + base := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", "", false, nil) + withPre := e.VerifyWithHeaders(map[string]interface{}{}, "1.2.3.4", "site", "ua", nil, "", "", false, pre) if base.CategoryScores["declared_ai"] != 0 { t.Fatalf("precondition: expected no declared_ai in base, got %v", base.CategoryScores["declared_ai"]) diff --git a/server-node/detection.js b/server-node/detection.js index 2b6433c..b52221d 100644 --- a/server-node/detection.js +++ b/server-node/detection.js @@ -403,14 +403,35 @@ function checkJA3Fingerprint(ja3Hash) { // and passed via a trusted header the server is configured to accept. // Configure with: TRUSTED_JA4_HEADERS=cf-ja4,x-tls-ja4 (comma-separated). -const KNOWN_BOT_JA4_HASHES = { - // Populate with observed automation fingerprints in production. - // JA4 format: t##d####_####..._#### - // Placeholders — administrators should add real fingerprints as they - // are collected from deployed environments. - // Example once identified: - // 't13d1516h2_8daaf6152771_02713d6af862': 'Go default stdlib TLS' -}; +// Observed automation fingerprints. Ships EMPTY, deliberately. +// +// JA4 format: t##d####__ +// +// # The placeholder that used to live here was wrong +// +// It read: +// +// 't13d1516h2_8daaf6152771_02713d6af862': 'Go default stdlib TLS' +// +// That is the canonical example from the JA4 specification, and it is **Chrome**, +// not Go. Measured against a real TLS listener with server-go's ComputeJA4: +// +// Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea +// Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 +// curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 +// node https t13i521000_b262b3658495_8e6e362c5eac +// python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac +// +// The middle section hashes the cipher list, i.e. the TLS stack itself. +// 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Had anyone uncommented that +// line, every Chrome visitor would have been scored as automation at 0.8/0.9. +// +// # Why it stays empty +// +// A static hash list is defeated by rotating a fingerprint, and these values +// drift with every browser release. Populate it only from abuse you actually saw +// on your own deployment, and record where each fingerprint came from. +const KNOWN_BOT_JA4_HASHES = {}; function getTrustedJA4HeaderNames() { const env = process.env.TRUSTED_JA4_HEADERS; diff --git a/server-python/detection.py b/server-python/detection.py index 9dc2eb2..95a37f8 100644 --- a/server-python/detection.py +++ b/server-python/detection.py @@ -404,13 +404,31 @@ def check_ja3_fingerprint(ja3_hash: Optional[str]) -> List[Dict]: # from the TLS ClientHello by the reverse proxy and passed via a trusted header # the server is configured to accept via TRUSTED_JA4_HEADERS env var. -KNOWN_BOT_JA4_HASHES: Dict[str, str] = { - # Populate with observed automation fingerprints in production. - # JA4 format: t##d####_####..._#### - # Placeholders — administrators should add real fingerprints as collected. - # Example once identified: - # "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS", -} +# Observed automation fingerprints. Ships EMPTY, deliberately. +# +# JA4 format: t##d####__ +# +# The placeholder that used to live here was wrong. It read: +# +# "t13d1516h2_8daaf6152771_02713d6af862": "Go default stdlib TLS" +# +# That is the canonical example from the JA4 specification, and it is *Chrome*, +# not Go. Measured against a real TLS listener with server-go's ComputeJA4: +# +# Chromium 141 t13i1515h2_8daaf6152771_806a8c22fdea +# Go stdlib t13i131000_f57a46bbacb6_e5728521abd4 +# curl t13i4906h2_0d8feac7bc37_7395dae3b2f3 +# node https t13i521000_b262b3658495_8e6e362c5eac +# python urllib t13i171000_ab0a1bf427ad_8e6e362c5eac +# +# The middle section hashes the cipher list, i.e. the TLS stack itself. +# 8daaf6152771 is Chrome's; Go's is f57a46bbacb6. Had anyone uncommented that +# line, every Chrome visitor would have been scored as automation at 0.8/0.9. +# +# A static hash list is defeated by rotating a fingerprint, and these values +# drift with every browser release. Populate it only from abuse you actually saw +# on your own deployment, and record where each fingerprint came from. +KNOWN_BOT_JA4_HASHES: Dict[str, str] = {} def get_trusted_ja4_header_names() -> List[str]: