diff --git a/.github/workflows/refresh-fronted-config.yml b/.github/workflows/refresh-fronted-config.yml index ac1d1ed1..283d7f21 100644 --- a/.github/workflows/refresh-fronted-config.yml +++ b/.github/workflows/refresh-fronted-config.yml @@ -2,8 +2,9 @@ name: Refresh embedded fronted.yaml.gz # kindling/fronted/fronted.yaml.gz is the last-resort fallback config used # when both the live fetch and on-disk cache miss. The canonical copy lives -# in getlantern/fronted and is updated daily by an external pipeline. Mirror -# that here so a fresh install bootstrap doesn't fall back to a stale copy. +# in getlantern/domainfront (getlantern/fronted is deprecated) and is +# updated daily by an external pipeline. Mirror that here so a fresh +# install bootstrap doesn't fall back to a stale copy. on: schedule: @@ -32,7 +33,7 @@ jobs: run: | curl -fsSL \ -o kindling/fronted/fronted.yaml.gz.new \ - https://raw.githubusercontent.com/getlantern/fronted/refs/heads/main/fronted.yaml.gz + https://raw.githubusercontent.com/getlantern/domainfront/refs/heads/main/fronted.yaml.gz gzip -t kindling/fronted/fronted.yaml.gz.new test -s kindling/fronted/fronted.yaml.gz.new @@ -59,7 +60,7 @@ jobs: title: "fronted: refresh embedded fronted.yaml.gz" body: | Automated daily refresh of `kindling/fronted/fronted.yaml.gz` - from `getlantern/fronted@main`. Safe to merge once CI passes. + from `getlantern/domainfront@main`. Safe to merge once CI passes. branch: chore/refresh-fronted-config delete-branch: true author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" diff --git a/backend/radiance.go b/backend/radiance.go index 8ea01106..ad6fabed 100644 --- a/backend/radiance.go +++ b/backend/radiance.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" @@ -34,6 +35,9 @@ import ( "github.com/getlantern/radiance/internal" "github.com/getlantern/radiance/issue" "github.com/getlantern/radiance/kindling" + "github.com/getlantern/radiance/kindling/fronted" + "github.com/getlantern/radiance/kindling/iran" + "github.com/getlantern/radiance/kindling/meek" "github.com/getlantern/radiance/log" "github.com/getlantern/radiance/servers" "github.com/getlantern/radiance/telemetry" @@ -77,6 +81,13 @@ type LocalBackend struct { stopURLTestListener context.CancelFunc urlTestMu sync.Mutex + + // meekProvider is non-nil only when the device is classified as + // likely in Iran. getBoxOptions reads it atomically so a slow + // scanner startup can't block VPN connects: the meek outbound is + // absent from the first connect and present once the scanner + // populates it. + meekProvider atomic.Pointer[meek.Provider] } type Options struct { @@ -87,6 +98,11 @@ type Options struct { // this should be the platform device ID on mobile devices, desktop platforms will generate their // own device ID and ignore this value DeviceID string + // MCC is the network Mobile Country Code reported by the cellular + // stack (Android: first 3 chars of TelephonyManager.getNetworkOperator()). + // Empty on WiFi-only, between-tower, or platforms that don't expose it. + // Used to gate activation of the heavier meek transport. + MCC string // User choice for telemetry consent TelemetryConsent bool PlatformInterface vpn.PlatformInterface @@ -135,6 +151,7 @@ func NewLocalBackend(ctx context.Context, opts Options) (*LocalBackend, error) { settings.Patch(settings.Settings{ settings.LocaleKey: opts.Locale, settings.DeviceIDKey: platformDeviceID, + settings.MCCKey: opts.MCC, settings.ConfigFetchDisabledKey: disableFetch, settings.TelemetryKey: opts.TelemetryConsent, }) @@ -205,6 +222,8 @@ func (r *LocalBackend) Start() { } }() + r.maybeStartMeek() + if settings.GetBool(settings.TelemetryKey) { if err := r.startTelemetry(); err != nil { slog.Error("Failed to start telemetry", "error", err) @@ -696,6 +715,33 @@ func (r *LocalBackend) runURLTestListener(ctx context.Context, storage vpn.URLTe } } +func (r *LocalBackend) maybeStartMeek() { + force := env.GetBool(env.ForceMeekOnly) + if !force && !iran.LikelyIran(settings.GetString(settings.MCCKey), iran.LocalTZName()) { + return + } + go func() { + dataDir := settings.GetString(settings.DataPathKey) + cfg, err := fronted.LoadCachedConfig(dataDir) + if err != nil { + slog.Warn("meek: failed to load fronted config", "err", err) + return + } + p, err := meek.NewProvider(meek.ProviderConfig{ + Config: cfg, + CacheFile: filepath.Join(dataDir, "meek_fronts_cache.json"), + }) + if err != nil { + slog.Warn("meek: NewProvider failed", "err", err) + return + } + p.Start(r.ctx) + r.meekProvider.Store(p) + r.shutdownFuncs = append(r.shutdownFuncs, func() error { return p.Close() }) + slog.Info("meek: scanner started", "forced", force) + }() +} + func (r *LocalBackend) flushURLTestResults(storage vpn.URLTestHistoryStorage) { results := make(map[string]servers.URLTestResult) for _, srv := range r.srvManager.AllServers() { @@ -737,6 +783,9 @@ func (r *LocalBackend) ConnectVPN(tag string) error { } func (r *LocalBackend) getBoxOptions() vpn.BoxOptions { + if env.GetBool(env.ForceMeekOnly) { + return r.meekOnlyBoxOptions() + } // ignore error, we can still connect with default options if config is not available for some reason cfg, _ := r.confHandler.GetConfig() bOptions := vpn.BoxOptions{ @@ -773,6 +822,37 @@ func (r *LocalBackend) getBoxOptions() vpn.BoxOptions { if len(seed) > 0 { bOptions.URLTestSeed = seed } + if p := r.meekProvider.Load(); p != nil { + if out, ok := meek.BuildOutbound("meek-fronted", meek.DefaultURL, p.FrontSpecs(3)); ok { + bOptions.MeekOutbound = &out + } + } + return bOptions +} + +// meekOnlyBoxOptions returns a stripped-down config in which meek is +// the sole outbound and InitialServer pins it. All API-provided +// outbounds, bandit configuration, and selector arms are omitted — +// VPN traffic must traverse meek or fail. Used when env.ForceMeekOnly +// is set (local testing). When the scanner hasn't produced fronts +// yet the meek outbound is absent; Connect will fail and the user +// retries after a few seconds. +func (r *LocalBackend) meekOnlyBoxOptions() vpn.BoxOptions { + bOptions := vpn.BoxOptions{ + BasePath: settings.GetString(settings.DataPathKey), + } + p := r.meekProvider.Load() + if p == nil { + slog.Warn("meek-only mode: provider not yet ready, returning empty options") + return bOptions + } + out, ok := meek.BuildOutbound("meek-fronted", meek.DefaultURL, p.FrontSpecs(3)) + if !ok { + slog.Warn("meek-only mode: scanner has no working fronts yet") + return bOptions + } + bOptions.MeekOutbound = &out + bOptions.InitialServer = out.Tag return bOptions } diff --git a/cmd/meek-client-smoke/main.go b/cmd/meek-client-smoke/main.go new file mode 100644 index 00000000..9e78de99 --- /dev/null +++ b/cmd/meek-client-smoke/main.go @@ -0,0 +1,248 @@ +// Command meek-client-smoke exercises the lantern-box meek client +// against the deployed meek-test server via the radiance fronted +// scanner. Prints httpbin's reported origin — equality with the +// known Linode IP proves traffic completed the full client→Akamai +// →Caddy→meek-server→microsocks→internet round trip. +// +// Run: +// go run ./cmd/meek-client-smoke +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + lbmeek "github.com/getlantern/lantern-box/protocol/meek" + + "github.com/getlantern/radiance/kindling/fronted" + rmeek "github.com/getlantern/radiance/kindling/meek" +) + +const ( + meekURL = "https://meek.dsa.akamai.getiantem.org/" + targetHost = "httpbin.org" + targetPort = 80 + expectedOrigin = "139.162.181.47" // Linode public IP +) + +func main() { + if err := run(); err != nil { + slog.Error("test failed", "err", err) + os.Exit(1) + } + fmt.Println("\n✅ end-to-end meek client smoke test PASSED") +} + +func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + slog.Info("step 1: load fronted config + start scanner") + dataDir, err := os.MkdirTemp("", "meek-client-smoke-*") + if err != nil { + return fmt.Errorf("mktemp: %w", err) + } + defer os.RemoveAll(dataDir) + + cfg, err := fronted.LoadCachedConfig(dataDir) + if err != nil { + return fmt.Errorf("load fronted config: %w", err) + } + + // Only sample Akamai fronts — our meek property is on Akamai, so + // CloudFront IPs would dial a CDN that doesn't host the meek server + // and the poll-response loop would hang on miss-routed requests. + // AkamaiSample matches the production meek provider default. + provider, err := rmeek.NewProvider(rmeek.ProviderConfig{ + Config: cfg, + CacheFile: filepath.Join(dataDir, "meek_fronts_cache.json"), + KnownSample: 0, + CloudFrontSample: 0, + AkamaiSample: 3, + }) + if err != nil { + return fmt.Errorf("new provider: %w", err) + } + defer provider.Close() + provider.Start(ctx) + + slog.Info("step 2: wait up to 30s for scanner to find working Akamai fronts") + var fronts []rmeek.FrontSpec + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + time.Sleep(1 * time.Second) + fronts = provider.FrontSpecs(3) + if len(fronts) > 0 { + break + } + } + if len(fronts) == 0 { + return errors.New("scanner found no working fronts in 30s") + } + slog.Info("got fronts", "count", len(fronts), "first_ip", fronts[0].IPAddress, "first_sni", fronts[0].SNI) + + slog.Info("step 3: build HTTPClient and dial meek server") + u, err := url.Parse(meekURL) + if err != nil { + return fmt.Errorf("parse meek url: %w", err) + } + httpClient := buildFrontedHTTPClient(fronts, 10*time.Second) + + conn, err := lbmeek.Dial(ctx, lbmeek.Config{ + URL: meekURL, + InnerHost: u.Host, + HTTPClient: httpClient, + PollInterval: 100 * time.Millisecond, + ReadTimeout: 30 * time.Second, + }) + if err != nil { + return fmt.Errorf("meek dial: %w", err) + } + defer conn.Close() + + slog.Info("step 4: SOCKS5 method-select") + if _, err := conn.Write([]byte{0x05, 0x01, 0x00}); err != nil { + return fmt.Errorf("write method-select: %w", err) + } + conn.SetReadDeadline(time.Now().Add(15 * time.Second)) + resp := make([]byte, 2) + if _, err := io.ReadFull(conn, resp); err != nil { + return fmt.Errorf("read method-select reply: %w", err) + } + if resp[0] != 0x05 || resp[1] != 0x00 { + return fmt.Errorf("method-select reply unexpected: %x", resp) + } + slog.Info("✅ SOCKS5 NO_AUTH accepted", "reply", fmt.Sprintf("%x", resp)) + + slog.Info("step 5: SOCKS5 CONNECT", "target", fmt.Sprintf("%s:%d", targetHost, targetPort)) + connectReq := []byte{0x05, 0x01, 0x00, 0x03, byte(len(targetHost))} + connectReq = append(connectReq, []byte(targetHost)...) + connectReq = append(connectReq, byte(targetPort>>8), byte(targetPort&0xff)) + if _, err := conn.Write(connectReq); err != nil { + return fmt.Errorf("write CONNECT: %w", err) + } + conn.SetReadDeadline(time.Now().Add(15 * time.Second)) + connectReply := make([]byte, 10) + if _, err := io.ReadFull(conn, connectReply); err != nil { + return fmt.Errorf("read CONNECT reply: %w", err) + } + if connectReply[0] != 0x05 || connectReply[1] != 0x00 { + return fmt.Errorf("CONNECT reply unexpected: %x", connectReply) + } + slog.Info("✅ SOCKS5 CONNECT succeeded", "reply", fmt.Sprintf("%x", connectReply)) + + slog.Info("step 6: HTTP GET /ip") + httpReq := fmt.Sprintf("GET /ip HTTP/1.0\r\nHost: %s\r\nConnection: close\r\n\r\n", targetHost) + if _, err := conn.Write([]byte(httpReq)); err != nil { + return fmt.Errorf("write HTTP request: %w", err) + } + conn.SetReadDeadline(time.Now().Add(20 * time.Second)) + var bodyBuf strings.Builder + buf := make([]byte, 4096) + for { + n, err := conn.Read(buf) + if n > 0 { + bodyBuf.Write(buf[:n]) + if strings.Contains(bodyBuf.String(), expectedOrigin+"\"") { + break + } + } + if err != nil { + break + } + } + body := bodyBuf.String() + slog.Info("✅ HTTP response received", "bytes", len(body)) + fmt.Println("--- HTTP response ---") + fmt.Println(body) + + if !strings.Contains(body, expectedOrigin) { + return fmt.Errorf("expected origin %q not in response body", expectedOrigin) + } + return nil +} + +// buildFrontedHTTPClient returns a client that dials a random front by IP +// and validates the served chain against the front's VerifyHostname rather +// than the request URL's host. +func buildFrontedHTTPClient(fronts []rmeek.FrontSpec, connectTimeout time.Duration) *http.Client { + if len(fronts) == 0 { + panic("buildFrontedHTTPClient: no fronts") + } + tr := &http.Transport{ + DialTLSContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + front := fronts[time.Now().UnixNano()%int64(len(fronts))] + addr := front.IPAddress + if !strings.Contains(addr, ":") { + addr = net.JoinHostPort(addr, "443") + } + dialCtx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + d := &net.Dialer{} + raw, err := d.DialContext(dialCtx, "tcp", addr) + if err != nil { + return nil, fmt.Errorf("tcp dial %s: %w", addr, err) + } + tlsCfg := &tls.Config{InsecureSkipVerify: true} + if front.SNI != "" { + tlsCfg.ServerName = front.SNI + } + verifyHost := front.VerifyHostname + if verifyHost == "" { + verifyHost = front.SNI + } + tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + return verifyChain(rawCerts, verifyHost) + } + conn := tls.Client(raw, tlsCfg) + if err := conn.HandshakeContext(dialCtx); err != nil { + raw.Close() + return nil, fmt.Errorf("tls handshake: %w", err) + } + return conn, nil + }, + DisableKeepAlives: false, + IdleConnTimeout: 90 * time.Second, + } + return &http.Client{Transport: tr, Timeout: 60 * time.Second} +} + +func verifyChain(rawCerts [][]byte, verifyHost string) error { + if len(rawCerts) == 0 { + return errors.New("no peer certs") + } + certs := make([]*x509.Certificate, 0, len(rawCerts)) + for _, raw := range rawCerts { + c, err := x509.ParseCertificate(raw) + if err != nil { + return fmt.Errorf("parse cert: %w", err) + } + certs = append(certs, c) + } + roots, err := x509.SystemCertPool() + if err != nil { + return fmt.Errorf("system cert pool: %w", err) + } + intermediates := x509.NewCertPool() + for _, c := range certs[1:] { + intermediates.AddCert(c) + } + _, err = certs[0].Verify(x509.VerifyOptions{ + DNSName: verifyHost, + Roots: roots, + Intermediates: intermediates, + }) + return err +} diff --git a/common/env/env.go b/common/env/env.go index 5b2dcba2..4b440a65 100644 --- a/common/env/env.go +++ b/common/env/env.go @@ -29,6 +29,11 @@ var ( Country _key = "RADIANCE_COUNTRY" FeatureOverrides _key = "RADIANCE_FEATURE_OVERRIDES" AppVersion _key = "RADIANCE_VERSION" + // ForceMeekOnly, when truthy, makes radiance start the meek front + // scanner regardless of region detection and routes all traffic + // through meek as the sole outbound. For local testing; never set + // in shipped builds. + ForceMeekOnly _key = "RADIANCE_FORCE_MEEK_ONLY" Testing _key = "RADIANCE_TESTING" diff --git a/common/settings/settings.go b/common/settings/settings.go index 23ef0c59..5369db44 100644 --- a/common/settings/settings.go +++ b/common/settings/settings.go @@ -33,6 +33,7 @@ const ( CountryCodeKey _key = "country_code" // string LocaleKey _key = "locale" // string DeviceIDKey _key = "device_id" // string/int + MCCKey _key = "mcc" // string // Application behavior related keys. TelemetryKey _key = "telemetry_enabled" // bool diff --git a/fronted/scanner/akamai.go b/fronted/scanner/akamai.go new file mode 100644 index 00000000..dbd391b4 --- /dev/null +++ b/fronted/scanner/akamai.go @@ -0,0 +1,227 @@ +package scanner + +import ( + "context" + "crypto/rand" + "fmt" + "math/big" + "net" +) + +// Resolver resolves a hostname to one or more IPv4 addresses. +// Implementations must not route DNS through the VPN tunnel — the OS / +// ISP resolver is the right path in IR because the ISP returns real +// Akamai IPs reachable from its own network and DoH/DoT endpoints are +// themselves blocked. +type Resolver interface { + LookupHost(ctx context.Context, host string) ([]string, error) +} + +// SystemResolver wraps the OS resolver. Use it for Akamai edge hostnames +// (a248.e.akamai.net and similar) which Iran's ISP resolvers return +// truthfully because Akamai hosts too much Iranian critical +// infrastructure to be blanket-blocked. +// +// Never use this for our own backend hostnames — those get poisoned. +type SystemResolver struct{} + +func (SystemResolver) LookupHost(ctx context.Context, host string) ([]string, error) { + r := &net.Resolver{} + addrs, err := r.LookupHost(ctx, host) + if err != nil { + return nil, fmt.Errorf("lookup %s: %w", host, err) + } + v4 := addrs[:0] + for _, a := range addrs { + ip := net.ParseIP(a) + if ip == nil { + continue + } + if v4ip := ip.To4(); v4ip != nil { + v4 = append(v4, v4ip.String()) + } + } + if len(v4) == 0 { + return nil, fmt.Errorf("lookup %s: no IPv4", host) + } + return v4, nil +} + +// AkamaiEdgeHostnames is the canonical Akamai edge hostname used by every +// masquerade in our existing fronted.yaml.gz Akamai provider. The IPs +// returned by the OS resolver for this hostname are geographically +// relevant to the client's network — exactly the per-(ISP, location) +// signal we want. Additional hostnames from the MahsaNG regex pattern +// can be appended to widen the candidate space. +var AkamaiEdgeHostnames = []string{ + "a248.e.akamai.net", +} + +// GenerateAkamaiHostnames produces n random hostnames matching the regex +// `a([1-9]|1[0-9])([0-9]{2})\.(dsc)?(b|d|g|g2|na|r|w7)\.akamai\.net`, +// matching the pattern Psiphon and MahsaNG use. The regex enumerates +// roughly 3,500 distinct hostnames; each is a valid Akamai edge that +// the OS resolver answers from the general edge pool. Fresh hostname per +// dial varies the outer SNI without changing which property is reached. +func GenerateAkamaiHostnames(n int) ([]string, error) { + if n <= 0 { + return nil, nil + } + out := make([]string, 0, n) + for i := 0; i < n; i++ { + h, err := randomAkamaiHostname() + if err != nil { + return out, err + } + out = append(out, h) + } + return out, nil +} + +func randomAkamaiHostname() (string, error) { + firstPart, err := pickInt(19) + if err != nil { + return "", err + } + first := firstPart + 1 + + rest, err := pickInt(100) + if err != nil { + return "", err + } + + dscFlip, err := pickInt(2) + if err != nil { + return "", err + } + dsc := "" + if dscFlip == 1 { + dsc = "dsc" + } + + suffixes := []string{"b", "d", "g", "g2", "na", "r", "w7"} + suf, err := pickInt(len(suffixes)) + if err != nil { + return "", err + } + + return fmt.Sprintf("a%d%02d.%s%s.akamai.net", first, rest, dsc, suffixes[suf]), nil +} + +func pickInt(n int) (int, error) { + v, err := rand.Int(rand.Reader, big.NewInt(int64(n))) + if err != nil { + return 0, fmt.Errorf("rand: %w", err) + } + return int(v.Int64()), nil +} + +// AkamaiCertHostname is the hostname every Akamai edge's default cert +// validates as (alongside *.akamaized.net, *.akamaihd.net, etc.). Used +// for post-handshake cert verification regardless of which hostname we +// looked up to discover the edge IP — the regex-generated hostnames +// (a1798.dscg.akamai.net, etc.) are useful for DNS-side discovery but +// aren't in the served cert's SANs. +const AkamaiCertHostname = "a248.e.akamai.net" + +// akamaiSNIsPerIP caps how many named-SNI candidates accompany each +// empty-SNI candidate per Akamai IP. Bare-SNI is the dominant working +// strategy in IR, so it stays as the first candidate per IP; named +// SNIs provide DPI cover for the periods where bare gets blocked. +const akamaiSNIsPerIP = 3 + +// AkamaiCandidates resolves the supplied hostnames and emits, for each +// distinct IPv4 returned, one Candidate with empty SNI plus up to +// akamaiSNIsPerIP additional Candidates with SNIs drawn at random from +// snis. VerifyHostname is AkamaiCertHostname for every entry — Akamai +// edges serve the same default cert regardless of outer SNI, so named +// SNIs are pure DPI cover. +// +// hostnames may be the canonical AkamaiEdgeHostnames (stable resolver +// IPs), MahsaNG-style regex hostnames (more IP diversity), or both. +// snis may be empty, in which case only bare-SNI candidates are emitted. +func AkamaiCandidates(ctx context.Context, hostnames, snis []string, resolver Resolver, testURL, innerHost string) ([]Candidate, error) { + if resolver == nil { + resolver = SystemResolver{} + } + if len(hostnames) == 0 { + hostnames = AkamaiEdgeHostnames + } + + var out []Candidate + var firstErr error + seenIP := make(map[string]bool) + for _, h := range hostnames { + ips, err := resolver.LookupHost(ctx, h) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + for _, ip := range ips { + if seenIP[ip] { + continue + } + seenIP[ip] = true + out = append(out, Candidate{ + Provider: "akamai", + Domain: h, + IPAddress: ip, + VerifyHostname: AkamaiCertHostname, + TestURL: testURL, + InnerHost: innerHost, + }) + picks, err := pickSNIs(snis, akamaiSNIsPerIP) + if err != nil { + return out, err + } + for _, s := range picks { + out = append(out, Candidate{ + Provider: "akamai", + Domain: h, + IPAddress: ip, + SNI: s, + VerifyHostname: AkamaiCertHostname, + TestURL: testURL, + InnerHost: innerHost, + }) + } + } + } + if len(out) == 0 && firstErr != nil { + return nil, firstErr + } + return out, nil +} + +// pickSNIs draws up to n entries without replacement from snis. +// Crypto-rand keeps the choice unpredictable so scans don't drift +// toward the same SNI set across clients. +func pickSNIs(snis []string, n int) ([]string, error) { + if n <= 0 || len(snis) == 0 { + return nil, nil + } + if n >= len(snis) { + out := make([]string, len(snis)) + copy(out, snis) + return out, nil + } + indices := make([]int, len(snis)) + for i := range indices { + indices[i] = i + } + for i := len(indices) - 1; i > 0; i-- { + j, err := rand.Int(rand.Reader, big.NewInt(int64(i+1))) + if err != nil { + return nil, fmt.Errorf("rand: %w", err) + } + jj := int(j.Int64()) + indices[i], indices[jj] = indices[jj], indices[i] + } + out := make([]string, n) + for i := 0; i < n; i++ { + out[i] = snis[indices[i]] + } + return out, nil +} diff --git a/fronted/scanner/akamai_test.go b/fronted/scanner/akamai_test.go new file mode 100644 index 00000000..03283af0 --- /dev/null +++ b/fronted/scanner/akamai_test.go @@ -0,0 +1,159 @@ +package scanner + +import ( + "context" + "errors" + "regexp" + "testing" +) + +type fakeResolver struct { + answers map[string][]string + err map[string]error +} + +func (f fakeResolver) LookupHost(_ context.Context, host string) ([]string, error) { + if err, ok := f.err[host]; ok { + return nil, err + } + if a, ok := f.answers[host]; ok { + return a, nil + } + return nil, errors.New("no answer") +} + +func TestAkamaiCandidates_Dedup(t *testing.T) { + r := fakeResolver{answers: map[string][]string{ + "a248.e.akamai.net": {"23.47.48.1", "23.47.48.2", "23.47.48.1"}, + }} + cands, err := AkamaiCandidates(context.Background(), nil, nil, r, "https://api.iantem.io/ping", "api.iantem.io") + if err != nil { + t.Fatalf("AkamaiCandidates: %v", err) + } + if len(cands) != 2 { + t.Errorf("len = %d; want 2 (dedup)", len(cands)) + } + for _, c := range cands { + if c.Provider != "akamai" { + t.Errorf("provider = %q", c.Provider) + } + if c.Domain != "a248.e.akamai.net" { + t.Errorf("Domain = %q", c.Domain) + } + } +} + +func TestAkamaiCandidates_MixesNamedSNIs(t *testing.T) { + r := fakeResolver{answers: map[string][]string{ + "a248.e.akamai.net": {"23.47.48.1", "23.47.48.2"}, + }} + snis := []string{"python.org", "pypi.org", "snapp.ir", "google.com", "aparat.com"} + cands, err := AkamaiCandidates(context.Background(), nil, snis, r, "https://api.iantem.io/ping", "api.iantem.io") + if err != nil { + t.Fatalf("AkamaiCandidates: %v", err) + } + if want := 2 * (1 + akamaiSNIsPerIP); len(cands) != want { + t.Errorf("len = %d; want %d", len(cands), want) + } + + byIP := map[string][]Candidate{} + for _, c := range cands { + byIP[c.IPAddress] = append(byIP[c.IPAddress], c) + if c.VerifyHostname != AkamaiCertHostname { + t.Errorf("VerifyHostname = %q; want %s", c.VerifyHostname, AkamaiCertHostname) + } + } + for ip, group := range byIP { + if group[0].SNI != "" { + t.Errorf("IP %s: first candidate SNI = %q; want empty", ip, group[0].SNI) + } + seen := map[string]bool{} + for _, c := range group[1:] { + if c.SNI == "" { + t.Errorf("IP %s: named candidate has empty SNI", ip) + } + if seen[c.SNI] { + t.Errorf("IP %s: SNI %q appears twice — should be without replacement", ip, c.SNI) + } + seen[c.SNI] = true + } + } +} + +func TestAkamaiCandidates_MultipleHostnames(t *testing.T) { + r := fakeResolver{answers: map[string][]string{ + "a248.e.akamai.net": {"23.47.48.1"}, + "a123.b.akamai.net": {"184.150.1.1"}, + }} + hostnames := []string{"a248.e.akamai.net", "a123.b.akamai.net"} + cands, err := AkamaiCandidates(context.Background(), hostnames, nil, r, "https://api.iantem.io/ping", "api.iantem.io") + if err != nil { + t.Fatalf("AkamaiCandidates: %v", err) + } + if len(cands) != 2 { + t.Errorf("len = %d; want 2", len(cands)) + } + domains := map[string]bool{} + for _, c := range cands { + domains[c.Domain] = true + } + if len(domains) != 2 { + t.Errorf("expected both hostnames, got %v", domains) + } +} + +func TestAkamaiCandidates_AllResolversFail(t *testing.T) { + r := fakeResolver{err: map[string]error{ + "a248.e.akamai.net": errors.New("dns blocked"), + }} + _, err := AkamaiCandidates(context.Background(), nil, nil, r, "https://api.iantem.io/ping", "api.iantem.io") + if err == nil { + t.Errorf("expected error when all lookups fail") + } +} + +func TestGenerateAkamaiHostnames_MatchesRegex(t *testing.T) { + pattern := regexp.MustCompile(`^a([1-9]|1[0-9])([0-9]{2})\.(dsc)?(b|d|g|g2|na|r|w7)\.akamai\.net$`) + hostnames, err := GenerateAkamaiHostnames(200) + if err != nil { + t.Fatalf("GenerateAkamaiHostnames: %v", err) + } + if len(hostnames) != 200 { + t.Errorf("len = %d; want 200", len(hostnames)) + } + for _, h := range hostnames { + if !pattern.MatchString(h) { + t.Errorf("hostname %q doesn't match Akamai edge pattern", h) + } + } +} + +func TestGenerateAkamaiHostnames_Variety(t *testing.T) { + hostnames, err := GenerateAkamaiHostnames(200) + if err != nil { + t.Fatalf("GenerateAkamaiHostnames: %v", err) + } + unique := map[string]bool{} + for _, h := range hostnames { + unique[h] = true + } + // 200 draws from ~3,500-name space, birthday paradox aside, should see >100 distinct. + if len(unique) < 100 { + t.Errorf("only %d distinct hostnames across 200 draws; want > 100", len(unique)) + } +} + +func TestAkamaiCandidates_PartialFailureStillReturns(t *testing.T) { + r := fakeResolver{ + answers: map[string][]string{"a248.e.akamai.net": {"23.47.48.1"}}, + err: map[string]error{"a999.z.akamai.net": errors.New("nxdomain")}, + } + hostnames := []string{"a248.e.akamai.net", "a999.z.akamai.net"} + cands, err := AkamaiCandidates(context.Background(), hostnames, nil, r, "https://api.iantem.io/ping", "api.iantem.io") + if err != nil { + t.Fatalf("expected nil err when at least one lookup succeeded, got %v", err) + } + if len(cands) != 1 { + t.Errorf("len = %d; want 1", len(cands)) + } +} diff --git a/fronted/scanner/cache.go b/fronted/scanner/cache.go new file mode 100644 index 00000000..188dfd99 --- /dev/null +++ b/fronted/scanner/cache.go @@ -0,0 +1,99 @@ +package scanner + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "time" +) + +const cacheSchemaVersion = 1 + +type cacheFile struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + Working []cacheEntry `json:"working"` +} + +type cacheEntry struct { + Candidate Candidate `json:"candidate"` + Latency time.Duration `json:"latency"` + Status int `json:"status"` + VerifiedAt time.Time `json:"verified_at"` +} + +// LoadCache reads a cache file and returns the working results. Returns +// (nil, nil) when the file doesn't exist — first-boot is not an error. +// Returns an error only on malformed contents. +// +// Entries older than ttl are filtered out so a stale cache from days +// ago doesn't seed the live pool with already-blocked IPs. ttl <= 0 +// disables the filter (load everything). +func LoadCache(path string, ttl time.Duration) ([]Result, error) { + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read cache: %w", err) + } + var f cacheFile + if err := json.Unmarshal(data, &f); err != nil { + return nil, fmt.Errorf("decode cache: %w", err) + } + if f.Version != cacheSchemaVersion { + return nil, fmt.Errorf("cache schema %d unsupported (want %d)", f.Version, cacheSchemaVersion) + } + + now := time.Now() + out := make([]Result, 0, len(f.Working)) + for _, e := range f.Working { + if ttl > 0 && now.Sub(e.VerifiedAt) > ttl { + continue + } + out = append(out, Result{ + Candidate: e.Candidate, + Latency: e.Latency, + Status: e.Status, + }) + } + return out, nil +} + +// SaveCache atomically writes the working results to path. The write is +// best-effort — a failed save is logged by the caller but doesn't +// affect runtime correctness. +func SaveCache(path string, working []Result) error { + now := time.Now() + f := cacheFile{ + Version: cacheSchemaVersion, + UpdatedAt: now, + Working: make([]cacheEntry, 0, len(working)), + } + for _, r := range working { + if !r.OK() { + continue + } + f.Working = append(f.Working, cacheEntry{ + Candidate: r.Candidate, + Latency: r.Latency, + Status: r.Status, + VerifiedAt: now, + }) + } + + data, err := json.MarshalIndent(f, "", " ") + if err != nil { + return fmt.Errorf("encode cache: %w", err) + } + + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("write cache: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("rename cache: %w", err) + } + return nil +} diff --git a/fronted/scanner/cache_test.go b/fronted/scanner/cache_test.go new file mode 100644 index 00000000..51098889 --- /dev/null +++ b/fronted/scanner/cache_test.go @@ -0,0 +1,92 @@ +package scanner + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestCache_SaveAndLoad(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "scanner_cache.json") + + working := []Result{ + {Candidate: Candidate{Provider: "akamai", Domain: "a248.e.akamai.net", IPAddress: "23.47.48.230"}, Latency: 95 * time.Millisecond, Status: 200}, + {Candidate: Candidate{Provider: "cloudfront", Domain: "aa1.awsstatic.com", IPAddress: "99.84.2.4"}, Latency: 110 * time.Millisecond, Status: 200}, + } + + if err := SaveCache(path, working); err != nil { + t.Fatalf("SaveCache: %v", err) + } + + got, err := LoadCache(path, 24*time.Hour) + if err != nil { + t.Fatalf("LoadCache: %v", err) + } + if len(got) != 2 { + t.Fatalf("loaded %d; want 2", len(got)) + } + if got[0].Candidate.Provider != "akamai" || got[1].Candidate.Provider != "cloudfront" { + t.Errorf("order or content lost: %#v", got) + } +} + +func TestCache_MissingFileIsNotError(t *testing.T) { + got, err := LoadCache("/nonexistent/path/that/cannot/exist", 24*time.Hour) + if err != nil { + t.Errorf("missing cache file should return (nil, nil); got err=%v", err) + } + if got != nil { + t.Errorf("expected nil results, got %v", got) + } +} + +func TestCache_TTLFiltersStaleEntries(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "scanner_cache.json") + + if err := SaveCache(path, []Result{ + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.2.3.4"}, Latency: 1 * time.Second, Status: 200}, + }); err != nil { + t.Fatalf("SaveCache: %v", err) + } + + got, err := LoadCache(path, 1*time.Nanosecond) + if err != nil { + t.Fatalf("LoadCache: %v", err) + } + if len(got) != 0 { + t.Errorf("TTL didn't filter stale entries: %v", got) + } +} + +func TestCache_SaveSkipsNonOKResults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "scanner_cache.json") + working := []Result{ + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.2.3.4"}, Status: 200}, + {Candidate: Candidate{Provider: "akamai", IPAddress: "5.6.7.8"}, Status: 403}, + } + if err := SaveCache(path, working); err != nil { + t.Fatalf("SaveCache: %v", err) + } + raw, _ := os.ReadFile(path) + if string(raw) == "" || len(raw) == 0 { + t.Fatalf("empty cache file") + } + got, _ := LoadCache(path, time.Hour) + if len(got) != 1 { + t.Errorf("expected 1 OK result saved, got %d", len(got)) + } +} + +func TestCache_WrongVersionIsError(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "scanner_cache.json") + os.WriteFile(path, []byte(`{"version":999,"working":[]}`), 0o644) + _, err := LoadCache(path, time.Hour) + if err == nil { + t.Errorf("expected version mismatch error") + } +} diff --git a/fronted/scanner/candidates.go b/fronted/scanner/candidates.go new file mode 100644 index 00000000..687fad3c --- /dev/null +++ b/fronted/scanner/candidates.go @@ -0,0 +1,137 @@ +package scanner + +import ( + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net/url" + + "github.com/getlantern/domainfront" +) + +// CandidatesFromConfig flattens a parsed domainfront config into a probe +// list. Each (provider, masquerade) pair becomes one Candidate; the +// provider's TestURL is the probe target and its host becomes the inner +// Host header. +// +// HostAliases on the provider are not expanded — TestURL points at the +// provider's ping endpoint which is already CDN-hosted, so the request +// reaches our backend through the front when the path works. +func CandidatesFromConfig(cfg *domainfront.Config) ([]Candidate, error) { + if cfg == nil { + return nil, errors.New("nil config") + } + var out []Candidate + for name, p := range cfg.Providers { + if p == nil { + continue + } + innerHost, err := innerHostFromTestURL(p.TestURL) + if err != nil { + return nil, fmt.Errorf("provider %q: %w", name, err) + } + providerVerify := "" + if p.VerifyHostname != nil { + providerVerify = *p.VerifyHostname + } + for _, m := range p.Masquerades { + if m == nil { + continue + } + c := Candidate{ + Provider: name, + Domain: m.Domain, + IPAddress: m.IpAddress, + SNI: m.SNI, + TestURL: p.TestURL, + InnerHost: innerHost, + } + if m.VerifyHostname != nil { + c.VerifyHostname = *m.VerifyHostname + } else { + c.VerifyHostname = providerVerify + } + out = append(out, c) + } + } + return out, nil +} + +func innerHostFromTestURL(testURL string) (string, error) { + if testURL == "" { + return "", errors.New("empty TestURL") + } + u, err := url.Parse(testURL) + if err != nil { + return "", fmt.Errorf("parse TestURL: %w", err) + } + if u.Host == "" { + return "", fmt.Errorf("TestURL %q has no host", testURL) + } + return u.Hostname(), nil +} + +// SNIsForProvider returns the distinct, non-empty outer-SNI candidates +// for the named provider in cfg. +// +// Both FrontingSNIs.ArbitrarySNIs and Masquerade.Domain are included: +// FrontingSNIs entries are manually curated and survive the scanner's +// daily Masquerade rebuild, so they must be probed alongside the +// rebuilt Masquerade pool. +func SNIsForProvider(cfg *domainfront.Config, provider string) []string { + if cfg == nil { + return nil + } + p := cfg.Providers[provider] + if p == nil { + return nil + } + seen := make(map[string]bool, len(p.Masquerades)) + var out []string + for _, sniCfg := range p.FrontingSNIs { + if sniCfg == nil || !sniCfg.UseArbitrarySNIs { + continue + } + for _, s := range sniCfg.ArbitrarySNIs { + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + } + } + for _, m := range p.Masquerades { + if m == nil || m.Domain == "" { + continue + } + if seen[m.Domain] { + continue + } + seen[m.Domain] = true + out = append(out, m.Domain) + } + return out +} + +// TrustedCAsPool builds an x509.CertPool from a domainfront.Config's +// TrustedCAs. Passes to Options.RootCAs so the probe verifies the front's +// cert chain against the same set domainfront uses in production. +func TrustedCAsPool(cfg *domainfront.Config) (*x509.CertPool, error) { + pool := x509.NewCertPool() + for _, ca := range cfg.TrustedCAs { + if ca == nil || ca.Cert == "" { + continue + } + block, _ := pem.Decode([]byte(ca.Cert)) + if block == nil { + return nil, fmt.Errorf("CA %q: PEM decode failed", ca.CommonName) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("CA %q: parse: %w", ca.CommonName, err) + } + pool.AddCert(cert) + } + return pool, nil +} diff --git a/fronted/scanner/cloudfront.go b/fronted/scanner/cloudfront.go new file mode 100644 index 00000000..c40a30d5 --- /dev/null +++ b/fronted/scanner/cloudfront.go @@ -0,0 +1,157 @@ +package scanner + +import ( + "bufio" + "crypto/rand" + _ "embed" + "encoding/binary" + "fmt" + "math/big" + "net" + "net/netip" + "strings" +) + +//go:embed cloudfront_prefixes.txt +var cloudFrontPrefixesRaw string + +// CloudFrontPrefixes returns the embedded CloudFront IPv4 prefix list. +// Edges anywhere in this range route by Host header, so any IP in any +// prefix is a viable outer dial target for an inner Host that points at +// our CloudFront distribution. +func CloudFrontPrefixes() ([]netip.Prefix, error) { + scanner := bufio.NewScanner(strings.NewReader(cloudFrontPrefixesRaw)) + var out []netip.Prefix + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + p, err := netip.ParsePrefix(line) + if err != nil { + return nil, fmt.Errorf("parse %q: %w", line, err) + } + out = append(out, p) + } + if err := scanner.Err(); err != nil { + return nil, err + } + if len(out) == 0 { + return nil, fmt.Errorf("no prefixes in embedded list") + } + return out, nil +} + +// CloudFrontCandidates produces n probe candidates by pairing IPs sampled +// from the embedded CloudFront IP range with masquerade domains randomly +// drawn from snis (used post-handshake for cert verification, not as +// outer SNI). +// +// Outer SNI is left empty. CloudFront's strict SNI/Host enforcement +// returns HTTP 421 when SNI and inner Host belong to different +// distributions; sending no SNI extension at all sidesteps that check +// (no SNI = nothing to mismatch) and lets the edge route by inner +// Host alone. Matches the sni: "" pattern in fronted.yaml.gz. +// +// snis is used as the post-handshake VerifyHostname — the served cert +// is expected to be valid for one of the listed masquerade domains. +func CloudFrontCandidates(n int, snis []string, testURL, innerHost string) ([]Candidate, error) { + if n <= 0 { + return nil, nil + } + if len(snis) == 0 { + return nil, fmt.Errorf("no outer SNIs supplied") + } + prefixes, err := CloudFrontPrefixes() + if err != nil { + return nil, err + } + + out := make([]Candidate, 0, n) + for i := 0; i < n; i++ { + ip, err := samplePrefix(prefixes) + if err != nil { + return out, err + } + sniIdx, err := rand.Int(rand.Reader, big.NewInt(int64(len(snis)))) + if err != nil { + return out, fmt.Errorf("rand: %w", err) + } + sni := snis[sniIdx.Int64()] + out = append(out, Candidate{ + Provider: "cloudfront", + Domain: sni, + // VerifyHostname is the inner Host — when no SNI is sent, + // CloudFront serves either the *.cloudfront.net default + // cert (which covers the inner Host by wildcard) or a + // customer-specific cert pinned to this edge's distribution. + // Verifying against the inner Host filters to the former, + // which is the case where our cross-distribution Host + // header routing actually works. + IPAddress: ip, + VerifyHostname: innerHost, + TestURL: testURL, + InnerHost: innerHost, + }) + } + return out, nil +} + +// samplePrefix picks a prefix weighted by its address count, then a +// uniform random IP inside it. Weighting matters because the CloudFront +// list mixes /14s with /27s — uniform-over-prefixes would massively +// over-represent the small ones. +func samplePrefix(prefixes []netip.Prefix) (string, error) { + if len(prefixes) == 0 { + return "", fmt.Errorf("no prefixes") + } + + weights := make([]*big.Int, len(prefixes)) + total := new(big.Int) + for i, p := range prefixes { + host := p.Bits() + bits := p.Addr().BitLen() - host + w := new(big.Int).Lsh(big.NewInt(1), uint(bits)) + weights[i] = w + total.Add(total, w) + } + + pick, err := rand.Int(rand.Reader, total) + if err != nil { + return "", fmt.Errorf("rand: %w", err) + } + + acc := new(big.Int) + for i, w := range weights { + acc.Add(acc, w) + if pick.Cmp(acc) < 0 { + return randomIPInPrefix(prefixes[i]) + } + } + return randomIPInPrefix(prefixes[len(prefixes)-1]) +} + +func randomIPInPrefix(p netip.Prefix) (string, error) { + if !p.Addr().Is4() { + return "", fmt.Errorf("v6 prefix not supported yet: %s", p) + } + host := p.Bits() + bits := 32 - host + if bits == 0 { + return p.Addr().String(), nil + } + cap := new(big.Int).Lsh(big.NewInt(1), uint(bits)) + pick, err := rand.Int(rand.Reader, cap) + if err != nil { + return "", fmt.Errorf("rand: %w", err) + } + + base := p.Addr().As4() + baseUint := binary.BigEndian.Uint32(base[:]) + offset := uint32(pick.Int64()) + addrUint := baseUint + offset + + var out [4]byte + binary.BigEndian.PutUint32(out[:], addrUint) + return net.IP(out[:]).String(), nil +} diff --git a/fronted/scanner/cloudfront_prefixes.txt b/fronted/scanner/cloudfront_prefixes.txt new file mode 100644 index 00000000..bceda3c9 --- /dev/null +++ b/fronted/scanner/cloudfront_prefixes.txt @@ -0,0 +1,211 @@ +# CloudFront IPv4 prefixes — extracted from AWS ip-ranges.json +# syncToken: 1778907425 +# createDate: 2026-05-16-04-57-05 +# count: 204 prefixes +# regenerated by hand: download https://ip-ranges.amazonaws.com/ip-ranges.json, +# filter service == CLOUDFRONT, write ip_prefix one-per-line, sort. + +108.138.0.0/15 +108.156.0.0/14 +111.13.171.128/26 +111.13.171.192/26 +111.13.185.32/27 +111.13.185.64/27 +116.129.226.0/25 +116.129.226.128/26 +118.193.97.128/25 +118.193.97.64/26 +119.147.182.0/25 +119.147.182.128/26 +120.232.236.0/25 +120.232.236.128/26 +120.253.240.192/26 +120.253.241.160/27 +120.253.245.128/26 +120.253.245.192/27 +120.52.12.64/26 +120.52.153.192/26 +120.52.22.96/27 +120.52.39.128/27 +13.113.196.64/26 +13.113.203.0/24 +13.124.199.0/24 +13.134.24.0/23 +13.134.94.0/23 +13.203.133.0/26 +13.210.67.128/26 +13.224.0.0/14 +13.228.69.0/24 +13.233.177.192/26 +13.249.0.0/16 +13.32.0.0/15 +13.35.0.0/16 +13.54.63.128/26 +13.59.250.0/26 +130.176.0.0/17 +130.176.128.0/18 +130.176.192.0/19 +130.176.224.0/20 +143.204.0.0/16 +144.220.0.0/16 +15.158.0.0/16 +15.188.184.0/24 +15.207.13.128/25 +15.207.213.128/25 +18.154.0.0/15 +18.160.0.0/15 +18.164.0.0/15 +18.172.0.0/15 +18.175.65.0/24 +18.175.66.0/24 +18.175.67.0/24 +18.192.142.0/23 +18.199.68.0/22 +18.199.72.0/22 +18.199.76.0/22 +18.200.212.0/23 +18.216.170.128/25 +18.229.220.192/26 +18.230.229.0/24 +18.230.230.0/25 +18.238.0.0/15 +18.244.0.0/15 +18.64.0.0/14 +18.68.0.0/16 +180.163.57.0/25 +180.163.57.128/26 +204.246.164.0/22 +204.246.168.0/22 +204.246.172.0/24 +204.246.173.0/24 +204.246.174.0/23 +204.246.176.0/20 +205.251.202.0/23 +205.251.204.0/23 +205.251.206.0/23 +205.251.208.0/20 +205.251.249.0/24 +205.251.250.0/23 +205.251.252.0/23 +205.251.254.0/24 +216.137.32.0/19 +23.228.212.0/24 +23.228.213.0/24 +23.228.214.0/24 +23.228.220.0/24 +23.228.221.0/24 +23.228.222.0/24 +23.228.223.0/24 +23.228.244.0/24 +23.234.192.0/18 +23.91.0.0/19 +24.110.32.0/19 +3.10.17.128/25 +3.101.158.0/23 +3.107.43.128/25 +3.107.44.0/25 +3.107.44.128/25 +3.11.53.0/24 +3.128.93.0/24 +3.134.215.0/24 +3.146.232.0/22 +3.147.164.0/22 +3.147.244.0/22 +3.160.0.0/14 +3.164.0.0/18 +3.164.128.0/17 +3.164.64.0/18 +3.165.0.0/16 +3.166.0.0/15 +3.168.0.0/14 +3.172.0.0/18 +3.172.64.0/18 +3.173.0.0/17 +3.173.128.0/18 +3.173.192.0/18 +3.174.0.0/15 +3.231.2.0/25 +3.234.232.224/27 +3.236.169.192/26 +3.236.48.0/23 +3.29.40.128/26 +3.29.40.192/26 +3.29.40.64/26 +3.29.57.0/26 +3.35.130.128/25 +34.195.252.0/24 +34.216.51.0/25 +34.223.12.224/27 +34.223.80.192/26 +34.226.14.0/24 +35.158.136.0/24 +35.162.63.192/26 +35.167.191.128/26 +35.93.168.0/23 +35.93.170.0/23 +35.93.172.0/23 +36.103.232.0/25 +36.103.232.128/26 +43.218.56.128/26 +43.218.56.192/26 +43.218.56.64/26 +43.218.71.0/26 +44.220.194.0/23 +44.220.196.0/23 +44.220.198.0/23 +44.220.200.0/23 +44.220.202.0/23 +44.222.66.0/24 +44.227.178.0/24 +44.234.108.128/25 +44.234.90.252/30 +47.129.82.0/24 +47.129.83.0/24 +47.129.84.0/24 +51.44.234.0/23 +51.44.236.0/23 +51.44.238.0/23 +51.74.192.0/18 +52.124.128.0/17 +52.15.127.128/26 +52.199.127.192/26 +52.212.248.0/26 +52.220.191.0/26 +52.222.128.0/17 +52.46.0.0/18 +52.47.139.0/24 +52.52.191.128/26 +52.56.127.0/25 +52.57.254.0/24 +52.66.194.128/26 +52.78.247.128/26 +52.82.128.0/19 +52.84.0.0/15 +54.182.0.0/16 +54.192.0.0/16 +54.230.0.0/17 +54.230.128.0/18 +54.230.200.0/21 +54.230.208.0/20 +54.230.224.0/19 +54.233.255.128/26 +54.239.128.0/18 +54.239.192.0/19 +54.240.128.0/18 +56.125.46.0/24 +56.125.47.0/32 +56.125.48.0/24 +57.182.253.0/24 +57.183.42.0/25 +58.254.138.0/25 +58.254.138.128/26 +64.252.128.0/18 +64.252.64.0/18 +65.8.0.0/16 +65.9.0.0/17 +65.9.128.0/18 +70.132.0.0/18 +71.152.0.0/17 +99.79.169.0/24 +99.84.0.0/16 +99.86.0.0/16 diff --git a/fronted/scanner/cloudfront_test.go b/fronted/scanner/cloudfront_test.go new file mode 100644 index 00000000..685a8601 --- /dev/null +++ b/fronted/scanner/cloudfront_test.go @@ -0,0 +1,109 @@ +package scanner + +import ( + "net" + "net/netip" + "testing" +) + +func TestCloudFrontPrefixes_NonEmpty(t *testing.T) { + p, err := CloudFrontPrefixes() + if err != nil { + t.Fatalf("CloudFrontPrefixes: %v", err) + } + if len(p) < 100 { + t.Errorf("got %d prefixes; want >= 100 (AWS publishes ~200)", len(p)) + } +} + +func TestSamplePrefix_HitsPrefix(t *testing.T) { + prefixes := []netip.Prefix{ + netip.MustParsePrefix("203.0.113.0/24"), + } + for i := 0; i < 50; i++ { + ip, err := samplePrefix(prefixes) + if err != nil { + t.Fatalf("samplePrefix: %v", err) + } + addr, err := netip.ParseAddr(ip) + if err != nil { + t.Fatalf("parse %q: %v", ip, err) + } + if !prefixes[0].Contains(addr) { + t.Errorf("sampled %s not in %s", ip, prefixes[0]) + } + } +} + +func TestSamplePrefix_WeightedByCount(t *testing.T) { + prefixes := []netip.Prefix{ + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/30"), + } + bigHits, smallHits := 0, 0 + for i := 0; i < 1000; i++ { + ip, err := samplePrefix(prefixes) + if err != nil { + t.Fatalf("samplePrefix: %v", err) + } + addr := netip.MustParseAddr(ip) + if prefixes[0].Contains(addr) { + bigHits++ + } else if prefixes[1].Contains(addr) { + smallHits++ + } + } + if bigHits <= smallHits { + t.Errorf("/24 hit %d, /30 hit %d — expected /24 dominant", bigHits, smallHits) + } +} + +func TestCloudFrontCandidates(t *testing.T) { + snis := []string{"aa1.awsstatic.com", "advertising.amazon.com", "abcmouse.com"} + cands, err := CloudFrontCandidates(30, snis, "https://api.iantem.io/ping", "api.iantem.io") + if err != nil { + t.Fatalf("CloudFrontCandidates: %v", err) + } + if len(cands) != 30 { + t.Errorf("len = %d; want 30", len(cands)) + } + + seenIP := map[string]int{} + sniHits := map[string]int{} + allowedSNI := map[string]bool{"aa1.awsstatic.com": true, "advertising.amazon.com": true, "abcmouse.com": true} + for _, c := range cands { + if c.Provider != "cloudfront" { + t.Errorf("provider = %q; want cloudfront", c.Provider) + } + if !allowedSNI[c.Domain] { + t.Errorf("Domain = %q; not in input SNI list", c.Domain) + } + if c.SNI != "" { + t.Errorf("SNI should be empty (no SNI sent); got %q", c.SNI) + } + if c.VerifyHostname != "api.iantem.io" { + t.Errorf("VerifyHostname = %q; want = InnerHost %q", c.VerifyHostname, "api.iantem.io") + } + if net.ParseIP(c.IPAddress) == nil { + t.Errorf("bad IP %q", c.IPAddress) + } + if c.InnerHost != "api.iantem.io" { + t.Errorf("InnerHost = %q; want api.iantem.io", c.InnerHost) + } + seenIP[c.IPAddress]++ + sniHits[c.Domain]++ + } + if len(seenIP) < 25 { + t.Errorf("only %d distinct IPs across 30 samples; want more variety", len(seenIP)) + } + if len(sniHits) < 2 { + t.Errorf("SNIs got %d unique hits; want at least 2 (random distribution)", len(sniHits)) + } +} + +func TestCloudFrontCandidates_NoSNIs(t *testing.T) { + _, err := CloudFrontCandidates(5, nil, "https://api.iantem.io/ping", "api.iantem.io") + if err == nil { + t.Errorf("expected error when snis is empty") + } +} diff --git a/fronted/scanner/integration_test.go b/fronted/scanner/integration_test.go new file mode 100644 index 00000000..1c0e6aea --- /dev/null +++ b/fronted/scanner/integration_test.go @@ -0,0 +1,145 @@ +package scanner + +import ( + "context" + "fmt" + "os" + "testing" + "time" +) + +// integrationGate enforces SCANNER_INTEGRATION=1 so unattended CI runs +// don't probe live CDNs. Run with: +// +// SCANNER_INTEGRATION=1 go test -count=1 -v -run TestLive ./fronted/scanner/... +func integrationGate(t *testing.T) { + t.Helper() + if os.Getenv("SCANNER_INTEGRATION") != "1" { + t.Skip("skipping live-network test; set SCANNER_INTEGRATION=1 to run") + } +} + +// Probe targets behind our Akamai and CloudFront fronts; URLs taken from +// the testurl fields in fronted.yaml.gz, the same probes domainfront +// uses to validate masquerades in production. +const ( + akamaiTestURL = "https://fronted-ping.dsa.akamai.getiantem.org/ping" + cloudfrontTestURL = "http://d157vud77ygy87.cloudfront.net/ping" +) + +func TestLive_AkamaiSystemResolver(t *testing.T) { + integrationGate(t) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + hostnames, err := GenerateAkamaiHostnames(8) + if err != nil { + t.Fatalf("GenerateAkamaiHostnames: %v", err) + } + hostnames = append(hostnames, AkamaiEdgeHostnames...) + + cands, err := AkamaiCandidates(ctx, hostnames, nil, SystemResolver{}, akamaiTestURL, "fronted-ping.dsa.akamai.getiantem.org") + if err != nil { + t.Fatalf("AkamaiCandidates: %v", err) + } + if len(cands) == 0 { + t.Fatal("no Akamai candidates resolved; system resolver may have failed") + } + + results := Scan(ctx, cands, Options{DialTimeout: 5 * time.Second, Concurrency: 4}) + working := RankWorking(results) + + report(t, "akamai", cands, results, working) + if len(working) == 0 { + t.Errorf("0 of %d Akamai candidates probed OK; expected at least 1", len(cands)) + } +} + +// TestLive_CloudFrontRandomIPs is diagnostic only — random IPs in the +// CloudFront range have low hit rate because each edge POP serves a +// subset of distributions. The probe correctly filters; the test +// reports the rate without asserting a floor. To validate the probe +// technique itself, see TestLive_CloudFrontKnownMasquerades. +func TestLive_CloudFrontRandomIPs(t *testing.T) { + integrationGate(t) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + snis := []string{ + "aa1.awsstatic.com", + "advertising.amazon.com", + "abcmouse.com", + "adsrvr.org", + } + cands, err := CloudFrontCandidates(40, snis, cloudfrontTestURL, "d157vud77ygy87.cloudfront.net") + if err != nil { + t.Fatalf("CloudFrontCandidates: %v", err) + } + + results := Scan(ctx, cands, Options{DialTimeout: 5 * time.Second, Concurrency: 8}) + working := RankWorking(results) + + report(t, "cloudfront-random", cands, results, working) +} + +// TestLive_CloudFrontKnownMasquerades probes a handful of pre-validated +// (IP, outer SNI) pairs from fronted.yaml.gz. Hit rate should be high +// because each pair was verified before being committed to the config. +// Use this to confirm the probe machinery works against CloudFront; +// random-IP discovery is a separate question covered by +// TestLive_CloudFrontRandomIPs. +func TestLive_CloudFrontKnownMasquerades(t *testing.T) { + integrationGate(t) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + known := []Candidate{ + {Provider: "cloudfront", Domain: "aa1.awsstatic.com", SNI: "aa1.awsstatic.com", IPAddress: "99.84.2.4", VerifyHostname: "aa1.awsstatic.com", TestURL: cloudfrontTestURL, InnerHost: "d157vud77ygy87.cloudfront.net"}, + {Provider: "cloudfront", Domain: "aa1.awsstatic.com", SNI: "aa1.awsstatic.com", IPAddress: "18.238.3.4", VerifyHostname: "aa1.awsstatic.com", TestURL: cloudfrontTestURL, InnerHost: "d157vud77ygy87.cloudfront.net"}, + {Provider: "cloudfront", Domain: "advertising.amazon.com", SNI: "advertising.amazon.com", IPAddress: "3.164.130.9", VerifyHostname: "advertising.amazon.com", TestURL: cloudfrontTestURL, InnerHost: "d157vud77ygy87.cloudfront.net"}, + {Provider: "cloudfront", Domain: "advertising.amazon.com", SNI: "advertising.amazon.com", IPAddress: "54.230.224.110", VerifyHostname: "advertising.amazon.com", TestURL: cloudfrontTestURL, InnerHost: "d157vud77ygy87.cloudfront.net"}, + {Provider: "cloudfront", Domain: "advertising.amazon.com", SNI: "advertising.amazon.com", IPAddress: "18.244.1.167", VerifyHostname: "advertising.amazon.com", TestURL: cloudfrontTestURL, InnerHost: "d157vud77ygy87.cloudfront.net"}, + } + results := Scan(ctx, known, Options{DialTimeout: 5 * time.Second, Concurrency: 4}) + working := RankWorking(results) + report(t, "cloudfront-known", known, results, working) + // Diagnostic only: pre-validated pairs may go stale as CloudFront + // re-shards distributions across POPs. The scanner correctly + // filtering stale entries is exactly its job. +} + +func report(t *testing.T, label string, cands []Candidate, results []Result, working []Result) { + t.Helper() + t.Logf("[%s] probed %d candidates, %d working (%.0f%%)", label, len(cands), len(working), 100*float64(len(working))/float64(len(cands))) + for i, r := range working { + if i >= 5 { + t.Logf("[%s] (… and %d more working)", label, len(working)-5) + break + } + t.Logf("[%s] OK %s ip=%s sni=%s latency=%s", label, r.Candidate.Provider, r.Candidate.IPAddress, r.Candidate.outerSNI(), r.Latency) + } + errs := map[string]int{} + for _, r := range results { + if r.OK() || r.Err == nil { + continue + } + errs[shortErr(r.Err)]++ + } + for kind, n := range errs { + t.Logf("[%s] error %q: %d", label, kind, n) + } +} + +func shortErr(err error) string { + s := err.Error() + if len(s) > 240 { + s = s[:240] + "…" + } + return s +} + +// Sanity check that compiles without integration gate so the file always builds. +var _ = fmt.Sprintf diff --git a/fronted/scanner/pool.go b/fronted/scanner/pool.go new file mode 100644 index 00000000..403d8436 --- /dev/null +++ b/fronted/scanner/pool.go @@ -0,0 +1,103 @@ +package scanner + +import ( + "context" + "errors" + "fmt" + + "github.com/getlantern/domainfront" +) + +// PoolOptions composes a candidate pool from the three feeder sources. +type PoolOptions struct { + Config *domainfront.Config + + KnownSample int + CloudFrontSample int + AkamaiSample int + + Resolver Resolver +} + +// BuildPool returns a probe pool combining (a) pre-validated masquerades +// from cfg, (b) random CloudFront IP × random masquerade-SNI pairs, and +// (c) Akamai hostnames generated from the MahsaNG/Psiphon regex and +// resolved via opts.Resolver. +// +// Probe target (TestURL, inner Host) for each candidate comes from its +// originating provider's TestURL in cfg. +// +// Sample sizes <= 0 disable the corresponding feeder. When AkamaiSample +// > 0, the canonical AkamaiEdgeHostnames are always included alongside +// the regex-generated draws — it's the highest-trust hostname in the +// pool. +// +// The raw-range feeders (Akamai DNS + CloudFront prefixes) produce +// per-scan-fresh IPs, which match Samim Mirhosseini's "different per +// ISP, location, time of day" observation. The Known feeder (pre- +// resolved IPs from fronted.yaml.gz) is opt-in via KnownSample > 0; +// it's higher-hit-rate when the YAML is current but goes stale faster +// than the raw range scans can self-heal. +func BuildPool(ctx context.Context, opts PoolOptions) ([]Candidate, error) { + if opts.Config == nil { + return nil, errors.New("BuildPool: nil Config") + } + + var cands []Candidate + if opts.KnownSample > 0 { + known, err := CandidatesFromConfig(opts.Config) + if err != nil { + return nil, fmt.Errorf("known masquerades: %w", err) + } + if opts.KnownSample < len(known) { + known = sampleN(known, opts.KnownSample) + } + cands = append(cands, known...) + } + + akamaiProv := opts.Config.Providers["akamai"] + if akamaiProv != nil && akamaiProv.TestURL != "" && opts.AkamaiSample > 0 { + innerHost, err := innerHostFromTestURL(akamaiProv.TestURL) + if err == nil { + hostnames := append([]string{}, AkamaiEdgeHostnames...) + more, err := GenerateAkamaiHostnames(opts.AkamaiSample) + if err == nil { + hostnames = append(hostnames, more...) + } + snis := SNIsForProvider(opts.Config, "akamai") + akCands, err := AkamaiCandidates(ctx, hostnames, snis, opts.Resolver, akamaiProv.TestURL, innerHost) + if err == nil { + cands = append(cands, akCands...) + } + } + } + + cfProv := opts.Config.Providers["cloudfront"] + if cfProv != nil && cfProv.TestURL != "" && opts.CloudFrontSample > 0 { + innerHost, err := innerHostFromTestURL(cfProv.TestURL) + if err == nil { + snis := SNIsForProvider(opts.Config, "cloudfront") + if len(snis) > 0 { + cfCands, err := CloudFrontCandidates(opts.CloudFrontSample, snis, cfProv.TestURL, innerHost) + if err == nil { + cands = append(cands, cfCands...) + } + } + } + } + + return cands, nil +} + +func sampleN(cands []Candidate, n int) []Candidate { + if n >= len(cands) { + return cands + } + out := make([]Candidate, len(cands)) + copy(out, cands) + for i := 0; i < n; i++ { + j, _ := pickInt(len(out) - i) + out[i], out[i+j] = out[i+j], out[i] + } + return out[:n] +} diff --git a/fronted/scanner/scanner.go b/fronted/scanner/scanner.go new file mode 100644 index 00000000..71128746 --- /dev/null +++ b/fronted/scanner/scanner.go @@ -0,0 +1,288 @@ +// Package scanner probes domain-fronting candidates from the user's +// network position to find which routes actually work end-to-end. +// +// A successful probe is a TCP+TLS handshake to a CDN edge IP using the +// candidate's outer SNI followed by an HTTPS GET to TestURL with +// InnerHost as the Host header that returns 2xx. Both legs must work: +// TLS-only success would only confirm the edge is reachable, not that +// the inner Host routes to our backend through that edge. +// +// The scanner is intended to run client-side so each user's results +// reflect their ISP, geography, and time of day — the variables Samim +// Mirhosseini identified as load-bearing for IR-specific fronting. +package scanner + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sort" + "sync" + "time" + + tls "github.com/refraction-networking/utls" +) + +// Candidate describes one (CDN edge, masquerade) pair to probe. +// +// SNI semantics matter and differ between providers. Empty SNI means +// "send no SNI extension at all" — Akamai edges return their default +// cert in that mode, which validates against Domain. Non-empty SNI is +// sent in the ClientHello — CloudFront edges serve cert content keyed +// to the SNI value, so the masquerade domain must be passed in SNI. +// Match the production domainfront dialer's behavior: leave SNI empty +// for Akamai-style entries; set it explicitly for CloudFront-style +// entries. +// +// Domain identifies the logical front and is the hostname the +// post-handshake cert chain is verified against when VerifyHostname +// isn't overridden. +type Candidate struct { + Provider string + Domain string + IPAddress string + SNI string + VerifyHostname string + TestURL string + InnerHost string +} + +func (c Candidate) outerSNI() string { + return c.SNI +} + +func (c Candidate) verify() string { + if c.VerifyHostname != "" { + return c.VerifyHostname + } + return c.Domain +} + +type Result struct { + Candidate Candidate + Latency time.Duration + Status int + Err error +} + +func (r Result) OK() bool { return r.Err == nil && r.Status >= 200 && r.Status < 300 } + +type Dialer interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) +} + +type Options struct { + Dialer Dialer + RootCAs *x509.CertPool + ClientHelloID tls.ClientHelloID + DialTimeout time.Duration + Concurrency int + // OnResult, when set, is called once per probe as it completes, + // from the probing goroutine. Multiple goroutines invoke it + // concurrently, so it must be safe for concurrent use. Lets callers + // consume working fronts as they're found rather than waiting for + // the whole scan. + OnResult func(Result) +} + +func (o *Options) defaults() { + if o.Dialer == nil { + o.Dialer = &net.Dialer{} + } + if o.ClientHelloID.Client == "" { + o.ClientHelloID = tls.HelloGolang + } + if o.DialTimeout <= 0 { + o.DialTimeout = 5 * time.Second + } + if o.Concurrency <= 0 { + o.Concurrency = 8 + } +} + +var errNoTestURL = errors.New("candidate has no TestURL") + +func Probe(ctx context.Context, c Candidate, opts Options) Result { + opts.defaults() + + start := time.Now() + res := Result{Candidate: c} + + if c.TestURL == "" { + res.Err = errNoTestURL + return res + } + if c.IPAddress == "" { + res.Err = errors.New("candidate has no IPAddress") + return res + } + + addr := c.IPAddress + if _, _, err := net.SplitHostPort(addr); err != nil { + addr = net.JoinHostPort(addr, "443") + } + + dialCtx, cancel := context.WithTimeout(ctx, opts.DialTimeout) + defer cancel() + + rawConn, err := opts.Dialer.DialContext(dialCtx, "tcp", addr) + if err != nil { + res.Latency = time.Since(start) + res.Err = fmt.Errorf("tcp: %w", err) + return res + } + + deadline := time.Now().Add(opts.DialTimeout) + _ = rawConn.SetDeadline(deadline) + + verifyHost := c.verify() + tlsConfig := &tls.Config{ + RootCAs: opts.RootCAs, + InsecureSkipVerify: true, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + return verifyCertChain(rawCerts, opts.RootCAs, verifyHost) + }, + } + if outer := c.outerSNI(); outer != "" { + tlsConfig.ServerName = outer + } + + tlsConn := tls.UClient(rawConn, tlsConfig, opts.ClientHelloID) + if err := tlsConn.HandshakeContext(dialCtx); err != nil { + rawConn.Close() + res.Latency = time.Since(start) + res.Err = fmt.Errorf("tls: %w", err) + return res + } + + req, err := buildProbeRequest(dialCtx, c) + if err != nil { + tlsConn.Close() + res.Latency = time.Since(start) + res.Err = err + return res + } + + tr := &http.Transport{ + DialTLSContext: func(context.Context, string, string) (net.Conn, error) { + return tlsConn, nil + }, + DisableKeepAlives: true, + } + client := &http.Client{Transport: tr, Timeout: opts.DialTimeout} + + resp, err := client.Do(req) + if err != nil { + tlsConn.Close() + res.Latency = time.Since(start) + res.Err = fmt.Errorf("http: %w", err) + return res + } + defer resp.Body.Close() + tr.CloseIdleConnections() + + res.Status = resp.StatusCode + res.Latency = time.Since(start) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + res.Err = fmt.Errorf("http status %d", resp.StatusCode) + } + return res +} + +func buildProbeRequest(ctx context.Context, c Candidate) (*http.Request, error) { + u, err := url.Parse(c.TestURL) + if err != nil { + return nil, fmt.Errorf("parse TestURL: %w", err) + } + // The outer connection is TLS on port 443 regardless of the + // TestURL scheme. http.Transport only routes via DialTLSContext + // (our pre-opened fronted TLS conn) for https URLs — if scheme + // stays http the request falls through to plain-text DNS + port + // 80, bypassing the front entirely. Some providers' testurls + // (CloudFront in fronted.yaml.gz) ship as http://. + u.Scheme = "https" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("new request: %w", err) + } + if c.InnerHost != "" { + req.Host = c.InnerHost + } + return req, nil +} + +// Scan probes candidates concurrently and returns one Result per +// candidate. Results retain input order so callers can correlate by +// index; sort by Latency or filter by OK() to rank. +func Scan(ctx context.Context, candidates []Candidate, opts Options) []Result { + opts.defaults() + + results := make([]Result, len(candidates)) + if len(candidates) == 0 { + return results + } + + sem := make(chan struct{}, opts.Concurrency) + var wg sync.WaitGroup + for i, c := range candidates { + wg.Add(1) + sem <- struct{}{} + go func(i int, c Candidate) { + defer wg.Done() + defer func() { <-sem }() + if err := ctx.Err(); err != nil { + results[i] = Result{Candidate: c, Err: err} + return + } + results[i] = Probe(ctx, c, opts) + if opts.OnResult != nil { + opts.OnResult(results[i]) + } + }(i, c) + } + wg.Wait() + return results +} + +// RankWorking returns the OK() results sorted by latency ascending. +func RankWorking(results []Result) []Result { + out := make([]Result, 0, len(results)) + for _, r := range results { + if r.OK() { + out = append(out, r) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Latency < out[j].Latency }) + return out +} + +func verifyCertChain(rawCerts [][]byte, roots *x509.CertPool, dnsName string) error { + if len(rawCerts) == 0 { + return errors.New("no certificates presented") + } + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return fmt.Errorf("parse leaf: %w", err) + } + opts := x509.VerifyOptions{ + Roots: roots, + CurrentTime: time.Now(), + DNSName: dnsName, + Intermediates: x509.NewCertPool(), + } + for i := 1; i < len(rawCerts); i++ { + c, err := x509.ParseCertificate(rawCerts[i]) + if err != nil { + return fmt.Errorf("parse intermediate %d: %w", i, err) + } + opts.Intermediates.AddCert(c) + } + if _, err := cert.Verify(opts); err != nil { + return fmt.Errorf("verify: %w", err) + } + return nil +} diff --git a/fronted/scanner/scanner_test.go b/fronted/scanner/scanner_test.go new file mode 100644 index 00000000..885cb896 --- /dev/null +++ b/fronted/scanner/scanner_test.go @@ -0,0 +1,355 @@ +package scanner + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/getlantern/domainfront" + tls "github.com/refraction-networking/utls" + stdtls "crypto/tls" +) + +func TestProbe_SuccessfulHandshakeAnd200(t *testing.T) { + srv, ca := newTLSEchoServer(t, "test.example", http.StatusOK) + t.Cleanup(srv.Close) + + host, port, err := net.SplitHostPort(srv.Listener.Addr().String()) + if err != nil { + t.Fatalf("split: %v", err) + } + + c := Candidate{ + Provider: "test", + Domain: "test.example", + IPAddress: host + ":" + port, + TestURL: fmt.Sprintf("https://%s/ping", srv.Listener.Addr().String()), + InnerHost: "test.example", + VerifyHostname: "test.example", + } + + res := Probe(context.Background(), c, Options{RootCAs: ca, DialTimeout: 2 * time.Second}) + + if !res.OK() { + t.Fatalf("probe failed: status=%d err=%v", res.Status, res.Err) + } + if res.Latency <= 0 { + t.Errorf("latency = %v; want > 0", res.Latency) + } +} + +func TestProbe_TCPConnectFails(t *testing.T) { + c := Candidate{ + Provider: "test", + Domain: "test.example", + IPAddress: "127.0.0.1:1", + TestURL: "https://test.example/ping", + InnerHost: "test.example", + } + res := Probe(context.Background(), c, Options{DialTimeout: 500 * time.Millisecond}) + if res.OK() { + t.Errorf("expected failure, got OK result") + } + if res.Err == nil { + t.Errorf("expected non-nil error") + } +} + +func TestProbe_TLSWrongHostname(t *testing.T) { + srv, ca := newTLSEchoServer(t, "test.example", http.StatusOK) + t.Cleanup(srv.Close) + + host, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + c := Candidate{ + Provider: "test", + Domain: "wrong.example", + IPAddress: host + ":" + port, + TestURL: fmt.Sprintf("https://%s/ping", srv.Listener.Addr().String()), + VerifyHostname: "wrong.example", + } + res := Probe(context.Background(), c, Options{RootCAs: ca, DialTimeout: 2 * time.Second}) + if res.OK() { + t.Errorf("expected hostname mismatch failure, got OK") + } +} + +func TestProbe_HTTP500NotOK(t *testing.T) { + srv, ca := newTLSEchoServer(t, "test.example", http.StatusInternalServerError) + t.Cleanup(srv.Close) + + host, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + c := Candidate{ + Provider: "test", + Domain: "test.example", + IPAddress: host + ":" + port, + TestURL: fmt.Sprintf("https://%s/ping", srv.Listener.Addr().String()), + VerifyHostname: "test.example", + } + res := Probe(context.Background(), c, Options{RootCAs: ca, DialTimeout: 2 * time.Second}) + if res.OK() { + t.Errorf("expected 5xx to fail OK()") + } + if res.Status != http.StatusInternalServerError { + t.Errorf("status = %d; want 500", res.Status) + } +} + +func TestScan_RanksByLatency(t *testing.T) { + srvFast, ca := newTLSEchoServer(t, "fast.example", http.StatusOK) + t.Cleanup(srvFast.Close) + srvSlow, _ := newTLSEchoServerWithCA(t, "slow.example", http.StatusOK, ca, 100*time.Millisecond) + t.Cleanup(srvSlow.Close) + + cands := []Candidate{ + { + Provider: "test", + Domain: "slow.example", + IPAddress: srvSlow.Listener.Addr().String(), + TestURL: fmt.Sprintf("https://%s/ping", srvSlow.Listener.Addr().String()), + VerifyHostname: "slow.example", + }, + { + Provider: "test", + Domain: "fast.example", + IPAddress: srvFast.Listener.Addr().String(), + TestURL: fmt.Sprintf("https://%s/ping", srvFast.Listener.Addr().String()), + VerifyHostname: "fast.example", + }, + { + Provider: "test", + Domain: "deadend.example", + IPAddress: "127.0.0.1:1", + TestURL: "https://deadend.example/ping", + }, + } + + results := Scan(context.Background(), cands, Options{RootCAs: ca, DialTimeout: 3 * time.Second}) + if len(results) != 3 { + t.Fatalf("len(results) = %d; want 3", len(results)) + } + + ranked := RankWorking(results) + if len(ranked) != 2 { + t.Fatalf("RankWorking returned %d; want 2", len(ranked)) + } + if ranked[0].Candidate.Domain != "fast.example" { + t.Errorf("rank[0] = %q; want fast.example", ranked[0].Candidate.Domain) + } + if ranked[1].Candidate.Domain != "slow.example" { + t.Errorf("rank[1] = %q; want slow.example", ranked[1].Candidate.Domain) + } +} + +func TestCandidatesFromConfig(t *testing.T) { + cfg := &domainfront.Config{ + Providers: map[string]*domainfront.Provider{ + "akamai": { + TestURL: "https://fronted-ping.dsa.akamai.getiantem.org/ping", + Masquerades: []*domainfront.Masquerade{ + {Domain: "a248.e.akamai.net", IpAddress: "23.47.48.230"}, + {Domain: "a248.e.akamai.net", IpAddress: "184.150.49.62"}, + }, + }, + "cloudfront": { + TestURL: "http://d157vud77ygy87.cloudfront.net/ping", + Masquerades: []*domainfront.Masquerade{ + {Domain: "aa1.awsstatic.com", IpAddress: "99.84.2.4"}, + }, + }, + }, + } + + cands, err := CandidatesFromConfig(cfg) + if err != nil { + t.Fatalf("CandidatesFromConfig: %v", err) + } + if len(cands) != 3 { + t.Fatalf("len(cands) = %d; want 3", len(cands)) + } + + byProvider := map[string]int{} + for _, c := range cands { + byProvider[c.Provider]++ + if c.InnerHost == "" { + t.Errorf("candidate %v has empty InnerHost", c) + } + if c.TestURL == "" { + t.Errorf("candidate %v has empty TestURL", c) + } + } + if byProvider["akamai"] != 2 || byProvider["cloudfront"] != 1 { + t.Errorf("provider distribution wrong: %v", byProvider) + } +} + +func TestCandidatesFromConfig_NilConfigReturnsError(t *testing.T) { + _, err := CandidatesFromConfig(nil) + if err == nil { + t.Errorf("expected error for nil config") + } +} + +func TestSNIsForProvider_UniqueAndOrdered(t *testing.T) { + cfg := &domainfront.Config{ + Providers: map[string]*domainfront.Provider{ + "cloudfront": { + Masquerades: []*domainfront.Masquerade{ + {Domain: "aa1.awsstatic.com", IpAddress: "1.1.1.1"}, + {Domain: "aa1.awsstatic.com", IpAddress: "1.1.1.2"}, // dup + {Domain: "advertising.amazon.com", IpAddress: "2.2.2.2"}, + {Domain: "", IpAddress: "3.3.3.3"}, // skip empty + }, + }, + }, + } + got := SNIsForProvider(cfg, "cloudfront") + if len(got) != 2 { + t.Fatalf("len = %d; want 2", len(got)) + } + want := map[string]bool{"aa1.awsstatic.com": true, "advertising.amazon.com": true} + for _, s := range got { + if !want[s] { + t.Errorf("unexpected SNI %q", s) + } + } +} + +func TestSNIsForProvider_MissingProvider(t *testing.T) { + cfg := &domainfront.Config{Providers: map[string]*domainfront.Provider{}} + if got := SNIsForProvider(cfg, "cloudfront"); got != nil { + t.Errorf("missing provider should yield nil, got %v", got) + } +} + +func TestSNIsForProvider_ReadsFrontingSNIs(t *testing.T) { + cfg := &domainfront.Config{ + Providers: map[string]*domainfront.Provider{ + "akamai": { + FrontingSNIs: map[string]*domainfront.SNIConfig{ + "default": { + UseArbitrarySNIs: true, + ArbitrarySNIs: []string{"crunchbase.com", "brightspace.com"}, + }, + "ir": { + UseArbitrarySNIs: true, + ArbitrarySNIs: []string{"snapp.ir", "pypi.org"}, + }, + "cn": { + UseArbitrarySNIs: false, + ArbitrarySNIs: []string{"01ny.cn"}, + }, + }, + Masquerades: []*domainfront.Masquerade{ + {Domain: "a248.e.akamai.net", IpAddress: "23.x.x.x"}, + }, + }, + }, + } + got := SNIsForProvider(cfg, "akamai") + want := map[string]bool{ + "crunchbase.com": true, + "brightspace.com": true, + "snapp.ir": true, + "pypi.org": true, + "a248.e.akamai.net": true, + } + if len(got) != len(want) { + t.Fatalf("got %d SNIs, want %d (%v)", len(got), len(want), got) + } + for _, s := range got { + if !want[s] { + t.Errorf("unexpected SNI %q (cn group with use_arbitrary=false should be excluded)", s) + } + } +} + +// --- helpers --- + +func newTLSEchoServer(t *testing.T, dnsName string, status int) (*httptest.Server, *x509.CertPool) { + t.Helper() + cert, pool := selfSignedCert(t, dnsName) + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte("ok")) + })) + srv.TLS = &stdtls.Config{Certificates: []stdtls.Certificate{cert}} + srv.StartTLS() + return srv, pool +} + +func newTLSEchoServerWithCA(t *testing.T, dnsName string, status int, pool *x509.CertPool, delay time.Duration) (*httptest.Server, *x509.CertPool) { + t.Helper() + cert, _ := selfSignedCertWithPool(t, dnsName, pool) + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if delay > 0 { + time.Sleep(delay) + } + w.WriteHeader(status) + _, _ = w.Write([]byte("ok")) + })) + srv.TLS = &stdtls.Config{Certificates: []stdtls.Certificate{cert}} + srv.StartTLS() + return srv, pool +} + +func selfSignedCert(t *testing.T, dnsName string) (stdtls.Certificate, *x509.CertPool) { + t.Helper() + return selfSignedCertWithPool(t, dnsName, x509.NewCertPool()) +} + +func selfSignedCertWithPool(t *testing.T, dnsName string, pool *x509.CertPool) (stdtls.Certificate, *x509.CertPool) { + t.Helper() + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("genkey: %v", err) + } + serial, _ := rand.Int(rand.Reader, big.NewInt(1<<62)) + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: dnsName}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{dnsName}, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &priv.PublicKey, priv) + if err != nil { + t.Fatalf("createcert: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsecert: %v", err) + } + pool.AddCert(cert) + + tlsCert := stdtls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: priv, + Leaf: cert, + } + return tlsCert, pool +} + +// Used by tls.HelloGolang in tests — guards against the utls import being +// hidden and the build succeeding by accident on a stdlib-tls fallback. +var _ = pem.Decode +var _ = tls.HelloGolang +var _ = errors.New diff --git a/fronted/scanner/service.go b/fronted/scanner/service.go new file mode 100644 index 00000000..04faf275 --- /dev/null +++ b/fronted/scanner/service.go @@ -0,0 +1,316 @@ +package scanner + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "log/slog" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/getlantern/domainfront" + tls "github.com/refraction-networking/utls" +) + +// ServiceConfig configures a scanner Service. +type ServiceConfig struct { + Config *domainfront.Config + + CacheFile string + + RefreshInterval time.Duration // default 1h + CacheTTL time.Duration // default 6h, matches Samim's "time of day" observation + MinWorkingFronts int // re-scan when working count drops below this; default 3 + + KnownSample int + CloudFrontSample int + AkamaiSample int + + Probe ProbeOptions + + Resolver Resolver + Logger *slog.Logger +} + +// ProbeOptions is the Scanner's view of scanner.Options — every probe +// in the service uses the same dialer / TLS settings. +type ProbeOptions struct { + Dialer Dialer + RootCAs *x509.CertPool + ClientHelloID tls.ClientHelloID + DialTimeout time.Duration + Concurrency int +} + +func (c *ServiceConfig) defaults() { + if c.RefreshInterval <= 0 { + c.RefreshInterval = 1 * time.Hour + } + if c.CacheTTL <= 0 { + c.CacheTTL = 6 * time.Hour + } + if c.MinWorkingFronts <= 0 { + c.MinWorkingFronts = 3 + } + if c.Probe.DialTimeout <= 0 { + c.Probe.DialTimeout = 5 * time.Second + } + if c.Probe.Concurrency <= 0 { + c.Probe.Concurrency = 8 + } + if c.Logger == nil { + c.Logger = slog.Default() + } +} + +// Service maintains a per-client working-front list, refreshing it on a +// schedule and on demand. Pick returns the next-best working front; +// ReportFailure demotes a front so subsequent Picks skip it and the +// next refresh runs sooner. +type Service struct { + cfg ServiceConfig + + mu sync.Mutex + working []Result + pickIdx int + failures map[string]int + + refreshSignal chan struct{} + refreshing atomic.Bool + + stop chan struct{} + stopOnce sync.Once + done chan struct{} + started atomic.Bool +} + +// NewService loads the on-disk cache if present and returns a Service +// ready to start. Start kicks off the background refresh loop. +func NewService(cfg ServiceConfig) (*Service, error) { + if cfg.Config == nil { + return nil, errors.New("scanner: ServiceConfig.Config required") + } + cfg.defaults() + + s := &Service{ + cfg: cfg, + failures: make(map[string]int), + refreshSignal: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + } + + if cfg.CacheFile != "" { + cached, err := LoadCache(cfg.CacheFile, cfg.CacheTTL) + if err != nil { + cfg.Logger.Warn("scanner: cache load failed", slog.Any("error", err)) + } else if len(cached) > 0 { + s.working = cached + cfg.Logger.Info("scanner: cache loaded", slog.Int("count", len(cached))) + } + } + return s, nil +} + +// Start runs an initial refresh and the periodic loop. Returns when ctx +// is canceled or Close is called. Safe to call once. +func (s *Service) Start(ctx context.Context) { + s.started.Store(true) + defer close(s.done) + go s.refresh(ctx) + + t := time.NewTicker(s.cfg.RefreshInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-s.stop: + return + case <-t.C: + go s.refresh(ctx) + case <-s.refreshSignal: + go s.refresh(ctx) + } + } +} + +// Close stops the background loop. Idempotent. Safe to call before +// Start — in that case it just marks the Service stopped without +// waiting on a loop that was never running. +func (s *Service) Close() error { + s.stopOnce.Do(func() { close(s.stop) }) + if s.started.Load() { + <-s.done + } + return nil +} + +// Working returns a snapshot of the current working front list ordered +// by latency. +func (s *Service) Working() []Result { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]Result, len(s.working)) + copy(out, s.working) + return out +} + +// Pick returns the next working front in round-robin order so all +// fronts get traffic instead of every dial pinning to the lowest- +// latency one (which is what would happen with naive head-of-list). +// Returns false when the working list is empty; callers should then +// either wait for a refresh or trigger one via Refresh. +func (s *Service) Pick() (Result, bool) { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.working) == 0 { + return Result{}, false + } + r := s.working[s.pickIdx%len(s.working)] + s.pickIdx++ + return r, true +} + +// ReportFailure tells the Service a front returned by Pick subsequently +// stopped working. Tracking is per-(provider, IP, SNI); after two +// failures within a refresh cycle the front is removed from the +// working list. A scheduled refresh is signaled when the working list +// drops below MinWorkingFronts. +func (s *Service) ReportFailure(c Candidate) { + key := failureKey(c) + s.mu.Lock() + s.failures[key]++ + count := s.failures[key] + if count >= 2 { + s.removeLocked(c) + } + lowWater := len(s.working) < s.cfg.MinWorkingFronts + s.mu.Unlock() + + if lowWater { + s.signalRefresh() + } +} + +// Refresh triggers an out-of-band scan, returning immediately. The +// refresh runs on the Service's goroutine; the resulting working list +// is observable via Working / Pick after it completes. Multiple calls +// while a refresh is already in flight are coalesced. +func (s *Service) Refresh() { s.signalRefresh() } + +func (s *Service) signalRefresh() { + select { + case s.refreshSignal <- struct{}{}: + default: + } +} + +func (s *Service) refresh(ctx context.Context) { + if !s.refreshing.CompareAndSwap(false, true) { + return + } + defer s.refreshing.Store(false) + + cands, err := BuildPool(ctx, PoolOptions{ + Config: s.cfg.Config, + KnownSample: s.cfg.KnownSample, + CloudFrontSample: s.cfg.CloudFrontSample, + AkamaiSample: s.cfg.AkamaiSample, + Resolver: s.cfg.Resolver, + }) + if err != nil { + s.cfg.Logger.Warn("scanner: build pool failed", slog.Any("error", err)) + return + } + if len(cands) == 0 { + s.cfg.Logger.Warn("scanner: empty pool, skipping scan") + return + } + + s.cfg.Logger.Info("scanner: scanning", slog.Int("candidates", len(cands))) + start := time.Now() + + // Publish each working front as its probe completes so Working/Pick + // return early winners instead of blocking on the full scan. The + // first hit of this scan supersedes the previous cycle: a re-scan + // keeps serving the old list until the first new result lands, so + // it never drops to empty mid-scan. + firstHit := true + onResult := func(r Result) { + if !r.OK() { + return + } + s.mu.Lock() + if firstHit { + s.working = s.working[:0] + s.pickIdx = 0 + s.failures = make(map[string]int) + firstHit = false + } + s.insertSortedLocked(r) + s.mu.Unlock() + } + + results := Scan(ctx, cands, Options{ + Dialer: s.cfg.Probe.Dialer, + RootCAs: s.cfg.Probe.RootCAs, + ClientHelloID: s.cfg.Probe.ClientHelloID, + DialTimeout: s.cfg.Probe.DialTimeout, + Concurrency: s.cfg.Probe.Concurrency, + OnResult: onResult, + }) + working := RankWorking(results) + elapsed := time.Since(start) + s.cfg.Logger.Info("scanner: scan complete", + slog.Int("working", len(working)), + slog.Int("total", len(results)), + slog.Duration("elapsed", elapsed), + ) + + // If no probe succeeded, firstHit never fired and the stale list is + // still live, so assigning here clears it to the (empty) canonical + // result. + s.mu.Lock() + s.working = working + s.pickIdx = 0 + s.failures = make(map[string]int) + s.mu.Unlock() + + if s.cfg.CacheFile != "" { + if err := SaveCache(s.cfg.CacheFile, working); err != nil { + s.cfg.Logger.Warn("scanner: cache save failed", slog.Any("error", err)) + } + } +} + +func (s *Service) removeLocked(c Candidate) { + key := failureKey(c) + filtered := s.working[:0] + for _, r := range s.working { + if failureKey(r.Candidate) == key { + continue + } + filtered = append(filtered, r) + } + s.working = filtered +} + +// insertSortedLocked inserts r into s.working keeping ascending-latency +// order, matching RankWorking so incremental publishing and the final +// bulk assign yield the same ordering. +func (s *Service) insertSortedLocked(r Result) { + i := sort.Search(len(s.working), func(i int) bool { + return s.working[i].Latency > r.Latency + }) + s.working = append(s.working, Result{}) + copy(s.working[i+1:], s.working[i:]) + s.working[i] = r +} + +func failureKey(c Candidate) string { + return fmt.Sprintf("%s|%s|%s", c.Provider, c.IPAddress, c.SNI) +} diff --git a/fronted/scanner/service_test.go b/fronted/scanner/service_test.go new file mode 100644 index 00000000..ae37e85b --- /dev/null +++ b/fronted/scanner/service_test.go @@ -0,0 +1,237 @@ +package scanner + +import ( + "context" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/getlantern/domainfront" +) + +func TestService_PickRoundRobin(t *testing.T) { + s := newServiceWithWorking(t, []Result{ + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.1.1.1"}, Status: 200}, + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.1.1.2"}, Status: 200}, + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.1.1.3"}, Status: 200}, + }) + + seen := map[string]int{} + for i := 0; i < 6; i++ { + r, ok := s.Pick() + if !ok { + t.Fatalf("Pick #%d returned !ok", i) + } + seen[r.Candidate.IPAddress]++ + } + for ip, n := range seen { + if n != 2 { + t.Errorf("expected each IP picked twice, %s got %d", ip, n) + } + } +} + +func TestService_PickEmptyReturnsFalse(t *testing.T) { + s := newServiceWithWorking(t, nil) + _, ok := s.Pick() + if ok { + t.Errorf("Pick on empty list should return false") + } +} + +func TestService_InsertSortedLockedMaintainsLatencyOrder(t *testing.T) { + s := newServiceWithWorking(t, nil) + for _, d := range []time.Duration{50, 10, 30, 20, 40} { + s.insertSortedLocked(Result{ + Candidate: Candidate{IPAddress: d.String()}, + Latency: d * time.Millisecond, + Status: 200, + }) + } + got := s.Working() + for i := 1; i < len(got); i++ { + if got[i-1].Latency > got[i].Latency { + t.Errorf("not sorted at %d: %v > %v", i, got[i-1].Latency, got[i].Latency) + } + } + if len(got) != 5 { + t.Errorf("len = %d; want 5", len(got)) + } +} + +func TestScan_OnResultCalledPerCandidate(t *testing.T) { + // Unroutable TEST-NET-1 addresses fail the TCP dial fast, so every + // probe completes quickly; we only assert OnResult fires once each. + cands := []Candidate{ + {Provider: "akamai", IPAddress: "192.0.2.1", TestURL: "https://x/ping", InnerHost: "x"}, + {Provider: "akamai", IPAddress: "192.0.2.2", TestURL: "https://x/ping", InnerHost: "x"}, + {Provider: "akamai", IPAddress: "192.0.2.3", TestURL: "https://x/ping", InnerHost: "x"}, + } + var mu sync.Mutex + count := 0 + results := Scan(context.Background(), cands, Options{ + DialTimeout: 500 * time.Millisecond, + Concurrency: 3, + OnResult: func(Result) { + mu.Lock() + count++ + mu.Unlock() + }, + }) + if len(results) != 3 { + t.Fatalf("results len = %d; want 3", len(results)) + } + if count != 3 { + t.Errorf("OnResult called %d times; want 3 (once per candidate)", count) + } +} + +func TestService_ReportFailureRemovesAfterTwo(t *testing.T) { + bad := Candidate{Provider: "akamai", IPAddress: "1.1.1.1"} + good := Candidate{Provider: "akamai", IPAddress: "2.2.2.2"} + s := newServiceWithWorking(t, []Result{ + {Candidate: bad, Status: 200}, + {Candidate: good, Status: 200}, + }) + + s.ReportFailure(bad) + if len(s.Working()) != 2 { + t.Errorf("first failure should not remove; got working=%d", len(s.Working())) + } + s.ReportFailure(bad) + if len(s.Working()) != 1 { + t.Errorf("second failure should remove; got working=%d", len(s.Working())) + } + if s.Working()[0].Candidate.IPAddress != "2.2.2.2" { + t.Errorf("wrong candidate remained: %v", s.Working()[0].Candidate) + } +} + +func TestService_ReportFailureSignalsRefreshAtLowWater(t *testing.T) { + s := newServiceWithWorking(t, []Result{ + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.1.1.1"}, Status: 200}, + }) + s.cfg.MinWorkingFronts = 2 + + s.ReportFailure(Candidate{Provider: "akamai", IPAddress: "1.1.1.1"}) + s.ReportFailure(Candidate{Provider: "akamai", IPAddress: "1.1.1.1"}) + + select { + case <-s.refreshSignal: + default: + t.Errorf("expected refresh signal after working dropped below MinWorkingFronts") + } +} + +func TestService_LoadsFromCacheOnConstruct(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "scanner_cache.json") + SaveCache(cachePath, []Result{ + {Candidate: Candidate{Provider: "akamai", IPAddress: "1.2.3.4"}, Latency: 50 * time.Millisecond, Status: 200}, + }) + + s, err := NewService(ServiceConfig{ + Config: &domainfront.Config{}, + CacheFile: cachePath, + }) + if err != nil { + t.Fatalf("NewService: %v", err) + } + w := s.Working() + if len(w) != 1 { + t.Errorf("expected 1 loaded from cache, got %d", len(w)) + } +} + +func TestService_NoConfigIsError(t *testing.T) { + _, err := NewService(ServiceConfig{}) + if err == nil { + t.Errorf("expected error when Config is nil") + } +} + +func TestBuildPool_KnownOnly(t *testing.T) { + cfg := &domainfront.Config{ + Providers: map[string]*domainfront.Provider{ + "akamai": { + TestURL: "https://akamai.test/ping", + Masquerades: []*domainfront.Masquerade{ + {Domain: "a248.e.akamai.net", IpAddress: "1.1.1.1"}, + {Domain: "a248.e.akamai.net", IpAddress: "1.1.1.2"}, + {Domain: "a248.e.akamai.net", IpAddress: "1.1.1.3"}, + }, + }, + }, + } + got, err := BuildPool(context.Background(), PoolOptions{ + Config: cfg, + KnownSample: 2, + }) + if err != nil { + t.Fatalf("BuildPool: %v", err) + } + if len(got) != 2 { + t.Errorf("expected 2 sampled, got %d", len(got)) + } +} + +func TestBuildPool_CloudFrontRawRange(t *testing.T) { + cfg := &domainfront.Config{ + Providers: map[string]*domainfront.Provider{ + "cloudfront": { + TestURL: "https://cf.test/ping", + Masquerades: []*domainfront.Masquerade{ + {Domain: "aa1.awsstatic.com", IpAddress: "99.84.2.4"}, + }, + }, + }, + } + got, err := BuildPool(context.Background(), PoolOptions{ + Config: cfg, + CloudFrontSample: 5, + }) + if err != nil { + t.Fatalf("BuildPool: %v", err) + } + if len(got) != 5 { + t.Errorf("expected 5 raw-range samples (KnownSample=0 skips known), got %d", len(got)) + } +} + +func TestBuildPool_KnownOptedIn(t *testing.T) { + cfg := &domainfront.Config{ + Providers: map[string]*domainfront.Provider{ + "cloudfront": { + TestURL: "https://cf.test/ping", + Masquerades: []*domainfront.Masquerade{ + {Domain: "aa1.awsstatic.com", IpAddress: "99.84.2.4"}, + {Domain: "advertising.amazon.com", IpAddress: "3.164.130.9"}, + }, + }, + }, + } + got, err := BuildPool(context.Background(), PoolOptions{ + Config: cfg, + KnownSample: 10, + CloudFrontSample: 3, + }) + if err != nil { + t.Fatalf("BuildPool: %v", err) + } + if len(got) != 5 { + t.Errorf("expected 2 known + 3 raw = 5, got %d", len(got)) + } +} + +// --- helpers --- + +func newServiceWithWorking(t *testing.T, working []Result) *Service { + t.Helper() + s, err := NewService(ServiceConfig{Config: &domainfront.Config{}}) + if err != nil { + t.Fatalf("NewService: %v", err) + } + s.working = working + return s +} diff --git a/fronted/scanner/timing_test.go b/fronted/scanner/timing_test.go new file mode 100644 index 00000000..a6af0dd8 --- /dev/null +++ b/fronted/scanner/timing_test.go @@ -0,0 +1,149 @@ +package scanner + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/getlantern/domainfront" +) + +// TestLive_TimeToFirstWorking measures real-world scan latency and +// time-to-first-working-front against live CDN infrastructure. Opt-in +// (SCANNER_INTEGRATION=1) — exercises the production probe path +// end-to-end. +// +// Reported metrics: +// - time-to-first-working: how soon after the scan starts does any +// candidate complete OK (the most operationally relevant number — +// this is how long the user waits before the first front is usable) +// - p50/p90 working-result latency: the per-probe RTT distribution +// - total scan wall time: when does the last probe finish +// - hit rate per feeder +func TestLive_TimeToFirstWorking(t *testing.T) { + integrationGate(t) + + cfg, err := loadProductionConfig(t) + if err != nil { + t.Fatalf("loadProductionConfig: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + // Raw-range-primary defaults: no pre-resolved YAML IPs, just fresh + // per-scan IPs from AWS CloudFront prefixes + DNS-resolved Akamai + // edges. + pool, err := BuildPool(ctx, PoolOptions{ + Config: cfg, + CloudFrontSample: 30, + AkamaiSample: 3, + }) + if err != nil { + t.Fatalf("BuildPool: %v", err) + } + t.Logf("pool: %d candidates (30 CloudFront-raw + Akamai-DNS-resolved from 4 hostnames)", len(pool)) + + rootCAs, err := TrustedCAsPool(cfg) + if err != nil { + t.Fatalf("TrustedCAsPool: %v", err) + } + + start := time.Now() + var firstWorking int64 + results := scanWithFirstHookCB(ctx, pool, Options{ + RootCAs: rootCAs, + Concurrency: 8, + DialTimeout: 5 * time.Second, + }, func() { + atomic.CompareAndSwapInt64(&firstWorking, 0, int64(time.Since(start))) + }) + elapsed := time.Since(start) + working := RankWorking(results) + + t.Logf("scan complete: %d/%d working in %s", len(working), len(results), elapsed.Round(time.Millisecond)) + if firstWorking > 0 { + t.Logf("time to first working front: %s", time.Duration(firstWorking).Round(time.Millisecond)) + } + + byProvider := map[string]struct{ ok, total int }{} + for _, r := range results { + stats := byProvider[r.Candidate.Provider] + stats.total++ + if r.OK() { + stats.ok++ + } + byProvider[r.Candidate.Provider] = stats + } + for prov, s := range byProvider { + t.Logf(" %s: %d/%d working (%.0f%%)", prov, s.ok, s.total, 100*float64(s.ok)/float64(s.total)) + } + + if len(working) > 0 { + latencies := make([]time.Duration, len(working)) + for i, r := range working { + latencies[i] = r.Latency + } + sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] }) + p50 := latencies[len(latencies)/2] + p90 := latencies[(len(latencies)*9)/10] + t.Logf("working-result latency: p50=%s p90=%s min=%s", p50.Round(time.Millisecond), p90.Round(time.Millisecond), latencies[0].Round(time.Millisecond)) + } + + if len(working) == 0 { + t.Errorf("0 working fronts after full scan; expected at least 1 against live CDN") + } +} + +// scanWithFirstHookCB is Scan with a callback fired exactly once on the +// first OK result. Used to time how soon the user could start using +// the scanner output rather than waiting for the full scan to finish. +func scanWithFirstHookCB(ctx context.Context, candidates []Candidate, opts Options, onFirst func()) []Result { + opts.defaults() + results := make([]Result, len(candidates)) + if len(candidates) == 0 { + return results + } + sem := make(chan struct{}, opts.Concurrency) + var wg sync.WaitGroup + var once sync.Once + for i, c := range candidates { + wg.Add(1) + sem <- struct{}{} + go func(i int, c Candidate) { + defer wg.Done() + defer func() { <-sem }() + if err := ctx.Err(); err != nil { + results[i] = Result{Candidate: c, Err: err} + return + } + r := Probe(ctx, c, opts) + if r.OK() && onFirst != nil { + once.Do(onFirst) + } + results[i] = r + }(i, c) + } + wg.Wait() + return results +} + +// loadProductionConfig reads the embedded radiance fronted.yaml.gz +// (the same config the radiance client uses in production) so the +// timing test exercises a realistic pool. +func loadProductionConfig(t *testing.T) (*domainfront.Config, error) { + t.Helper() + _, thisFile, _, _ := runtime.Caller(0) + path := filepath.Join(filepath.Dir(thisFile), "..", "..", "kindling", "fronted", "fronted.yaml.gz") + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return domainfront.ParseConfig(raw) +} diff --git a/go.mod b/go.mod index 09000312..0474bb90 100644 --- a/go.mod +++ b/go.mod @@ -29,10 +29,10 @@ require ( github.com/getlantern/amp v0.0.0-20260305201851-782bc8045e58 github.com/getlantern/common v1.2.1-0.20260326210434-cb69537aaf46 github.com/getlantern/dnstt v0.0.0-20260112160750-05100563bd0d - github.com/getlantern/domainfront v0.0.0-20260419161617-0bff0b2169f4 + github.com/getlantern/domainfront v0.0.0-20260526192615-fdc839bc10ed github.com/getlantern/keepcurrent v0.0.0-20260422161259-54a4d9a93694 github.com/getlantern/kindling v0.0.0-20260516120759-a9712f95df03 - github.com/getlantern/lantern-box v0.0.82 + github.com/getlantern/lantern-box v0.0.83-0.20260524155143-c467035b6497 github.com/getlantern/pluriconfig v0.0.0-20251126214241-8cc8bc561535 github.com/getlantern/publicip v0.0.0-20260328175246-2c460fe80c6b github.com/getlantern/semconv v0.0.0-20260327040646-21845dda05cb @@ -44,6 +44,7 @@ require ( github.com/knadh/koanf/parsers/json v1.0.0 github.com/knadh/koanf/providers/rawbytes v1.0.0 github.com/knadh/koanf/v2 v2.3.0 + github.com/refraction-networking/utls v1.8.2 github.com/sagernet/sing v0.7.18 github.com/sagernet/sing-box v1.12.22 github.com/stretchr/testify v1.11.1 @@ -181,7 +182,6 @@ require ( github.com/prometheus-community/pro-bing v0.4.0 // indirect github.com/protolambda/ctxlock v0.1.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect - github.com/refraction-networking/utls v1.8.2 // indirect github.com/refraction-networking/water v0.7.1-alpha // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect diff --git a/go.sum b/go.sum index 738b3197..441d8b0e 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 h1:oEZYEpZo28Wd github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201/go.mod h1:Y9WZUHEb+mpra02CbQ/QczLUe6f0Dezxaw5DCJlJQGo= github.com/getlantern/dnstt v0.0.0-20260112160750-05100563bd0d h1:TrauJ2jdJqOAHyQB5wIL0kWN/dipqKagERE1I/TRVSY= github.com/getlantern/dnstt v0.0.0-20260112160750-05100563bd0d/go.mod h1:LA7cwZQtgXxBJdSJDj2ZgQNo/UY3Qa7nxNxzOuMMIyw= -github.com/getlantern/domainfront v0.0.0-20260419161617-0bff0b2169f4 h1:/Q9FJvKPyuXfH6tfA+C+t9/AbvGWs3Yp9iqI74FYvb4= -github.com/getlantern/domainfront v0.0.0-20260419161617-0bff0b2169f4/go.mod h1:nsdIvgenGUqPKnRFjkssbfxnV/WYWyC0c/t15qGym/A= +github.com/getlantern/domainfront v0.0.0-20260526192615-fdc839bc10ed h1:M7ND7KQ3JLEXo/wV4mogdb8BQRt4q3j7iq5sEakee4I= +github.com/getlantern/domainfront v0.0.0-20260526192615-fdc839bc10ed/go.mod h1:nsdIvgenGUqPKnRFjkssbfxnV/WYWyC0c/t15qGym/A= github.com/getlantern/errors v1.0.4 h1:i2iR1M9GKj4WuingpNqJ+XQEw6i6dnAgKAmLj6ZB3X0= github.com/getlantern/errors v1.0.4/go.mod h1:/Foq8jtSDGP8GOXzAjeslsC4Ar/3kB+UiQH+WyV4pzY= github.com/getlantern/golog v0.0.0-20230503153817-8e72de7e0a65 h1:NlQedYmPI3pRAXJb+hLVVDGqfvvXGRPV8vp7XOjKAZ0= @@ -248,8 +248,8 @@ github.com/getlantern/keepcurrent v0.0.0-20260422161259-54a4d9a93694 h1:iLWm6S/4 github.com/getlantern/keepcurrent v0.0.0-20260422161259-54a4d9a93694/go.mod h1:ag5g9aWUw2FJcX5RVRpJ9EBQBy5yJuy2WXDouIn/m4w= github.com/getlantern/kindling v0.0.0-20260516120759-a9712f95df03 h1:dUTN7mnTTBcSvsURNs1rTlyKrD1uXUEPqxEZDfl+hb4= github.com/getlantern/kindling v0.0.0-20260516120759-a9712f95df03/go.mod h1:TGTxpoNVwc8Be4qkBNtf5oj2psJaEIZEq47GOPS7zkA= -github.com/getlantern/lantern-box v0.0.82 h1:hCXqpCxLOQNxYtQZQDYVh3aj3t8NqSBqJjCn2mIBtK0= -github.com/getlantern/lantern-box v0.0.82/go.mod h1:wJhPQKdnwD6qW/ghAfzsrj/IfHZbvFSAfr52+Tu6dbw= +github.com/getlantern/lantern-box v0.0.83-0.20260524155143-c467035b6497 h1:yXtbk9i03UD7/S5NYoMjKqE+LfuzPs/t0S3SDTesr6Q= +github.com/getlantern/lantern-box v0.0.83-0.20260524155143-c467035b6497/go.mod h1:wJhPQKdnwD6qW/ghAfzsrj/IfHZbvFSAfr52+Tu6dbw= github.com/getlantern/lantern-water v0.0.0-20260317143726-e0ee64a11d90 h1:P9JX1yAu2uq3b5YiT0sLtHkTrkZuttV8gPZh81nUuag= github.com/getlantern/lantern-water v0.0.0-20260317143726-e0ee64a11d90/go.mod h1:3JpJgwi4KEI6rS9loOAvcBp+F2jP65d0tTg2GQcTPBU= github.com/getlantern/ops v0.0.0-20231025133620-f368ab734534 h1:3BwvWj0JZzFEvNNiMhCu4bf60nqcIuQpTYb00Ezm1ag= diff --git a/kindling/fronted/fronted.go b/kindling/fronted/fronted.go index 6eab3b9f..d88f0ea6 100644 --- a/kindling/fronted/fronted.go +++ b/kindling/fronted/fronted.go @@ -24,7 +24,7 @@ import ( const ( tracerName = "github.com/getlantern/radiance/kindling/fronted" - configURL = "https://raw.githubusercontent.com/getlantern/fronted/refs/heads/main/fronted.yaml.gz" + configURL = "https://raw.githubusercontent.com/getlantern/domainfront/refs/heads/main/fronted.yaml.gz" initialFetchTime = 30 * time.Second // configCacheName holds the last successfully fetched config so the next // startup can bootstrap when configURL is unreachable. @@ -147,3 +147,11 @@ func loadCachedConfig(path string, fetchErr error) (*domainfront.Config, error) slog.Warn("using embedded fronted config", "fetch_err", fetchErr) return cfg, nil } + +// LoadCachedConfig returns the *domainfront.Config from the on-disk +// cache under dataDir, falling back to the embedded copy. Skips the +// live fetch, for consumers that want the config without taking on a +// full domainfront.Client lifecycle. +func LoadCachedConfig(dataDir string) (*domainfront.Config, error) { + return loadCachedConfig(filepath.Join(dataDir, configCacheName), nil) +} diff --git a/kindling/fronted/fronted.yaml.gz b/kindling/fronted/fronted.yaml.gz index c59c5056..057098a3 100644 Binary files a/kindling/fronted/fronted.yaml.gz and b/kindling/fronted/fronted.yaml.gz differ diff --git a/kindling/iran/detect.go b/kindling/iran/detect.go new file mode 100644 index 00000000..934b8be7 --- /dev/null +++ b/kindling/iran/detect.go @@ -0,0 +1,34 @@ +// Package iran provides device-local detection of whether the user is +// currently in Iran. Designed to run without any network access since +// the Lantern API may be unreachable exactly when the heuristic is +// needed. Classification is imperfect; callers should expose a manual +// override. +package iran + +import "time" + +// IranMCC is the ITU-T E.212 Mobile Country Code for Iran. +const IranMCC = "432" + +// IranTZName is the IANA timezone identifier for Iran. +const IranTZName = "Asia/Tehran" + +// LikelyIran reports whether on-device signals suggest the user is in +// Iran. When mcc is non-empty it is authoritative and overrides +// tzName; when empty (WiFi-only, no signal, iOS 16+) the function +// falls back to tzName alone. +func LikelyIran(mcc, tzName string) bool { + if mcc != "" { + return mcc == IranMCC + } + return tzName == IranTZName +} + +// LocalTZName returns the process's system timezone in IANA name +// form, or "" when no zone could be determined. +func LocalTZName() string { + if time.Local == nil { + return "" + } + return time.Local.String() +} diff --git a/kindling/iran/detect_test.go b/kindling/iran/detect_test.go new file mode 100644 index 00000000..014162c6 --- /dev/null +++ b/kindling/iran/detect_test.go @@ -0,0 +1,87 @@ +package iran + +import "testing" + +func TestLikelyIran(t *testing.T) { + cases := []struct { + name string + mcc string + tzName string + want bool + }{ + { + name: "MCC=432 (Iranian network) alone", + mcc: "432", + tzName: "", + want: true, + }, + { + name: "MCC=432 overrides non-Tehran TZ (Iranian roaming in from US)", + mcc: "432", + tzName: "America/Los_Angeles", + want: true, + }, + { + name: "MCC=310 (US network) overrides Tehran TZ (diaspora user)", + mcc: "310", + tzName: IranTZName, + want: false, + }, + { + name: "MCC=262 (Germany) overrides Tehran TZ (Iranian student in Berlin)", + mcc: "262", + tzName: IranTZName, + want: false, + }, + { + name: "MCC=424 (UAE) overrides Tehran TZ", + mcc: "424", + tzName: IranTZName, + want: false, + }, + { + name: "no MCC, TZ=Tehran (iOS / WiFi-only in Iran)", + mcc: "", + tzName: IranTZName, + want: true, + }, + { + name: "no MCC, TZ=non-Tehran", + mcc: "", + tzName: "America/Los_Angeles", + want: false, + }, + { + name: "no MCC, TZ=UTC (containerized / default)", + mcc: "", + tzName: "UTC", + want: false, + }, + { + name: "no MCC, no TZ", + mcc: "", + tzName: "", + want: false, + }, + { + name: "empty-but-not-nil MCC treated as absent", + mcc: "", + tzName: IranTZName, + want: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := LikelyIran(tc.mcc, tc.tzName); got != tc.want { + t.Errorf("LikelyIran(%q, %q) = %v, want %v", + tc.mcc, tc.tzName, got, tc.want) + } + }) + } +} + +// LocalTZName depends on the test host's timezone, so we can only +// assert the contract: it returns a string and never panics. +func TestLocalTZName_NoPanic(t *testing.T) { + _ = LocalTZName() +} diff --git a/kindling/meek/provider.go b/kindling/meek/provider.go new file mode 100644 index 00000000..7ec5645b --- /dev/null +++ b/kindling/meek/provider.go @@ -0,0 +1,205 @@ +// Package meek wires radiance's fronted/scanner output into the +// sing-box meek outbound config shape. A Provider holds a scanner +// Service, samples its current working list, and produces FrontSpec +// JSON entries suitable for inclusion in a sing-box meek outbound +// configuration. +package meek + +import ( + "context" + "errors" + "log/slog" + "net" + "time" + + "github.com/getlantern/domainfront" + sbo "github.com/sagernet/sing-box/option" + + "github.com/getlantern/radiance/bypass" + "github.com/getlantern/radiance/fronted/scanner" +) + +var errNilConfig = errors.New("meek: ProviderConfig.Config is nil") + +// MeekOutboundOptions mirrors lantern-box/option.MeekOutboundOptions +// kept local to avoid version-coupling radiance to lantern-box's release +// cadence. The JSON tags are identical, so once lantern-box ships meek +// and radiance bumps the dep, drop this copy and import the upstream +// type directly. +type MeekOutboundOptions struct { + sbo.DialerOptions + + URL string `json:"url"` + Fronts []FrontSpec `json:"fronts"` + Header map[string]string `json:"header,omitempty"` + + PollIntervalMs int `json:"poll_interval_ms,omitempty"` + MaxBodyBytes int `json:"max_body_bytes,omitempty"` + SessionIDLen int `json:"session_id_len,omitempty"` + ConnectTimeout string `json:"connect_timeout,omitempty"` + ReadTimeout string `json:"read_timeout,omitempty"` +} + +// MeekOutboundType is the sing-box outbound type string. Matches +// lantern-box/constant.TypeMeek. Stringified so radiance doesn't have +// to import a version of lantern-box that registers meek. +const MeekOutboundType = "meek" + +// DefaultURL is the inner Host header sent through the meek tunnel. +// It is never resolved or dialed; callers supply the real SNI and dial +// target via FrontSpec. +const DefaultURL = "https://meek.dsa.akamai.getiantem.org/" + +// BuildOutbound returns a sing-box outbound for the meek transport with +// the given tag, meek-server URL, and front pool. The returned Outbound +// can be appended directly to O.Options.Outbounds; selector groups can +// reference it by tag. +// +// Returns ok=false when fronts is empty — without at least one front, +// the meek outbound has nothing to dial and would fail at first use. +// Callers should skip injection in that case. +func BuildOutbound(tag, url string, fronts []FrontSpec) (sbo.Outbound, bool) { + if len(fronts) == 0 { + return sbo.Outbound{}, false + } + return sbo.Outbound{ + Type: MeekOutboundType, + Tag: tag, + Options: &MeekOutboundOptions{ + URL: url, + Fronts: fronts, + }, + }, true +} + +// FrontSpec mirrors lantern-box/option.FrontSpec; kept local to avoid +// version-coupling radiance's release cadence to lantern-box's. +type FrontSpec struct { + IPAddress string `json:"ip_address"` + SNI string `json:"sni,omitempty"` + VerifyHostname string `json:"verify_hostname,omitempty"` +} + +// ProviderConfig configures the bridge between scanner and meek +// outbound. Defaults are tuned for IR usage: a 1h refresh interval is +// short enough to react to CDN block churn, a 6h cache TTL means a +// reboot loads the recent working list rather than re-scanning cold. +// +// Discovery is raw-range-primary by default: fresh IPs from the AWS +// CloudFront prefix list and DNS-resolved Akamai edges produce +// per-(ISP, location, time-of-day) candidates rather than the same +// baked list every user sees. fronted.yaml.gz is consulted only for +// outer-SNI pools, trusted CAs, and host-alias mappings (not its +// pre-resolved IPs) unless KnownSample > 0. +type ProviderConfig struct { + Config *domainfront.Config + CacheFile string + + RefreshInterval time.Duration + CacheTTL time.Duration + KnownSample int + CloudFrontSample int + AkamaiSample int + + Logger *slog.Logger +} + +func (c *ProviderConfig) defaults() { + if c.CloudFrontSample == 0 { + c.CloudFrontSample = 30 + } + if c.AkamaiSample == 0 { + c.AkamaiSample = 3 + } + if c.Logger == nil { + c.Logger = slog.Default() + } +} + +// Provider runs a scanner Service over the supplied domainfront config +// and exposes the working-front list as []FrontSpec for the meek +// outbound. +type Provider struct { + service *scanner.Service +} + +// NewProvider constructs a Provider. The scanner is configured to dial +// through radiance/bypass so its probes don't loop through the active +// VPN TUN; cert validation uses the trusted-CA pool from cfg. Call +// Start to begin background scanning. +func NewProvider(cfg ProviderConfig) (*Provider, error) { + if cfg.Config == nil { + return nil, errNilConfig + } + cfg.defaults() + + rootCAs, err := scanner.TrustedCAsPool(cfg.Config) + if err != nil { + return nil, err + } + + svc, err := scanner.NewService(scanner.ServiceConfig{ + Config: cfg.Config, + CacheFile: cfg.CacheFile, + RefreshInterval: cfg.RefreshInterval, + CacheTTL: cfg.CacheTTL, + KnownSample: cfg.KnownSample, + CloudFrontSample: cfg.CloudFrontSample, + AkamaiSample: cfg.AkamaiSample, + Probe: scanner.ProbeOptions{ + Dialer: bypassDialer{}, + RootCAs: rootCAs, + }, + Logger: cfg.Logger, + }) + if err != nil { + return nil, err + } + return &Provider{service: svc}, nil +} + +// Start kicks off the background refresh loop and returns immediately. +// The loop runs until ctx is canceled or Close is called. +func (p *Provider) Start(ctx context.Context) { + go p.service.Start(ctx) +} + +// Close stops the background loop. Idempotent. +func (p *Provider) Close() error { return p.service.Close() } + +// FrontSpecs returns up to n working fronts in the meek-outbound JSON +// shape. n <= 0 returns all. The list is ordered by ascending latency. +func (p *Provider) FrontSpecs(n int) []FrontSpec { + return resultsToFrontSpecs(p.service.Working(), n) +} + +func resultsToFrontSpecs(working []scanner.Result, n int) []FrontSpec { + if n > 0 && n < len(working) { + working = working[:n] + } + out := make([]FrontSpec, 0, len(working)) + for _, r := range working { + out = append(out, FrontSpec{ + IPAddress: r.Candidate.IPAddress, + SNI: r.Candidate.SNI, + VerifyHostname: r.Candidate.VerifyHostname, + }) + } + return out +} + +// ReportFailure passes a meek dial failure back to the scanner so the +// underlying front gets dropped after enough failures and the next +// refresh runs sooner. spec.IPAddress is the load-bearing key. +func (p *Provider) ReportFailure(spec FrontSpec) { + p.service.ReportFailure(scanner.Candidate{ + IPAddress: spec.IPAddress, + SNI: spec.SNI, + }) +} + +type bypassDialer struct{} + +func (bypassDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return bypass.DialContext(ctx, network, addr) +} diff --git a/kindling/meek/provider_test.go b/kindling/meek/provider_test.go new file mode 100644 index 00000000..4d01b597 --- /dev/null +++ b/kindling/meek/provider_test.go @@ -0,0 +1,95 @@ +package meek + +import ( + "testing" + "time" + + "github.com/getlantern/domainfront" + "github.com/getlantern/radiance/fronted/scanner" +) + +func TestNewProvider_NilConfigErrors(t *testing.T) { + _, err := NewProvider(ProviderConfig{}) + if err == nil { + t.Errorf("expected error for nil Config") + } +} + +func TestNewProvider_OK(t *testing.T) { + p, err := NewProvider(ProviderConfig{Config: &domainfront.Config{}}) + if err != nil { + t.Fatalf("NewProvider: %v", err) + } + t.Cleanup(func() { _ = p.Close() }) + if got := p.FrontSpecs(5); got == nil { + t.Errorf("FrontSpecs returned nil; want empty slice") + } +} + +func TestResultsToFrontSpecs_PreservesOrderAndShape(t *testing.T) { + working := []scanner.Result{ + {Candidate: scanner.Candidate{Provider: "akamai", IPAddress: "23.47.48.230", VerifyHostname: "a248.e.akamai.net"}, Latency: 50 * time.Millisecond, Status: 200}, + {Candidate: scanner.Candidate{Provider: "cloudfront", IPAddress: "99.84.2.4", SNI: "aa1.awsstatic.com", VerifyHostname: "aa1.awsstatic.com"}, Latency: 110 * time.Millisecond, Status: 200}, + } + got := resultsToFrontSpecs(working, 0) + if len(got) != 2 { + t.Fatalf("len = %d; want 2", len(got)) + } + if got[0].IPAddress != "23.47.48.230" || got[0].SNI != "" || got[0].VerifyHostname != "a248.e.akamai.net" { + t.Errorf("akamai mapping wrong: %+v", got[0]) + } + if got[1].IPAddress != "99.84.2.4" || got[1].SNI != "aa1.awsstatic.com" { + t.Errorf("cloudfront mapping wrong: %+v", got[1]) + } +} + +func TestBuildOutbound_EmptyFrontsReturnsFalse(t *testing.T) { + _, ok := BuildOutbound("meek", "https://meek.example/meek", nil) + if ok { + t.Errorf("expected ok=false when fronts is empty") + } +} + +func TestBuildOutbound_ShapesOutbound(t *testing.T) { + fronts := []FrontSpec{ + {IPAddress: "1.2.3.4", VerifyHostname: "a248.e.akamai.net"}, + {IPAddress: "99.84.2.4", SNI: "aa1.awsstatic.com", VerifyHostname: "aa1.awsstatic.com"}, + } + out, ok := BuildOutbound("meek-fronted", "https://meek.example/", fronts) + if !ok { + t.Fatalf("BuildOutbound returned ok=false") + } + if out.Type != "meek" { + t.Errorf("Type = %q; want meek", out.Type) + } + if out.Tag != "meek-fronted" { + t.Errorf("Tag = %q", out.Tag) + } + mo, isMeek := out.Options.(*MeekOutboundOptions) + if !isMeek { + t.Fatalf("Options is %T; want *MeekOutboundOptions", out.Options) + } + if mo.URL != "https://meek.example/" { + t.Errorf("URL = %q", mo.URL) + } + if len(mo.Fronts) != 2 { + t.Errorf("Fronts length = %d; want 2", len(mo.Fronts)) + } +} + +func TestResultsToFrontSpecs_LimitsToN(t *testing.T) { + working := []scanner.Result{ + {Candidate: scanner.Candidate{IPAddress: "1.1.1.1"}, Status: 200}, + {Candidate: scanner.Candidate{IPAddress: "2.2.2.2"}, Status: 200}, + {Candidate: scanner.Candidate{IPAddress: "3.3.3.3"}, Status: 200}, + } + if got := resultsToFrontSpecs(working, 0); len(got) != 3 { + t.Errorf("n=0 should return all; got %d", len(got)) + } + if got := resultsToFrontSpecs(working, 2); len(got) != 2 { + t.Errorf("n=2 should return 2; got %d", len(got)) + } + if got := resultsToFrontSpecs(working, 10); len(got) != 3 { + t.Errorf("n>len should return all; got %d", len(got)) + } +} diff --git a/vpn/boxoptions.go b/vpn/boxoptions.go index cda33b58..d5806c63 100644 --- a/vpn/boxoptions.go +++ b/vpn/boxoptions.go @@ -83,6 +83,12 @@ type BoxOptions struct { // prior latency results survive across tunnel close/open. Keyed by // outbound/endpoint tag. URLTestSeed map[string]adapter.URLTestHistory `json:"-"` + // MeekOutbound is an optional client-built outbound (typically the + // domain-fronted meek transport). When non-nil, buildOptions appends + // it to Outbounds and includes its Tag in the selector groups so it + // participates in auto/manual routing alongside API-supplied ones. + // Nil = no-op; safe to leave unset. + MeekOutbound *O.Outbound `json:"-"` } // baseOpts returns the minimum sing-box options required for the tunnel to @@ -341,6 +347,11 @@ func buildOptions(bOptions BoxOptions) (O.Options, error) { } tags := mergeAndCollectTags(&opts, &bOptions.Options) + if bOptions.MeekOutbound != nil && bOptions.MeekOutbound.Tag != "" { + opts.Outbounds = append(opts.Outbounds, *bOptions.MeekOutbound) + tags = append(tags, bOptions.MeekOutbound.Tag) + slog.Info("Injected meek outbound", slog.String("tag", bOptions.MeekOutbound.Tag)) + } initial := bOptions.InitialServer if initial == "" || initial == AutoSelectTag { opts.Experimental.ClashAPI.DefaultMode = AutoSelectTag diff --git a/vpn/boxoptions_test.go b/vpn/boxoptions_test.go index 6b4ef59c..e2b212ef 100644 --- a/vpn/boxoptions_test.go +++ b/vpn/boxoptions_test.go @@ -218,6 +218,56 @@ func testConfig(t *testing.T) config.Config { return cfg } +func TestBuildOptions_MeekInjection(t *testing.T) { + options, _ := testBoxOptions(t) + + meek := &O.Outbound{ + Type: "meek", + Tag: "meek-fronted", + } + opts, err := buildOptions(BoxOptions{ + BasePath: t.TempDir(), + Options: options, + MeekOutbound: meek, + }) + require.NoError(t, err) + + var foundMeek bool + for _, o := range opts.Outbounds { + if o.Tag == "meek-fronted" && o.Type == "meek" { + foundMeek = true + break + } + } + assert.True(t, foundMeek, "meek outbound should be present in Outbounds") + + var autoSelector *O.Outbound + for i, o := range opts.Outbounds { + if o.Tag == AutoSelectTag { + autoSelector = &opts.Outbounds[i] + break + } + } + require.NotNil(t, autoSelector, "auto selector should exist") + autoOpts, ok := autoSelector.Options.(*lbO.MutableURLTestOutboundOptions) + require.True(t, ok, "auto selector Options should be MutableURLTestOutboundOptions") + assert.Contains(t, autoOpts.Outbounds, "meek-fronted", "auto selector should reference meek tag") +} + +func TestBuildOptions_MeekOmittedWhenNil(t *testing.T) { + options, _ := testBoxOptions(t) + + opts, err := buildOptions(BoxOptions{ + BasePath: t.TempDir(), + Options: options, + }) + require.NoError(t, err) + + for _, o := range opts.Outbounds { + assert.NotEqual(t, "meek", o.Type, "no meek outbound should be present when MeekOutbound is nil") + } +} + func testBoxOptions(t *testing.T) (O.Options, []string) { cfg := testConfig(t) var tags []string