Web Source

go.mod

@@ -138,10 +138,15 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/go-crypto v1.1.5 // indirect
+	github.com/PuerkitoBio/goquery v1.11.0 // indirect
 	github.com/STARRY-S/zip v0.2.1 // indirect
 	github.com/alecthomas/chroma/v2 v2.14.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/antchfx/htmlquery v1.3.5 // indirect
+	github.com/antchfx/xmlquery v1.5.0 // indirect
+	github.com/antchfx/xpath v1.3.5 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect
@@ -158,6 +163,7 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.24.4 // indirect
 	github.com/bodgit/plumbing v1.3.0 // indirect
 	github.com/bodgit/sevenzip v1.6.0 // indirect
 	github.com/bodgit/windows v1.0.1 // indirect
@@ -204,12 +210,14 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/gocolly/colly/v2 v2.3.0 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/go-github/v72 v72.0.0 // indirect
 	github.com/google/go-querystring v1.2.0 // indirect
@@ -226,6 +234,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jpillora/s3 v1.1.4 // indirect
+	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
@@ -256,6 +265,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/nlnwa/whatwg-url v0.6.2 // indirect
 	github.com/nwaples/rardecode/v2 v2.2.1 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -270,6 +280,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect
+	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/segmentio/asm v1.2.1 // indirect
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
@@ -280,6 +291,7 @@ require (
 	github.com/sorairolake/lzip-go v0.3.5 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
 	github.com/tetratelabs/wazero v1.9.0 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -314,6 +326,7 @@ require (
 	golang.org/x/mod v0.30.0 // indirect
 	golang.org/x/sys v0.39.0 // indirect
 	golang.org/x/term v0.38.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect

main.go

@@ -276,6 +276,15 @@ var (
 	jsonEnumeratorScan  = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
 	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
 
+	webScan         = cli.Command("web", "Scan websites for leaked credentials.")
+	webUrls         = webScan.Flag("url", "URL to scan. Repeat the flag for multiple targets, e.g. --url https://a.com --url https://b.com. Supports http:// and https://.").Required().Strings()
+	webCrawl        = webScan.Flag("crawl", "Follow links found on each page. Without this flag only the seed URL(s) are scanned.").Default("false").Bool()
+	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when --crawl is enabled. 1 = seed; 2 = one level deeper; 0 = unlimited.").Default("1").Int()
+	webDelay        = webScan.Flag("delay", "Seconds to wait between requests to the same domain. Increase this to reduce load on the target server.").Default("1").Int()
+	webTimeout      = webScan.Flag("timeout", "Seconds to spend crawling URLs before aborting. Total time shared across all URLs when multiple --url flags are given.").Default("30").Int()
+	webUserAgent    = webScan.Flag("user-agent", "User-Agent header to send with each request. Defaults to a TruffleHog identifier if not set.").String()
+	webIgnoreRobots = webScan.Flag("ignore-robots", "Ignore robots.txt restrictions. Only use this if you have explicit permission to crawl the target site.").Default("false").Bool()
+
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
 )
@@ -1156,6 +1165,26 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		} else {
 			refs = []sources.JobProgressRef{ref}
 		}
+	case webScan.FullCommand():
+		if len(*webUrls) == 0 {
+			return scanMetrics, fmt.Errorf("invalid config: you must specify at least one url")
+		}
+
+		cfg := sources.WebConfig{
+			URLs:         *webUrls,
+			Crawl:        *webCrawl,
+			Depth:        *webDepth,
+			Delay:        *webDelay,
+			Timeout:      *webTimeout,
+			UserAgent:    *webUserAgent,
+			IgnoreRobots: *webIgnoreRobots,
+		}
+
+		if ref, err := eng.ScanWeb(ctx, cfg); err != nil {
+			return scanMetrics, fmt.Errorf("failed to scan web: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
+		}
 	default:
 		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
 	}

pkg/engine/web.go

@@ -0,0 +1,42 @@
+package engine
+
+import (
+	"runtime"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/web"
+)
+
+// ScanWeb scans a given web connection.
+func (e *Engine) ScanWeb(ctx context.Context, c sources.WebConfig) (sources.JobProgressRef, error) {
+	connection := &sourcespb.Web{
+		Urls:         c.URLs,
+		Crawl:        c.Crawl,
+		Depth:        int64(c.Depth),
+		Delay:        int64(c.Delay),
+		Timeout:      int64(c.Timeout),
+		UserAgent:    c.UserAgent,
+		IgnoreRobots: c.IgnoreRobots,
+	}
+
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		ctx.Logger().Error(err, "failed to marshal web connection")
+		return sources.JobProgressRef{}, err
+	}
+
+	sourceName := "trufflehog - web"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, web.SourceType)
+
+	webSource := &web.Source{}
+	if err := webSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+		return sources.JobProgressRef{}, err
+	}
+	return e.sourceManager.EnumerateAndScan(ctx, sourceName, webSource)
+}