diff --git a/go.mod b/go.mod index e075fea6105e..f063588c7388 100644 --- a/go.mod +++ b/go.mod @@ -138,10 +138,15 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.5 // indirect + github.com/PuerkitoBio/goquery v1.11.0 // indirect github.com/STARRY-S/zip v0.2.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/brotli v1.1.1 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/antchfx/htmlquery v1.3.5 // indirect + github.com/antchfx/xmlquery v1.5.0 // indirect + github.com/antchfx/xpath v1.3.5 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect @@ -158,6 +163,7 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.24.4 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.0 // indirect github.com/bodgit/windows v1.0.1 // indirect @@ -204,12 +210,14 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/goccy/go-json v0.10.3 // indirect + github.com/gocolly/colly/v2 v2.3.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/go-github/v72 v72.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect @@ -226,6 +234,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jpillora/s3 v1.1.4 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect github.com/klauspost/compress v1.18.0 // indirect @@ -256,6 +265,7 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nlnwa/whatwg-url v0.6.2 // indirect github.com/nwaples/rardecode/v2 v2.2.1 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -270,6 +280,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/shirou/gopsutil/v3 v3.23.12 // indirect @@ -280,6 +291,7 @@ require ( github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -314,6 +326,7 @@ require ( golang.org/x/mod v0.30.0 // indirect golang.org/x/sys v0.39.0 // indirect golang.org/x/term v0.38.0 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect diff --git a/go.sum b/go.sum index 540dbc6bacb3..9c6307c7e797 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProtonMail/go-crypto v1.1.5 h1:eoAQfK2dwL+tFSFpr7TbOaPNUbPiJj4fLYwwGE1FQO4= github.com/ProtonMail/go-crypto v1.1.5/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= +github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/TheZeroSlave/zapsentry v1.23.0 h1:TKyzfEL7LRlRr+7AvkukVLZ+jZPC++ebCUv7ZJHl1AU= @@ -98,8 +100,16 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0= +github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA= +github.com/antchfx/xmlquery v1.5.0 h1:uAi+mO40ZWfyU6mlUBxRVvL6uBNZ6LMU4M3+mQIBV4c= +github.com/antchfx/xmlquery v1.5.0/go.mod h1:lJfWRXzYMK1ss32zm1GQV3gMIW/HFey3xDZmkP1SuNc= +github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ= +github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -159,6 +169,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0= github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ= +github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoGXLwLQOZE= +github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= @@ -348,6 +361,10 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= +github.com/gocolly/colly/v2 v2.3.0 h1:HSFh0ckbgVd2CSGRE+Y/iA4goUhGROJwyQDCMXGFBWM= +github.com/gocolly/colly/v2 v2.3.0/go.mod h1:Qp54s/kQbwCQvFVx8KzKCSTXVJ1wWT4QeAKEu33x1q8= github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -366,6 +383,8 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -380,6 +399,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= @@ -478,6 +499,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -587,6 +610,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nlnwa/whatwg-url v0.6.2 h1:jU61lU2ig4LANydbEJmA2nPrtCGiKdtgT0rmMd2VZ/Q= +github.com/nlnwa/whatwg-url v0.6.2/go.mod h1:x0FPXJzzOEieQtsBT/AKvbiBbQ46YlL6Xa7m02M1ECk= github.com/nwaples/rardecode/v2 v2.2.1 h1:DgHK/O/fkTQEKBJxBMC5d9IU8IgauifbpG78+rZJMnI= github.com/nwaples/rardecode/v2 v2.2.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -653,6 +678,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f h1:MvTmaQdww/z0Q4wrYjDSCcZ78NoftLQyHBSLW/Cx79Y= github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg= github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI= github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= @@ -709,6 +736,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0 h1:BBwJUs9xBpt1uOfO+yAr2pYW75MsyzuO/o70HTPnhe4= @@ -833,6 +862,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -864,6 +898,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -886,7 +924,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -904,6 +949,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -946,12 +996,24 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -963,6 +1025,11 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -998,6 +1065,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1019,6 +1089,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1056,6 +1128,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index d01a5fb50db9..a7f1e33105eb 100644 --- a/main.go +++ b/main.go @@ -276,6 +276,15 @@ var ( jsonEnumeratorScan = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.") jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings() + webScan = cli.Command("web", "Scan websites for leaked credentials.") + webUrls = webScan.Flag("url", "URL to scan. Repeat the flag for multiple targets, e.g. --url https://a.com --url https://b.com. Supports http:// and https://.").Required().Strings() + webCrawl = webScan.Flag("crawl", "Follow links found on each page. Without this flag only the seed URL(s) are scanned.").Default("false").Bool() + webDepth = webScan.Flag("depth", "Maximum link depth to follow when --crawl is enabled. 1 = seed; 2 = one level deeper; 0 = unlimited.").Default("1").Int() + webDelay = webScan.Flag("delay", "Seconds to wait between requests to the same domain. Increase this to reduce load on the target server.").Default("1").Int() + webTimeout = webScan.Flag("timeout", "Seconds to spend crawling URLs before aborting. Total time shared across all URLs when multiple --url flags are given.").Default("30").Int() + webUserAgent = webScan.Flag("user-agent", "User-Agent header to send with each request. Defaults to a TruffleHog identifier if not set.").String() + webIgnoreRobots = webScan.Flag("ignore-robots", "Ignore robots.txt restrictions. Only use this if you have explicit permission to crawl the target site.").Default("false").Bool() + analyzeCmd = analyzer.Command(cli) usingTUI = false ) @@ -1156,6 +1165,26 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, } else { refs = []sources.JobProgressRef{ref} } + case webScan.FullCommand(): + if len(*webUrls) == 0 { + return scanMetrics, fmt.Errorf("invalid config: you must specify at least one url") + } + + cfg := sources.WebConfig{ + URLs: *webUrls, + Crawl: *webCrawl, + Depth: *webDepth, + Delay: *webDelay, + Timeout: *webTimeout, + UserAgent: *webUserAgent, + IgnoreRobots: *webIgnoreRobots, + } + + if ref, err := eng.ScanWeb(ctx, cfg); err != nil { + return scanMetrics, fmt.Errorf("failed to scan web: %v", err) + } else { + refs = []sources.JobProgressRef{ref} + } default: return scanMetrics, fmt.Errorf("invalid command: %s", cmd) } diff --git a/pkg/engine/web.go b/pkg/engine/web.go new file mode 100644 index 000000000000..310633e45586 --- /dev/null +++ b/pkg/engine/web.go @@ -0,0 +1,42 @@ +package engine + +import ( + "runtime" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/web" +) + +// ScanWeb scans a given web connection. +func (e *Engine) ScanWeb(ctx context.Context, c sources.WebConfig) (sources.JobProgressRef, error) { + connection := &sourcespb.Web{ + Urls: c.URLs, + Crawl: c.Crawl, + Depth: int64(c.Depth), + Delay: int64(c.Delay), + Timeout: int64(c.Timeout), + UserAgent: c.UserAgent, + IgnoreRobots: c.IgnoreRobots, + } + + var conn anypb.Any + err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) + if err != nil { + ctx.Logger().Error(err, "failed to marshal web connection") + return sources.JobProgressRef{}, err + } + + sourceName := "trufflehog - web" + sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, web.SourceType) + + webSource := &web.Source{} + if err := webSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { + return sources.JobProgressRef{}, err + } + return e.sourceManager.EnumerateAndScan(ctx, sourceName, webSource) +} diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.go b/pkg/pb/source_metadatapb/source_metadata.pb.go index a457dee1e500..b7f39588de71 100644 --- a/pkg/pb/source_metadatapb/source_metadata.pb.go +++ b/pkg/pb/source_metadatapb/source_metadata.pb.go @@ -3582,6 +3582,85 @@ func (x *JSONEnumerator) GetMetadata() string { return "" } +type Web struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + PageTitle string `protobuf:"bytes,2,opt,name=page_title,json=pageTitle,proto3" json:"page_title,omitempty"` + ContentType string `protobuf:"bytes,3,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"` + Depth int64 `protobuf:"varint,4,opt,name=depth,proto3" json:"depth,omitempty"` + Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"` +} + +func (x *Web) Reset() { + *x = Web{} + if protoimpl.UnsafeEnabled { + mi := &file_source_metadata_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Web) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Web) ProtoMessage() {} + +func (x *Web) ProtoReflect() protoreflect.Message { + mi := &file_source_metadata_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Web.ProtoReflect.Descriptor instead. +func (*Web) Descriptor() ([]byte, []int) { + return file_source_metadata_proto_rawDescGZIP(), []int{37} +} + +func (x *Web) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *Web) GetPageTitle() string { + if x != nil { + return x.PageTitle + } + return "" +} + +func (x *Web) GetContentType() string { + if x != nil { + return x.ContentType + } + return "" +} + +func (x *Web) GetDepth() int64 { + if x != nil { + return x.Depth + } + return 0 +} + +func (x *Web) GetTimestamp() string { + if x != nil { + return x.Timestamp + } + return "" +} + type MetaData struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3625,13 +3704,14 @@ type MetaData struct { // *MetaData_Stdin // *MetaData_SlackContinuous // *MetaData_JsonEnumerator + // *MetaData_Web Data isMetaData_Data `protobuf_oneof:"data"` } func (x *MetaData) Reset() { *x = MetaData{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[37] + mi := &file_source_metadata_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3644,7 +3724,7 @@ func (x *MetaData) String() string { func (*MetaData) ProtoMessage() {} func (x *MetaData) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[37] + mi := &file_source_metadata_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3657,7 +3737,7 @@ func (x *MetaData) ProtoReflect() protoreflect.Message { // Deprecated: Use MetaData.ProtoReflect.Descriptor instead. func (*MetaData) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{37} + return file_source_metadata_proto_rawDescGZIP(), []int{38} } func (m *MetaData) GetData() isMetaData_Data { @@ -3919,6 +3999,13 @@ func (x *MetaData) GetJsonEnumerator() *JSONEnumerator { return nil } +func (x *MetaData) GetWeb() *Web { + if x, ok := x.GetData().(*MetaData_Web); ok { + return x.Web + } + return nil +} + type isMetaData_Data interface { isMetaData_Data() } @@ -4067,6 +4154,10 @@ type MetaData_JsonEnumerator struct { JsonEnumerator *JSONEnumerator `protobuf:"bytes,36,opt,name=jsonEnumerator,proto3,oneof"` } +type MetaData_Web struct { + Web *Web `protobuf:"bytes,37,opt,name=web,proto3,oneof"` +} + func (*MetaData_Azure) isMetaData_Data() {} func (*MetaData_Bitbucket) isMetaData_Data() {} @@ -4139,6 +4230,8 @@ func (*MetaData_SlackContinuous) isMetaData_Data() {} func (*MetaData_JsonEnumerator) isMetaData_Data() {} +func (*MetaData_Web) isMetaData_Data() {} + var File_source_metadata_proto protoreflect.FileDescriptor var file_source_metadata_proto_rawDesc = []byte{ @@ -4597,7 +4690,16 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x28, 0x09, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x22, 0x2c, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, 0x0f, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x8d, 0x01, + 0x0a, 0x03, 0x57, 0x65, 0x62, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, + 0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x67, + 0x65, 0x54, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x70, + 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x12, + 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xe9, 0x0f, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, @@ -4721,45 +4823,47 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x0e, 0x6a, 0x73, 0x6f, 0x6e, 0x45, 0x6e, 0x75, - 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, - 0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, - 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, - 0x76, 0x61, 0x74, 0x65, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, - 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x2a, - 0xc2, 0x03, 0x0a, 0x13, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, - 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x50, 0x41, - 0x52, 0x41, 0x4d, 0x45, 0x54, 0x45, 0x52, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x51, - 0x55, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, - 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x03, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, - 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x44, 0x41, - 0x54, 0x41, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, - 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x52, 0x41, 0x57, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, - 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x55, 0x52, 0x4c, 0x5f, 0x45, - 0x4e, 0x43, 0x4f, 0x44, 0x45, 0x44, 0x10, 0x06, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, - 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x47, 0x52, 0x41, 0x50, 0x48, 0x51, 0x4c, - 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x43, - 0x52, 0x49, 0x50, 0x54, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, - 0x54, 0x5f, 0x55, 0x52, 0x4c, 0x10, 0x09, 0x12, 0x18, 0x0a, 0x14, 0x45, 0x4e, 0x56, 0x49, 0x52, - 0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, - 0x0a, 0x12, 0x18, 0x0a, 0x14, 0x46, 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x41, 0x55, 0x54, 0x48, - 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x46, - 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0c, 0x12, 0x15, - 0x0a, 0x11, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x43, 0x52, - 0x49, 0x50, 0x54, 0x10, 0x0d, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, 0x1c, - 0x0a, 0x18, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, - 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d, - 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x10, 0x10, 0x12, - 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, - 0x45, 0x52, 0x10, 0x11, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x77, 0x65, 0x62, 0x18, 0x25, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x65, 0x62, 0x48, 0x00, 0x52, 0x03, 0x77, 0x65, + 0x62, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73, + 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10, 0x01, + 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, + 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x2a, 0xc2, 0x03, 0x0a, 0x13, 0x50, 0x6f, + 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x50, 0x4f, 0x53, + 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, + 0x54, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x50, 0x41, 0x52, 0x41, 0x4d, 0x45, 0x54, 0x45, + 0x52, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x41, + 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x12, + 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, + 0x10, 0x03, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, + 0x44, 0x59, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, 0x14, + 0x0a, 0x10, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x52, + 0x41, 0x57, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, + 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x55, 0x52, 0x4c, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x45, 0x44, + 0x10, 0x06, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, + 0x44, 0x59, 0x5f, 0x47, 0x52, 0x41, 0x50, 0x48, 0x51, 0x4c, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, + 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x08, + 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x55, 0x52, 0x4c, 0x10, + 0x09, 0x12, 0x18, 0x0a, 0x14, 0x45, 0x4e, 0x56, 0x49, 0x52, 0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54, + 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0a, 0x12, 0x18, 0x0a, 0x14, 0x46, + 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x46, 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, + 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0c, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4c, 0x4c, + 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0d, 0x12, + 0x17, 0x0a, 0x13, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x41, + 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, 0x1c, 0x0a, 0x18, 0x43, 0x4f, 0x4c, 0x4c, + 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, + 0x53, 0x45, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x10, 0x10, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53, + 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x11, 0x42, 0x43, + 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, + 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, + 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, + 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4775,7 +4879,7 @@ func file_source_metadata_proto_rawDescGZIP() []byte { } var file_source_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 38) +var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 39) var file_source_metadata_proto_goTypes = []interface{}{ (Visibility)(0), // 0: source_metadata.Visibility (PostmanLocationType)(0), // 1: source_metadata.PostmanLocationType @@ -4816,8 +4920,9 @@ var file_source_metadata_proto_goTypes = []interface{}{ (*Stdin)(nil), // 36: source_metadata.Stdin (*SlackContinuous)(nil), // 37: source_metadata.SlackContinuous (*JSONEnumerator)(nil), // 38: source_metadata.JSONEnumerator - (*MetaData)(nil), // 39: source_metadata.MetaData - (*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp + (*Web)(nil), // 39: source_metadata.Web + (*MetaData)(nil), // 40: source_metadata.MetaData + (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp } var file_source_metadata_proto_depIdxs = []int32{ 0, // 0: source_metadata.Github.visibility:type_name -> source_metadata.Visibility @@ -4828,7 +4933,7 @@ var file_source_metadata_proto_depIdxs = []int32{ 18, // 5: source_metadata.Forager.pypi:type_name -> source_metadata.PyPi 0, // 6: source_metadata.AzureRepos.visibility:type_name -> source_metadata.Visibility 1, // 7: source_metadata.Postman.location_type:type_name -> source_metadata.PostmanLocationType - 40, // 8: source_metadata.Vector.timestamp:type_name -> google.protobuf.Timestamp + 41, // 8: source_metadata.Vector.timestamp:type_name -> google.protobuf.Timestamp 32, // 9: source_metadata.Webhook.vector:type_name -> source_metadata.Vector 0, // 10: source_metadata.SlackContinuous.visibility:type_name -> source_metadata.Visibility 2, // 11: source_metadata.MetaData.azure:type_name -> source_metadata.Azure @@ -4867,11 +4972,12 @@ var file_source_metadata_proto_depIdxs = []int32{ 36, // 44: source_metadata.MetaData.stdin:type_name -> source_metadata.Stdin 37, // 45: source_metadata.MetaData.slackContinuous:type_name -> source_metadata.SlackContinuous 38, // 46: source_metadata.MetaData.jsonEnumerator:type_name -> source_metadata.JSONEnumerator - 47, // [47:47] is the sub-list for method output_type - 47, // [47:47] is the sub-list for method input_type - 47, // [47:47] is the sub-list for extension type_name - 47, // [47:47] is the sub-list for extension extendee - 0, // [0:47] is the sub-list for field type_name + 39, // 47: source_metadata.MetaData.web:type_name -> source_metadata.Web + 48, // [48:48] is the sub-list for method output_type + 48, // [48:48] is the sub-list for method input_type + 48, // [48:48] is the sub-list for extension type_name + 48, // [48:48] is the sub-list for extension extendee + 0, // [0:48] is the sub-list for field type_name } func init() { file_source_metadata_proto_init() } @@ -5325,6 +5431,18 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Web); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_source_metadata_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetaData); i { case 0: return &v.state @@ -5345,7 +5463,7 @@ func file_source_metadata_proto_init() { file_source_metadata_proto_msgTypes[31].OneofWrappers = []interface{}{ (*Webhook_Vector)(nil), } - file_source_metadata_proto_msgTypes[37].OneofWrappers = []interface{}{ + file_source_metadata_proto_msgTypes[38].OneofWrappers = []interface{}{ (*MetaData_Azure)(nil), (*MetaData_Bitbucket)(nil), (*MetaData_Circleci)(nil), @@ -5382,6 +5500,7 @@ func file_source_metadata_proto_init() { (*MetaData_Stdin)(nil), (*MetaData_SlackContinuous)(nil), (*MetaData_JsonEnumerator)(nil), + (*MetaData_Web)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -5389,7 +5508,7 @@ func file_source_metadata_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_source_metadata_proto_rawDesc, NumEnums: 2, - NumMessages: 38, + NumMessages: 39, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.validate.go b/pkg/pb/source_metadatapb/source_metadata.pb.validate.go index 1cb13f369b59..26f77c28dc14 100644 --- a/pkg/pb/source_metadatapb/source_metadata.pb.validate.go +++ b/pkg/pb/source_metadatapb/source_metadata.pb.validate.go @@ -4349,6 +4349,114 @@ var _ interface { ErrorName() string } = JSONEnumeratorValidationError{} +// Validate checks the field values on Web with the rules defined in the proto +// definition for this message. If any rules are violated, the first error +// encountered is returned, or nil if there are no violations. +func (m *Web) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Web with the rules defined in the +// proto definition for this message. If any rules are violated, the result is +// a list of violation errors wrapped in WebMultiError, or nil if none found. +func (m *Web) ValidateAll() error { + return m.validate(true) +} + +func (m *Web) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Url + + // no validation rules for PageTitle + + // no validation rules for ContentType + + // no validation rules for Depth + + // no validation rules for Timestamp + + if len(errors) > 0 { + return WebMultiError(errors) + } + + return nil +} + +// WebMultiError is an error wrapping multiple validation errors returned by +// Web.ValidateAll() if the designated constraints aren't met. +type WebMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m WebMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m WebMultiError) AllErrors() []error { return m } + +// WebValidationError is the validation error returned by Web.Validate if the +// designated constraints aren't met. +type WebValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e WebValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e WebValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e WebValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e WebValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e WebValidationError) ErrorName() string { return "WebValidationError" } + +// Error satisfies the builtin error interface +func (e WebValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sWeb.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = WebValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = WebValidationError{} + // Validate checks the field values on MetaData with the rules defined in the // proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. @@ -5848,6 +5956,47 @@ func (m *MetaData) validate(all bool) error { } } + case *MetaData_Web: + if v == nil { + err := MetaDataValidationError{ + field: "Data", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetWeb()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, MetaDataValidationError{ + field: "Web", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, MetaDataValidationError{ + field: "Web", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetWeb()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return MetaDataValidationError{ + field: "Web", + reason: "embedded message failed validation", + cause: err, + } + } + } + default: _ = v // ensures v is used } diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go index 37424d7eee7c..55eefc9de7f7 100644 --- a/pkg/pb/sourcespb/sources.pb.go +++ b/pkg/pb/sourcespb/sources.pb.go @@ -71,6 +71,7 @@ const ( SourceType_SOURCE_TYPE_STDIN SourceType = 40 SourceType_SOURCE_TYPE_SLACK_CONTINUOUS SourceType = 41 SourceType_SOURCE_TYPE_JSON_ENUMERATOR SourceType = 42 + SourceType_SOURCE_TYPE_WEB SourceType = 43 ) // Enum value maps for SourceType. @@ -119,6 +120,7 @@ var ( 40: "SOURCE_TYPE_STDIN", 41: "SOURCE_TYPE_SLACK_CONTINUOUS", 42: "SOURCE_TYPE_JSON_ENUMERATOR", + 43: "SOURCE_TYPE_WEB", } SourceType_value = map[string]int32{ "SOURCE_TYPE_AZURE_STORAGE": 0, @@ -164,6 +166,7 @@ var ( "SOURCE_TYPE_STDIN": 40, "SOURCE_TYPE_SLACK_CONTINUOUS": 41, "SOURCE_TYPE_JSON_ENUMERATOR": 42, + "SOURCE_TYPE_WEB": 43, } ) @@ -5100,6 +5103,101 @@ func (x *JSONEnumerator) GetPaths() []string { return nil } +type Web struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Urls []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"` + Crawl bool `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"` + Depth int64 `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"` + Delay int64 `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"` + Timeout int64 `protobuf:"varint,5,opt,name=timeout,proto3" json:"timeout,omitempty"` + UserAgent string `protobuf:"bytes,6,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"` + IgnoreRobots bool `protobuf:"varint,7,opt,name=ignore_robots,json=ignoreRobots,proto3" json:"ignore_robots,omitempty"` +} + +func (x *Web) Reset() { + *x = Web{} + if protoimpl.UnsafeEnabled { + mi := &file_sources_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Web) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Web) ProtoMessage() {} + +func (x *Web) ProtoReflect() protoreflect.Message { + mi := &file_sources_proto_msgTypes[41] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Web.ProtoReflect.Descriptor instead. +func (*Web) Descriptor() ([]byte, []int) { + return file_sources_proto_rawDescGZIP(), []int{41} +} + +func (x *Web) GetUrls() []string { + if x != nil { + return x.Urls + } + return nil +} + +func (x *Web) GetCrawl() bool { + if x != nil { + return x.Crawl + } + return false +} + +func (x *Web) GetDepth() int64 { + if x != nil { + return x.Depth + } + return 0 +} + +func (x *Web) GetDelay() int64 { + if x != nil { + return x.Delay + } + return 0 +} + +func (x *Web) GetTimeout() int64 { + if x != nil { + return x.Timeout + } + return 0 +} + +func (x *Web) GetUserAgent() string { + if x != nil { + return x.UserAgent + } + return "" +} + +func (x *Web) GetIgnoreRobots() bool { + if x != nil { + return x.IgnoreRobots + } + return false +} + var File_sources_proto protoreflect.FileDescriptor var file_sources_proto_rawDesc = []byte{ @@ -5892,101 +5990,114 @@ var file_sources_proto_rawDesc = []byte{ 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x2a, 0xc4, 0x09, - 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, - 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, - 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, - 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, - 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0xb9, 0x01, + 0x0a, 0x03, 0x57, 0x65, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x61, + 0x77, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x63, 0x72, 0x61, 0x77, 0x6c, 0x12, + 0x14, 0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, + 0x64, 0x65, 0x70, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x74, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x74, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, + 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, + 0x6f, 0x72, 0x65, 0x52, 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x2a, 0xd9, 0x09, 0x0a, 0x0a, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, + 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, + 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, + 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, + 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, + 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, - 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, - 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, - 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, - 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, - 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, - 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, - 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, - 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, - 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, - 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, - 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, - 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, - 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, - 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, - 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, - 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, - 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, - 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, - 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, - 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, - 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, - 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, - 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, - 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, - 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, - 0x4f, 0x53, 0x10, 0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, - 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, - 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, - 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x45, 0x4c, 0x41, 0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, - 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, - 0x55, 0x47, 0x47, 0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, - 0x55, 0x42, 0x5f, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, - 0x25, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, - 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, - 0x28, 0x12, 0x20, 0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, - 0x53, 0x10, 0x29, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, - 0x4f, 0x52, 0x10, 0x2a, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, - 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87, 0x01, - 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, - 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20, 0x0a, - 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, - 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, - 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, - 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, - 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, + 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, + 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, + 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, + 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, + 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, + 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, + 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, + 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, + 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12, + 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, + 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, + 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, + 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, + 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, + 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, + 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, + 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, + 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, + 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f, + 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, + 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, 0x0a, 0x19, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, 0x41, 0x53, + 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, 0x17, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, 0x47, 0x49, + 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x45, + 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, 0x16, 0x0a, + 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x4e, + 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, 0x41, 0x4c, + 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, 0x20, 0x0a, + 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, + 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, 0x29, 0x12, + 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, + 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10, 0x2a, + 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, + 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, + 0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87, + 0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f, + 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20, + 0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, + 0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, + 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, + 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, + 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, + 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -6002,7 +6113,7 @@ func file_sources_proto_rawDescGZIP() []byte { } var file_sources_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 41) +var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 42) var file_sources_proto_goTypes = []interface{}{ (SourceType)(0), // 0: sources.SourceType (BitbucketInstallationType)(0), // 1: sources.BitbucketInstallationType @@ -6049,80 +6160,81 @@ var file_sources_proto_goTypes = []interface{}{ (*Stdin)(nil), // 42: sources.Stdin (*SlackContinuous)(nil), // 43: sources.SlackContinuous (*JSONEnumerator)(nil), // 44: sources.JSONEnumerator - (*durationpb.Duration)(nil), // 45: google.protobuf.Duration - (*anypb.Any)(nil), // 46: google.protobuf.Any - (*credentialspb.BasicAuth)(nil), // 47: credentials.BasicAuth - (*credentialspb.Unauthenticated)(nil), // 48: credentials.Unauthenticated - (*credentialspb.Oauth2)(nil), // 49: credentials.Oauth2 - (*credentialspb.KeySecret)(nil), // 50: credentials.KeySecret - (*credentialspb.CloudEnvironment)(nil), // 51: credentials.CloudEnvironment - (*credentialspb.SSHAuth)(nil), // 52: credentials.SSHAuth - (*credentialspb.GitHubApp)(nil), // 53: credentials.GitHubApp - (*credentialspb.GoogleDriveDWD)(nil), // 54: credentials.GoogleDriveDWD - (*credentialspb.AWSSessionTokenSecret)(nil), // 55: credentials.AWSSessionTokenSecret - (*credentialspb.SlackTokens)(nil), // 56: credentials.SlackTokens - (*credentialspb.Header)(nil), // 57: credentials.Header - (*credentialspb.ClientCredentials)(nil), // 58: credentials.ClientCredentials - (*timestamppb.Timestamp)(nil), // 59: google.protobuf.Timestamp + (*Web)(nil), // 45: sources.Web + (*durationpb.Duration)(nil), // 46: google.protobuf.Duration + (*anypb.Any)(nil), // 47: google.protobuf.Any + (*credentialspb.BasicAuth)(nil), // 48: credentials.BasicAuth + (*credentialspb.Unauthenticated)(nil), // 49: credentials.Unauthenticated + (*credentialspb.Oauth2)(nil), // 50: credentials.Oauth2 + (*credentialspb.KeySecret)(nil), // 51: credentials.KeySecret + (*credentialspb.CloudEnvironment)(nil), // 52: credentials.CloudEnvironment + (*credentialspb.SSHAuth)(nil), // 53: credentials.SSHAuth + (*credentialspb.GitHubApp)(nil), // 54: credentials.GitHubApp + (*credentialspb.GoogleDriveDWD)(nil), // 55: credentials.GoogleDriveDWD + (*credentialspb.AWSSessionTokenSecret)(nil), // 56: credentials.AWSSessionTokenSecret + (*credentialspb.SlackTokens)(nil), // 57: credentials.SlackTokens + (*credentialspb.Header)(nil), // 58: credentials.Header + (*credentialspb.ClientCredentials)(nil), // 59: credentials.ClientCredentials + (*timestamppb.Timestamp)(nil), // 60: google.protobuf.Timestamp } var file_sources_proto_depIdxs = []int32{ - 45, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration - 46, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any - 47, // 2: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth - 48, // 3: sources.Artifactory.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 4: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth - 48, // 5: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated - 49, // 6: sources.Bitbucket.oauth:type_name -> credentials.Oauth2 - 47, // 7: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth + 46, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration + 47, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any + 48, // 2: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth + 49, // 3: sources.Artifactory.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 4: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth + 49, // 5: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated + 50, // 6: sources.Bitbucket.oauth:type_name -> credentials.Oauth2 + 48, // 7: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth 1, // 8: sources.Bitbucket.installation_type:type_name -> sources.BitbucketInstallationType - 48, // 9: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 10: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth + 49, // 9: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 10: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth 3, // 11: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope - 48, // 12: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 13: sources.Docker.basic_auth:type_name -> credentials.BasicAuth - 50, // 14: sources.ECR.access_key:type_name -> credentials.KeySecret - 48, // 15: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated - 51, // 16: sources.GCS.adc:type_name -> credentials.CloudEnvironment - 49, // 17: sources.GCS.oauth:type_name -> credentials.Oauth2 - 47, // 18: sources.Git.basic_auth:type_name -> credentials.BasicAuth - 48, // 19: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated - 52, // 20: sources.Git.ssh_auth:type_name -> credentials.SSHAuth - 49, // 21: sources.GitLab.oauth:type_name -> credentials.Oauth2 - 47, // 22: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth - 53, // 23: sources.GitHub.github_app:type_name -> credentials.GitHubApp - 48, // 24: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 25: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth - 53, // 26: sources.GitHubRealtime.github_app:type_name -> credentials.GitHubApp - 48, // 27: sources.GitHubRealtime.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 28: sources.GitHubRealtime.basic_auth:type_name -> credentials.BasicAuth - 49, // 29: sources.GoogleDrive.oauth:type_name -> credentials.Oauth2 - 54, // 30: sources.GoogleDrive.dwd:type_name -> credentials.GoogleDriveDWD - 48, // 31: sources.Huggingface.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 32: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth - 48, // 33: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated - 49, // 34: sources.JIRA.oauth:type_name -> credentials.Oauth2 + 49, // 12: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 13: sources.Docker.basic_auth:type_name -> credentials.BasicAuth + 51, // 14: sources.ECR.access_key:type_name -> credentials.KeySecret + 49, // 15: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated + 52, // 16: sources.GCS.adc:type_name -> credentials.CloudEnvironment + 50, // 17: sources.GCS.oauth:type_name -> credentials.Oauth2 + 48, // 18: sources.Git.basic_auth:type_name -> credentials.BasicAuth + 49, // 19: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated + 53, // 20: sources.Git.ssh_auth:type_name -> credentials.SSHAuth + 50, // 21: sources.GitLab.oauth:type_name -> credentials.Oauth2 + 48, // 22: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth + 54, // 23: sources.GitHub.github_app:type_name -> credentials.GitHubApp + 49, // 24: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 25: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth + 54, // 26: sources.GitHubRealtime.github_app:type_name -> credentials.GitHubApp + 49, // 27: sources.GitHubRealtime.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 28: sources.GitHubRealtime.basic_auth:type_name -> credentials.BasicAuth + 50, // 29: sources.GoogleDrive.oauth:type_name -> credentials.Oauth2 + 55, // 30: sources.GoogleDrive.dwd:type_name -> credentials.GoogleDriveDWD + 49, // 31: sources.Huggingface.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 32: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth + 49, // 33: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated + 50, // 34: sources.JIRA.oauth:type_name -> credentials.Oauth2 2, // 35: sources.JIRA.installation_type:type_name -> sources.JiraInstallationType - 48, // 36: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated - 48, // 37: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated - 50, // 38: sources.S3.access_key:type_name -> credentials.KeySecret - 48, // 39: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated - 51, // 40: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment - 55, // 41: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret - 56, // 42: sources.Slack.tokens:type_name -> credentials.SlackTokens - 47, // 43: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth - 48, // 44: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 45: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth - 57, // 46: sources.Jenkins.header:type_name -> credentials.Header - 48, // 47: sources.Jenkins.unauthenticated:type_name -> credentials.Unauthenticated - 58, // 48: sources.Teams.authenticated:type_name -> credentials.ClientCredentials - 49, // 49: sources.Teams.oauth:type_name -> credentials.Oauth2 - 48, // 50: sources.Forager.unauthenticated:type_name -> credentials.Unauthenticated - 59, // 51: sources.Forager.since:type_name -> google.protobuf.Timestamp - 56, // 52: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens - 49, // 53: sources.Sharepoint.oauth:type_name -> credentials.Oauth2 - 49, // 54: sources.AzureRepos.oauth:type_name -> credentials.Oauth2 - 48, // 55: sources.Postman.unauthenticated:type_name -> credentials.Unauthenticated - 57, // 56: sources.Webhook.header:type_name -> credentials.Header + 49, // 36: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated + 49, // 37: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated + 51, // 38: sources.S3.access_key:type_name -> credentials.KeySecret + 49, // 39: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated + 52, // 40: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment + 56, // 41: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret + 57, // 42: sources.Slack.tokens:type_name -> credentials.SlackTokens + 48, // 43: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth + 49, // 44: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 45: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth + 58, // 46: sources.Jenkins.header:type_name -> credentials.Header + 49, // 47: sources.Jenkins.unauthenticated:type_name -> credentials.Unauthenticated + 59, // 48: sources.Teams.authenticated:type_name -> credentials.ClientCredentials + 50, // 49: sources.Teams.oauth:type_name -> credentials.Oauth2 + 49, // 50: sources.Forager.unauthenticated:type_name -> credentials.Unauthenticated + 60, // 51: sources.Forager.since:type_name -> google.protobuf.Timestamp + 57, // 52: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens + 50, // 53: sources.Sharepoint.oauth:type_name -> credentials.Oauth2 + 50, // 54: sources.AzureRepos.oauth:type_name -> credentials.Oauth2 + 49, // 55: sources.Postman.unauthenticated:type_name -> credentials.Unauthenticated + 58, // 56: sources.Webhook.header:type_name -> credentials.Header 39, // 57: sources.Webhook.vector:type_name -> sources.Vector 58, // [58:58] is the sub-list for method output_type 58, // [58:58] is the sub-list for method input_type @@ -6629,6 +6741,18 @@ func file_sources_proto_init() { return nil } } + file_sources_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Web); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_sources_proto_msgTypes[1].OneofWrappers = []interface{}{ (*Artifactory_BasicAuth)(nil), @@ -6780,7 +6904,7 @@ func file_sources_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_sources_proto_rawDesc, NumEnums: 4, - NumMessages: 41, + NumMessages: 42, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go index eca7b0d6a469..12252d102699 100644 --- a/pkg/pb/sourcespb/sources.pb.validate.go +++ b/pkg/pb/sourcespb/sources.pb.validate.go @@ -7238,3 +7238,113 @@ var _ interface { Cause() error ErrorName() string } = JSONEnumeratorValidationError{} + +// Validate checks the field values on Web with the rules defined in the proto +// definition for this message. If any rules are violated, the first error +// encountered is returned, or nil if there are no violations. +func (m *Web) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Web with the rules defined in the +// proto definition for this message. If any rules are violated, the result is +// a list of violation errors wrapped in WebMultiError, or nil if none found. +func (m *Web) ValidateAll() error { + return m.validate(true) +} + +func (m *Web) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Crawl + + // no validation rules for Depth + + // no validation rules for Delay + + // no validation rules for Timeout + + // no validation rules for UserAgent + + // no validation rules for IgnoreRobots + + if len(errors) > 0 { + return WebMultiError(errors) + } + + return nil +} + +// WebMultiError is an error wrapping multiple validation errors returned by +// Web.ValidateAll() if the designated constraints aren't met. +type WebMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m WebMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m WebMultiError) AllErrors() []error { return m } + +// WebValidationError is the validation error returned by Web.Validate if the +// designated constraints aren't met. +type WebValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e WebValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e WebValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e WebValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e WebValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e WebValidationError) ErrorName() string { return "WebValidationError" } + +// Error satisfies the builtin error interface +func (e WebValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sWeb.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = WebValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = WebValidationError{} diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index 88b11b85a5ea..b2c84c0e3c55 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -482,6 +482,37 @@ type JSONEnumeratorConfig struct { Paths []string } +// WebConfig defines the configuration for the web source. +type WebConfig struct { + // URLs are the seed URLs to scan. At least one is required. + // Each URL is crawled independently with its own collector. + URLs []string + + // Crawl controls whether links discovered on each page are followed. + // When false, only the seed URLs themselves are scanned. + Crawl bool + + // Depth is the maximum number of link hops to follow when Crawl is true. + // 1 = seed; 2 = one level deeper; 0 = unlimited. + Depth int + + // Delay is the number of seconds to wait between requests to the same + // domain. Increase this to reduce load on the target server. + Delay int + + // Timeout is the maximum number of seconds to spend crawling + // seeded URLs before aborting. Defaults to 30 seconds if unset or zero. + Timeout int + + // UserAgent is the User-Agent header sent with each request. + // Defaults to a TruffleHog identifier if empty. + UserAgent string + + // IgnoreRobots disables robots.txt enforcement when true. + // Only enable this if you have explicit permission to crawl the target site. + IgnoreRobots bool +} + // Progress is used to update job completion progress across sources. type Progress struct { mut sync.Mutex diff --git a/pkg/sources/web/README.md b/pkg/sources/web/README.md new file mode 100644 index 000000000000..987c68f099e7 --- /dev/null +++ b/pkg/sources/web/README.md @@ -0,0 +1,91 @@ +# TruffleHog Web Source + +Crawls and scans websites for secrets and sensitive information using the [Colly](http://go-colly.org/) web scraping framework. + +## Configuration + +### Required + +| Flag | Description | +|------|-------------| +| `--url` | URL to scan. Repeat for multiple targets: `--url https://a.com --url https://b.com`. Supports `http://` and `https://`. | + +### Optional + +| Flag | Default | Description | +|------|---------|-------------| +| `--crawl` | `false` | Follow links found on each page. Without this flag only the seed URL(s) are scanned. | +| `--depth` | `1` | Maximum link depth to follow when `--crawl` is enabled. `1` = seed; `2` = one level deeper; `0` = unlimited. Has no effect without `--crawl`. | +| `--delay` | `1` | Seconds to wait between requests to the same domain. Increase this to reduce load on the target server. | +| `--timeout` | `30` | Seconds to spend crawling each URL before aborting. Applied independently per URL. | +| `--user-agent` | TruffleHog identifier | User-Agent header sent with each request. | +| `--ignore-robots` | `false` | Ignore `robots.txt` restrictions. Only enable this if you have explicit permission to crawl the target site. | + +## Usage + +**Scan a single page (no crawling)** +```bash +trufflehog web --url https://example.com +``` + +**Scan a page and its direct links** +```bash +trufflehog web --url https://example.com --crawl --depth 2 --delay 2 +``` + +**Scan multiple URLs two levels deep** +```bash +trufflehog web \ + --url https://example.com \ + --url https://app.example.com \ + --crawl \ + --depth 3 \ + --delay 1 +``` + +**Scan with a short per-URL timeout** +```bash +trufflehog web --url https://example.com --crawl --depth 2 --timeout 60 +``` + +## Behavior + +### Domain scope + +The crawler visits the exact domain provided and all of its subdomains. External links to other domains are always skipped. + +### Depth counting + +Depth is counted in hops from the seed URL. The seed itself is hop 1; pages linked directly from it are hop 2, and so on. Setting `--depth 0` with `--crawl` enables unlimited traversal. + +### Robots.txt + +Robots.txt rules are respected by default. Disabling this with `--ignore-robots` should only be done with explicit permission from the site owner. + +## Output + +Each crawled page produces a chunk with the following metadata: + +| Field | Description | +|-------|-------------| +| `url` | Full URL of the crawled page | +| `page_title` | Text content of the `` element | +| `depth` | Number of hops from the seed URL | +| `content_type` | MIME type of the response | +| `timestamp` | Crawl time in UTC RFC3339 format | + +Example: +```json +{ + "url": "https://example.com/about", + "page_title": "About Us", + "depth": 1, + "content_type": "text/html; charset=utf-8", + "timestamp": "2026-03-27T17:26:34Z" +} +``` + +## Limitations + +- Only the target domain and its subdomains are crawled; external links are skipped by design. +- No support for authenticated pages (login-gated content). diff --git a/pkg/sources/web/metrics.go b/pkg/sources/web/metrics.go new file mode 100644 index 000000000000..9f8d85e2a00c --- /dev/null +++ b/pkg/sources/web/metrics.go @@ -0,0 +1,17 @@ +package web + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" +) + +var ( + webUrlsScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "web_urls_scanned", + Help: "Total number of URLs scanned.", + }, + []string{"source_name", "job_id"}) +) diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go new file mode 100644 index 000000000000..304ae90b4f6c --- /dev/null +++ b/pkg/sources/web/web.go @@ -0,0 +1,300 @@ +package web + +import ( + "bytes" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/gocolly/colly/v2" + "golang.org/x/net/html" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +const SourceType = sourcespb.SourceType_SOURCE_TYPE_WEB + +type Source struct { + name string + sourceId sources.SourceID + jobId sources.JobID + verify bool + concurrency int + conn sourcespb.Web + + sources.Progress + sources.CommonSourceUnitUnmarshaller +} + +// Ensure the Source satisfies the interfaces at compile time +var _ sources.Source = (*Source)(nil) +var _ sources.SourceUnitUnmarshaller = (*Source)(nil) + +func (s *Source) Type() sourcespb.SourceType { return SourceType } +func (s *Source) SourceID() sources.SourceID { return s.sourceId } +func (s *Source) JobID() sources.JobID { return s.jobId } + +// Init initializes the source. +func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, + verify bool, connection *anypb.Any, concurrency int, +) error { + s.name = name + s.sourceId = sourceId + s.jobId = jobId + s.verify = verify + s.concurrency = concurrency + // If s.concurrency is 0, use 1 + // TODO: make it configurable + if s.concurrency == 0 { + s.concurrency = 1 + } + + if err := anypb.UnmarshalTo(connection, &s.conn, proto.UnmarshalOptions{}); err != nil { + return fmt.Errorf("error unmarshalling connection: %w", err) + } + + // Use the user-provided User-Agent if set; otherwise fall back to a default that identifies TruffleHog. + if s.conn.GetUserAgent() == "" { + ctx.Logger().Info("No user agent set; using default", "user-agent", "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)") + s.conn.UserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)" + } + + // The 30-second timeout is a safety net + if s.conn.GetTimeout() <= 0 { + s.conn.Timeout = 30 + } + + if s.conn.GetIgnoreRobots() { + ctx.Logger().Info("Warning: Robots.txt is ignored. Only use this if you have permission to crawl the target site.") + } + + // validations + if len(s.conn.GetUrls()) == 0 { + return errors.New("no URL provided") + } + + // Validate URLs format + for _, u := range s.conn.GetUrls() { + if _, err := url.Parse(u); err != nil { + return fmt.Errorf("invalid URL %q: %w", u, err) + } + } + + // metrics + jobIDStr := fmt.Sprint(s.jobId) + webUrlsScanned.WithLabelValues(s.name, jobIDStr).Set(0) + + return nil +} + +// Chunks emits data over a channel that is decoded and scanned for secrets. +func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error { + jobIDStr := fmt.Sprint(s.jobId) + + // Create a new context with timeout. + crawlCtx, cancel := context.WithTimeout(ctx, time.Duration(s.conn.GetTimeout())*time.Second) + defer cancel() + + eg, _ := errgroup.WithContext(crawlCtx) + + for _, u := range s.conn.GetUrls() { + ctx.Logger().V(5).Info("Processing Url", "url", u) + webUrlsScanned.WithLabelValues(s.name, jobIDStr).Inc() + eg.Go(func() error { + return s.crawlURL(crawlCtx, u, chunksChan) + }) + } + + if err := eg.Wait(); err != nil { + ctx.Logger().Error(err, "One or more crawls failed") + return err + } + + ctx.Logger().Info("All crawls completed") + return nil +} + +func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *sources.Chunk) error { + // Add static crawl configuration to the context so that all subsequent logs include these fields. + ctx = context.WithValues(ctx, + "url", seedURL, + "user_agent", s.conn.GetUserAgent(), + "ignore_robots", s.conn.GetIgnoreRobots(), + ) + + parsedURL, err := url.Parse(seedURL) + if err != nil { + return fmt.Errorf("invalid URL %q: %w", seedURL, err) + } + + // docs: http://go-colly.org/docs/introduction/configuration/ + collector := colly.NewCollector( + colly.UserAgent(s.conn.GetUserAgent()), + colly.Async(true), + ) + + // Apply depth limit only when crawling is enabled and a positive depth is set. + if s.conn.GetCrawl() && s.conn.GetDepth() > 0 { + collector.MaxDepth = int(s.conn.GetDepth()) + } + + // By default, the crawler respects robots.txt rules. Setting IgnoreRobotsTxt to true overrides this behavior. + // Users can enable this only when they have explicit permission to crawl the target site. + collector.IgnoreRobotsTxt = s.conn.GetIgnoreRobots() + + if err := collector.Limit(&colly.LimitRule{ + DomainGlob: "*", + Parallelism: s.concurrency, + Delay: time.Duration(s.conn.GetDelay()) * time.Second, + }); err != nil { + return fmt.Errorf("failed to limit rules to the colly collector: %w", err) + } + + // request validations + collector.OnRequest(func(r *colly.Request) { + host := r.URL.Hostname() + if host != parsedURL.Hostname() && !strings.HasSuffix(host, "."+parsedURL.Hostname()) { + ctx.Logger().V(5).Info("blocked by domain filter", "url", r.URL.String()) + r.Abort() + } + }) + + // Set up callbacks + collector.OnResponse(func(r *colly.Response) { + ctx.Logger().Info("Response received") + if err := s.processChunk(ctx, r, chunksChan); err != nil { + ctx.Logger().Error(err, "error processing page") + } + }) + collector.OnError(func(r *colly.Response, err error) { + ctx.Logger().Error(err, "error fetching page", "url", r.Request.URL) + }) + + // Follow links only when crawling is explicitly enabled. + if s.conn.GetCrawl() { + collector.OnHTML("a[href]", func(e *colly.HTMLElement) { + link := e.Request.AbsoluteURL(e.Attr("href")) + if link == "" { + return + } + + if err := e.Request.Visit(link); err != nil { + if _, ok := err.(*colly.AlreadyVisitedError); !ok { + ctx.Logger().V(5).Info("Skipping link", "url", link, "reason", err) + } + } + }) + + // Also enqueue linked JavaScript files - a common location for hardcoded secrets. + collector.OnHTML("script[src]", func(e *colly.HTMLElement) { + src := e.Request.AbsoluteURL(e.Attr("src")) + if src == "" { + return + } + + if err := e.Request.Visit(src); err != nil { + if _, ok := err.(*colly.AlreadyVisitedError); !ok { + ctx.Logger().V(5).Info("Skipping script", "url", src, "reason", err) + } + } + }) + } + + // Create a channel to signal when the crawl is done. + done := make(chan struct{}) + go func() { + ctx.Logger().Info("Starting crawl") + if err := collector.Visit(seedURL); err != nil { + ctx.Logger().Error(err, "Visit failed") + } + collector.Wait() // blocks until all requests finish + close(done) + }() + + // Wait for either crawl to finish or context cancellation. + select { + case <-done: + ctx.Logger().Info("Crawl finished normally") + return nil + case <-ctx.Done(): + ctx.Logger().Info("Context cancelled or timeout reached") + <-done // Wait for goroutine to finish cleanup + return ctx.Err() + } +} + +func (s *Source) processChunk(ctx context.Context, data *colly.Response, chunksChan chan *sources.Chunk) error { + pageTitle := extractPageTitle(data.Body) + + ctx.Logger().V(5).WithValues("page_title", pageTitle).Info("Processing web chunk") + + // Create a chunk from the response body. + chunk := &sources.Chunk{ + Data: data.Body, + SourceType: s.Type(), + SourceName: s.name, + SourceID: s.SourceID(), + JobID: s.JobID(), + SourceVerify: s.verify, + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Web{ + Web: &source_metadatapb.Web{ + Url: data.Request.URL.String(), + PageTitle: pageTitle, + Depth: int64(data.Request.Depth), + ContentType: data.Headers.Get("Content-Type"), + Timestamp: time.Now().UTC().Format(time.RFC3339), + }, + }, + }, + } + + return common.CancellableWrite(ctx, chunksChan, chunk) +} + +// extractPageTitle parses an HTML document and returns the text content of the +// first <title> element, with leading and trailing whitespace trimmed. +// Returns an empty string if the body is empty, cannot be parsed, or contains +// no <title> element. +func extractPageTitle(body []byte) string { + doc, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return "" + } + + var title string + + // f is a recursive depth-first walker over the HTML node tree. + // It is declared as a variable first so that the closure can reference + // itself when recursing into child nodes. + var f func(*html.Node) + f = func(n *html.Node) { + if title != "" { + return // already found, skip the rest of the tree + } + // We are only interested in element nodes (e.g. <title>, <div>). + // Text, comment, and doctype nodes are skipped by this check. + if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { + // <title> content is always a single text node directly inside + // the element. n.FirstChild.Data holds the raw string value. + title = strings.TrimSpace(n.FirstChild.Data) + return + } + + // Recurse into child nodes to continue the depth-first traversal. + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + return title +} diff --git a/pkg/sources/web/web_test.go b/pkg/sources/web/web_test.go new file mode 100644 index 000000000000..07ae70be2468 --- /dev/null +++ b/pkg/sources/web/web_test.go @@ -0,0 +1,431 @@ +package web + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// helper: marshal a *sourcespb.Web config into an *anypb.Any and Init a Source. +func initSource(t *testing.T, cfg *sourcespb.Web, concurrency int) *Source { + t.Helper() + conn := &anypb.Any{} + require.NoError(t, conn.MarshalFrom(cfg)) + s := &Source{} + require.NoError(t, s.Init(context.TODO(), "test-source", 0, 0, false, conn, concurrency)) + return s +} + +// helper: run Chunks and collect all emitted chunks. +func collectChunks(t *testing.T, s *Source) []*sources.Chunk { + t.Helper() + chunksChan := make(chan *sources.Chunk, 16) + var wg sync.WaitGroup + var chunks []*sources.Chunk + wg.Add(1) + go func() { + defer wg.Done() + for c := range chunksChan { + chunks = append(chunks, c) + } + }() + require.NoError(t, s.Chunks(context.TODO(), chunksChan)) + close(chunksChan) + wg.Wait() + return chunks +} + +// Init validation + +func TestInit_NoURL(t *testing.T) { + conn := &anypb.Any{} + require.NoError(t, conn.MarshalFrom(&sourcespb.Web{})) + s := &Source{} + err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1) + assert.Error(t, err) +} + +func TestInit_DefaultUserAgent(t *testing.T) { + conn := &anypb.Any{} + require.NoError(t, conn.MarshalFrom(&sourcespb.Web{ + Urls: []string{"http://example.com"}, + })) + s := &Source{} + require.NoError(t, s.Init(context.TODO(), "test", 0, 0, false, conn, 1)) + assert.Contains(t, s.conn.GetUserAgent(), "trufflehog") +} + +func TestInit_CustomUserAgent(t *testing.T) { + conn := &anypb.Any{} + require.NoError(t, conn.MarshalFrom(&sourcespb.Web{ + Urls: []string{"http://example.com"}, + UserAgent: "my-bot/1.0", + })) + s := &Source{} + require.NoError(t, s.Init(context.TODO(), "test", 0, 0, false, conn, 1)) + assert.Equal(t, "my-bot/1.0", s.conn.GetUserAgent()) +} + +func TestInit_ZeroConcurrencyDefaultsToOne(t *testing.T) { + conn := &anypb.Any{} + require.NoError(t, conn.MarshalFrom(&sourcespb.Web{ + Urls: []string{"http://example.com"}, + })) + s := &Source{} + require.NoError(t, s.Init(context.TODO(), "test", 0, 0, false, conn, 0)) + assert.Equal(t, 1, s.concurrency) +} + +// Happy path + +func TestChunks_HappyPath(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprint(w, `<!DOCTYPE html><html><head><title>Test PageHello, world!`) + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1) + chunks := collectChunks(t, s) + + require.Equal(t, 1, len(chunks)) + chunk := chunks[0] + + assert.Contains(t, string(chunk.Data), "Hello, world!") + + meta, ok := chunk.SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + require.True(t, ok, "expected web metadata") + assert.Equal(t, "Test Page", meta.Web.PageTitle) + assert.Equal(t, "text/html; charset=utf-8", meta.Web.ContentType) + assert.NotEmpty(t, meta.Web.Url) + assert.NotEmpty(t, meta.Web.Timestamp) +} + +func TestChunks_TimestampIsRFC3339(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `Tbody`) + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1) + chunks := collectChunks(t, s) + require.Equal(t, 1, len(chunks)) + + meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + _, err := time.Parse(time.RFC3339, meta.Web.Timestamp) + assert.NoError(t, err, "timestamp must be valid RFC3339") +} + +// Page title extraction + +func TestExtractPageTitle_Normal(t *testing.T) { + body := []byte(` Hello World `) + assert.Equal(t, "Hello World", extractPageTitle(body)) +} + +func TestExtractPageTitle_Missing(t *testing.T) { + body := []byte(`no title`) + assert.Equal(t, "", extractPageTitle(body)) +} + +func TestExtractPageTitle_Empty(t *testing.T) { + assert.Equal(t, "", extractPageTitle([]byte{})) +} + +func TestExtractPageTitle_MalformedHTML(t *testing.T) { + // html.Parse is lenient; just confirm it doesn't panic and returns something sensible. + body := []byte(`Partial`) + title := extractPageTitle(body) + // Go's html.Parse fills in missing closing tags, so the title is still parsed. + assert.Equal(t, "Partial", title) +} + +func TestExtractPageTitle_NonHTML(t *testing.T) { + // Binary / JSON bodies must not panic. + body := []byte(`{"secret": "abc123"}`) + _ = extractPageTitle(body) // just confirm no panic +} + +// Depth / crawl behaviour + +// TestChunks_NoCrawl confirms that when Crawl=false only the seed page is +// fetched, even if the page contains an internal link. +func TestChunks_NoCrawl(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `<html><head><title>Root + page 2 + `) + }) + mux.HandleFunc("/page2", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Page 2Secret`) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + // Crawl=false - only the seed URL is visited. + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: false}, 1) + chunks := collectChunks(t, s) + + assert.Equal(t, 1, len(chunks), "expected only the root page when Crawl=false") + meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + assert.Equal(t, "Root", meta.Web.PageTitle) +} + +// TestChunks_CrawlDepth1 confirms that with Crawl=true and Depth=1 the crawler +// visits the seed page and its direct links, but not grandchildren. +func TestChunks_CrawlDepth1(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Root + child + `) + }) + mux.HandleFunc("/child", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Child + grandchild + `) + }) + mux.HandleFunc("/grandchild", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Grandchilddeep`) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 2}, 1) + chunks := collectChunks(t, s) + + titles := make(map[string]bool) + for _, c := range chunks { + meta := c.SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + titles[meta.Web.PageTitle] = true + } + + assert.True(t, titles["Root"], "root page should be crawled") + assert.True(t, titles["Child"], "child page should be crawled at depth 1") + assert.False(t, titles["Grandchild"], "grandchild should NOT be crawled at depth 1") +} + +// TestChunks_CrawlDepth2 confirms depth-2 traversal reaches grandchildren. +func TestChunks_CrawlDepth2(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Root + child + `) + }) + mux.HandleFunc("/child", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Child + grandchild + `) + }) + mux.HandleFunc("/grandchild", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Grandchilddeep`) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 3}, 1) + chunks := collectChunks(t, s) + + titles := make(map[string]bool) + for _, c := range chunks { + meta := c.SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + titles[meta.Web.PageTitle] = true + } + + assert.True(t, titles["Root"]) + assert.True(t, titles["Child"]) + assert.True(t, titles["Grandchild"]) +} + +// TestChunks_CrossDomainLinksIgnored ensures that links pointing to a different +// hostname are not followed. httptest.NewServer always binds to 127.0.0.1, so +// we cannot spin up a second "external" server on a different IP in a portable +// way. Instead we embed a link to an unreachable external hostname directly in +// the page HTML and assert that no chunk with that hostname is ever produced. +func TestChunks_CrossDomainLinksIgnored(t *testing.T) { + const externalURL = "http://external.example.invalid/secret" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprintf(w, `Seed + external link + `, externalURL) + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 1}, 1) + chunks := collectChunks(t, s) + + for _, c := range chunks { + meta := c.SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + assert.NotContains(t, meta.Web.Url, "external.example.invalid", + "cross-domain URL must not appear in chunks") + } +} + +// Content types + +func TestChunks_PlainTextPage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "API_KEY=supersecret") + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1) + chunks := collectChunks(t, s) + + require.Equal(t, 1, len(chunks)) + assert.Contains(t, string(chunks[0].Data), "supersecret") + + meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web) + assert.Equal(t, "text/plain", meta.Web.ContentType) +} + +func TestChunks_JSONPage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"token":"ghp_supersecret"}`) + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1) + chunks := collectChunks(t, s) + + require.Equal(t, 1, len(chunks)) + assert.Contains(t, string(chunks[0].Data), "ghp_supersecret") +} + +// Error / edge cases + +func TestChunks_ServerReturns404(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1) + // colly treats 4xx as errors and fires OnError; no chunk should be emitted. + chunks := collectChunks(t, s) + assert.Empty(t, chunks, "a 404 response should not produce a chunk") +} + +func TestChunks_EmptyBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + // no body written + })) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1) + chunks := collectChunks(t, s) + // An empty but 200-OK response should still produce a chunk (body == ""). + require.Equal(t, 1, len(chunks)) + assert.Empty(t, chunks[0].Data) +} + +// Duplicate link deduplication + +// TestChunks_DuplicateLinksVisitedOnce ensures that the same URL appearing +// multiple times in a page is only fetched once. +func TestChunks_DuplicateLinksVisitedOnce(t *testing.T) { + hitCount := 0 + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Root + link1 + link2 + link3 + `) + }) + mux.HandleFunc("/dup", func(w http.ResponseWriter, r *http.Request) { + hitCount++ + fmt.Fprint(w, `Dupdup`) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 0}, 1) + _ = collectChunks(t, s) + + assert.Equal(t, 1, hitCount, "duplicate links should only be fetched once") +} + +// Robots.txt + +func TestChunks_RobotsTxtRespected(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "User-agent: *\nDisallow: /secret\n") + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Root + secret + `) + }) + secretVisited := false + mux.HandleFunc("/secret", func(w http.ResponseWriter, r *http.Request) { + secretVisited = true + fmt.Fprint(w, `Secretprivate`) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{ + Urls: []string{srv.URL}, Crawl: true, Depth: 1, IgnoreRobots: false, + }, 1) + _ = collectChunks(t, s) + + assert.False(t, secretVisited, "/secret must not be crawled when disallowed by robots.txt") +} + +func TestChunks_IgnoreRobotsTxt(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "User-agent: *\nDisallow: /secret\n") + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Root + secret + `) + }) + secretVisited := false + mux.HandleFunc("/secret", func(w http.ResponseWriter, r *http.Request) { + secretVisited = true + fmt.Fprint(w, `Secretprivate`) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + s := initSource(t, &sourcespb.Web{ + Urls: []string{srv.URL}, Crawl: true, Depth: 2, IgnoreRobots: true, + }, 1) + _ = collectChunks(t, s) + + assert.True(t, secretVisited, "/secret should be crawled when IgnoreRobots=true") +} diff --git a/proto/source_metadata.proto b/proto/source_metadata.proto index 40846c1f7e04..2916f9e7348a 100644 --- a/proto/source_metadata.proto +++ b/proto/source_metadata.proto @@ -392,6 +392,14 @@ message JSONEnumerator { string metadata = 1; } +message Web { + string url = 1; + string page_title = 2; + string content_type = 3; + int64 depth = 4; + string timestamp = 5; +} + message MetaData { oneof data { Azure azure = 1; @@ -430,5 +438,6 @@ message MetaData { Stdin stdin = 34; SlackContinuous slackContinuous = 35; JSONEnumerator jsonEnumerator = 36; + Web web = 37; } } diff --git a/proto/sources.proto b/proto/sources.proto index cf64caca8687..ce2f28faf3e2 100644 --- a/proto/sources.proto +++ b/proto/sources.proto @@ -55,6 +55,7 @@ enum SourceType { SOURCE_TYPE_STDIN = 40; SOURCE_TYPE_SLACK_CONTINUOUS = 41; SOURCE_TYPE_JSON_ENUMERATOR = 42; + SOURCE_TYPE_WEB = 43; } message LocalSource { @@ -554,3 +555,13 @@ message SlackContinuous { message JSONEnumerator { repeated string paths = 1; } + +message Web { + repeated string urls = 1; + bool crawl = 2; + int64 depth = 3; + int64 delay = 4; + int64 timeout = 5; + string user_agent = 6; + bool ignore_robots = 7; +}