diff --git a/.github/workflows/auto-merge-dependabot.yml b/.github/workflows/auto-merge-dependabot.yml index 1f60f78c..f8e358c0 100644 --- a/.github/workflows/auto-merge-dependabot.yml +++ b/.github/workflows/auto-merge-dependabot.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2.4.0 + uses: dependabot/fetch-metadata@v3.1.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 878f30ce..31e9fde8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,23 +17,23 @@ jobs: steps: - name: checkout sources - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: push: true platforms: linux/amd64,linux/arm/v7,linux/arm64/v8,linux/386,linux/ppc64le diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b659900e..a991d639 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,15 +8,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: "stable" - name: Install Task - uses: arduino/setup-task@v2 + uses: go-task/setup-task@v2 with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d6fcd95b..f6eaa1c0 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -8,14 +8,14 @@ jobs: timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - - uses: actions/setup-go@v6 + - uses: actions/setup-go@v7 with: go-version: "stable" - name: golangci-lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@v9 with: version: latest args: --timeout=5m diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml index d59735f8..a3f103ad 100644 --- a/.github/workflows/hadolint.yml +++ b/.github/workflows/hadolint.yml @@ -12,8 +12,8 @@ jobs: name: hadolint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: hadolint/hadolint-action@v3.2.0 + - uses: actions/checkout@v7 + - uses: hadolint/hadolint-action@v3.5.0 with: dockerfile: Dockerfile # DL3007: Using latest is prone to errors if the image will ever update. Pin the version explicitly to a release tag diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87f70332..69a549c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 0 @@ -21,12 +21,12 @@ jobs: run: git fetch --force --tags - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: "stable" - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v6.4.0 + uses: goreleaser/goreleaser-action@v7.2.3 with: distribution: goreleaser version: latest diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index edeff029..d24ebbad 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: ref: dev - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: "stable" @@ -28,7 +28,7 @@ jobs: go mod tidy - name: Commit and push changes - uses: stefanzweifel/git-auto-commit-action@v5 + uses: stefanzweifel/git-auto-commit-action@v7 with: commit_message: "chore: update dependencies [automated]" branch: dev diff --git a/.github/workflows/vhs.yml b/.github/workflows/vhs.yml index 2b23a88a..f46b6f72 100644 --- a/.github/workflows/vhs.yml +++ b/.github/workflows/vhs.yml @@ -3,6 +3,10 @@ on: push: paths: - vhs/**.tape + schedule: + # every week + - cron: "0 0 * * 0" + workflow_dispatch: permissions: contents: write @@ -11,15 +15,15 @@ jobs: vhs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: "stable" - name: Install Task - uses: arduino/setup-task@v2 + uses: go-task/setup-task@v2 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -29,14 +33,18 @@ jobs: - name: Build linux run: task linux - - name: Install deps + - name: Install vhs deps run: | sudo apt update sudo apt install -y ffmpeg ttyd - - uses: charmbracelet/vhs-action@v2 - with: - path: "vhs/gobuster_dir.tape" + - name: Install vhs + run: | + go install github.com/charmbracelet/vhs@latest + + - name: Generate vhs gif + run: | + vhs vhs/gobuster_dir.tape -o vhs/gobuster_dir.gif - name: commit and push changes run: | @@ -44,4 +52,4 @@ jobs: git config user.email "<>" git add vhs/*.gif git commit -m "update vhs gifs" || echo "no changes to commit" - git push origin master + git push diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml index d58ce73a..9693d7df 100644 --- a/.github/workflows/yamllint.yml +++ b/.github/workflows/yamllint.yml @@ -13,7 +13,7 @@ jobs: name: yamllint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - uses: karancode/yamllint-github-action@master with: # fail on warnings and errors diff --git a/.gitignore b/.gitignore index 3bbfe4b5..a8bf3b57 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ config.json gobuster *.txt dist/ +wordlist \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 51495629..859d2f4e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,7 +23,6 @@ linters: - gocheckcompilerdirectives - gochecknoinits - gochecksumtype - - goconst - gocritic - gomoddirectives - goprintffuncname @@ -159,7 +158,6 @@ linters: - linters: - bodyclose - errcheck - - goconst - gosec - noctx - wrapcheck diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..fbab1c28 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +`main.go` is the executable entry point. Command-line wiring and mode-specific argument handling live under `cli/` (for example, `cli/dir/` and `cli/dns/`). Shared scanning infrastructure is in `libgobuster/`; implementations are split into top-level packages such as `gobusterdir/`, `gobusterdns/`, `gobusterfuzz/`, `gobustervhost/`, `gobusters3/`, `gobustergcs/`, and `gobustertftp/`. Keep mode options, results, and execution logic within the corresponding package. Tests sit beside production code as `*_test.go`. Demo recordings and their source scripts are under `vhs/`; CI definitions are in `.github/workflows/`. + +## Build, Test, and Development Commands + +- `task build` runs dependency cleanup, formatting, vetting, and then builds a static Linux AMD64 `gobuster` binary. +- `task test` runs `go test -race -cover ./...` after dependency and formatting checks. +- `task check` applies `go fmt`, applies `gofumpt`, and runs `go vet ./...`. +- `task lint` runs `golangci-lint` across all packages and verifies that `go.mod`/`go.sum` remain tidy. +- `task windows` cross-compiles `gobuster.exe`; `task linux` builds the default Linux target. +- `go run . dir -u https://example.com -w wordlist.txt` runs a local development build. + +Task commands may update module or formatting files. Review the resulting diff before committing. + +## Coding Style & Naming Conventions + +Follow idiomatic Go and let `gofumpt` determine formatting (tabs for Go indentation). Package names are short and lowercase. Exported identifiers use `PascalCase`; internal identifiers use `camelCase`. Match existing mode types such as `Options`, `Result`, and `GobusterDir`, and keep CLI concerns separate from scanning logic. Add context to returned errors and avoid unnecessary global state. + +## Testing Guidelines + +Use Go's standard `testing` package and name files `*_test.go` with functions such as `TestParseExtensions`. Prefer table-driven tests for option parsing and multiple edge cases. Run `task test` before opening a pull request; the race detector and coverage report are part of the normal suite. There is no fixed coverage threshold, but new behavior and regressions should receive focused tests. + +## Commit & Pull Request Guidelines + +History favors brief, imperative subjects (for example, `fix go vet` or `update dependencies`); use a clear scope when useful, such as `dns: handle wildcard timeout`. Keep commits focused and avoid mixing refactors with behavioral fixes. Pull requests should explain the problem and solution, list validation commands, link relevant issues, and call out user-visible CLI changes. Include terminal output or updated VHS assets when output formatting changes; screenshots are otherwise unnecessary. diff --git a/Dockerfile b/Dockerfile index 68fcc2d2..7c8fea8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,14 +9,18 @@ RUN go build -a -o gobuster -trimpath FROM alpine:latest +ARG UID=1000 +ARG GID=1000 + RUN apk add --no-cache ca-certificates \ && rm -rf /var/cache/* RUN mkdir -p /app \ - && adduser -D gobuster \ + && addgroup -g ${GID} gobuster \ + && adduser -D gobuster -u ${UID} -G gobuster \ && chown -R gobuster:gobuster /app -USER gobuster +USER ${UID}:${GID} WORKDIR /app COPY --from=build-env /src/gobuster . diff --git a/README.md b/README.md index b00383c1..fb22a14c 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,13 @@ gobuster dir -u https://example.com -w wordlist.txt -l # Filter by status codes gobuster dir -u https://example.com -w wordlist.txt -s 200,301,302 + +# Filter using a regex against the response body +# This can be handy for websites that return status code 200 for everything, but the html contains an error message +gobuster dir -u https://example.com -w wordlist.txt -re "error\shello" + +# Filter using a regex but inverted against the response body +gobuster dir -u https://example.com -w wordlist.txt -rei "(?i)\berror\b" ``` #### 🔍 DNS Mode (`dns`) @@ -344,6 +351,18 @@ _Remember: Always test responsibly and with proper authorization._
+3.8.3 + +## 3.8.3 + +- Add option to filter body by regex +- Add option to save response bodies +- Allow comma in Header values passed via the CLI + +
+ +
+ 3.8.2 ## 3.8.2 diff --git a/Taskfile.yml b/Taskfile.yml index 06fc2f9e..00a25b67 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -19,6 +19,7 @@ tasks: - go fmt ./... - go tool gofumpt -l -w . - go vet ./... + - go fix ./... build: aliases: [default] diff --git a/cli/dir/dir.go b/cli/dir/dir.go index ca46f331..c8861df9 100644 --- a/cli/dir/dir.go +++ b/cli/dir/dir.go @@ -3,6 +3,7 @@ package dir import ( "errors" "fmt" + "regexp" internalcli "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobusterdir" @@ -36,11 +37,19 @@ func getFlags() []cli.Flag { &cli.BoolFlag{Name: "discover-backup", Aliases: []string{"db"}, Value: false, Usage: "Upon finding a file search for backup files by appending multiple backup extensions"}, &cli.StringFlag{Name: "exclude-length", Aliases: []string{"xl"}, Usage: "exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206"}, &cli.BoolFlag{Name: "force", Value: false, Usage: "Continue even if the prechecks fail. Please only use this if you know what you are doing, it can lead to unexpected results."}, + &cli.StringFlag{Name: "regex", Aliases: []string{"re"}, Usage: "Use regex to filter the results, by inspecting the content of the response body. When using this option be sure to set the status-codes and status-codes-blacklist options accordingly. The regex check is done after the status code checks. Only responses matching the regex will be displayed."}, + &cli.StringFlag{Name: "regex-invert", Aliases: []string{"rei"}, Usage: "Use regex to filter the results, but inverted, by inspecting the content of the response body. When using this option be sure to set the status-codes and status-codes-blacklist options accordingly. The regex check is done after the status code checks. Only responses NOT matching the regex will be displayed."}, }...) return flags } func run(c *cli.Context) error { + globalOpts, err := internalcli.ParseGlobalOptions(c) + if err != nil { + return err + } + log := libgobuster.NewLogger(globalOpts.Debug) + pluginOpts := gobusterdir.NewOptions() httpOptions, err := internalcli.ParseCommonHTTPOptions(c) @@ -101,12 +110,26 @@ func run(c *cli.Context) error { } pluginOpts.ExcludeLengthParsed = ret4 - globalOpts, err := internalcli.ParseGlobalOptions(c) - if err != nil { - return err + if c.IsSet("regex") && c.IsSet("regex-invert") { + return errors.New("regex and regex-invert are mutually exclusive, please set only one") } - log := libgobuster.NewLogger(globalOpts.Debug) + if c.IsSet("regex") && c.String("regex") != "" { + regex, err := regexp.Compile(c.String("regex")) + if err != nil { + return fmt.Errorf("invalid value for regex: %w", err) + } + pluginOpts.Regex = regex + } + + if c.IsSet("regex-invert") && c.String("regex-invert") != "" { + regex, err := regexp.Compile(c.String("regex-invert")) + if err != nil { + return fmt.Errorf("invalid value for regex-invert: %w", err) + } + pluginOpts.Regex = regex + pluginOpts.RegexInvert = true + } plugin, err := gobusterdir.New(&globalOpts, pluginOpts, log) if err != nil { @@ -114,8 +137,7 @@ func run(c *cli.Context) error { } if err := internalcli.Gobuster(c.Context, &globalOpts, plugin, log); err != nil { - var wErr *gobusterdir.WildcardError - if errors.As(err, &wErr) { + if wErr, ok := errors.AsType[*gobusterdir.WildcardError](err); ok { return fmt.Errorf("%w. To continue please exclude the status code or the length", wErr) } log.Debugf("%#v", err) diff --git a/cli/dns/dns.go b/cli/dns/dns.go index bca6914e..f4c3dad6 100644 --- a/cli/dns/dns.go +++ b/cli/dns/dns.go @@ -72,8 +72,7 @@ func run(c *cli.Context) error { log := libgobuster.NewLogger(globalOpts.Debug) if err := internalcli.Gobuster(c.Context, &globalOpts, plugin, log); err != nil { - var wErr *gobusterdns.WildcardError - if errors.As(err, &wErr) { + if wErr, ok := errors.AsType[*gobusterdns.WildcardError](err); ok { return fmt.Errorf("%w. To force processing of Wildcard DNS, specify the '--wildcard' switch", wErr) } log.Debugf("%#v", err) diff --git a/cli/gobuster.go b/cli/gobuster.go index 3841db39..a2b44093 100644 --- a/cli/gobuster.go +++ b/cli/gobuster.go @@ -112,6 +112,7 @@ func progressWorker(ctx context.Context, g *libgobuster.Gobuster, wg *sync.WaitG defer wg.Done() tick := time.NewTicker(cliProgressUpdate) + defer tick.Stop() for { select { diff --git a/cli/options.go b/cli/options.go index 7f035744..4f26f306 100644 --- a/cli/options.go +++ b/cli/options.go @@ -129,7 +129,8 @@ func CommonHTTPOptions() []cli.Flag { &cli.BoolFlag{Name: "follow-redirect", Aliases: []string{"r"}, Value: false, Usage: "Follow redirects"}, &cli.StringSliceFlag{Name: "headers", Aliases: []string{"H"}, Usage: "Specify HTTP headers, -H 'Header1: val1' -H 'Header2: val2'"}, &cli.BoolFlag{Name: "no-canonicalize-headers", Aliases: []string{"nch"}, Value: false, Usage: "Do not canonicalize HTTP header names. If set header names are sent as is"}, - &cli.StringFlag{Name: "method", Aliases: []string{"m"}, Value: "GET", Usage: "the password to the p12 file"}, + &cli.StringFlag{Name: "method", Aliases: []string{"m"}, Value: "GET", Usage: "Specify HTTP method"}, + &cli.StringFlag{Name: "body-output-dir", Usage: "Directory to store response bodies"}, }...) flags = append(flags, BasicHTTPOptions()...) return flags @@ -209,6 +210,14 @@ func ParseCommonHTTPOptions(c *cli.Context) (libgobuster.HTTPOptions, error) { opts.Headers = append(opts.Headers, header) } + if c.IsSet("body-output-dir") { + opts.BodyOutputDir = c.String("body-output-dir") + err = os.MkdirAll(opts.BodyOutputDir, 0o755) + if err != nil { + return opts, fmt.Errorf("could not create body output dir %q: %w", opts.BodyOutputDir, err) + } + } + return opts, nil } @@ -241,6 +250,9 @@ func ParseGlobalOptions(c *cli.Context) (libgobuster.Options, error) { opts.Delay = c.Duration("delay") opts.Threads = c.Int("threads") + if opts.Threads <= 0 { + return opts, errors.New("threads must be bigger than 0") + } opts.WordlistOffset = c.Int("wordlist-offset") if opts.Wordlist == "-" && opts.WordlistOffset > 0 { return opts, errors.New("wordlist-offset is not supported when reading from STDIN") @@ -274,7 +286,7 @@ func ParseGlobalOptions(c *cli.Context) (libgobuster.Options, error) { opts.DiscoverPatternFile = c.String("discover-pattern") if opts.DiscoverPatternFile != "" { - if _, err := os.Stat(opts.PatternFile); os.IsNotExist(err) { + if _, err := os.Stat(opts.DiscoverPatternFile); os.IsNotExist(err) { return opts, fmt.Errorf("discover pattern file %q does not exist: %w", opts.DiscoverPatternFile, err) } discoverPatternFile, err := os.Open(opts.DiscoverPatternFile) diff --git a/go.mod b/go.mod index aa3bb380..ecf478c6 100644 --- a/go.mod +++ b/go.mod @@ -1,31 +1,30 @@ module github.com/OJ/gobuster/v3 -go 1.25 +go 1.27.0 require ( - github.com/fatih/color v1.18.0 + github.com/fatih/color v1.19.0 github.com/google/uuid v1.6.0 - github.com/pin/tftp/v3 v3.1.0 + github.com/pin/tftp/v3 v3.2.0 github.com/urfave/cli/v2 v2.27.7 go.uber.org/automaxprocs v1.6.0 - golang.org/x/term v0.37.0 - software.sslmate.com/src/go-pkcs12 v0.6.0 + golang.org/x/term v0.45.0 + software.sslmate.com/src/go-pkcs12 v0.7.3 ) require ( github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/google/go-cmp v0.7.0 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/mod v0.27.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/tools v0.36.0 // indirect - mvdan.cc/gofumpt v0.9.0 // indirect + golang.org/x/crypto v0.56.0 // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/tools v0.49.0 // indirect + mvdan.cc/gofumpt v0.11.0 // indirect ) tool mvdan.cc/gofumpt diff --git a/go.sum b/go.sum index 7d3c5f60..05e54828 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= -github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= -github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= -github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -14,18 +14,18 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/pin/tftp/v3 v3.1.0 h1:rQaxd4pGwcAJnpId8zC+O2NX3B2/NscjDZQaqEjuE7c= -github.com/pin/tftp/v3 v3.1.0/go.mod h1:xwQaN4viYL019tM4i8iecm++5cGxSqen6AJEOEyEI0w= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/pin/tftp/v3 v3.2.0 h1:q6K5G6T0TA7e3wDJsB/7VpD3iaWwVEJD/nEuh3q9Sk0= +github.com/pin/tftp/v3 v3.2.0/go.mod h1:qc5ySXB5aOS1H6ULneqB4g5nshqV1CgeV/l/M6rEDms= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= @@ -36,28 +36,23 @@ github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAz github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -mvdan.cc/gofumpt v0.9.0 h1:W0wNHMSvDBDIyZsm3nnGbVfgp5AknzBrGJnfLCy501w= -mvdan.cc/gofumpt v0.9.0/go.mod h1:3xYtNemnKiXaTh6R4VtlqDATFwBbdXI8lJvH/4qk7mw= -software.sslmate.com/src/go-pkcs12 v0.6.0 h1:f3sQittAeF+pao32Vb+mkli+ZyT+VwKaD014qFGq6oU= -software.sslmate.com/src/go-pkcs12 v0.6.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= +mvdan.cc/gofumpt v0.11.0 h1:0H01XB95PnN2QgCSR9ELdZyTlJqNZ7181B0BTMh5VZc= +mvdan.cc/gofumpt v0.11.0/go.mod h1:BeT5wCsOJt6J9zT2MZIOGszjUHzFkn1/l9g6xAzqsXo= +software.sslmate.com/src/go-pkcs12 v0.7.3 h1:JBQD3FDqYjTeyDAeZQklj2ar88ykBLtALloPJHyAauU= +software.sslmate.com/src/go-pkcs12 v0.7.3/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/gobusterdir/gobusterdir.go b/gobusterdir/gobusterdir.go index 4f1d8f37..bd683711 100644 --- a/gobusterdir/gobusterdir.go +++ b/gobusterdir/gobusterdir.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "syscall" "text/tabwriter" @@ -40,7 +41,7 @@ func (e *WildcardError) Error() string { } else { addInfo = fmt.Sprintf("%s => %d (Length: %d)", e.url, e.statusCode, e.length) } - return fmt.Sprintf("the server returns a status code that matches the provided options for non existing urls. %s. Please exclude the response length or the status code or set the wildcard option.", addInfo) + return fmt.Sprintf("the server returns a status code that matches the provided options for non existing urls. %s. Please exclude the response length (or as a range), the status code or set the force option (but expect false positives).", addInfo) } // GobusterDir is the main type to implement the interface @@ -86,6 +87,7 @@ func New(globalopts *libgobuster.Options, opts *OptionsDir, logger *libgobuster. NoCanonicalizeHeaders: opts.NoCanonicalizeHeaders, Cookies: opts.Cookies, Method: opts.Method, + BodyOutputDir: opts.BodyOutputDir, } h, err := libgobuster.NewHTTPClient(&httpOpts, logger) @@ -102,6 +104,18 @@ func (d *GobusterDir) Name() string { return "directory enumeration" } +func (d *GobusterDir) regexBodyIsAMatch(body []byte) bool { + switch { + case d.options.Regex != nil && body != nil: + if d.options.RegexInvert { + return !d.options.Regex.Match(body) + } + return d.options.Regex.Match(body) + default: + return true + } +} + // PreRun is the pre run implementation of gobusterdir func (d *GobusterDir) PreRun(ctx context.Context, pr *libgobuster.Progress) error { // add trailing slash @@ -139,7 +153,7 @@ func (d *GobusterDir) PreRun(ctx context.Context, pr *libgobuster.Progress) erro url.Path = fmt.Sprintf("%s/", url.Path) } - wildcardResp, wildcardLength, wildcardHeader, _, err := d.http.Request(ctx, url, libgobuster.RequestOptions{}) + wildcardResp, wildcardLength, wildcardHeader, wildcardBody, err := d.http.Request(ctx, url, libgobuster.RequestOptions{ReturnBody: true}) if err != nil { var retErr error switch { @@ -170,10 +184,16 @@ func (d *GobusterDir) PreRun(ctx context.Context, pr *libgobuster.Progress) erro switch { case d.options.StatusCodesBlacklistParsed.Length() > 0: if !d.options.StatusCodesBlacklistParsed.Contains(wildcardResp) { + if d.regexBodyIsAMatch(wildcardBody) { + return nil + } return &WildcardError{url: url.String(), statusCode: wildcardResp, length: wildcardLength, location: wildcardHeader.Get("Location")} } case d.options.StatusCodesParsed.Length() > 0: if d.options.StatusCodesParsed.Contains(wildcardResp) { + if d.regexBodyIsAMatch(wildcardBody) { + return nil + } return &WildcardError{url: url.String(), statusCode: wildcardResp, length: wildcardLength, location: wildcardHeader.Get("Location")} } default: @@ -252,9 +272,16 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li var statusCode int var size int64 var header http.Header + var body []byte + + requestOptions := libgobuster.RequestOptions{} + if d.options.Regex != nil || d.options.BodyOutputDir != "" { + requestOptions.ReturnBody = true + } + for i := 1; i <= tries; i++ { var err error - statusCode, size, header, _, err = d.http.Request(ctx, url, libgobuster.RequestOptions{}) + statusCode, size, header, body, err = d.http.Request(ctx, url, requestOptions) if err != nil { // check if it's a timeout and if we should try again and try again // otherwise the timeout error is raised @@ -280,17 +307,29 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li break } + if d.options.BodyOutputDir != "" && body != nil { + fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(entity, "/"), statusCode)) + fpath := filepath.Join(d.options.BodyOutputDir, fname) + err := os.WriteFile(fpath, body, 0o600) // nolint:gosec + if err != nil { + progress.MessageChan <- libgobuster.Message{ + Level: libgobuster.LevelError, + Message: fmt.Sprintf("Could not write body to file %s: %v", fpath, err), + } + } + } + if statusCode != 0 { resultStatus := false switch { case d.options.StatusCodesBlacklistParsed.Length() > 0: if !d.options.StatusCodesBlacklistParsed.Contains(statusCode) { - resultStatus = true + resultStatus = d.regexBodyIsAMatch(body) } case d.options.StatusCodesParsed.Length() > 0: if d.options.StatusCodesParsed.Contains(statusCode) { - resultStatus = true + resultStatus = d.regexBodyIsAMatch(body) } default: return nil, errors.New("StatusCodes and StatusCodesBlacklist are both not set which should not happen") @@ -448,6 +487,18 @@ func (d *GobusterDir) GetConfigString() (string, error) { } } + if o.Regex != nil { + if o.RegexInvert { + if _, err := fmt.Fprintf(tw, "[+] Regex Inverted:\t%s\n", o.Regex.String()); err != nil { + return "", err + } + } else { + if _, err := fmt.Fprintf(tw, "[+] Regex:\t%s\n", o.Regex.String()); err != nil { + return "", err + } + } + } + if _, err := fmt.Fprintf(tw, "[+] Timeout:\t%s\n", o.Timeout.String()); err != nil { return "", err } diff --git a/gobusterdir/options.go b/gobusterdir/options.go index 3e1bdb59..1785b3ce 100644 --- a/gobusterdir/options.go +++ b/gobusterdir/options.go @@ -1,6 +1,8 @@ package gobusterdir import ( + "regexp" + "github.com/OJ/gobuster/v3/libgobuster" ) @@ -22,6 +24,8 @@ type OptionsDir struct { ExcludeLength string ExcludeLengthParsed libgobuster.Set[int] Force bool + Regex *regexp.Regexp + RegexInvert bool } // NewOptions returns a new initialized OptionsDir diff --git a/gobusterdns/gobusterdns.go b/gobusterdns/gobusterdns.go index 20c91b34..ad303e26 100644 --- a/gobusterdns/gobusterdns.go +++ b/gobusterdns/gobusterdns.go @@ -137,16 +137,16 @@ func (d *GobusterDNS) ProcessWord(ctx context.Context, word string, progress *li if !d.isWildcard || !d.wildcardIps.ContainsAny(ips) { result := Result{ Subdomain: strings.TrimSuffix(subdomain, "."), - } - result.IPs = ips + IPs: ips, + } if d.options.CheckCNAME { cname, err := d.dnsLookupCname(ctx, subdomain) if err == nil { result.CNAME = cname } else { var wErr *net.DNSError - if !errors.As(err, &wErr) && !wErr.IsNotFound { + if !errors.As(err, &wErr) || !wErr.IsNotFound { // host not found is the expected error here, send all other errors to the error channel progress.ErrorChan <- err } diff --git a/gobusterfuzz/gobusterfuzz.go b/gobusterfuzz/gobusterfuzz.go index a5cb028f..f30c55d0 100644 --- a/gobusterfuzz/gobusterfuzz.go +++ b/gobusterfuzz/gobusterfuzz.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "syscall" "text/tabwriter" @@ -52,13 +53,15 @@ func New(globalopts *libgobuster.Options, opts *OptionsFuzz, logger *libgobuster } basicOptions := libgobuster.BasicHTTPOptions{ - Proxy: opts.Proxy, - Timeout: opts.Timeout, - UserAgent: opts.UserAgent, - NoTLSValidation: opts.NoTLSValidation, - RetryOnTimeout: opts.RetryOnTimeout, - RetryAttempts: opts.RetryAttempts, - TLSCertificate: opts.TLSCertificate, + Proxy: opts.Proxy, + Timeout: opts.Timeout, + UserAgent: opts.UserAgent, + NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, + TLSCertificate: opts.TLSCertificate, + LocalAddr: opts.LocalAddr, + TLSRenegotiation: opts.TLSRenegotiation, } httpOpts := libgobuster.HTTPOptions{ @@ -70,6 +73,7 @@ func New(globalopts *libgobuster.Options, opts *OptionsFuzz, logger *libgobuster NoCanonicalizeHeaders: opts.NoCanonicalizeHeaders, Cookies: opts.Cookies, Method: opts.Method, + BodyOutputDir: opts.BodyOutputDir, } h, err := libgobuster.NewHTTPClient(&httpOpts, logger) @@ -139,6 +143,10 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l requestOptions.UpdatedBasicAuthPassword = strings.ReplaceAll(d.options.Password, FuzzKeyword, word) } + if d.options.BodyOutputDir != "" { + requestOptions.ReturnBody = true + } + // add some debug output if d.globalopts.Debug { progress.MessageChan <- libgobuster.Message{ @@ -156,9 +164,10 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l var statusCode int var size int64 var responseHeaders http.Header + var body []byte for i := 1; i <= tries; i++ { var err error - statusCode, size, responseHeaders, _, err = d.http.Request(ctx, url, requestOptions) + statusCode, size, responseHeaders, body, err = d.http.Request(ctx, url, requestOptions) if err != nil { // check if it's a timeout and if we should try again and try again // otherwise the timeout error is raised @@ -185,6 +194,18 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l break } + if d.options.BodyOutputDir != "" && body != nil { + fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(word, "/"), statusCode)) + fpath := filepath.Join(d.options.BodyOutputDir, fname) + err := os.WriteFile(fpath, body, 0o600) // nolint:gosec + if err != nil { + progress.MessageChan <- libgobuster.Message{ + Level: libgobuster.LevelError, + Message: fmt.Sprintf("Could not write body to file %s: %v", fpath, err), + } + } + } + if statusCode != 0 { resultStatus := true diff --git a/gobustergcs/gobustersgcs.go b/gobustergcs/gobustersgcs.go index 54bda375..01f68b0a 100644 --- a/gobustergcs/gobustersgcs.go +++ b/gobustergcs/gobustersgcs.go @@ -43,13 +43,15 @@ func New(globalopts *libgobuster.Options, opts *OptionsGCS, logger *libgobuster. } basicOptions := libgobuster.BasicHTTPOptions{ - Proxy: opts.Proxy, - Timeout: opts.Timeout, - UserAgent: opts.UserAgent, - NoTLSValidation: opts.NoTLSValidation, - RetryOnTimeout: opts.RetryOnTimeout, - RetryAttempts: opts.RetryAttempts, - TLSCertificate: opts.TLSCertificate, + Proxy: opts.Proxy, + Timeout: opts.Timeout, + UserAgent: opts.UserAgent, + NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, + TLSCertificate: opts.TLSCertificate, + LocalAddr: opts.LocalAddr, + TLSRenegotiation: opts.TLSRenegotiation, } httpOpts := libgobuster.HTTPOptions{ @@ -166,7 +168,7 @@ func (s *GobusterGCS) ProcessWord(ctx context.Context, word string, progress *li if s.options.ShowFiles { // get status - var result map[string]interface{} + var result map[string]any err := json.Unmarshal(body, &result) if err != nil { return nil, fmt.Errorf("could not parse response json: %w", err) diff --git a/gobustervhost/gobustervhost.go b/gobustervhost/gobustervhost.go index f2bae76d..113d7744 100644 --- a/gobustervhost/gobustervhost.go +++ b/gobustervhost/gobustervhost.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "sync" "syscall" @@ -45,13 +46,15 @@ func New(globalopts *libgobuster.Options, opts *OptionsVhost, logger *libgobuste } basicOptions := libgobuster.BasicHTTPOptions{ - Proxy: opts.Proxy, - Timeout: opts.Timeout, - UserAgent: opts.UserAgent, - NoTLSValidation: opts.NoTLSValidation, - RetryOnTimeout: opts.RetryOnTimeout, - RetryAttempts: opts.RetryAttempts, - TLSCertificate: opts.TLSCertificate, + Proxy: opts.Proxy, + Timeout: opts.Timeout, + UserAgent: opts.UserAgent, + NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, + TLSCertificate: opts.TLSCertificate, + LocalAddr: opts.LocalAddr, + TLSRenegotiation: opts.TLSRenegotiation, } httpOpts := libgobuster.HTTPOptions{ @@ -63,6 +66,7 @@ func New(globalopts *libgobuster.Options, opts *OptionsVhost, logger *libgobuste NoCanonicalizeHeaders: opts.NoCanonicalizeHeaders, Cookies: opts.Cookies, Method: opts.Method, + BodyOutputDir: opts.BodyOutputDir, } h, err := libgobuster.NewHTTPClient(&httpOpts, logger) @@ -197,6 +201,18 @@ func (v *GobusterVhost) ProcessWord(ctx context.Context, word string, progress * break } + if v.options.BodyOutputDir != "" && body != nil { + fname := libgobuster.SanitizeFilename(fmt.Sprintf("%s_%d.html", strings.Trim(word, "/"), statusCode)) + fpath := filepath.Join(v.options.BodyOutputDir, fname) + err := os.WriteFile(fpath, body, 0o600) // nolint:gosec + if err != nil { + progress.MessageChan <- libgobuster.Message{ + Level: libgobuster.LevelError, + Message: fmt.Sprintf("Could not write body to file %s: %v", fpath, err), + } + } + } + // subdomain must not match default vhost and non existent vhost // or verbose mode is enabled found := body != nil && !bytes.Equal(body, v.normalBody) && !bytes.Equal(body, v.abnormalBody) diff --git a/libgobuster/helpers.go b/libgobuster/helpers.go index 4acd82f3..3d85fe77 100644 --- a/libgobuster/helpers.go +++ b/libgobuster/helpers.go @@ -7,9 +7,11 @@ import ( "fmt" "io" "os" + "path/filepath" "regexp" "strconv" "strings" + "unicode" ) // Set is a set of Ts @@ -68,15 +70,14 @@ func (set *Set[T]) Stringify() string { return strings.Join(values, ",") } -// this method is much faster than lineCounter_slow but has the following errors: -// - empty files are reported as 1 line -// - files only containing a newline are reported as 1 line -// - also counts lines with comments +// This method is much faster than lineCounterSlow. It counts physical lines, +// including blank and comment lines; the feeder adjusts blank lines later. func lineCounter(r io.Reader) (int, error) { buf := make([]byte, 32*1024) - count := 1 + count := 0 lineSep := []byte{'\n'} var lastChar byte + var hasData bool for { c, err := r.Read(buf) @@ -84,14 +85,15 @@ func lineCounter(r io.Reader) (int, error) { // store last character received if we got any bytes if c > 0 { - lastChar = buf[c-1] + hasData = true + lastChar = buf[c-1] // nolint:gosec } switch { case errors.Is(err, io.EOF): - // account for trailing new line - if lastChar == '\n' { - count-- + // A final line without a newline still counts. + if hasData && lastChar != '\n' { + count++ } return count, nil @@ -132,7 +134,7 @@ func ParseExtensions(extensions string) (Set[string], error) { return ret, nil } - for _, e := range strings.Split(extensions, ",") { + for e := range strings.SplitSeq(extensions, ",") { e = strings.TrimSpace(e) // remove leading . from extensions ret.Add(strings.TrimPrefix(e, ".")) @@ -172,7 +174,7 @@ func ParseCommaSeparatedInt(inputString string) (Set[int], error) { return ret, nil } - for _, part := range strings.Split(inputString, ",") { + for part := range strings.SplitSeq(inputString, ",") { part = strings.TrimSpace(part) // check for range if strings.Contains(part, "-") { @@ -207,3 +209,71 @@ func ParseCommaSeparatedInt(inputString string) (Set[int], error) { } return ret, nil } + +// Windows reserved characters: < > : " | ? * and control characters (0-31) +var filenameInvalidChars = regexp.MustCompile(`[<>:"|?*\x00-\x1f]`) + +// sanitizeFilename removes or replaces invalid characters from a filename +// to make it safe for use on Windows, macOS, and Linux filesystems +func SanitizeFilename(filename string) string { + if filename == "" { + return "unnamed" + } + + // Remove leading/trailing whitespace + filename = strings.TrimSpace(filename) + + // Replace path separators and other problematic characters + filename = strings.ReplaceAll(filename, "/", "_") + filename = strings.ReplaceAll(filename, "\\", "_") + + filename = filenameInvalidChars.ReplaceAllString(filename, "_") + + // Remove non-printable Unicode characters + filename = strings.Map(func(r rune) rune { + if unicode.IsPrint(r) { + return r + } + return '_' + }, filename) + + // Windows reserved names (case-insensitive) + reservedNames := []string{ + "CON", "PRN", "AUX", "NUL", + "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", + } + + // Check if filename (without extension) is a reserved name + nameOnly := strings.TrimSuffix(filename, filepath.Ext(filename)) + for _, reserved := range reservedNames { + if strings.EqualFold(nameOnly, reserved) { + filename = "_" + filename + break + } + } + + // Remove trailing dots and spaces (Windows requirement) + filename = strings.TrimRight(filename, ". ") + + // Ensure filename isn't empty after sanitization + if filename == "" { + filename = "unnamed" + } + + filename = filepath.Base(filename) + + // Limit length to 255 characters (common filesystem limit) + if len(filename) > 255 { + ext := filepath.Ext(filename) + base := strings.TrimSuffix(filename, ext) + maxBase := 255 - len(ext) + if maxBase > 0 { + filename = base[:maxBase] + ext + } else { + filename = filename[:255] + } + } + + return filename +} diff --git a/libgobuster/helpers_test.go b/libgobuster/helpers_test.go index f2f39d04..ca29a8a6 100644 --- a/libgobuster/helpers_test.go +++ b/libgobuster/helpers_test.go @@ -152,7 +152,7 @@ func TestSetStringify(t *testing.T) { // order is random for _, i := range v2 { if !strings.Contains(z, strconv.Itoa(i)) { - t.Fatalf("Did not find value %q in %q", i, z) + t.Fatalf("Did not find value %d in %q", i, z) } } } @@ -168,9 +168,9 @@ func TestLineCounter(t *testing.T) { {"3 Lines", "TestString\nTest\n1234", 3}, {"Trailing newline", "TestString\nTest\n1234\n", 3}, {"3 Lines cr lf", "TestString\r\nTest\r\n1234", 3}, - {"Empty", "", 1}, // these are wrong, but I've found no good way to handle those - {"Empty 2", "\n", 1}, // these are wrong, but I've found no good way to handle those - {"Empty 3", "\r\n", 1}, // these are wrong, but I've found no good way to handle those + {"Empty", "", 0}, + {"Empty 2", "\n", 1}, + {"Empty 3", "\r\n", 1}, } for _, x := range tt { t.Run(x.testName, func(t *testing.T) { @@ -392,3 +392,173 @@ func BenchmarkParseCommaSeparatedInt(b *testing.B) { }) } } + +func TestSanitizeFilename(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "empty string", + input: "", + expected: "unnamed", + }, + { + name: "normal filename", + input: "test.txt", + expected: "test.txt", + }, + { + name: "filename with spaces", + input: " test file.txt ", + expected: "test file.txt", + }, + { + name: "filename with path separators", + input: "folder/test\\file.txt", + expected: "folder_test_file.txt", + }, + { + name: "filename with Windows invalid characters", + input: "testname:with|invalid?chars*.txt", + expected: "test_file_name_with_invalid_chars_.txt", + }, + { + name: "filename with control characters", + input: "test\x00file\x1fname.txt", + expected: "test_file_name.txt", + }, + { + name: "filename with non-printable Unicode", + input: "test\u200bfile\u2028name.txt", + expected: "test_file_name.txt", + }, + { + name: "Windows reserved name - CON", + input: "CON.txt", + expected: "_CON.txt", + }, + { + name: "Windows reserved name - PRN (lowercase)", + input: "prn.log", + expected: "_prn.log", + }, + { + name: "Windows reserved name - COM1", + input: "COM1", + expected: "_COM1", + }, + { + name: "Windows reserved name - LPT9", + input: "lpt9.dat", + expected: "_lpt9.dat", + }, + { + name: "filename with reserved name as part of longer name", + input: "CONfig.txt", + expected: "CONfig.txt", + }, + { + name: "filename with trailing dots and spaces", + input: "test.txt.. ", + expected: "test.txt", + }, + { + name: "filename that becomes empty after sanitization", + input: "... ", + expected: "unnamed", + }, + { + name: "very long filename", + input: strings.Repeat("a", 300) + ".txt", + expected: strings.Repeat("a", 251) + ".txt", // 255 total + }, + { + name: "long filename with long extension", + input: strings.Repeat("a", 250) + "." + strings.Repeat("b", 10), + expected: strings.Repeat("a", 244) + "." + strings.Repeat("b", 10), // 255 total + }, + { + name: "long filename where extension is too long", + input: "test." + strings.Repeat("b", 260), + expected: ("test." + strings.Repeat("b", 260))[:255], + }, + { + name: "whitespace only", + input: " \t\n ", + expected: "unnamed", + }, + { + name: "mixed invalid characters and reserved name", + input: "aux|withchars.log", + expected: "aux_with_invalid_chars.log", + }, + { + name: "reserved name", + input: "AUX.log", + expected: "_AUX.log", + }, + { + name: "Unicode filename", + input: "тест файл.txt", + expected: "тест файл.txt", + }, + { + name: "filename with quotes", + input: `"quoted filename".txt`, + expected: "_quoted filename_.txt", + }, + { + name: "filename with pipe character", + input: "file|with|pipes.txt", + expected: "file_with_pipes.txt", + }, + { + name: "filename with question marks", + input: "what?is?this?.txt", + expected: "what_is_this_.txt", + }, + { + name: "filename with asterisks", + input: "wild*card*name*.txt", + expected: "wild_card_name_.txt", + }, + { + name: "only extension", + input: ".hidden", + expected: ".hidden", + }, + { + name: "reserved name without extension", + input: "NUL", + expected: "_NUL", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SanitizeFilename(tt.input) + if result != tt.expected { + t.Errorf("sanitizeFilename(%q) = %q, want %q", tt.input, result, tt.expected) + } + + // Additional validation: ensure result is safe + if len(result) > 255 { + t.Errorf("sanitizeFilename(%q) returned filename too long: %d characters", tt.input, len(result)) + } + + if strings.ContainsAny(result, `<>:"|?*`) { + t.Errorf("sanitizeFilename(%q) still contains invalid characters: %q", tt.input, result) + } + + if strings.Contains(result, "/") || strings.Contains(result, "\\") { + t.Errorf("sanitizeFilename(%q) still contains path separators: %q", tt.input, result) + } + + if result == "" { + t.Errorf("sanitizeFilename(%q) returned empty string", tt.input) + } + }) + } +} diff --git a/libgobuster/http.go b/libgobuster/http.go index 9a80e0a5..896b8e6b 100644 --- a/libgobuster/http.go +++ b/libgobuster/http.go @@ -13,6 +13,8 @@ import ( "strings" ) +const defaultMaxResponseBodySize int64 = 10 * 1024 * 1024 + // HTTPHeader holds a single key value pair of a HTTP header type HTTPHeader struct { Name string @@ -32,6 +34,7 @@ type HTTPClient struct { method string host string logger *Logger + maxResponseBodySize int64 } // RequestOptions is used to pass options to a single individual request @@ -125,6 +128,7 @@ func NewHTTPClient(opt *HTTPOptions, logger *Logger) (*HTTPClient, error) { } } client.logger = logger + client.maxResponseBodySize = defaultMaxResponseBodySize return &client, nil } @@ -144,10 +148,13 @@ func (client *HTTPClient) Request(ctx context.Context, fullURL url.URL, opts Req var body []byte var length int64 if opts.ReturnBody { - body, err = io.ReadAll(resp.Body) + body, err = io.ReadAll(io.LimitReader(resp.Body, client.maxResponseBodySize+1)) if err != nil { return 0, 0, nil, nil, fmt.Errorf("could not read body %w", err) } + if int64(len(body)) > client.maxResponseBodySize { + return 0, 0, nil, nil, fmt.Errorf("response body exceeds maximum size of %d bytes", client.maxResponseBodySize) + } length = int64(len(body)) } else { // DO NOT REMOVE! @@ -226,10 +233,9 @@ func (client *HTTPClient) makeRequest(ctx context.Context, fullURL url.URL, opts client.logger.Debugf("%s", dump) } - resp, err := client.client.Do(req) + resp, err := client.client.Do(req) // nolint:gosec if err != nil { - var ue *url.Error - if errors.As(err, &ue) { + if ue, ok := errors.AsType[*url.Error](err); ok { if strings.HasPrefix(ue.Err.Error(), "x509") { return nil, fmt.Errorf("invalid certificate: %w", ue.Err) } diff --git a/libgobuster/http_test.go b/libgobuster/http_test.go index 84804610..3e9cbf7e 100644 --- a/libgobuster/http_test.go +++ b/libgobuster/http_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strings" "testing" ) @@ -79,6 +80,26 @@ func TestRequest(t *testing.T) { } } +func TestRequestRejectsOversizedBody(t *testing.T) { + t.Parallel() + h := httpServerT(t, strings.Repeat("x", 11*1024*1024)) + defer h.Close() + + c, err := NewHTTPClient(&HTTPOptions{}, NewLogger(false)) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(h.URL) + if err != nil { + t.Fatal(err) + } + + _, _, _, _, err = c.Request(t.Context(), *u, RequestOptions{ReturnBody: true}) + if err == nil || !strings.Contains(err.Error(), "response body exceeds maximum size") { + t.Fatalf("expected response size error, got %v", err) + } +} + func BenchmarkRequestWithoutBody(b *testing.B) { r, err := randomString(10000) if err != nil { diff --git a/libgobuster/libgobuster.go b/libgobuster/libgobuster.go index 1e803d62..91f1cc1d 100644 --- a/libgobuster/libgobuster.go +++ b/libgobuster/libgobuster.go @@ -15,6 +15,8 @@ import ( // PATTERN is the pattern for wordlist replacements in pattern file const PATTERN = "{GOBUSTER}" +const maxWordlistLineSize = 10 * 1024 * 1024 + // SetupFunc is the "setup" function prototype for implementations type SetupFunc func(*Gobuster) error @@ -120,8 +122,9 @@ func (g *Gobuster) feeder(ctx context.Context, guessChan chan<- *Guess, words [] feed(ctx, guessChan, words, discoverOnSuccess) } -func (g *Gobuster) feedWordlist(ctx context.Context, guessChan chan<- *Guess, wordlist *Wordlist, wg *sync.WaitGroup) { +func (g *Gobuster) feedWordlist(ctx context.Context, guessChan chan<- *Guess, wordlist *Wordlist, scanDone chan<- error, wg *sync.WaitGroup) { defer wg.Done() + defer func() { scanDone <- wordlist.scanner.Err() }() for wordlist.scanner.Scan() { // Prioritize stopping when the context is done @@ -183,7 +186,9 @@ func (g *Gobuster) getWordlist(wordlist io.ReadSeeker) (*Wordlist, error) { if g.Opts.Wordlist == "-" { // Read directly from stdin - return &Wordlist{scanner: bufio.NewScanner(os.Stdin), guessesPerLine: guessesPerLine, isStream: true}, nil + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, 64*1024), maxWordlistLineSize) + return &Wordlist{scanner: scanner, guessesPerLine: guessesPerLine, isStream: true}, nil } lines, err := lineCounter(wordlist) @@ -207,6 +212,7 @@ func (g *Gobuster) getWordlist(wordlist io.ReadSeeker) (*Wordlist, error) { } wordlistScanner := bufio.NewScanner(wordlist) + wordlistScanner.Buffer(make([]byte, 64*1024), maxWordlistLineSize) // skip lines for range g.Opts.WordlistOffset { @@ -242,6 +248,7 @@ func (g *Gobuster) Run(ctx context.Context) error { guessChan := make(chan *Guess, g.Opts.Threads*3) successChan := make(chan *Guess) + scanDone := make(chan error, 1) var f io.ReadSeekCloser if g.Opts.Wordlist != "-" { // stdin case is handled inside getWordlist @@ -265,8 +272,10 @@ func (g *Gobuster) Run(ctx context.Context) error { } feederGroup.Add(1) - go g.feedWordlist(feederCtx, guessChan, wordlist, &feederGroup) + go g.feedWordlist(feederCtx, guessChan, wordlist, scanDone, &feederGroup) + wordlistFinished := false + var scanErr error ListenForMore: for { // Prioritize stopping when the context is done @@ -279,6 +288,11 @@ ListenForMore: select { case <-ctx.Done(): break ListenForMore + case scanErr = <-scanDone: + wordlistFinished = true + if scanErr != nil { + break ListenForMore + } case successGuess := <-successChan: // Add more guesses based on the results of previous attempts if successGuess.discoverOnSuccess { @@ -301,7 +315,7 @@ ListenForMore: // reported, this is well ordered without the timeout, however it would // exert a lot of lock pressure during the run to keep doing this in a // hot loop - if g.Progress.RequestsExpected() == g.Progress.RequestsIssued() { + if wordlistFinished && g.Progress.RequestsExpected() == g.Progress.RequestsIssued() { // All the expected requests have completed, there is no pending or // in-progress work. If moreWordsChan was buffered we would need to // check it again here to ensure no pending work was added while we @@ -316,8 +330,8 @@ ListenForMore: feederGroup.Wait() workerGroup.Wait() - if err := wordlist.scanner.Err(); err != nil { - return err + if scanErr != nil { + return scanErr } return nil diff --git a/libgobuster/libgobuster_test.go b/libgobuster/libgobuster_test.go new file mode 100644 index 00000000..8f65b9ca --- /dev/null +++ b/libgobuster/libgobuster_test.go @@ -0,0 +1,55 @@ +package libgobuster + +import ( + "context" + "os" + "strings" + "testing" + "time" +) + +type testPlugin struct{} + +func (testPlugin) Name() string { return "test" } +func (testPlugin) PreRun(context.Context, *Progress) error { return nil } +func (testPlugin) ProcessWord(context.Context, string, *Progress) (Result, error) { + return nil, nil //nolint:nilnil // A test plugin intentionally produces no result. +} +func (testPlugin) AdditionalWords(string) []string { return nil } +func (testPlugin) AdditionalWordsLen() int { return 0 } +func (testPlugin) AdditionalSuccessWords(string) []string { return nil } +func (testPlugin) GetConfigString() (string, error) { return "", nil } + +func TestRunRejectsEmptyWordlist(t *testing.T) { + t.Parallel() + wordlist := t.TempDir() + "/empty.txt" + if err := os.WriteFile(wordlist, nil, 0o600); err != nil { + t.Fatal(err) + } + + g, err := NewGobuster(&Options{Threads: 1, Wordlist: wordlist}, testPlugin{}, NewLogger(false)) + if err != nil { + t.Fatal(err) + } + if err := g.Run(t.Context()); err == nil { + t.Fatal("expected an empty wordlist error") + } +} + +func TestRunReturnsOversizedWordlistLineError(t *testing.T) { + t.Parallel() + wordlist := t.TempDir() + "/large.txt" + if err := os.WriteFile(wordlist, []byte(strings.Repeat("x", maxWordlistLineSize+1)), 0o600); err != nil { + t.Fatal(err) + } + + g, err := NewGobuster(&Options{Threads: 1, Wordlist: wordlist}, testPlugin{}, NewLogger(false)) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if err := g.Run(ctx); err == nil { + t.Fatal("expected an oversized wordlist line error") + } +} diff --git a/libgobuster/options_http.go b/libgobuster/options_http.go index f4dcbbe6..e50e497a 100644 --- a/libgobuster/options_http.go +++ b/libgobuster/options_http.go @@ -23,7 +23,7 @@ type BasicHTTPOptions struct { // HTTPOptions is the struct to pass in all http options to Gobuster type HTTPOptions struct { BasicHTTPOptions - Password string + Password string // nolint:gosec URL *url.URL Username string Cookies string @@ -31,4 +31,5 @@ type HTTPOptions struct { NoCanonicalizeHeaders bool FollowRedirect bool Method string + BodyOutputDir string } diff --git a/libgobuster/version.go b/libgobuster/version.go index 69134730..51a56265 100644 --- a/libgobuster/version.go +++ b/libgobuster/version.go @@ -8,7 +8,7 @@ import ( const ( // VERSION contains the current gobuster version - VERSION = "3.8.2" + VERSION = "3.8.3" ) func GetVersion() string { diff --git a/main.go b/main.go index 08ce1ae1..bb300dd5 100644 --- a/main.go +++ b/main.go @@ -54,6 +54,7 @@ func main() { s3.Command(), gcs.Command(), }, + DisableSliceFlagSeparator: true, // needed so we can specify ',' in slice flags. Otherwise urfave/cli splits on ',' } err := app.Run(os.Args) diff --git a/vhs/gobuster_dir.tape b/vhs/gobuster_dir.tape index 2b4e8ecb..04766c3a 100644 --- a/vhs/gobuster_dir.tape +++ b/vhs/gobuster_dir.tape @@ -55,7 +55,7 @@ # Hide Hide the subsequent commands from the output # Show Show the subsequent commands in the output -Output vhs/gobuster_dir.gif +Output gobuster_dir.gif # Require gobuster diff --git a/vhs/server.go b/vhs/server.go index 749fa97f..c1a5e4be 100644 --- a/vhs/server.go +++ b/vhs/server.go @@ -41,8 +41,8 @@ func main() { })}) x.routes = append(x.routes, &route{regexp.MustCompile(`^/`), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - if _, err := w.Write([]byte(r.URL.Path)); err != nil { - log.Fatal(err.Error()) + if _, err := w.Write([]byte(r.URL.Path)); err != nil { // nolint:gosec + log.Fatal(err.Error()) // nolint:gosec } })})