feat: add -no-classify to skip ML page-type model download

README.md

@@ -153,6 +153,7 @@ FILTERS:
    -fc, -filter-code string               filter response with specified status code (-fc 403,401)
    -fpt, -filter-page-type string[]       filter response with specified page type (e.g. -fpt login,captcha,parked)
    -fep, -filter-error-page               [DEPRECATED: use -fpt] filter response with ML based error page detection
+   -nc, -no-classify                      disable ML page-type classification (skips ~92MB model download with -json/-csv)
    -fd, -filter-duplicates                filter out near-duplicate responses (only first response is retained)
    -fl, -filter-length string             filter response with specified content length (-fl 23,33)
    -flc, -filter-line-count string        filter response body with specified line count (-flc 423,532)

runner/options.go

@@ -204,6 +204,9 @@ type Options struct {
 	// Deprecated: use OutputFilterPageType with "error" instead.
 	OutputFilterErrorPage bool
 	OutputFilterPageType      goflags.StringSlice
+	// NoClassify disables ML page-type classification (and its model download).
+	// Useful with -json/-csv when KnowledgeBase/PageType is not needed.
+	NoClassify                bool
 	FilterOutDuplicates       bool
 	OutputFilterContentLength string
 	InputRawRequest           string
@@ -456,6 +459,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
 		flagSet.StringSliceVarP(&options.OutputFilterPageType, "filter-page-type", "fpt", nil, "filter response with specified page type (e.g. -fpt login,captcha,parked)", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "[DEPRECATED: use -fpt] filter response with ML based error page detection"),
+		flagSet.BoolVarP(&options.NoClassify, "no-classify", "nc", false, "disable ML page-type classification (skips ~92MB model download with -json/-csv)"),
 		flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses (only first response is retained)"),
 		flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
 		flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
@@ -720,6 +724,16 @@ func (options *Options) HasMatcherOrFilter() bool {
 		options.OutputFilterResponseTime != ""
 }
 
+// ShouldInitPageClassifier reports whether the ML page-type classifier should be loaded.
+// -json/-csv historically always initialized it for KnowledgeBase/PageType enrichment;
+// -no-classify opts out so those formats do not download the ~92MB model when unused.
+func (options *Options) ShouldInitPageClassifier() bool {
+	if options.NoClassify {
+		return false
+	}
+	return options.JSONOutput || options.CSVOutput || len(options.OutputFilterPageType) > 0
+}
+
 func (options *Options) ValidateOptions() error {
 	if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
 		return fmt.Errorf("file '%s' does not exist", options.InputFile)
@@ -854,6 +868,10 @@ func (options *Options) ValidateOptions() error {
 		options.OutputCDN = "true"
 	}
 
+	if options.NoClassify && (len(options.OutputFilterPageType) > 0 || options.OutputFilterErrorPage) {
+		return fmt.Errorf("-no-classify cannot be combined with -filter-page-type/-fpt or -filter-error-page/-fep")
+	}
+
 	if !stringsutil.EqualFoldAny(options.Protocol, string(httpxcommon.UNKNOWN), string(httpxcommon.HTTP11)) {
 		return fmt.Errorf("invalid protocol: %s", options.Protocol)
 	}

runner/runner.go

@@ -431,7 +431,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	runner.simHashes = gcache.New[uint64, []string](1000).ARC().Build()
-	if options.JSONOutput || options.CSVOutput || len(options.OutputFilterPageType) > 0 {
+	if options.ShouldInitPageClassifier() {
 		ditClassifier, err := dit.New()
 		if err != nil {
 			gologger.Warning().Msgf("Could not initialize page classifier: %s", err)

runner/runner_test.go

@@ -515,6 +515,40 @@ func TestOptions_hasMatcherOrFilter(t *testing.T) {
 	}
 }
 
+func TestShouldInitPageClassifier(t *testing.T) {
+	tests := []struct {
+		name     string
+		options  Options
+		expected bool
+	}{
+		{name: "default", options: Options{}, expected: false},
+		{name: "json", options: Options{JSONOutput: true}, expected: true},
+		{name: "csv", options: Options{CSVOutput: true}, expected: true},
+		{name: "filter page type", options: Options{OutputFilterPageType: []string{"login"}}, expected: true},
+		{name: "json with no-classify", options: Options{JSONOutput: true, NoClassify: true}, expected: false},
+		{name: "csv with no-classify", options: Options{CSVOutput: true, NoClassify: true}, expected: false},
+		{name: "no-classify alone", options: Options{NoClassify: true}, expected: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expected, tc.options.ShouldInitPageClassifier())
+		})
+	}
+}
+
+func TestNoClassifyConflictsWithPageTypeFilter(t *testing.T) {
+	opts := Options{
+		NoClassify:           true,
+		OutputFilterPageType: []string{"error"},
+		Protocol:             "http11",
+		Threads:              50,
+	}
+	err := opts.ValidateOptions()
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "-no-classify")
+}
+
 func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) {
 	dir := t.TempDir()
 	opts := &Options{
@@ -529,8 +563,8 @@ func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) {
 func TestStoreResponse_withMatcherSetsFlag(t *testing.T) {
 	dir := t.TempDir()
 	opts := &Options{
-		StoreResponse:       true,
-		StoreResponseDir:    dir,
+		StoreResponse:         true,
+		StoreResponseDir:      dir,
 		OutputMatchStatusCode: "200",
 	}
 	err := opts.ValidateOptions()