[do-not-merge] Waf Challenge Collection

.gitignore

@@ -31,3 +31,6 @@ workspace.code-workspace
 waf-check/dataset/*
 waf-check/output/*
 waf-check/waf-check
+
+## local dev tooling (e.g. nektos/act for running CI locally)
+bin/

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-ai-crawlers.yaml

@@ -0,0 +1,32 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-ai-crawlers
+# Skip the challenge for verified AI crawlers (matched against their published IP/rDNS ranges).
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/gptbot.json")
+      apply:
+        - ExemptFromChallenge("gptbot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/openai-searchbot.json")
+      apply:
+        - ExemptFromChallenge("openai-searchbot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/openai-chatgpt-user.json")
+      apply:
+        - ExemptFromChallenge("openai-chatgpt-user")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/perplexitybot.json")
+      apply:
+        - ExemptFromChallenge("perplexitybot")
+
+# Declares the datafiles so crowdsec downloads them.
+data:
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/gptbot.json
+    dest_file: legit_bots/gptbot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/openai-searchbot.json
+    dest_file: legit_bots/openai-searchbot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/openai-chatgpt-user.json
+    dest_file: legit_bots/openai-chatgpt-user.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/perplexitybot.json
+    dest_file: legit_bots/perplexitybot.json
+    type: bots

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-api.yaml

@@ -0,0 +1,15 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-api
+# Let programmatic/API endpoints through without a challenge: it would break with legitimate clients 
+# that cannot solve the challenge.
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: >-
+        req.URL.Path in ['/api', '/graphql'] ||
+        req.URL.Path startsWith '/api/' || req.URL.Path startsWith '/wp-json/' ||
+        req.URL.Path startsWith '/graphql/' || req.URL.Path startsWith '/rest/' ||
+        req.URL.Path startsWith '/v1/' || req.URL.Path startsWith '/v2/' ||
+        req.URL.Path startsWith '/oauth/' || req.URL.Path startsWith '/.netlify/functions/' ||
+        req.URL.Path startsWith '/_next/data/'
+      apply:
+        - ExemptFromChallenge("api")

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-crawler-files.yaml

@@ -0,0 +1,10 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-crawler-files
+# Let well-known crawler files fly through without a challenge.
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: >-
+        req.URL.Path in ['/robots.txt', '/ads.txt', '/app-ads.txt', '/sitemap.xml', '/apple-app-site-association', '/security.txt', '/humans.txt'] ||
+        req.URL.Path startsWith '/.well-known/'
+      apply:
+        - ExemptFromChallenge("crawler-files")

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-feeds.yaml

@@ -0,0 +1,10 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-feeds
+# Let syndication feed endpoints fly through without a challenge.
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: >-
+        req.URL.Path in ['/feed', '/rss', '/atom.xml', '/feed.xml', '/rss.xml', '/index.xml', '/feed.atom'] ||
+        req.URL.Path startsWith '/feed/' || req.URL.Path startsWith '/rss/'
+      apply:
+        - ExemptFromChallenge("feeds")

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-monitoring.yaml

@@ -0,0 +1,32 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-monitoring
+# Skip the challenge for verified uptime/monitoring bots (matched against their published IP/rDNS ranges).
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/uptimerobot.json")
+      apply:
+        - ExemptFromChallenge("uptimerobot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/cookiebot.json")
+      apply:
+        - ExemptFromChallenge("cookiebot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/datadog.json")
+      apply:
+        - ExemptFromChallenge("datadog")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/pagerduty.json")
+      apply:
+        - ExemptFromChallenge("pagerduty")
+
+# Declares the datafiles so crowdsec downloads them.
+data:
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/uptimerobot.json
+    dest_file: legit_bots/uptimerobot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/cookiebot.json
+    dest_file: legit_bots/cookiebot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/datadog.json
+    dest_file: legit_bots/datadog.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/pagerduty.json
+    dest_file: legit_bots/pagerduty.json
+    type: bots

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-search-engines.yaml

@@ -0,0 +1,74 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-search-engines
+# Skip the challenge for verified search-engine crawlers (matched against their published IP/rDNS ranges).
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/googlebot.json")
+      apply:
+        - ExemptFromChallenge("googlebot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/bingbot.json")
+      apply:
+        - ExemptFromChallenge("bingbot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/applebot.json")
+      apply:
+        - ExemptFromChallenge("applebot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/amazonbot.json")
+      apply:
+        - ExemptFromChallenge("amazonbot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/yandex.json")
+      apply:
+        - ExemptFromChallenge("yandex")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/baidu.json")
+      apply:
+        - ExemptFromChallenge("baidu")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/yahoo.json")
+      apply:
+        - ExemptFromChallenge("yahoo")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/sogou.json")
+      apply:
+        - ExemptFromChallenge("sogou")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/qwant.json")
+      apply:
+        - ExemptFromChallenge("qwant")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/babbar.json")
+      apply:
+        - ExemptFromChallenge("babbar")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/duckduckbot.json")
+      apply:
+        - ExemptFromChallenge("duckduckbot")
+
+# Declares the datafiles so crowdsec downloads them.
+data:
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/googlebot.json
+    dest_file: legit_bots/googlebot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/bingbot.json
+    dest_file: legit_bots/bingbot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/applebot.json
+    dest_file: legit_bots/applebot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/amazonbot.json
+    dest_file: legit_bots/amazonbot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/yandex.json
+    dest_file: legit_bots/yandex.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/baidu.json
+    dest_file: legit_bots/baidu.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/yahoo.json
+    dest_file: legit_bots/yahoo.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/sogou.json
+    dest_file: legit_bots/sogou.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/qwant.json
+    dest_file: legit_bots/qwant.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/babbar.json
+    dest_file: legit_bots/babbar.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/duckduckbot.json
+    dest_file: legit_bots/duckduckbot.json
+    type: bots

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-social.yaml

@@ -0,0 +1,38 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-social
+# Skip the challenge for verified social / link-preview bots (matched against their published IP/rDNS ranges).
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/meta.json")
+      apply:
+        - ExemptFromChallenge("meta")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/discord.json")
+      apply:
+        - ExemptFromChallenge("discord")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/telegram.json")
+      apply:
+        - ExemptFromChallenge("telegram")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/twitterbot.json")
+      apply:
+        - ExemptFromChallenge("twitterbot")
+    - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/pinterest.json")
+      apply:
+        - ExemptFromChallenge("pinterest")
+
+# Declares the datafiles so crowdsec downloads them.
+data:
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/meta.json
+    dest_file: legit_bots/meta.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/discord.json
+    dest_file: legit_bots/discord.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/telegram.json
+    dest_file: legit_bots/telegram.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/twitterbot.json
+    dest_file: legit_bots/twitterbot.json
+    type: bots
+  - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/pinterest.json
+    dest_file: legit_bots/pinterest.json
+    type: bots

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-static.yaml

@@ -0,0 +1,10 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-static
+# Let static assets and browser metadata files fly through without a challenge
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: >-
+        req.URL.Path matches '\\.(css|js|mjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|woff2?|ttf|otf|eot|mp4|m4v|webm|mov|mp3|m4a|wav|ogg|oga|ogv|flac|aac|webmanifest)$' ||
+        req.URL.Path in ['/manifest.json', '/browserconfig.xml']
+      apply:
+        - ExemptFromChallenge("static")

appsec-configs/crowdsecurity/appsec-bot-challenge-exclude-webhooks.yaml

@@ -0,0 +1,8 @@
+name: crowdsecurity/appsec-bot-challenge-exclude-webhooks
+# Let third-party webhook callbacks fly through without a challenge.
+# Do not edit this file to add your own exclusions, create a custom config.
+inband:
+  pre_eval:
+    - filter: "req.URL.Path startsWith '/webhooks/'"
+      apply:
+        - ExemptFromChallenge("webhooks")

appsec-configs/crowdsecurity/appsec-bot-challenge-simple.yaml

@@ -0,0 +1,14 @@
+name: crowdsecurity/appsec-bot-challenge-simple
+inband:
+  post_eval:
+    # Challenge every request. Requests a pre_eval exclusion config flagged via
+    # ExemptFromChallenge() (verified bots, well-known paths) are skipped by
+    # SendChallenge itself, so no explicit bot check is needed here.
+    - filter: "true"
+      apply:
+        - SendChallenge()
+  on_challenge_submit:
+    # Reject submissions whose fingerprint fast-detect flags a known bot.
+    - filter: "fingerprint.IsBot()"
+      apply:
+        - RejectSubmission("known bot (fast bot detection)")

collections/crowdsecurity/appsec-bot-challenge.md

@@ -0,0 +1,43 @@
+# AppSec Bot Challenge
+
+This collection enables CrowdSec AppSec **challenge mode** for bot detection: visitors are served a
+lightweight proof-of-work + browser-fingerprint challenge, and clients flagged as known bots by the
+fast bot detection are rejected.
+
+What it ships:
+
+ - `crowdsecurity/appsec-bot-challenge-simple`: challenges incoming requests and rejects challenge
+   submissions whose fingerprint trips the bot detection.
+ - Verified-good-bot exclusion configs that exempt known bots from the challenge, split by family so you
+   can enable only the ones you need. Each config both declares the datafiles and exempts a match:
+   - `crowdsecurity/appsec-bot-challenge-exclude-search-engines`: googlebot, bingbot, applebot, amazonbot,
+     yandex, baidu, yahoo, sogou, qwant, babbar, duckduckbot
+   - `crowdsecurity/appsec-bot-challenge-exclude-ai-crawlers`: gptbot, openai-searchbot, openai-chatgpt-user,
+     perplexitybot
+   - `crowdsecurity/appsec-bot-challenge-exclude-social`: meta, discord, telegram, twitterbot, pinterest
+   - `crowdsecurity/appsec-bot-challenge-exclude-monitoring`: uptimerobot, cookiebot, datadog, pagerduty
+
+   A bot is exempted **only** when it can be network-verified (dns or ip range).
+ - Path exclusion configs that let machine-facing or non-navigational requests through without a
+   challenge:
+   - `appsec-bot-challenge-exclude-crawler-files` — `/robots.txt`, `/.well-known/*`, `/security.txt`, …
+   - `appsec-bot-challenge-exclude-feeds` — RSS/Atom feed paths
+   - `appsec-bot-challenge-exclude-webhooks` — third-party webhook paths
+   - `appsec-bot-challenge-exclude-static` — static assets and media
+   - `appsec-bot-challenge-exclude-api` — programmatic endpoints
+ - Scenarios that alert on detected bots and on challenge abuse.
+ - A user-friendly alert context exposing `fsid`, OS, and the bot signals that fired.
+
+## Enabling bot challenge
+
+Add the `crowdsecurity/appsec-bot-*` appsec-configs to your WAF acquisition, plus any of the
+well-known-path exclusion configs you need:
+
+```yaml
+appsec_configs:
+ - crowdsecurity/appsec-bot-*
+labels:
+  type: appsec
+listen_addr: 127.0.0.1:7422
+source: appsec
+```

collections/crowdsecurity/appsec-bot-challenge.yaml

@@ -0,0 +1,32 @@
+name: crowdsecurity/appsec-bot-challenge
+parsers:
+  - crowdsecurity/appsec-logs
+  - crowdsecurity/appsec-bot-detection-logs
+appsec-configs:
+  - crowdsecurity/appsec-bot-challenge-simple
+  - crowdsecurity/appsec-bot-challenge-exclude-crawler-files
+  - crowdsecurity/appsec-bot-challenge-exclude-feeds
+  - crowdsecurity/appsec-bot-challenge-exclude-webhooks
+  - crowdsecurity/appsec-bot-challenge-exclude-static
+  - crowdsecurity/appsec-bot-challenge-exclude-api
+  - crowdsecurity/appsec-bot-challenge-exclude-search-engines
+  - crowdsecurity/appsec-bot-challenge-exclude-ai-crawlers
+  - crowdsecurity/appsec-bot-challenge-exclude-social
+  - crowdsecurity/appsec-bot-challenge-exclude-monitoring
+scenarios:
+  - crowdsecurity/appsec-bot-detected
+  - crowdsecurity/appsec-bot-challenge-too-many-requests
+  - crowdsecurity/appsec-bot-challenge-too-many-submissions
+  - crowdsecurity/appsec-bot-challenge-request-with-no-submission
+contexts:
+  - crowdsecurity/appsec_base
+  - crowdsecurity/appsec-bot-detection
+description: "AppSec bot detection: challenge mode with fast-detect bot rejection and well-known-path exclusions"
+labels:
+  label: "WAF - Bot Challenge"
+author: crowdsecurity
+tags:
+  - waf
+  - appsec
+  - http
+  - bot

contexts/crowdsecurity/appsec-bot-detection.yaml

@@ -0,0 +1,18 @@
+# User-friendly context for AppSec bot-detection / challenge events.
+# detected_signals uses the fingerprint helper BotSignals(), which returns the
+# names of the fast-detection signals that fired; the alert-context engine
+# renders the []string as a list. The nil guard covers "requested" events that
+# carry no fingerprint.
+context:
+  fingerprint_id:
+    - evt.Meta.fsid
+  operating_system:
+    - evt.Meta.os
+  user_agent:
+    - evt.Meta.http_user_agent
+  bot_detected:
+    - evt.Meta.fingerprint_bot
+  challenge_event:
+    - evt.Meta.challenge_event
+  detected_signals:
+    - 'evt.Unmarshaled.fingerprint != nil ? evt.Unmarshaled.fingerprint.BotSignals() : ""'

parsers/s01-parse/crowdsecurity/appsec-bot-detection-logs.yaml

@@ -0,0 +1,36 @@
+onsuccess: next_stage
+format: 3.0
+#debug: true
+filter: "evt.Parsed.program == 'appsec' && evt.Parsed.source == 'crowdsec-appsec-challenge'"
+name: crowdsecurity/appsec-bot-detection-logs
+description: "Parse AppSec bot-detection / challenge events"
+statics:
+  - meta: service
+    value: appsec
+  - meta: log_type
+    value: appsec-challenge
+  - meta: source_ip
+    expression: "evt.Parsed.source_ip"
+  - meta: target_host
+    expression: "evt.Parsed.target_host"
+  - meta: request_uuid
+    expression: "evt.Parsed.req_uuid"
+  - meta: target_uri
+    expression: "evt.Parsed.target_uri"
+  - meta: challenge_event
+    expression: evt.Parsed.challenge_event
+  - meta: challenge_difficulty
+    expression: evt.Parsed.challenge_difficulty
+  - meta: challenge_fail_reason
+    expression: evt.Parsed.challenge_fail_reason
+  - meta: fsid
+    expression: evt.Parsed.fsid
+  - meta: fingerprint_bot
+    expression: evt.Parsed.fingerprint_bot
+  - meta: http_user_agent
+    expression: evt.Parsed.user_agent
+  # Friendly OS straight from the fingerprint helper (high-entropy client-hint
+  # platform, falling back to navigator.platform). The nil guard covers
+  # "requested" challenge events, which carry no fingerprint.
+  - meta: os
+    expression: 'evt.Unmarshaled.fingerprint != nil ? evt.Unmarshaled.fingerprint.Platform() : ""'