[do-not-merge] Waf Challenge Collection

appsec-configs/crowdsecurity/challenge-exclude-crawler-files.yaml

@@ -0,0 +1,11 @@
+name: crowdsecurity/challenge-exclude-crawler-files
+# Let well-known crawler files fly through without a challenge. SetLegitimateBot() is
+# strictly per-request (never persisted to a cookie or across requests), so this cannot
+# be abused to whitelist a session for other paths.
+inband:
+  pre_eval:
+    - filter: >-
+        req.URL.Path in ['/robots.txt', '/ads.txt', '/app-ads.txt', '/sitemap.xml'] ||
+        req.URL.Path startsWith '/.well-known/'
+      apply:
+        - SetLegitimateBot()

appsec-configs/crowdsecurity/challenge-exclude-feeds.yaml

@@ -0,0 +1,10 @@
+name: crowdsecurity/challenge-exclude-feeds
+# Let syndication feed endpoints fly through without a challenge. SetLegitimateBot() is
+# strictly per-request (never persisted to a cookie or across requests).
+inband:
+  pre_eval:
+    - filter: >-
+        req.URL.Path in ['/feed', '/rss', '/atom.xml'] ||
+        req.URL.Path startsWith '/feed/' || req.URL.Path startsWith '/rss/'
+      apply:
+        - SetLegitimateBot()

appsec-configs/crowdsecurity/challenge-exclude-webhooks.yaml

@@ -0,0 +1,8 @@
+name: crowdsecurity/challenge-exclude-webhooks
+# Let third-party webhook callbacks fly through without a challenge. SetLegitimateBot() is
+# strictly per-request (never persisted to a cookie or across requests).
+inband:
+  pre_eval:
+    - filter: "req.URL.Path in ['/webhooks/stripe', '/webhooks/github', '/webhooks/sendgrid']"
+      apply:
+        - SetLegitimateBot()

appsec-configs/crowdsecurity/simple-bot-challenge.yaml

@@ -0,0 +1,14 @@
+name: crowdsecurity/simple-bot-challenge
+inband:
+  post_eval:
+    # Skip the challenge for verified good bots (forward-confirmed rDNS) and for requests a
+    # well-known-path exclusion config flagged via SetLegitimateBot(). The flag is per-request only.
+    - filter: "!IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)"
+      apply:
+        - SendChallenge()
+  on_challenge_submit:
+    # Reject submissions whose fingerprint fast-detect flags a known bot. The resulting "rejected"
+    # event feeds crowdsecurity/appsec-bot-detected (alert only, no active ban).
+    - filter: "fingerprint.IsBot()"
+      apply:
+        - RejectSubmission("known bot (fast bot detection)")

collections/crowdsecurity/appsec-bot-challenge.md

@@ -0,0 +1,35 @@
+# AppSec Bot Challenge
+
+This collection enables CrowdSec AppSec **challenge mode** for bot detection: visitors are served a
+lightweight proof-of-work + browser-fingerprint challenge, and clients flagged as known bots by the
+fast bot detection are rejected.
+
+What it ships:
+
+ - `crowdsecurity/simple-bot-challenge`: challenges incoming requests, skips verified good bots
+   (forward-confirmed rDNS), and rejects challenge submissions whose fingerprint trips the fast bot
+   detection.
+ - Well-known-path exclusion configs (`challenge-exclude-crawler-files`, `-feeds`, `-webhooks`) that
+   let machine-facing endpoints (`/robots.txt`, `/.well-known/*`, feeds, webhooks) through without a
+   challenge. The exclusion is strictly per-request and is never persisted to a cookie or across
+   requests, so it cannot be abused to whitelist a session for other paths.
+ - A dedicated bot-detection parser tracking the fingerprint session id (`fsid`) and OS.
+ - Scenarios that alert on detected bots and on challenge abuse.
+ - A user-friendly alert context exposing `fsid`, OS, and the bot signals that fired.
+
+## Enabling bot challenge
+
+Add the `crowdsecurity/simple-bot-challenge` appsec-config to your WAF acquisition, plus any of the
+well-known-path exclusion configs you need:
+
+```yaml
+appsec_configs:
+ - crowdsecurity/simple-bot-challenge
+ - crowdsecurity/challenge-exclude-crawler-files
+ - crowdsecurity/challenge-exclude-feeds
+ - crowdsecurity/challenge-exclude-webhooks
+labels:
+  type: appsec
+listen_addr: 127.0.0.1:7422
+source: appsec
+```

collections/crowdsecurity/appsec-bot-challenge.yaml

@@ -0,0 +1,26 @@
+name: crowdsecurity/appsec-bot-challenge
+parsers:
+  - crowdsecurity/appsec-logs
+  - crowdsecurity/appsec-bot-detection-logs
+appsec-configs:
+  - crowdsecurity/simple-bot-challenge
+  - crowdsecurity/challenge-exclude-crawler-files
+  - crowdsecurity/challenge-exclude-feeds
+  - crowdsecurity/challenge-exclude-webhooks
+scenarios:
+  - crowdsecurity/appsec-bot-detected
+  - crowdsecurity/appsec-challenge-too-many-requests
+  - crowdsecurity/appsec-challenge-too-many-submissions
+  - crowdsecurity/appsec-challenge-request-with-no-submission
+contexts:
+  - crowdsecurity/appsec_base
+  - crowdsecurity/appsec_bot_detection
+description: "AppSec bot detection: challenge mode with fast-detect bot rejection and well-known-path exclusions"
+labels:
+  label: "WAF - Bot Challenge"
+author: crowdsecurity
+tags:
+  - waf
+  - appsec
+  - http
+  - bot

contexts/crowdsecurity/appsec_bot_detection.yaml

@@ -0,0 +1,23 @@
+# User-friendly context for AppSec bot-detection / challenge events.
+# Each detected_signals line emits a readable signal name only when its flag is set,
+# so the console shows a compact list of what tripped the fast bot detection.
+context:
+  fingerprint_id:
+    - evt.Meta.fsid
+  operating_system:
+    - evt.Meta.os
+  user_agent:
+    - evt.Meta.http_user_agent
+  bot_detected:
+    - evt.Meta.fingerprint_bot
+  challenge_event:
+    - evt.Meta.challenge_event
+  detected_signals:
+    - "evt.Unmarshaled.fingerprint.Bot.Webdriver ? 'webdriver' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.Selenium ? 'selenium' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.CDP ? 'chrome-devtools-protocol' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.Playwright ? 'playwright' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.BotUserAgent ? 'bot-user-agent' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.MissingChromeObject ? 'headless-chrome' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.PlatformMismatch ? 'platform-mismatch' : ''"
+    - "evt.Unmarshaled.fingerprint.Bot.GPUMismatch ? 'gpu-mismatch' : ''"

parsers/s01-parse/crowdsecurity/appsec-bot-detection-logs.yaml

@@ -0,0 +1,42 @@
+onsuccess: next_stage
+format: 3.0
+#debug: true
+filter: "evt.Parsed.program == 'appsec' && evt.Parsed.source == 'crowdsec-appsec-challenge'"
+name: crowdsecurity/appsec-bot-detection-logs
+description: "Parse AppSec bot-detection / challenge events"
+statics:
+  - meta: service
+    value: appsec
+  - meta: log_type
+    value: appsec-challenge
+  - meta: source_ip
+    expression: "evt.Parsed.source_ip"
+  - meta: target_host
+    expression: "evt.Parsed.target_host"
+  - meta: request_uuid
+    expression: "evt.Parsed.req_uuid"
+  - meta: target_uri
+    expression: "evt.Parsed.target_uri"
+  - meta: challenge_event
+    expression: evt.Parsed.challenge_event
+  - meta: challenge_difficulty
+    expression: evt.Parsed.challenge_difficulty
+  - meta: challenge_fail_reason
+    expression: evt.Parsed.challenge_fail_reason
+  - meta: fsid
+    expression: evt.Parsed.fsid
+  - meta: fingerprint_bot
+    expression: evt.Parsed.fingerprint_bot
+  - meta: http_user_agent
+    expression: evt.Parsed.user_agent
+  # raw navigator.platform, kept for precise matching
+  - meta: platform
+    expression: evt.Parsed.platform
+  # user-friendly OS label derived from the raw platform string
+  - meta: os
+    expression: |
+      evt.Parsed.platform contains "Win" ? "Windows" :
+      (evt.Parsed.platform contains "Mac" ? "macOS" :
+      (evt.Parsed.platform contains "Linux" ? "Linux" :
+      (evt.Parsed.platform contains "iPhone" || evt.Parsed.platform contains "iPad" ? "iOS" :
+      (evt.Parsed.platform contains "Android" ? "Android" : evt.Parsed.platform))))

parsers/s01-parse/crowdsecurity/appsec-logs.yaml

@@ -1,7 +1,7 @@
 onsuccess: next_stage
 format: 3.0
 #debug: true
-filter: "evt.Parsed.program == 'appsec'"
+filter: "evt.Parsed.program == 'appsec' && evt.Parsed.source != 'crowdsec-appsec-challenge'"
 name: crowdsecurity/appsec-logs
 description: "Parse Appsec events"
 statics:

scenarios/crowdsecurity/appsec-bot-detected.yaml

@@ -0,0 +1,17 @@
+type: trigger
+format: 3.0
+#debug: true
+name: crowdsecurity/appsec-bot-detected
+description: "A known bot was detected and rejected by the AppSec challenge (fast bot detection)"
+filter: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'rejected'"
+groupby: evt.Meta.source_ip
+blackhole: 1m
+labels:
+  service: http
+  confidence: 1
+  spoofable: 0
+  classification:
+    - attack.T1071
+  label: "Bot detected by CrowdSec AppSec challenge"
+  behavior: "http:bot"
+  remediation: false

scenarios/crowdsecurity/challenge-request-with-no-submission.yaml

@@ -0,0 +1,20 @@
+type: counter
+format: 3.0
+#debug: true
+name: crowdsecurity/appsec-challenge-request-with-no-submission
+description: "Client made too many request to challenge page"
+filter: "evt.Meta.log_type == 'appsec-challenge' && (evt.Meta.challenge_event == 'requested' || evt.Meta.challenge_event == 'submitted')"
+cancel_on: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'submitted'"
+duration: 60s
+capacity: -1
+cache_size: 1
+groupby: evt.Meta.source_ip
+blackhole: 10s
+labels:
+  service: http
+  confidence: 0
+  spoofable: 0
+  classification:
+    - attack.T1110
+  label: "Blocked by CrowdSec AppSec"
+  behavior: "http:exploit"

scenarios/crowdsecurity/challenge-too-many-request.yaml

@@ -0,0 +1,19 @@
+type: leaky
+format: 3.0
+#debug: true
+name: crowdsecurity/appsec-challenge-too-many-requests
+description: "Client made too many request to challenge page"
+filter: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'requested'"
+leakspeed: "60s"
+capacity: 5
+groupby: evt.Meta.source_ip
+blackhole: 1m
+labels:
+  service: http
+  confidence: 0
+  spoofable: 0
+  classification:
+    - attack.T1110
+  label: "Blocked by CrowdSec AppSec"
+  behavior: "http:exploit"
+  remediation: true

scenarios/crowdsecurity/challenge-too-many-submit.yaml

@@ -0,0 +1,19 @@
+type: leaky
+format: 3.0
+#debug: true
+name: crowdsecurity/appsec-challenge-too-many-submissions
+description: "Client made too many submissions for challenge"
+filter: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'submitted'"
+leakspeed: "60s"
+capacity: 5
+groupby: evt.Meta.source_ip
+blackhole: 1m
+labels:
+  service: http
+  confidence: 0
+  spoofable: 0
+  classification:
+    - attack.T1110
+  label: "Blocked by CrowdSec AppSec"
+  behavior: "http:exploit"
+  remediation: true