diff --git a/appsec-configs/crowdsecurity/challenge-exclude-crawler-files.yaml b/appsec-configs/crowdsecurity/challenge-exclude-crawler-files.yaml new file mode 100644 index 00000000000..f166f120346 --- /dev/null +++ b/appsec-configs/crowdsecurity/challenge-exclude-crawler-files.yaml @@ -0,0 +1,11 @@ +name: crowdsecurity/challenge-exclude-crawler-files +# Let well-known crawler files fly through without a challenge. SetLegitimateBot() is +# strictly per-request (never persisted to a cookie or across requests), so this cannot +# be abused to whitelist a session for other paths. +inband: + pre_eval: + - filter: >- + req.URL.Path in ['/robots.txt', '/ads.txt', '/app-ads.txt', '/sitemap.xml'] || + req.URL.Path startsWith '/.well-known/' + apply: + - SetLegitimateBot() diff --git a/appsec-configs/crowdsecurity/challenge-exclude-feeds.yaml b/appsec-configs/crowdsecurity/challenge-exclude-feeds.yaml new file mode 100644 index 00000000000..178c865b323 --- /dev/null +++ b/appsec-configs/crowdsecurity/challenge-exclude-feeds.yaml @@ -0,0 +1,10 @@ +name: crowdsecurity/challenge-exclude-feeds +# Let syndication feed endpoints fly through without a challenge. SetLegitimateBot() is +# strictly per-request (never persisted to a cookie or across requests). +inband: + pre_eval: + - filter: >- + req.URL.Path in ['/feed', '/rss', '/atom.xml'] || + req.URL.Path startsWith '/feed/' || req.URL.Path startsWith '/rss/' + apply: + - SetLegitimateBot() diff --git a/appsec-configs/crowdsecurity/challenge-exclude-webhooks.yaml b/appsec-configs/crowdsecurity/challenge-exclude-webhooks.yaml new file mode 100644 index 00000000000..3e625628f5c --- /dev/null +++ b/appsec-configs/crowdsecurity/challenge-exclude-webhooks.yaml @@ -0,0 +1,8 @@ +name: crowdsecurity/challenge-exclude-webhooks +# Let third-party webhook callbacks fly through without a challenge. SetLegitimateBot() is +# strictly per-request (never persisted to a cookie or across requests). +inband: + pre_eval: + - filter: "req.URL.Path in ['/webhooks/stripe', '/webhooks/github', '/webhooks/sendgrid']" + apply: + - SetLegitimateBot() diff --git a/appsec-configs/crowdsecurity/simple-bot-challenge.yaml b/appsec-configs/crowdsecurity/simple-bot-challenge.yaml new file mode 100644 index 00000000000..259d32bdf00 --- /dev/null +++ b/appsec-configs/crowdsecurity/simple-bot-challenge.yaml @@ -0,0 +1,14 @@ +name: crowdsecurity/simple-bot-challenge +inband: + post_eval: + # Skip the challenge for verified good bots (forward-confirmed rDNS) and for requests a + # well-known-path exclusion config flagged via SetLegitimateBot(). The flag is per-request only. + - filter: "!IsLegitimateBot(req.RemoteAddr, req.UserAgent(), req.URL.Path)" + apply: + - SendChallenge() + on_challenge_submit: + # Reject submissions whose fingerprint fast-detect flags a known bot. The resulting "rejected" + # event feeds crowdsecurity/appsec-bot-detected (alert only, no active ban). + - filter: "fingerprint.IsBot()" + apply: + - RejectSubmission("known bot (fast bot detection)") diff --git a/collections/crowdsecurity/appsec-bot-challenge.md b/collections/crowdsecurity/appsec-bot-challenge.md new file mode 100644 index 00000000000..5b52cc0acf5 --- /dev/null +++ b/collections/crowdsecurity/appsec-bot-challenge.md @@ -0,0 +1,35 @@ +# AppSec Bot Challenge + +This collection enables CrowdSec AppSec **challenge mode** for bot detection: visitors are served a +lightweight proof-of-work + browser-fingerprint challenge, and clients flagged as known bots by the +fast bot detection are rejected. + +What it ships: + + - `crowdsecurity/simple-bot-challenge`: challenges incoming requests, skips verified good bots + (forward-confirmed rDNS), and rejects challenge submissions whose fingerprint trips the fast bot + detection. + - Well-known-path exclusion configs (`challenge-exclude-crawler-files`, `-feeds`, `-webhooks`) that + let machine-facing endpoints (`/robots.txt`, `/.well-known/*`, feeds, webhooks) through without a + challenge. The exclusion is strictly per-request and is never persisted to a cookie or across + requests, so it cannot be abused to whitelist a session for other paths. + - A dedicated bot-detection parser tracking the fingerprint session id (`fsid`) and OS. + - Scenarios that alert on detected bots and on challenge abuse. + - A user-friendly alert context exposing `fsid`, OS, and the bot signals that fired. + +## Enabling bot challenge + +Add the `crowdsecurity/simple-bot-challenge` appsec-config to your WAF acquisition, plus any of the +well-known-path exclusion configs you need: + +```yaml +appsec_configs: + - crowdsecurity/simple-bot-challenge + - crowdsecurity/challenge-exclude-crawler-files + - crowdsecurity/challenge-exclude-feeds + - crowdsecurity/challenge-exclude-webhooks +labels: + type: appsec +listen_addr: 127.0.0.1:7422 +source: appsec +``` diff --git a/collections/crowdsecurity/appsec-bot-challenge.yaml b/collections/crowdsecurity/appsec-bot-challenge.yaml new file mode 100644 index 00000000000..058352dd4c5 --- /dev/null +++ b/collections/crowdsecurity/appsec-bot-challenge.yaml @@ -0,0 +1,26 @@ +name: crowdsecurity/appsec-bot-challenge +parsers: + - crowdsecurity/appsec-logs + - crowdsecurity/appsec-bot-detection-logs +appsec-configs: + - crowdsecurity/simple-bot-challenge + - crowdsecurity/challenge-exclude-crawler-files + - crowdsecurity/challenge-exclude-feeds + - crowdsecurity/challenge-exclude-webhooks +scenarios: + - crowdsecurity/appsec-bot-detected + - crowdsecurity/appsec-challenge-too-many-requests + - crowdsecurity/appsec-challenge-too-many-submissions + - crowdsecurity/appsec-challenge-request-with-no-submission +contexts: + - crowdsecurity/appsec_base + - crowdsecurity/appsec_bot_detection +description: "AppSec bot detection: challenge mode with fast-detect bot rejection and well-known-path exclusions" +labels: + label: "WAF - Bot Challenge" +author: crowdsecurity +tags: + - waf + - appsec + - http + - bot diff --git a/contexts/crowdsecurity/appsec_bot_detection.yaml b/contexts/crowdsecurity/appsec_bot_detection.yaml new file mode 100644 index 00000000000..bbe99f516fd --- /dev/null +++ b/contexts/crowdsecurity/appsec_bot_detection.yaml @@ -0,0 +1,23 @@ +# User-friendly context for AppSec bot-detection / challenge events. +# Each detected_signals line emits a readable signal name only when its flag is set, +# so the console shows a compact list of what tripped the fast bot detection. +context: + fingerprint_id: + - evt.Meta.fsid + operating_system: + - evt.Meta.os + user_agent: + - evt.Meta.http_user_agent + bot_detected: + - evt.Meta.fingerprint_bot + challenge_event: + - evt.Meta.challenge_event + detected_signals: + - "evt.Unmarshaled.fingerprint.Bot.Webdriver ? 'webdriver' : ''" + - "evt.Unmarshaled.fingerprint.Bot.Selenium ? 'selenium' : ''" + - "evt.Unmarshaled.fingerprint.Bot.CDP ? 'chrome-devtools-protocol' : ''" + - "evt.Unmarshaled.fingerprint.Bot.Playwright ? 'playwright' : ''" + - "evt.Unmarshaled.fingerprint.Bot.BotUserAgent ? 'bot-user-agent' : ''" + - "evt.Unmarshaled.fingerprint.Bot.MissingChromeObject ? 'headless-chrome' : ''" + - "evt.Unmarshaled.fingerprint.Bot.PlatformMismatch ? 'platform-mismatch' : ''" + - "evt.Unmarshaled.fingerprint.Bot.GPUMismatch ? 'gpu-mismatch' : ''" diff --git a/parsers/s01-parse/crowdsecurity/appsec-bot-detection-logs.yaml b/parsers/s01-parse/crowdsecurity/appsec-bot-detection-logs.yaml new file mode 100644 index 00000000000..8595c6ea565 --- /dev/null +++ b/parsers/s01-parse/crowdsecurity/appsec-bot-detection-logs.yaml @@ -0,0 +1,42 @@ +onsuccess: next_stage +format: 3.0 +#debug: true +filter: "evt.Parsed.program == 'appsec' && evt.Parsed.source == 'crowdsec-appsec-challenge'" +name: crowdsecurity/appsec-bot-detection-logs +description: "Parse AppSec bot-detection / challenge events" +statics: + - meta: service + value: appsec + - meta: log_type + value: appsec-challenge + - meta: source_ip + expression: "evt.Parsed.source_ip" + - meta: target_host + expression: "evt.Parsed.target_host" + - meta: request_uuid + expression: "evt.Parsed.req_uuid" + - meta: target_uri + expression: "evt.Parsed.target_uri" + - meta: challenge_event + expression: evt.Parsed.challenge_event + - meta: challenge_difficulty + expression: evt.Parsed.challenge_difficulty + - meta: challenge_fail_reason + expression: evt.Parsed.challenge_fail_reason + - meta: fsid + expression: evt.Parsed.fsid + - meta: fingerprint_bot + expression: evt.Parsed.fingerprint_bot + - meta: http_user_agent + expression: evt.Parsed.user_agent + # raw navigator.platform, kept for precise matching + - meta: platform + expression: evt.Parsed.platform + # user-friendly OS label derived from the raw platform string + - meta: os + expression: | + evt.Parsed.platform contains "Win" ? "Windows" : + (evt.Parsed.platform contains "Mac" ? "macOS" : + (evt.Parsed.platform contains "Linux" ? "Linux" : + (evt.Parsed.platform contains "iPhone" || evt.Parsed.platform contains "iPad" ? "iOS" : + (evt.Parsed.platform contains "Android" ? "Android" : evt.Parsed.platform)))) diff --git a/parsers/s01-parse/crowdsecurity/appsec-logs.yaml b/parsers/s01-parse/crowdsecurity/appsec-logs.yaml index 185d5bf54fe..243981c6a6b 100644 --- a/parsers/s01-parse/crowdsecurity/appsec-logs.yaml +++ b/parsers/s01-parse/crowdsecurity/appsec-logs.yaml @@ -1,7 +1,7 @@ onsuccess: next_stage format: 3.0 #debug: true -filter: "evt.Parsed.program == 'appsec'" +filter: "evt.Parsed.program == 'appsec' && evt.Parsed.source != 'crowdsec-appsec-challenge'" name: crowdsecurity/appsec-logs description: "Parse Appsec events" statics: diff --git a/scenarios/crowdsecurity/appsec-bot-detected.yaml b/scenarios/crowdsecurity/appsec-bot-detected.yaml new file mode 100644 index 00000000000..8c7d6a3cd3a --- /dev/null +++ b/scenarios/crowdsecurity/appsec-bot-detected.yaml @@ -0,0 +1,17 @@ +type: trigger +format: 3.0 +#debug: true +name: crowdsecurity/appsec-bot-detected +description: "A known bot was detected and rejected by the AppSec challenge (fast bot detection)" +filter: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'rejected'" +groupby: evt.Meta.source_ip +blackhole: 1m +labels: + service: http + confidence: 1 + spoofable: 0 + classification: + - attack.T1071 + label: "Bot detected by CrowdSec AppSec challenge" + behavior: "http:bot" + remediation: false diff --git a/scenarios/crowdsecurity/challenge-request-with-no-submission.yaml b/scenarios/crowdsecurity/challenge-request-with-no-submission.yaml new file mode 100644 index 00000000000..c1c0a11eb0f --- /dev/null +++ b/scenarios/crowdsecurity/challenge-request-with-no-submission.yaml @@ -0,0 +1,20 @@ +type: counter +format: 3.0 +#debug: true +name: crowdsecurity/appsec-challenge-request-with-no-submission +description: "Client made too many request to challenge page" +filter: "evt.Meta.log_type == 'appsec-challenge' && (evt.Meta.challenge_event == 'requested' || evt.Meta.challenge_event == 'submitted')" +cancel_on: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'submitted'" +duration: 60s +capacity: -1 +cache_size: 1 +groupby: evt.Meta.source_ip +blackhole: 10s +labels: + service: http + confidence: 0 + spoofable: 0 + classification: + - attack.T1110 + label: "Blocked by CrowdSec AppSec" + behavior: "http:exploit" diff --git a/scenarios/crowdsecurity/challenge-too-many-request.yaml b/scenarios/crowdsecurity/challenge-too-many-request.yaml new file mode 100644 index 00000000000..795bc607045 --- /dev/null +++ b/scenarios/crowdsecurity/challenge-too-many-request.yaml @@ -0,0 +1,19 @@ +type: leaky +format: 3.0 +#debug: true +name: crowdsecurity/appsec-challenge-too-many-requests +description: "Client made too many request to challenge page" +filter: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'requested'" +leakspeed: "60s" +capacity: 5 +groupby: evt.Meta.source_ip +blackhole: 1m +labels: + service: http + confidence: 0 + spoofable: 0 + classification: + - attack.T1110 + label: "Blocked by CrowdSec AppSec" + behavior: "http:exploit" + remediation: true diff --git a/scenarios/crowdsecurity/challenge-too-many-submit.yaml b/scenarios/crowdsecurity/challenge-too-many-submit.yaml new file mode 100644 index 00000000000..47ab5fe1c96 --- /dev/null +++ b/scenarios/crowdsecurity/challenge-too-many-submit.yaml @@ -0,0 +1,19 @@ +type: leaky +format: 3.0 +#debug: true +name: crowdsecurity/appsec-challenge-too-many-submissions +description: "Client made too many submissions for challenge" +filter: "evt.Meta.log_type == 'appsec-challenge' && evt.Meta.challenge_event == 'submitted'" +leakspeed: "60s" +capacity: 5 +groupby: evt.Meta.source_ip +blackhole: 1m +labels: + service: http + confidence: 0 + spoofable: 0 + classification: + - attack.T1110 + label: "Blocked by CrowdSec AppSec" + behavior: "http:exploit" + remediation: true