diff --git a/stream/cloudflare-rules-list/.env.sample b/stream/cloudflare-rules-list/.env.sample new file mode 100644 index 00000000000..13da8bd4ae3 --- /dev/null +++ b/stream/cloudflare-rules-list/.env.sample @@ -0,0 +1,23 @@ +# OpenCTI +OPENCTI_URL=http://localhost:8080 +OPENCTI_TOKEN=ChangeMe + +# Connector +CONNECTOR_ID=ChangeMe +CONNECTOR_NAME=Cloudflare Rules List +CONNECTOR_SCOPE=cloudflare +CONNECTOR_LOG_LEVEL=info +CONNECTOR_LIVE_STREAM_ID=live +CONNECTOR_LIVE_STREAM_LISTEN_DELETE=true +CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES=true + +# Cloudflare +CLOUDFLARE_ACCOUNT_ID=ChangeMe +# API token with 'Account > Account Filter Lists > Edit' permission +CLOUDFLARE_API_TOKEN=ChangeMe +# ID of the existing Cloudflare Rules List (IP kind) to sync into +CLOUDFLARE_LIST_ID=ChangeMe +# Base URL of the Cloudflare API. Override only for testing or a compatible gateway. +# CLOUDFLARE_API_BASE_URL=https://api.cloudflare.com/client/v4 +# Minimum interval between snapshot uploads (ISO-8601: 'PT30M', 'PT1H', 'PT1H30M') +CLOUDFLARE_SYNC_INTERVAL=PT1H diff --git a/stream/cloudflare-rules-list/Dockerfile b/stream/cloudflare-rules-list/Dockerfile new file mode 100644 index 00000000000..5cea3c1e2da --- /dev/null +++ b/stream/cloudflare-rules-list/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-alpine + +ENV CONNECTOR_TYPE=STREAM \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# connectors-sdk is installed from a git URL, so git/build tooling is needed at build time. +# libmagic is a runtime dependency of pycti (python-magic) and must remain installed. +COPY src /opt/opencti-connector-cloudflare-rules-list +RUN apk --no-cache add git build-base libffi-dev openssl-dev libmagic && \ + cd /opt/opencti-connector-cloudflare-rules-list && \ + pip3 install --no-cache-dir --upgrade pip && \ + pip3 install --no-cache-dir -r requirements.txt && \ + apk del git build-base + +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/stream/cloudflare-rules-list/README.md b/stream/cloudflare-rules-list/README.md new file mode 100644 index 00000000000..06dd778b646 --- /dev/null +++ b/stream/cloudflare-rules-list/README.md @@ -0,0 +1,158 @@ +# OpenCTI Cloudflare Rules List Connector + +A **STREAM** connector that pushes **IPv4** threat-intelligence indicators and +observables from OpenCTI into a +[Cloudflare Rules List](https://developers.cloudflare.com/waf/tools/lists/), +where they can be referenced from WAF custom rules, firewall rules, and other +Cloudflare security configurations. + +## Table of Contents + +- [Introduction](#introduction) +- [Requirements](#requirements) +- [Configuration variables](#configuration-variables) +- [Deployment](#deployment) +- [Behavior](#behavior) +- [Capabilities and limitations](#capabilities-and-limitations) +- [Development](#development) + +## Introduction + +Cloudflare Rules Lists are reusable collections of IP addresses scoped to a +Cloudflare account. This connector subscribes to an OpenCTI live stream and +keeps a target Rules List (of **IP** kind) in sync with the IPv4 indicators and +`IPv4-Addr` observables known to OpenCTI. + +Cloudflare's list-items API follows a **replace/snapshot** model: each upload +replaces the full contents of the list. The connector therefore maintains an +in-memory snapshot of all known IPv4 values and pushes the complete snapshot at +most once per `CLOUDFLARE_SYNC_INTERVAL`. + +## Requirements + +- OpenCTI Platform 7.x. The connector pins `pycti==7.260728.0`; because pycti + follows the platform's CalVer scheme, its major version must match your + OpenCTI platform. Align the pin with your platform version if it differs. +- A Cloudflare account and an existing Rules List of **IP** kind +- A Cloudflare API token with the `Account > Account Filter Lists > Edit` + permission +- Python 3.11 or 3.12 (for local runs; the SDK requires `< 3.13`) + +## Configuration variables + +Configuration is read (in precedence order) from environment variables, then a +`config.yml` or `.env` file next to the connector, then field defaults. + +### OpenCTI configuration + +| Parameter | config.yml | Docker env var | Mandatory | Description | +| ------------- | --------------- | ---------------- | --------- | ---------------------------------------- | +| OpenCTI URL | `opencti.url` | `OPENCTI_URL` | Yes | The base URL of the OpenCTI instance. | +| OpenCTI Token | `opencti.token` | `OPENCTI_TOKEN` | Yes | The API token to connect to OpenCTI. | + +### Base connector configuration + +| Parameter | config.yml | Docker env var | Default | Mandatory | Description | +| ------------------ | ---------------------------- | ---------------------------- | --------------------- | --------- | --------------------------------------------------------------------------- | +| Connector ID | `connector.id` | `CONNECTOR_ID` | / | Yes | A unique `UUIDv4` identifier for this connector instance. | +| Connector Name | `connector.name` | `CONNECTOR_NAME` | `Cloudflare Rules List` | No | Name of the connector as shown in OpenCTI. | +| Connector Scope | `connector.scope` | `CONNECTOR_SCOPE` | `cloudflare` | No | The scope of the stream connector (comma-separated). | +| Log Level | `connector.log_level` | `CONNECTOR_LOG_LEVEL` | `error` | No | `debug`, `info`, `warn`, `warning`, or `error`. | +| Live Stream ID | `connector.live_stream_id` | `CONNECTOR_LIVE_STREAM_ID` | / | Yes | The ID of the OpenCTI live stream to connect to (e.g. `live`). | +| Listen Delete | `connector.live_stream_listen_delete` | `CONNECTOR_LIVE_STREAM_LISTEN_DELETE` | `true` | No | Whether to receive delete events from the live stream. Must be `true` for the connector to drop removed IPs from the list. | +| No Dependencies | `connector.live_stream_no_dependencies` | `CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES` | `true` | No | Whether to ignore object dependencies when processing live-stream events. | + +### Cloudflare configuration + +| Parameter | config.yml | Docker env var | Default | Mandatory | Description | +| -------------------- | ---------------------- | ----------------------- | ------- | --------- | --------------------------------------------------------------------- | +| Account ID | `cloudflare.account_id`| `CLOUDFLARE_ACCOUNT_ID` | / | Yes | Cloudflare account ID that owns the Rules List. | +| API Token | `cloudflare.api_token` | `CLOUDFLARE_API_TOKEN` | / | Yes | API token with `Account > Account Filter Lists > Edit` permission. | +| List ID | `cloudflare.list_id` | `CLOUDFLARE_LIST_ID` | / | Yes | ID of the existing Rules List (IP kind) to sync into. | +| API Base URL | `cloudflare.api_base_url` | `CLOUDFLARE_API_BASE_URL` | `https://api.cloudflare.com/client/v4` | No | Base URL of the Cloudflare API. Override only for testing or a compatible gateway. | +| Sync Interval | `cloudflare.sync_interval` | `CLOUDFLARE_SYNC_INTERVAL` | `PT1H` | No | Minimum interval between snapshot uploads (ISO-8601 duration: `PT30M`, `PT1H`, `PT1H30M`, etc.). | + +> The Rules List must already exist and be of **IP** kind. Create it under +> Cloudflare → Manage Account → Configurations → Lists, then copy its ID. + +## Deployment + +### Docker Deployment + +Build the image and start the container: + +```shell +docker compose up -d +# -or, if you've made local changes- +docker compose up -d --build +``` + +### Manual Deployment + +Create a `config.yml` from the sample, fill in your values, then: + +```shell +cd src +pip install -r requirements.txt +python main.py +``` + +`git` must be available when installing requirements because `connectors-sdk` is +fetched from its repository. + +## Behavior + +1. **Verify** the configured Cloudflare Rules List exists (and log its kind). +2. **Full sync** on startup: load all IPv4 indicators and `IPv4-Addr` + observables from OpenCTI into an in-memory snapshot, then immediately push + that snapshot to Cloudflare. +3. **Listen** to the OpenCTI live stream — cache IPv4 values on create/update, + drop them on delete. +4. **Replace** the entire Cloudflare list with the current snapshot, then poll + the resulting bulk operation to completion. This push is triggered by + live-stream events and throttled to **at most once per + `CLOUDFLARE_SYNC_INTERVAL`** — an idle stream produces no uploads even after + the interval elapses. + +IPv4 values are extracted from three shapes: STIX indicators with an +`[ipv4-addr:value = '...']` pattern, STIX SCOs with `type: "ipv4-addr"`, and +OpenCTI observables with `entity_type: "IPv4-Addr"`. + +Each entry written to the Cloudflare list is tagged with a comment of the form +`OpenCTI: `, recording the source OpenCTI object ID. + +## Capabilities and limitations + +- **IPv4 only.** IPv6 addresses, domains, URLs, file hashes, and every other + indicator/observable type are ignored. Indicators match only when their STIX + pattern is `[ipv4-addr:value = '...']` (a single address, or a CIDR inside the + quotes); compound patterns contribute only their first IPv4 value. +- **No built-in score, label, confidence, or marking filtering.** Every IPv4 the + connector sees is pushed to the list. To restrict *which* entities reach the + connector, scope the OpenCTI **live stream definition** (e.g. by score or + labels) — the connector itself applies no filtering. +- **The startup full sync is not stream-filtered.** It loads *all* IPv4 + indicators and `IPv4-Addr` observables from the platform via the OpenCTI API, + regardless of the live stream's filters. Only the incremental live updates + honor the stream definition, so a filtered stream and the full sync can + disagree on what belongs in the list. +- **State is in-memory.** The snapshot is rebuilt by a full sync on every + restart; nothing is persisted locally. +- **Removals happen on delete events only.** A revoked or expired indicator is + dropped from the list when its delete event arrives, which requires + `CONNECTOR_LIVE_STREAM_LISTEN_DELETE=true`. +- **The Cloudflare list is owned by the connector.** Because each sync *replaces* + the entire list, any items added to it outside the connector are overwritten on + the next push. Use a dedicated list. + +## Development + +Run the test suite (requires Python 3.11/3.12): + +```shell +pip install -r tests/test-requirements.txt pytest-cov +pytest tests/ --cov=cloudflare_rules_list --cov-report=term-missing +``` + +`pytest-cov` is installed explicitly above (it is intentionally not listed in +`tests/test-requirements.txt`). diff --git a/stream/cloudflare-rules-list/__metadata__/connector_manifest.json b/stream/cloudflare-rules-list/__metadata__/connector_manifest.json new file mode 100644 index 00000000000..7f48e95eb3b --- /dev/null +++ b/stream/cloudflare-rules-list/__metadata__/connector_manifest.json @@ -0,0 +1,26 @@ +{ + "title": "Cloudflare Rules List", + "slug": "cloudflare-rules-list", + "description": "Cloudflare Rules Lists are reusable collections of IP addresses that can be referenced from WAF custom rules, firewall rules, and other security configurations across a Cloudflare account.\n\nThe integration of Cloudflare with OpenCTI enables the automatic dissemination of IPv4 threat-intelligence indicators and observables into a Cloudflare Rules List. The connector consumes IPv4 indicators from an OpenCTI live stream, maintains an in-memory snapshot, and periodically replaces the contents of a target Cloudflare Rules List (IP kind) using Cloudflare's bulk list-items API documented at https://developers.cloudflare.com/api/resources/rules/subresources/lists/.", + "short_description": "Push IPv4 threat-intelligence indicators from OpenCTI into a Cloudflare Rules List for use in WAF and firewall rules.", + "logo": "stream/cloudflare-rules-list/__metadata__/logo.png", + "use_cases": [ + "Detection & Response Enablement" + ], + "solution_categories": [ + "Network Security" + ], + "license_type": "Commercial", + "contact": null, + "verified": false, + "last_verified_date": null, + "playbook_supported": false, + "max_confidence_level": 50, + "support_version": ">= 6.7.5", + "subscription_link": "https://developers.cloudflare.com/waf/tools/lists/", + "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/stream/cloudflare-rules-list", + "manager_supported": false, + "container_version": "rolling", + "container_image": "opencti/connector-cloudflare-rules-list", + "container_type": "STREAM" +} diff --git a/stream/cloudflare-rules-list/__metadata__/logo.png b/stream/cloudflare-rules-list/__metadata__/logo.png new file mode 100644 index 00000000000..01ce9d3028b Binary files /dev/null and b/stream/cloudflare-rules-list/__metadata__/logo.png differ diff --git a/stream/cloudflare-rules-list/docker-compose.yml b/stream/cloudflare-rules-list/docker-compose.yml new file mode 100644 index 00000000000..92d9df8daea --- /dev/null +++ b/stream/cloudflare-rules-list/docker-compose.yml @@ -0,0 +1,24 @@ +services: + connector-cloudflare-rules-list: + image: opencti/connector-cloudflare-rules-list:latest + container_name: opencti-connector-cloudflare-rules-list + restart: unless-stopped + environment: + # OpenCTI + - OPENCTI_URL=${OPENCTI_URL} + - OPENCTI_TOKEN=${OPENCTI_TOKEN} + + # Connector + - CONNECTOR_ID=${CONNECTOR_ID} + - CONNECTOR_NAME=Cloudflare Rules List + - CONNECTOR_SCOPE=cloudflare + - CONNECTOR_LOG_LEVEL=${CONNECTOR_LOG_LEVEL:-info} + - CONNECTOR_LIVE_STREAM_ID=${CONNECTOR_LIVE_STREAM_ID:-live} + - CONNECTOR_LIVE_STREAM_LISTEN_DELETE=true + - CONNECTOR_LIVE_STREAM_NO_DEPENDENCIES=true + + # Cloudflare + - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID} + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + - CLOUDFLARE_LIST_ID=${CLOUDFLARE_LIST_ID} + - CLOUDFLARE_SYNC_INTERVAL=${CLOUDFLARE_SYNC_INTERVAL:-PT1H} diff --git a/stream/cloudflare-rules-list/entrypoint.sh b/stream/cloudflare-rules-list/entrypoint.sh new file mode 100644 index 00000000000..b450492dd12 --- /dev/null +++ b/stream/cloudflare-rules-list/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +cd /opt/opencti-connector-cloudflare-rules-list +python3 main.py diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/__init__.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/__init__.py new file mode 100644 index 00000000000..53116f14c93 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/__init__.py @@ -0,0 +1,11 @@ +"""OpenCTI -> Cloudflare Rules List stream connector (v2). + +Pushes IPv4 threat-intelligence indicators from OpenCTI into a Cloudflare +Rules List, built on the modern OpenCTI ``connectors-sdk`` + Pydantic-settings +architecture (OpenCTI 7.x / pycti 7.x). +""" + +from .connector import Connector +from .settings import ConnectorSettings + +__all__ = ["Connector", "ConnectorSettings"] diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/client.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/client.py new file mode 100644 index 00000000000..25b03962fe3 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/client.py @@ -0,0 +1,172 @@ +"""Cloudflare Rules Lists API client. + +Documentation: +https://developers.cloudflare.com/api/resources/rules/subresources/lists/ +""" + +import json +import time +from typing import Any, Optional, Union + +import requests +from pydantic import HttpUrl + + +class CloudflareAPIError(Exception): + """Exception raised for Cloudflare API errors.""" + + +class CloudflareRulesListClient: + """Client for the Cloudflare Rules Lists API.""" + + BASE_URL = "https://api.cloudflare.com/client/v4" + + def __init__( + self, + account_id: str, + api_token: str, + timeout: int = 60, + base_url: Optional[Union[str, HttpUrl]] = None, + ): + """Initialize the client. + + Args: + account_id: Cloudflare account ID. + api_token: API token (Bearer auth). + timeout: Default request timeout in seconds. + base_url: Override the Cloudflare API base URL (for testing or a + compatible gateway). Accepts a ``str`` or a pydantic ``HttpUrl``. + Defaults to the public Cloudflare API. + """ + self.account_id = account_id + self.api_token = api_token + self.timeout = timeout + self.base_url = (str(base_url) if base_url else self.BASE_URL).rstrip("/") + self._session = requests.Session() + self._session.verify = True # explicit for security scanners + self._session.headers.update( + { + "Authorization": f"Bearer {self.api_token}", + "Content-Type": "application/json", + } + ) + + def _make_request( + self, + method: str, + endpoint: str, + data: Optional[Any] = None, + timeout: Optional[int] = None, + ) -> dict: + """Make an API request to Cloudflare and return the parsed JSON body.""" + url = f"{self.base_url}/accounts/{self.account_id}{endpoint}" + request_timeout = timeout or self.timeout + + try: + response = self._session.request( + method=method, + url=url, + json=data, + timeout=request_timeout, + ) + response.raise_for_status() + except requests.exceptions.RequestException as exc: + error_msg = str(exc) + err_response = getattr(exc, "response", None) + if err_response is not None: + try: + error_data = err_response.json() + if "errors" in error_data: + error_msg = str(error_data["errors"]) + except (json.JSONDecodeError, ValueError): + error_msg = err_response.text + raise CloudflareAPIError(f"API request failed: {error_msg}") from exc + + try: + return response.json() + except (json.JSONDecodeError, ValueError) as exc: + raise CloudflareAPIError( + f"Invalid JSON in Cloudflare response: {response.text[:200]}" + ) from exc + + def list_lists(self) -> list[dict]: + """List all rules lists in the account.""" + response = self._make_request("GET", "/rules/lists") + return response.get("result", []) + + def get_list(self, list_id: str) -> dict: + """Get a specific list's metadata.""" + response = self._make_request("GET", f"/rules/lists/{list_id}") + return response.get("result", {}) + + def get_list_items(self, list_id: str, cursor: Optional[str] = None) -> dict: + """Get a page of items from a list.""" + endpoint = f"/rules/lists/{list_id}/items" + if cursor: + endpoint += f"?cursor={cursor}" + return self._make_request("GET", endpoint) + + def get_all_list_items(self, list_id: str) -> list[dict]: + """Get all items from a list, following pagination cursors.""" + all_items: list[dict] = [] + cursor = None + + while True: + response = self.get_list_items(list_id, cursor) + all_items.extend(response.get("result", [])) + + result_info = response.get("result_info", {}) + cursor = result_info.get("cursors", {}).get("after") + if not cursor: + break + + return all_items + + def replace_list_items(self, list_id: str, items: list[dict]) -> dict: + """Replace ALL items in a list with the provided items (snapshot). + + Args: + list_id: The list ID. + items: Items in the kind-specific format, e.g. for an IP list: + ``[{"ip": "192.0.2.1"}, {"ip": "10.0.0.0/8"}]``. + + Returns: + Operation result, including an ``operation_id`` for the async bulk job. + """ + response = self._make_request( + "PUT", f"/rules/lists/{list_id}/items", data=items, timeout=300 + ) + return response.get("result", {}) + + def get_bulk_operation(self, operation_id: str) -> dict: + """Get the status of a bulk operation.""" + response = self._make_request( + "GET", f"/rules/lists/bulk_operations/{operation_id}" + ) + return response.get("result", {}) + + def wait_for_operation( + self, operation_id: str, timeout: int = 300, poll_interval: int = 2 + ) -> dict: + """Block until a bulk operation completes. + + Raises: + CloudflareAPIError: If the operation fails or times out. + """ + start_time = time.monotonic() + + while True: + status = self.get_bulk_operation(operation_id) + state = status.get("status") + + if state == "completed": + return status + if state == "failed": + raise CloudflareAPIError( + f"Bulk operation failed: {status.get('error')}" + ) + + if time.monotonic() - start_time > timeout: + raise CloudflareAPIError(f"Bulk operation timed out after {timeout}s") + + time.sleep(poll_interval) diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/connector.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/connector.py new file mode 100644 index 00000000000..6d3124f5480 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/connector.py @@ -0,0 +1,246 @@ +"""OpenCTI -> Cloudflare Rules List stream connector. + +Listens to the OpenCTI live stream for IPv4 indicators/observables, maintains an +in-memory snapshot of all known IPv4 values, and periodically pushes the full +snapshot to a Cloudflare Rules List (snapshot/replace model). +""" + +import json +import re +import sys +import time +from typing import Optional + +from pycti import OpenCTIConnectorHelper + +from .client import CloudflareAPIError, CloudflareRulesListClient +from .settings import ConnectorSettings + +# STIX pattern for an IPv4 indicator: [ipv4-addr:value = '...'] +_IPV4_PATTERN_RE = re.compile(r"\[ipv4-addr:value\s*=\s*'([^']+)'\]", re.IGNORECASE) + + +class Connector: + """OpenCTI connector for Cloudflare Rules Lists (IPv4).""" + + def __init__( + self, + helper: OpenCTIConnectorHelper, + config: ConnectorSettings, + client: CloudflareRulesListClient, + ): + self.helper = helper + self.config = config + self.client = client + self.logger = helper.connector_logger + + self.list_id = config.cloudflare.list_id + self.sync_interval = config.cloudflare.sync_interval.total_seconds() + + # Snapshot of IPv4 values keyed by OpenCTI id. + self._indicator_cache: dict[str, str] = {} + self._last_sync_time = 0.0 + + # ------------------------------------------------------------------ # + # IPv4 extraction + # ------------------------------------------------------------------ # + def _extract_ipv4(self, data: dict) -> Optional[str]: + """Return the IPv4 value from an OpenCTI/STIX object, or None. + + Two representations reach this method with different type keys: + * Live-stream / STIX shape uses the lowercase STIX ``type`` field + (``"indicator"``, ``"ipv4-addr"``). + * OpenCTI API objects (full sync via ``helper.api.*.list``) use the + capitalized ``entity_type`` field (``"Indicator"``, ``"IPv4-Addr"``) + and leave ``type`` unset. + + Both indicator shapes carry a STIX ``pattern``; both observable shapes + carry the address in ``value`` / ``observable_value``. + """ + stix_type = data.get("type", "") + entity_type = data.get("entity_type", "") + + # Indicator (stream: type=="indicator"; API: entity_type=="Indicator"). + if stix_type == "indicator" or entity_type == "Indicator": + pattern = data.get("pattern", "") + match = _IPV4_PATTERN_RE.search(pattern) if pattern else None + return match.group(1) if match else None + + # IPv4 observable (stream: type=="ipv4-addr"; API: entity_type=="IPv4-Addr"). + observable_type = entity_type or stix_type + if observable_type in ("ipv4-addr", "IPv4-Addr"): + return data.get("value") or data.get("observable_value") + + return None + + @staticmethod + def _object_id(data: dict) -> Optional[str]: + """Return the OpenCTI id for a stream/STIX object.""" + return data.get("id") or data.get("x_opencti_id") + + # ------------------------------------------------------------------ # + # Stream handling + # ------------------------------------------------------------------ # + def process_message(self, msg) -> None: + """Callback for each OpenCTI live-stream event.""" + try: + try: + data = json.loads(msg.data)["data"] + except (json.JSONDecodeError, KeyError, TypeError): + self.logger.warning("Could not parse stream message data") + return + + # The initial catch-up event type may be "message"; treat as create. + event_type = msg.event if getattr(msg, "event", None) else "create" + if event_type == "message": + event_type = "create" + + if event_type in ("create", "update"): + self._handle_upsert(data) + elif event_type == "delete": + self._handle_delete(data) + except (KeyboardInterrupt, SystemExit): + self.logger.info("Connector stopped") + sys.exit(0) + except Exception as exc: # noqa: BLE001 - never let the stream die + self.logger.error( + "Error processing stream message", meta={"error": str(exc)} + ) + + def _handle_upsert(self, data: dict) -> None: + value = self._extract_ipv4(data) + if not value: + return + + indicator_id = self._object_id(data) + if not indicator_id: + return + + self._indicator_cache[indicator_id] = value + self.logger.debug( + "Cached IPv4 indicator", meta={"id": indicator_id, "value": value} + ) + self._check_sync() + + def _handle_delete(self, data: dict) -> None: + indicator_id = self._object_id(data) + if indicator_id and indicator_id in self._indicator_cache: + del self._indicator_cache[indicator_id] + self.logger.debug("Removed indicator from cache", meta={"id": indicator_id}) + self._check_sync() + + # ------------------------------------------------------------------ # + # Sync to Cloudflare + # ------------------------------------------------------------------ # + def _check_sync(self) -> None: + """Sync to Cloudflare if the configured interval has elapsed.""" + if time.monotonic() - self._last_sync_time >= self.sync_interval: + self._sync_to_cloudflare() + + def _sync_to_cloudflare(self) -> None: + """Push the full IPv4 snapshot to the Cloudflare Rules List.""" + if not self._indicator_cache: + # Nothing to push -- do not open the throttle window, otherwise the + # first real indicator to arrive could be delayed by up to + # sync_interval before it is synced. + self.logger.info("No indicators to sync") + return + + self._last_sync_time = time.monotonic() + + self.logger.info( + "Syncing indicators to Cloudflare", + meta={"count": len(self._indicator_cache), "list_id": self.list_id}, + ) + + items = [ + {"ip": value, "comment": f"OpenCTI: {indicator_id}"} + for indicator_id, value in self._indicator_cache.items() + ] + + try: + result = self.client.replace_list_items(self.list_id, items) + operation_id = result.get("operation_id") + if operation_id: + self.logger.info( + "Bulk operation started", meta={"operation_id": operation_id} + ) + final_status = self.client.wait_for_operation(operation_id) + self.logger.info( + "Snapshot uploaded", + meta={"count": len(items), "status": final_status.get("status")}, + ) + else: + self.logger.info("Snapshot uploaded", meta={"count": len(items)}) + except CloudflareAPIError as exc: + self.logger.error("Failed to sync to Cloudflare", meta={"error": str(exc)}) + + # ------------------------------------------------------------------ # + # Full sync (startup) + # ------------------------------------------------------------------ # + def _full_sync(self) -> None: + """Load all IPv4 indicators and observables from OpenCTI, then sync.""" + self.logger.info("Starting full sync from OpenCTI") + self._indicator_cache = {} + + indicators = self.helper.api.indicator.list(getAll=True) + self.logger.info( + "Fetched indicators from OpenCTI", meta={"count": len(indicators)} + ) + for indicator in indicators: + value = self._extract_ipv4(indicator) + indicator_id = indicator.get("id") + if value and indicator_id: + self._indicator_cache[indicator_id] = value + + try: + observables = self.helper.api.stix_cyber_observable.list( + types=["IPv4-Addr"], getAll=True + ) + for observable in observables: + value = self._extract_ipv4(observable) + obs_id = observable.get("id") + if value and obs_id: + self._indicator_cache[obs_id] = value + except Exception as exc: # noqa: BLE001 + self.logger.warning( + "Could not fetch IPv4 observables", meta={"error": str(exc)} + ) + + self.logger.info( + "Loaded IPv4 indicators for sync", + meta={"count": len(self._indicator_cache)}, + ) + self._sync_to_cloudflare() + + # ------------------------------------------------------------------ # + # Lifecycle + # ------------------------------------------------------------------ # + def run(self) -> None: + """Verify the target list, full-sync, then listen to the live stream.""" + self.logger.info("Starting Cloudflare Rules List connector") + + # Verify the configured list exists before doing any work. + try: + list_info = self.client.get_list(self.list_id) + self.logger.info( + "Using Cloudflare list", + meta={ + "name": list_info.get("name"), + "id": self.list_id, + "kind": list_info.get("kind"), + }, + ) + except CloudflareAPIError as exc: + self.logger.error( + "Could not find Cloudflare list", + meta={"list_id": self.list_id, "error": str(exc)}, + ) + raise + + try: + self._full_sync() + except Exception as exc: # noqa: BLE001 + self.logger.error("Initial full sync failed", meta={"error": str(exc)}) + + self.helper.listen_stream(message_callback=self.process_message) diff --git a/stream/cloudflare-rules-list/src/cloudflare_rules_list/settings.py b/stream/cloudflare-rules-list/src/cloudflare_rules_list/settings.py new file mode 100644 index 00000000000..0766bee3096 --- /dev/null +++ b/stream/cloudflare-rules-list/src/cloudflare_rules_list/settings.py @@ -0,0 +1,93 @@ +"""Pydantic-settings configuration models for the connector. + +Configuration is loaded (in precedence order) from environment variables, then +``config.yml`` or ``.env`` next to the connector, then field defaults -- the +same precedence the legacy ``get_config_variable`` helper used. + +Environment variable mapping (nested by the first underscore): + + OPENCTI_URL -> opencti.url + OPENCTI_TOKEN -> opencti.token + CONNECTOR_ID -> connector.id + CONNECTOR_NAME -> connector.name + CONNECTOR_SCOPE -> connector.scope + CONNECTOR_LIVE_STREAM_ID -> connector.live_stream_id + CLOUDFLARE_ACCOUNT_ID -> cloudflare.account_id + CLOUDFLARE_API_TOKEN -> cloudflare.api_token + CLOUDFLARE_LIST_ID -> cloudflare.list_id + CLOUDFLARE_API_BASE_URL -> cloudflare.api_base_url + CLOUDFLARE_SYNC_INTERVAL -> cloudflare.sync_interval +""" + +from datetime import timedelta + +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseStreamConnectorConfig, + ListFromString, +) +from pydantic import Field, HttpUrl, SecretStr + + +class _StreamConnectorConfig(BaseStreamConnectorConfig): + """Stream connector namespace (``connector.*``). + + Inherits ``type`` (forced to ``STREAM``), ``log_level``, + ``live_stream_id``, ``live_stream_listen_delete`` and + ``live_stream_no_dependencies`` from the SDK base class. + """ + + id: str = Field( + default="e5b2b078-39d5-4fde-9944-c53c91fbd9d1", + description="The UUID of the connector as shown in OpenCTI.", + ) + name: str = Field( + default="Cloudflare Rules List", + description="The name of the connector as shown in OpenCTI.", + ) + scope: ListFromString = Field( + default=["cloudflare"], + description="Connector scope (comma-separated).", + ) + + +class CloudflareConfig(BaseConfigModel): + """Cloudflare namespace (``cloudflare.*``).""" + + account_id: str = Field( + description="Cloudflare account ID that owns the Rules List.", + ) + api_token: SecretStr = Field( + description=( + "Cloudflare API token with the 'Account > Account Filter Lists > " + "Edit' permission." + ), + ) + list_id: str = Field( + description="ID of the existing Cloudflare Rules List (IP kind) to sync into.", + ) + api_base_url: HttpUrl = Field( + default=HttpUrl("https://api.cloudflare.com/client/v4"), + description=( + "Base URL of the Cloudflare API. Override only for testing against a " + "mock server or a Cloudflare-compatible gateway." + ), + ) + sync_interval: timedelta = Field( + default=timedelta(hours=1), + description=( + "Minimum interval between snapshot uploads to Cloudflare. " + "Accepts ISO-8601 duration like 'PT30M', 'PT1H', 'PT1H30M', etc." + ), + ) + + +class ConnectorSettings(BaseConnectorSettings): + """Top-level settings. + + ``opencti`` (url, token) is provided by :class:`BaseConnectorSettings`. + """ + + connector: _StreamConnectorConfig = Field(default_factory=_StreamConnectorConfig) + cloudflare: CloudflareConfig = Field(default_factory=CloudflareConfig) diff --git a/stream/cloudflare-rules-list/src/config.yml.sample b/stream/cloudflare-rules-list/src/config.yml.sample new file mode 100644 index 00000000000..6f733bc8168 --- /dev/null +++ b/stream/cloudflare-rules-list/src/config.yml.sample @@ -0,0 +1,25 @@ +opencti: + url: 'http://localhost:8080' + token: 'ChangeMe' + +connector: + id: 'ChangeMe' # Unique connector UUID + name: 'Cloudflare Rules List' + scope: 'cloudflare' + log_level: 'info' # debug | info | warn | error + live_stream_id: 'live' # OpenCTI live stream ID + live_stream_listen_delete: true + live_stream_no_dependencies: true + +cloudflare: + # Cloudflare account ID that owns the Rules List. + account_id: 'ChangeMe' + # API token with 'Account > Account Filter Lists > Edit' permission. + api_token: 'ChangeMe' + # ID of the existing Cloudflare Rules List (IP kind) to sync into. + list_id: 'ChangeMe' + # Base URL of the Cloudflare API. Override only for testing or a compatible gateway. + # api_base_url: 'https://api.cloudflare.com/client/v4' + # Minimum interval between snapshot uploads to Cloudflare. + # Accepts ISO-8601 duration strings like 'PT30M', 'PT1H', 'PT1H30M', etc. + sync_interval: 'PT1H' diff --git a/stream/cloudflare-rules-list/src/main.py b/stream/cloudflare-rules-list/src/main.py new file mode 100644 index 00000000000..9a672b529f8 --- /dev/null +++ b/stream/cloudflare-rules-list/src/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Entry point for the OpenCTI -> Cloudflare Rules List stream connector (v2).""" + +import traceback + +from cloudflare_rules_list import Connector, ConnectorSettings +from cloudflare_rules_list.client import CloudflareRulesListClient +from pycti import OpenCTIConnectorHelper + +if __name__ == "__main__": + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + client = CloudflareRulesListClient( + account_id=settings.cloudflare.account_id, + api_token=settings.cloudflare.api_token.get_secret_value(), + base_url=settings.cloudflare.api_base_url, + ) + connector = Connector(helper=helper, config=settings, client=client) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/stream/cloudflare-rules-list/src/requirements.txt b/stream/cloudflare-rules-list/src/requirements.txt new file mode 100644 index 00000000000..f5cd5f2fcf8 --- /dev/null +++ b/stream/cloudflare-rules-list/src/requirements.txt @@ -0,0 +1,12 @@ +# OpenCTI Python client (CalVer; must be compatible with your OpenCTI 7.x platform) +pycti==7.260728.0 + +# Modern connector SDK (Pydantic-settings base classes, shared types) +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk + +# Configuration +pydantic~=2.11.5 +pydantic-settings~=2.9.1 + +# HTTP client +requests>=2.31.0,<3.0.0 diff --git a/stream/cloudflare-rules-list/tests/__init__.py b/stream/cloudflare-rules-list/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/stream/cloudflare-rules-list/tests/conftest.py b/stream/cloudflare-rules-list/tests/conftest.py new file mode 100644 index 00000000000..5ee8fc0e226 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) diff --git a/stream/cloudflare-rules-list/tests/test-requirements.txt b/stream/cloudflare-rules-list/tests/test-requirements.txt new file mode 100644 index 00000000000..96e880d6dfa --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test-requirements.txt @@ -0,0 +1,3 @@ +-r ../src/requirements.txt +pytest +pytest-mock diff --git a/stream/cloudflare-rules-list/tests/test_client.py b/stream/cloudflare-rules-list/tests/test_client.py new file mode 100644 index 00000000000..7d65c759e61 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_client.py @@ -0,0 +1,196 @@ +from unittest.mock import MagicMock + +import pytest +import requests +from cloudflare_rules_list.client import CloudflareAPIError, CloudflareRulesListClient + + +@pytest.fixture +def client(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="token-1", timeout=10) + c._session = MagicMock() + return c + + +def _response(json_body=None, raise_exc=None): + resp = MagicMock() + resp.json.return_value = json_body or {} + if raise_exc is not None: + resp.raise_for_status.side_effect = raise_exc + return resp + + +def test_auth_header_is_set(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="secret") + assert c._session.headers["Authorization"] == "Bearer secret" + + +def test_default_base_url(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="secret") + assert c.base_url == CloudflareRulesListClient.BASE_URL + + +def test_base_url_override_strips_trailing_slash(): + c = CloudflareRulesListClient( + account_id="acc-1", + api_token="secret", + base_url="https://gateway.example/client/v4/", + ) + assert c.base_url == "https://gateway.example/client/v4" + + +def test_session_verify_is_true(): + c = CloudflareRulesListClient(account_id="acc-1", api_token="secret") + assert c._session.verify is True + + +def test_make_request_uses_overridden_base_url(): + c = CloudflareRulesListClient( + account_id="acc-1", api_token="secret", base_url="http://mock:9999/v4" + ) + c._session = MagicMock() + c._session.request.return_value = _response({"result": {}}) + c._make_request("GET", "/rules/lists/abc") + assert ( + c._session.request.call_args.kwargs["url"] + == "http://mock:9999/v4/accounts/acc-1/rules/lists/abc" + ) + + +def test_make_request_success(client): + client._session.request.return_value = _response({"result": {"ok": True}}) + result = client._make_request("GET", "/rules/lists/abc") + assert result == {"result": {"ok": True}} + args, kwargs = client._session.request.call_args + assert kwargs["url"].endswith("/accounts/acc-1/rules/lists/abc") + + +def test_make_request_raises_on_non_json_success(client): + # 2xx response whose body is not JSON (e.g. an HTML challenge page) must be + # wrapped as CloudflareAPIError, not surface a bare ValueError. + resp = _response() + resp.json.side_effect = ValueError("no json") + resp.text = "challenge" + client._session.request.return_value = resp + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "Invalid JSON" in str(exc.value) + + +def test_make_request_uses_custom_timeout(client): + client._session.request.return_value = _response({"result": {}}) + client._make_request("PUT", "/x", data=[], timeout=300) + assert client._session.request.call_args.kwargs["timeout"] == 300 + + +def test_make_request_raises_with_structured_errors(client): + err = requests.exceptions.HTTPError("400") + err.response = MagicMock() + err.response.json.return_value = {"errors": [{"code": 10001, "message": "bad"}]} + client._session.request.return_value = _response(raise_exc=err) + + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "10001" in str(exc.value) + + +def test_make_request_raises_with_text_body(client): + err = requests.exceptions.HTTPError("500") + err.response = MagicMock() + err.response.json.side_effect = ValueError("not json") + err.response.text = "Internal Server Error" + client._session.request.return_value = _response(raise_exc=err) + + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "Internal Server Error" in str(exc.value) + + +def test_make_request_raises_without_response(client): + err = requests.exceptions.ConnectionError("no network") + client._session.request.return_value = _response(raise_exc=err) + with pytest.raises(CloudflareAPIError) as exc: + client._make_request("GET", "/x") + assert "no network" in str(exc.value) + + +def test_list_lists(client): + client._session.request.return_value = _response({"result": [{"id": "l1"}]}) + assert client.list_lists() == [{"id": "l1"}] + + +def test_get_list(client): + client._session.request.return_value = _response({"result": {"id": "l1"}}) + assert client.get_list("l1") == {"id": "l1"} + + +def test_get_list_items_with_cursor(client): + client._session.request.return_value = _response({"result": []}) + client.get_list_items("l1", cursor="CURSOR") + assert "cursor=CURSOR" in client._session.request.call_args.kwargs["url"] + + +def test_get_all_list_items_follows_pagination(client): + page1 = _response( + {"result": [{"ip": "1.1.1.1"}], "result_info": {"cursors": {"after": "C2"}}} + ) + page2 = _response({"result": [{"ip": "2.2.2.2"}], "result_info": {"cursors": {}}}) + client._session.request.side_effect = [page1, page2] + + items = client.get_all_list_items("l1") + assert items == [{"ip": "1.1.1.1"}, {"ip": "2.2.2.2"}] + + +def test_replace_list_items(client): + client._session.request.return_value = _response({"result": {"operation_id": "op"}}) + result = client.replace_list_items("l1", [{"ip": "1.1.1.1"}]) + assert result == {"operation_id": "op"} + assert client._session.request.call_args.kwargs["method"] == "PUT" + + +def test_get_bulk_operation(client): + client._session.request.return_value = _response( + {"result": {"status": "completed"}} + ) + assert client.get_bulk_operation("op") == {"status": "completed"} + + +def test_wait_for_operation_completed(client, monkeypatch): + monkeypatch.setattr("cloudflare_rules_list.client.time.monotonic", lambda: 0.0) + client._session.request.return_value = _response( + {"result": {"status": "completed"}} + ) + assert client.wait_for_operation("op") == {"status": "completed"} + + +def test_wait_for_operation_failed(client, monkeypatch): + monkeypatch.setattr("cloudflare_rules_list.client.time.monotonic", lambda: 0.0) + client._session.request.return_value = _response( + {"result": {"status": "failed", "error": "boom"}} + ) + with pytest.raises(CloudflareAPIError) as exc: + client.wait_for_operation("op") + assert "boom" in str(exc.value) + + +def test_wait_for_operation_times_out(client, monkeypatch): + times = iter([0.0, 0.0, 100.0]) + monkeypatch.setattr( + "cloudflare_rules_list.client.time.monotonic", lambda: next(times) + ) + monkeypatch.setattr("cloudflare_rules_list.client.time.sleep", lambda _: None) + client._session.request.return_value = _response({"result": {"status": "pending"}}) + + with pytest.raises(CloudflareAPIError) as exc: + client.wait_for_operation("op", timeout=10) + assert "timed out" in str(exc.value) + + +def test_wait_for_operation_polls_until_complete(client, monkeypatch): + monkeypatch.setattr("cloudflare_rules_list.client.time.monotonic", lambda: 0.0) + monkeypatch.setattr("cloudflare_rules_list.client.time.sleep", lambda _: None) + client._session.request.side_effect = [ + _response({"result": {"status": "pending"}}), + _response({"result": {"status": "completed"}}), + ] + assert client.wait_for_operation("op") == {"status": "completed"} diff --git a/stream/cloudflare-rules-list/tests/test_connector.py b/stream/cloudflare-rules-list/tests/test_connector.py new file mode 100644 index 00000000000..8f2c6d04513 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_connector.py @@ -0,0 +1,301 @@ +import json +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from cloudflare_rules_list.client import CloudflareAPIError +from cloudflare_rules_list.connector import Connector + + +# --------------------------------------------------------------------------- # +# Fixtures +# --------------------------------------------------------------------------- # +@pytest.fixture +def helper(): + h = MagicMock() + h.connector_logger = MagicMock() + return h + + +@pytest.fixture +def client(): + return MagicMock() + + +@pytest.fixture +def config(): + return SimpleNamespace( + cloudflare=SimpleNamespace( + list_id="list-123", sync_interval=timedelta(hours=1) + ), + ) + + +@pytest.fixture +def connector(helper, config, client): + conn = Connector(helper=helper, config=config, client=client) + # Force sync to trigger on demand in tests. + conn.sync_interval = 0 + return conn + + +# --------------------------------------------------------------------------- # +# _extract_ipv4 +# --------------------------------------------------------------------------- # +def test_extract_ipv4_from_indicator_pattern(connector): + data = {"type": "indicator", "pattern": "[ipv4-addr:value = '192.0.2.1']"} + assert connector._extract_ipv4(data) == "192.0.2.1" + + +def test_extract_ipv4_from_api_indicator(connector): + # OpenCTI API (full-sync) shape: entity_type=="Indicator", no lowercase type. + data = {"entity_type": "Indicator", "pattern": "[ipv4-addr:value = '192.0.2.30']"} + assert connector._extract_ipv4(data) == "192.0.2.30" + + +def test_extract_ipv4_from_api_indicator_no_match(connector): + data = {"entity_type": "Indicator", "pattern": "[domain-name:value = 'evil.com']"} + assert connector._extract_ipv4(data) is None + + +def test_extract_ipv4_from_indicator_no_match(connector): + data = {"type": "indicator", "pattern": "[domain-name:value = 'evil.com']"} + assert connector._extract_ipv4(data) is None + + +def test_extract_ipv4_from_indicator_empty_pattern(connector): + assert connector._extract_ipv4({"type": "indicator"}) is None + + +def test_extract_ipv4_from_stix_sco(connector): + data = {"type": "ipv4-addr", "value": "203.0.113.5"} + assert connector._extract_ipv4(data) == "203.0.113.5" + + +def test_extract_ipv4_from_opencti_observable(connector): + data = {"entity_type": "IPv4-Addr", "observable_value": "198.51.100.7"} + assert connector._extract_ipv4(data) == "198.51.100.7" + + +def test_extract_ipv4_unrelated_type(connector): + assert connector._extract_ipv4({"type": "domain-name", "value": "x"}) is None + + +# --------------------------------------------------------------------------- # +# _object_id +# --------------------------------------------------------------------------- # +def test_object_id_prefers_id(connector): + assert connector._object_id({"id": "a", "x_opencti_id": "b"}) == "a" + + +def test_object_id_falls_back_to_x_opencti_id(connector): + assert connector._object_id({"x_opencti_id": "b"}) == "b" + + +def test_object_id_none(connector): + assert connector._object_id({}) is None + + +# --------------------------------------------------------------------------- # +# process_message routing +# --------------------------------------------------------------------------- # +def _msg(event, payload): + return SimpleNamespace(event=event, data=json.dumps({"data": payload})) + + +def test_process_message_create_caches(connector): + connector.process_message( + _msg("create", {"id": "ind-1", "type": "ipv4-addr", "value": "1.1.1.1"}) + ) + assert connector._indicator_cache["ind-1"] == "1.1.1.1" + + +def test_process_message_message_event_treated_as_create(connector): + connector.process_message( + _msg("message", {"id": "ind-2", "type": "ipv4-addr", "value": "2.2.2.2"}) + ) + assert connector._indicator_cache["ind-2"] == "2.2.2.2" + + +def test_process_message_delete_removes(connector): + connector._indicator_cache["ind-3"] = "3.3.3.3" + connector.process_message(_msg("delete", {"id": "ind-3", "type": "ipv4-addr"})) + assert "ind-3" not in connector._indicator_cache + + +def test_process_message_bad_json_warns(connector): + connector.process_message(SimpleNamespace(event="create", data="not-json")) + connector.logger.warning.assert_called_once() + + +def test_process_message_missing_data_key_warns(connector): + connector.process_message(SimpleNamespace(event="create", data=json.dumps({}))) + connector.logger.warning.assert_called_once() + + +def test_process_message_exits_on_keyboard_interrupt(connector, monkeypatch): + monkeypatch.setattr( + connector, "_handle_upsert", MagicMock(side_effect=KeyboardInterrupt) + ) + with pytest.raises(SystemExit): + connector.process_message( + _msg("create", {"id": "x", "type": "ipv4-addr", "value": "9.9.9.9"}) + ) + + +def test_process_message_swallows_unexpected_errors(connector, monkeypatch): + monkeypatch.setattr( + connector, "_handle_upsert", MagicMock(side_effect=RuntimeError("boom")) + ) + connector.process_message( + _msg("create", {"id": "x", "type": "ipv4-addr", "value": "9.9.9.9"}) + ) + connector.logger.error.assert_called_once() + + +# --------------------------------------------------------------------------- # +# upsert / delete +# --------------------------------------------------------------------------- # +def test_handle_upsert_ignores_without_value(connector): + connector._handle_upsert({"id": "i", "type": "domain-name"}) + assert connector._indicator_cache == {} + + +def test_handle_upsert_ignores_without_id(connector): + connector._handle_upsert({"type": "ipv4-addr", "value": "1.2.3.4"}) + assert connector._indicator_cache == {} + + +def test_handle_delete_unknown_id_is_noop(connector): + connector._handle_delete({"id": "unknown"}) + assert connector._indicator_cache == {} + + +# --------------------------------------------------------------------------- # +# _check_sync / _sync_to_cloudflare +# --------------------------------------------------------------------------- # +def test_check_sync_does_not_sync_before_interval(connector, monkeypatch): + connector.sync_interval = 9999 + connector._last_sync_time = 0.0 + monkeypatch.setattr("cloudflare_rules_list.connector.time.monotonic", lambda: 1.0) + sync = MagicMock() + monkeypatch.setattr(connector, "_sync_to_cloudflare", sync) + connector._check_sync() + sync.assert_not_called() + + +def test_sync_to_cloudflare_empty_cache(connector): + connector._sync_to_cloudflare() + connector.client.replace_list_items.assert_not_called() + connector.logger.info.assert_any_call("No indicators to sync") + + +def test_sync_to_cloudflare_empty_cache_does_not_open_throttle(connector, monkeypatch): + # An empty push must NOT stamp _last_sync_time, or the first real indicator + # could be delayed by up to sync_interval before it is synced. + monkeypatch.setattr("cloudflare_rules_list.connector.time.monotonic", lambda: 123.0) + connector._last_sync_time = 0.0 + connector._indicator_cache = {} + connector._sync_to_cloudflare() + assert connector._last_sync_time == 0.0 + + +def test_sync_to_cloudflare_with_operation(connector): + connector._indicator_cache = {"ind-1": "1.1.1.1"} + connector.client.replace_list_items.return_value = {"operation_id": "op-1"} + connector.client.wait_for_operation.return_value = {"status": "completed"} + + connector._sync_to_cloudflare() + + connector.client.replace_list_items.assert_called_once_with( + "list-123", [{"ip": "1.1.1.1", "comment": "OpenCTI: ind-1"}] + ) + connector.client.wait_for_operation.assert_called_once_with("op-1") + + +def test_sync_to_cloudflare_without_operation_id(connector): + connector._indicator_cache = {"ind-1": "1.1.1.1"} + connector.client.replace_list_items.return_value = {} + connector._sync_to_cloudflare() + connector.client.wait_for_operation.assert_not_called() + + +def test_sync_to_cloudflare_handles_api_error(connector): + connector._indicator_cache = {"ind-1": "1.1.1.1"} + connector.client.replace_list_items.side_effect = CloudflareAPIError("nope") + connector._sync_to_cloudflare() + connector.logger.error.assert_called_once() + + +# --------------------------------------------------------------------------- # +# _full_sync +# --------------------------------------------------------------------------- # +def test_full_sync_loads_indicators_and_observables(connector): + # Use the real OpenCTI API object shape: entity_type (capitalized), no + # lowercase STIX `type`. This mirrors helper.api.*.list() output so a + # regression in _extract_ipv4's indicator handling is caught here. + connector.helper.api.indicator.list.return_value = [ + { + "id": "ind-1", + "entity_type": "Indicator", + "pattern": "[ipv4-addr:value = '1.1.1.1']", + }, + { + "id": "ind-2", + "entity_type": "Indicator", + "pattern": "[domain-name:value = 'x']", + }, + ] + connector.helper.api.stix_cyber_observable.list.return_value = [ + {"id": "obs-1", "entity_type": "IPv4-Addr", "observable_value": "8.8.8.8"}, + ] + connector.client.replace_list_items.return_value = {} + + connector._full_sync() + + assert connector._indicator_cache == {"ind-1": "1.1.1.1", "obs-1": "8.8.8.8"} + connector.client.replace_list_items.assert_called_once() + + +def test_full_sync_handles_observable_error(connector): + connector.helper.api.indicator.list.return_value = [] + connector.helper.api.stix_cyber_observable.list.side_effect = RuntimeError("boom") + connector.client.replace_list_items.return_value = {} + + connector._full_sync() + + connector.logger.warning.assert_called_once() + + +# --------------------------------------------------------------------------- # +# run +# --------------------------------------------------------------------------- # +def test_run_verifies_list_then_listens(connector, monkeypatch): + connector.client.get_list.return_value = {"name": "blocklist", "kind": "ip"} + full_sync = MagicMock() + monkeypatch.setattr(connector, "_full_sync", full_sync) + + connector.run() + + connector.client.get_list.assert_called_once_with("list-123") + full_sync.assert_called_once() + connector.helper.listen_stream.assert_called_once_with( + message_callback=connector.process_message + ) + + +def test_run_raises_when_list_missing(connector): + connector.client.get_list.side_effect = CloudflareAPIError("404") + with pytest.raises(CloudflareAPIError): + connector.run() + connector.helper.listen_stream.assert_not_called() + + +def test_run_continues_when_full_sync_fails(connector, monkeypatch): + connector.client.get_list.return_value = {"name": "blocklist", "kind": "ip"} + monkeypatch.setattr( + connector, "_full_sync", MagicMock(side_effect=RuntimeError("boom")) + ) + connector.run() + connector.helper.listen_stream.assert_called_once() diff --git a/stream/cloudflare-rules-list/tests/test_main.py b/stream/cloudflare-rules-list/tests/test_main.py new file mode 100644 index 00000000000..56fe993223b --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_main.py @@ -0,0 +1,81 @@ +from typing import Any +from unittest.mock import MagicMock + +import pytest +from cloudflare_rules_list import Connector, ConnectorSettings +from cloudflare_rules_list.client import CloudflareRulesListClient +from pycti import OpenCTIConnectorHelper + + +@pytest.fixture +def mock_opencti_connector_helper(monkeypatch): + """Mock all heavy dependencies of OpenCTIConnectorHelper (calls to OpenCTI).""" + module_import_path = "pycti.connector.opencti_connector_helper" + monkeypatch.setattr(f"{module_import_path}.killProgramHook", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.sched.scheduler", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.ConnectorInfo", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIApiClient", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIConnector", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIMetricHandler", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.PingAlive", MagicMock()) + + +class StubConnectorSettings(ConnectorSettings): + """`ConnectorSettings` subclass returning a fixed, valid config dict.""" + + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "cloudflare", + "log_level": "error", + "live_stream_id": "live", + "live_stream_listen_delete": True, + "live_stream_no_dependencies": True, + }, + "cloudflare": { + "account_id": "acc-1", + "api_token": "secret", + "list_id": "list-1", + "sync_interval": "PT1H", + }, + } + ) + + +def test_connector_settings_is_instantiated(): + settings = StubConnectorSettings() + assert isinstance(settings, ConnectorSettings) + assert isinstance(settings.to_helper_config(), dict) + + +def test_opencti_connector_helper_is_instantiated(mock_opencti_connector_helper): + settings = StubConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + + assert helper.opencti_url == "http://localhost:8080/" + assert helper.opencti_token == "test-token" + assert helper.connect_id == "connector-id" + assert helper.connect_name == "Test Connector" + assert helper.connect_scope == "cloudflare" + assert helper.log_level == "ERROR" + assert helper.connect_live_stream_id == "live" + + +def test_connector_is_instantiated(mock_opencti_connector_helper): + settings = StubConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + client = CloudflareRulesListClient( + account_id=settings.cloudflare.account_id, + api_token=settings.cloudflare.api_token.get_secret_value(), + ) + connector = Connector(helper=helper, config=settings, client=client) + + assert connector.config == settings + assert connector.helper == helper + assert connector.list_id == "list-1" + assert connector.sync_interval == 3600 diff --git a/stream/cloudflare-rules-list/tests/test_settings.py b/stream/cloudflare-rules-list/tests/test_settings.py new file mode 100644 index 00000000000..8bc6d391758 --- /dev/null +++ b/stream/cloudflare-rules-list/tests/test_settings.py @@ -0,0 +1,178 @@ +from datetime import timedelta +from typing import Any + +import pytest +from cloudflare_rules_list import ConnectorSettings +from connectors_sdk import BaseConfigModel, ConfigValidationError + + +@pytest.mark.parametrize( + "settings_dict", + [ + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "cloudflare", + "log_level": "error", + "live_stream_id": "live", + "live_stream_listen_delete": True, + "live_stream_no_dependencies": True, + }, + "cloudflare": { + "account_id": "ChangeMe", + "api_token": "ChangeMe", + "list_id": "ChangeMe", + "sync_interval": "PT30M", + }, + }, + id="full_valid_settings_dict", + ), + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "scope": "cloudflare", + "log_level": "error", + "live_stream_id": "live", + }, + "cloudflare": { + "account_id": "ChangeMe", + "api_token": "ChangeMe", + "list_id": "ChangeMe", + }, + }, + id="minimal_valid_settings_dict", + ), + ], +) +def test_settings_should_accept_valid_input(settings_dict): + """`ConnectorSettings` accepts valid input and exposes typed namespaces.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler(settings_dict) + + settings = FakeConnectorSettings() + assert isinstance(settings.opencti, BaseConfigModel) is True + assert isinstance(settings.connector, BaseConfigModel) is True + assert isinstance(settings.cloudflare, BaseConfigModel) is True + + +def test_settings_defaults_are_applied(): + """Optional connector fields fall back to their declared defaults.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "live_stream_id": "live", + }, + "cloudflare": { + "account_id": "acc", + "api_token": "secret", + "list_id": "list", + }, + } + ) + + settings = FakeConnectorSettings() + assert settings.connector.name == "Cloudflare Rules List" + assert settings.connector.scope == ["cloudflare"] + assert settings.cloudflare.sync_interval == timedelta(hours=1) + assert ( + str(settings.cloudflare.api_base_url).rstrip("/") + == "https://api.cloudflare.com/client/v4" + ) + + +def test_settings_api_token_is_secret(): + """The Cloudflare API token is wrapped as a SecretStr and not leaked in repr.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": { + "account_id": "acc", + "api_token": "super-secret", + "list_id": "list", + }, + } + ) + + settings = FakeConnectorSettings() + assert settings.cloudflare.api_token.get_secret_value() == "super-secret" + assert "super-secret" not in repr(settings.cloudflare.api_token) + + +def test_to_helper_config_returns_dict(): + """`to_helper_config` (inherited) returns a dict for OpenCTIConnectorHelper.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": { + "account_id": "acc", + "api_token": "secret", + "list_id": "list", + }, + } + ) + + settings = FakeConnectorSettings() + assert isinstance(settings.to_helper_config(), dict) + + +@pytest.mark.parametrize( + "settings_dict", + [ + pytest.param({}, id="empty_settings_dict"), + pytest.param( + { + "opencti": {"url": "http://localhost:PORT", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": { + "account_id": "acc", + "api_token": "secret", + "list_id": "list", + }, + }, + id="invalid_opencti_url", + ), + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "live_stream_id": "live"}, + "cloudflare": {"account_id": "acc"}, + }, + id="missing_cloudflare_fields", + ), + ], +) +def test_settings_should_raise_when_invalid_input(settings_dict): + """`ConnectorSettings` raises a validation error on invalid/incomplete input.""" + + class FakeConnectorSettings(ConnectorSettings): + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler(settings_dict) + + with pytest.raises(ConfigValidationError) as err: + FakeConnectorSettings() + assert "Error validating configuration" in str(err)