diff --git a/external-import/domaintools-feeds/.dockerignore b/external-import/domaintools-feeds/.dockerignore new file mode 100644 index 00000000000..c5d360e969a --- /dev/null +++ b/external-import/domaintools-feeds/.dockerignore @@ -0,0 +1,9 @@ +__metadata__ +**/__pycache__ +**/__docs__ +**/.venv +**/venv +**/logs +**/config.yml +**/*.egg-info +**/*.gql diff --git a/external-import/domaintools-feeds/Dockerfile b/external-import/domaintools-feeds/Dockerfile new file mode 100644 index 00000000000..77042ab0de0 --- /dev/null +++ b/external-import/domaintools-feeds/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-alpine +ENV CONNECTOR_TYPE=EXTERNAL_IMPORT + +# Copy the connector +COPY src /opt/opencti-connector-domaintools-feeds + +# Install Python modules +# hadolint ignore=DL3003 +RUN apk update && apk upgrade && \ + apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev + +RUN cd /opt/opencti-connector-domaintools-feeds && \ + pip3 install --no-cache-dir -r requirements.txt && \ + apk del git build-base + +# Expose and entrypoint +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/external-import/domaintools-feeds/README.md b/external-import/domaintools-feeds/README.md new file mode 100644 index 00000000000..f4281ae62c2 --- /dev/null +++ b/external-import/domaintools-feeds/README.md @@ -0,0 +1,161 @@ +# OpenCTI DomainTools Feeds Connector + +| Status | Date | Comment | +|------------------|------------|---------| +| Partner Verified | - | - | + +## Table of Contents + +- [OpenCTI DomainTools Feeds Connector](#opencti-domaintools-feeds-connector) + - [Introduction](#introduction) + - [Installation](#installation) + - [Requirements](#requirements) + - [Configuration variables](#configuration-variables) + - [OpenCTI environment variables](#opencti-environment-variables) + - [Base connector environment variables](#base-connector-environment-variables) + - [DomainTools extra parameters environment variables](#domaintools-extra-parameters-environment-variables) + - [Deployment](#deployment) + - [Docker Deployment](#docker-deployment) + - [Manual Deployment](#manual-deployment) + - [Usage](#usage) + - [Behavior](#behavior) + - [Debugging](#debugging) + - [Additional information](#additional-information) + +## Introduction + +This connector fetches data from DomainTools Threat Feeds (via the Real-time Feed API) and imports it into OpenCTI as Observables. Each feed entry is mapped to a Structured Threat Information Expression (STIX) 2.1 Observable object, enriched with metadata such as severity, entity state, platform, audit logs, etc. Created objects carry a Traffic Light Protocol (TLP) marking for downstream sharing. + +## Installation + +### Requirements + +- OpenCTI Platform version >= 6.x +- DomainTools API credential + +## Configuration variables + +There are a number of configuration options, which are set either in `docker-compose.yml` (for Docker) or in `config.yml` (for manual deployment). + +Below are the parameters you'll need to set for OpenCTI: + +### OpenCTI environment variables + +| Parameter | config.yml `opencti` | Docker environment variable | Default | Mandatory | Description | +|---------------|----------------------|-----------------------------|---------|-----------|------------------------------------------------------| +| OpenCTI URL | `url` | `OPENCTI_URL` | / | Yes | The URL of the OpenCTI platform. | +| OpenCTI Token | `token` | `OPENCTI_TOKEN` | / | Yes | The default admin token set in the OpenCTI platform. | + +### Base connector environment variables + +Below are the parameters you'll need to set for running the connector properly: + +| Parameter | config.yml `connector` | Docker environment variable | Default | Mandatory | Description | +|-----------------|------------------------|-----------------------------|---------|-----------|------------------------------------------------------------------------------------------| +| Connector ID | `id` | `CONNECTOR_ID` | / | Yes | A unique `UUIDv4` identifier for this connector instance. | +| Connector Name | `name` | `CONNECTOR_NAME` | / | Yes | Name of the connector. | +| Connector Scope | `scope` | `CONNECTOR_SCOPE` | stix2 | Yes | The scope or type of data the connector is importing, either a MIME type or Stix Object. | +| Log Level | `log_level` | `CONNECTOR_LOG_LEVEL` | error | No | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`. | +| Duration Period | `duration_period` | `CONNECTOR_DURATION_PERIOD` | PT1H | Yes | The period of time between two connector runs (ISO 8601 duration format). | + +### DomainTools extra parameters environment variables + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|-------------------------|--------------------------------|----------------------------------|---------|-----------|---------------------------------------| +| API base URL | domaintools.api_base_url | `DOMAINTOOLS_API_BASE_URL` | https://api.domaintools.com | Yes | DomainTools API base URL | +| API key | domaintools.api_key | `DOMAINTOOLS_API_KEY` | | Yes | DomainTools API key | +| Feed Type | domaintools.feed_type | `DOMAINTOOLS_FEED_TYPE` | nod | Yes | Type of feed to ingest (domainhotlist, domainrisk, nod, nad, noh, domaindiscovery) | +| Session ID | domaintools.session_id | `DOMAINTOOLS_SESSION_ID` | | No | A unique identifier for the session, used for resuming data retrieval from the last point. | +| TLP Level | domaintools.tlp_level | `DOMAINTOOLS_TLP_LEVEL` | clear | No | TLP marking for created STIX objects. | +| Domain Filter | domaintools.domain | `DOMAINTOOLS_DOMAIN` | | No | Filter for an exact domain or a domain substring by prefixing or suffixing your string with *. | +| Top Results | domaintools.top | `DOMAINTOOLS_TOP` | | No | Limits the number of results in the response payload. | + +## Deployment + +### Docker Deployment + +Before building the Docker container, you need to set the version of pycti in `requirements.txt` equal to whatever version of OpenCTI you're running. Example, `pycti==6.5.1`. If you don't, it will take the latest version, but sometimes the OpenCTI SDK fails to initialize. + +Build a Docker Image using the provided `Dockerfile`. + +Example: + +Register connector in the **main** OpenCTI `docker-compose.yml`: + +```yaml + connector-domaintools: + image: opencti/connector-domaintools:latest + environment: + - OPENCTI_URL=http://opencti:8080 + - OPENCTI_TOKEN=changeme + - CONNECTOR_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + - CONNECTOR_NAME=DomainTools Feeds + - CONNECTOR_SCOPE=stix2 + - CONNECTOR_LOG_LEVEL=error + - CONNECTOR_DURATION_PERIOD=PT5M + - DOMAINTOOLS_API_BASE_URL=https://api.domaintools.com + - DOMAINTOOLS_API_KEY=changeme + - DOMAINTOOLS_FEED_TYPE=nod + - DOMAINTOOLS_SESSION_ID=OpenCTI-NOD + - DOMAINTOOLS_TLP_LEVEL=clear + restart: always +``` + +Make sure to replace the environment variables in `docker-compose.yml` with the appropriate configurations for your environment. Then, start the docker container with the provided `docker-compose.yml`. + +```shell +docker compose up -d +# -d for detached +``` + +### Manual Deployment + +Create a file `config.yml` based on the provided `config.yml.sample`. + +Replace the configuration variables (especially the "**ChangeMe**" variables) with the appropriate configurations for your environment. + +Install the required python dependencies (preferably in a virtual environment): + +```shell +pip3 install -r requirements.txt +``` + +Then, start the connector from the `src` directory: + +```shell +python3 main.py +``` + +## Usage + +After installation, the connector should require minimal interaction to use, and should update automatically at a regular interval specified in your `docker-compose.yml` or `config.yml` in `duration_period`. + +However, if you would like to force an immediate download of a new batch of data, navigate to: + +**Data management → Ingestion → Connectors** in the OpenCTI platform. + +Find the "DomainTools Feeds" connector, and click on the refresh button to reset the connector's state and force a new download of data by re-running the connector. + +## Behavior + +- Fetches data from DomainTools Feeds paginated by `session_id` +- Converts each feed entry into a STIX 2.1 Observable object +- Bundles and sends the STIX objects to OpenCTI + +## Debugging + +The connector can be debugged by setting the appropriate log level. Note that logging messages can be added using `self.helper.connector_logger.{LOG_LEVEL}("Sample message")`, for example, `self.helper.connector_logger.error("An error message")`. + +Set `CONNECTOR_LOG_LEVEL=debug` for verbose logging. Log output includes: + +- API call details and retry behavior +- Data count fetched per run +- Page-by-page fetch progress +- STIX conversion trace per entry +- Bundle send status + +## Additional information + +- For the risk feeds (`domainhotlist`, `domainrisk`), each entry becomes a STIX 2.1 Indicator with a domain-name pattern, a confidence derived from the overall risk score, and per-category risk labels (phishing, malware, spam, proximity). +- For the observation feeds (`nod`, `nad`, `noh`, `domaindiscovery`), each entry becomes a STIX 2.1 Domain-Name observable labeled with its feed type. +- Every object is attributed to a DomainTools author identity and tagged with the configured TLP marking. \ No newline at end of file diff --git a/external-import/domaintools-feeds/__metadata__/connector_manifest.json b/external-import/domaintools-feeds/__metadata__/connector_manifest.json new file mode 100644 index 00000000000..8ab518fc671 --- /dev/null +++ b/external-import/domaintools-feeds/__metadata__/connector_manifest.json @@ -0,0 +1,22 @@ +{ + "title": "DomainTools Feeds", + "slug": "domaintools-feeds", + "description": "Import DomainTools Feeds", + "short_description": "Import DomainTools Feeds", + "logo": null, + "use_cases": ["Infrastructure & Attack Surface Visibility", "Detection & Response Enablement"], + "solution_categories": ["Threat Intelligence Feed"], + "contact": null, + "license_type": null, + "verified": true, + "last_verified_date": null, + "playbook_supported": false, + "max_confidence_level": 100, + "support_version": ">=6.8.12", + "subscription_link": null, + "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/external-import/domaintools-feeds", + "manager_supported": false, + "container_version": "rolling", + "container_image": "opencti/connector-domaintools-feeds", + "container_type": "EXTERNAL_IMPORT" +} \ No newline at end of file diff --git a/external-import/domaintools-feeds/__metadata__/logo.png b/external-import/domaintools-feeds/__metadata__/logo.png new file mode 100644 index 00000000000..c94ed391094 Binary files /dev/null and b/external-import/domaintools-feeds/__metadata__/logo.png differ diff --git a/external-import/domaintools-feeds/config.yml.sample b/external-import/domaintools-feeds/config.yml.sample new file mode 100644 index 00000000000..8499ff1edf5 --- /dev/null +++ b/external-import/domaintools-feeds/config.yml.sample @@ -0,0 +1,20 @@ +opencti: + url: 'http://localhost' + token: 'ChangeMe' + +connector: + type: 'EXTERNAL_IMPORT' + id: 'ChangeMe' + name: 'DomainTools Feeds' # optional + scope: 'stix2' + log_level: 'error' # optional (default: 'error') + duration_period: 'PT5M' # # optional, interval in ISO-8601 format between two runs of the connector (default: 'PT1H') + +domaintools: + api_base_url: 'https://api.domaintools.com' + api_key: 'ChangeMe' + feed_type: 'ChangeMe' # domainhotlist, domainrisk, nod, nad, noh, domaindiscovery + session_id: 'ChangeMe' # + # domain: *bank* # optional + # top: 10 # optional + tlp_level: 'clear' # optional, available values: 'clear', 'white', 'green', 'amber', 'amber+strict', 'red' (default: 'clear') \ No newline at end of file diff --git a/external-import/domaintools-feeds/docker-compose.yml b/external-import/domaintools-feeds/docker-compose.yml new file mode 100644 index 00000000000..2e77bb89a72 --- /dev/null +++ b/external-import/domaintools-feeds/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3' +services: + opencti-connector-domaintools-feeds: + image: opencti/opencti-connector-domaintools-feeds:latest + environment: + # Generic parameters (connection with OpenCTI) + - OPENCTI_URL=http://localhost + - OPENCTI_TOKEN=CHANGEME + # Common parameters for connectors of type EXTERNAL_IMPORT + - CONNECTOR_ID=CHANGEME + - CONNECTOR_NAME=CHANGEME # optional + - CONNECTOR_SCOPE=CHANGEME + - CONNECTOR_LOG_LEVEL=error # optional (default: 'error') + - CONNECTOR_DURATION_PERIOD=CHANGEME # optional, interval in ISO-8601 format between two runs of the connector (default: 'PT1H') + - DOMAINTOOLS_API_BASE_URL=https://api.domaintools.com + - DOMAINTOOLS_API_KEY=ChangeMe + - DOMAINTOOLS_FEED_TYPE=ChangeMe # domainhotlist, domainrisk, nod, nad, noh, domaindiscovery + - DOMAINTOOLS_SESSION_ID=ChangeMe # + # - DOMAINTOOLS_DOMAIN=*bank* # optional + # - DOMAINTOOLS_TOP=10 # optional + - DOMAINTOOLS_TLP_LEVEL=clear + restart: always diff --git a/external-import/domaintools-feeds/entrypoint.sh b/external-import/domaintools-feeds/entrypoint.sh new file mode 100644 index 00000000000..e3675d8929f --- /dev/null +++ b/external-import/domaintools-feeds/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Go to the right directory +cd /opt/opencti-connector-domaintools-feeds + +# Launch the worker +python3 main.py \ No newline at end of file diff --git a/external-import/domaintools-feeds/src/client/__init__.py b/external-import/domaintools-feeds/src/client/__init__.py new file mode 100644 index 00000000000..30c04e41ff4 --- /dev/null +++ b/external-import/domaintools-feeds/src/client/__init__.py @@ -0,0 +1,5 @@ +from client.api_client import DomainToolsClient + +__all__ = [ + "DomainToolsClient", +] diff --git a/external-import/domaintools-feeds/src/client/api_client.py b/external-import/domaintools-feeds/src/client/api_client.py new file mode 100644 index 00000000000..4cd1dca9149 --- /dev/null +++ b/external-import/domaintools-feeds/src/client/api_client.py @@ -0,0 +1,81 @@ +import requests +from pycti import OpenCTIConnectorHelper +from pydantic import HttpUrl + + +class DomainToolsClient: + def __init__( + self, + helper: OpenCTIConnectorHelper, + base_url: HttpUrl, + api_key: str, + feed_type: str, + ): + """ + Initialize the client with necessary configuration. + For log purpose, the connector's helper CAN be injected. + Other arguments CAN be added (e.g. `api_key`) if necessary. + + Args: + helper (OpenCTIConnectorHelper): The helper of the connector. Used for logs. + base_url (str): The external API base URL. + api_key (str): The API key to authenticate the connector to the external API. + """ + self.helper = helper + + self.base_url = f"{base_url}/v1/feed/{feed_type}/" + self.feed_type = feed_type + # Define headers in session and update when needed + headers = {"x-api-key": api_key} + self.session = requests.Session() + self.session.headers.update(headers) + + def _request_data(self, api_url: str, params=None): + """ + Internal method to handle API requests + :return: Response in JSON format + """ + try: + response = self.session.get(api_url, params=params) + + self.helper.connector_logger.info( + "[API] HTTP Get Request to endpoint", {"url_path": api_url} + ) + + response.raise_for_status() + return response + + except requests.RequestException as err: + error_msg = "[API] Error while fetching data: " + self.helper.connector_logger.error( + error_msg, {"url_path": api_url, "error": str(err)} + ) + return None + + def get_entities(self, params=None) -> list: + """Fetch feed entries from the DomainTools Threat Feeds API. + Args: + params: Optional query parameters (e.g., sessionID, domain filter, top). + Returns: + A list of feed entries (one per line) as strings. + """ + try: + # =========================== + # === Add your code below === + # =========================== + + response = self._request_data(self.base_url, params=params) + if response is None: + raise RuntimeError("Failed to fetch DomainTools feed entries") + return response.text.splitlines() + + # return response.json() + # =========================== + # === Add your code above === + # =========================== + + # raise NotImplementedError + + except Exception as err: + self.helper.connector_logger.error(err) + return [] diff --git a/external-import/domaintools-feeds/src/connector/__init__.py b/external-import/domaintools-feeds/src/connector/__init__.py new file mode 100644 index 00000000000..f0984d65aaf --- /dev/null +++ b/external-import/domaintools-feeds/src/connector/__init__.py @@ -0,0 +1,7 @@ +from connector.connector import DomainToolsFeedsConnector +from connector.settings import ConnectorSettings + +__all__ = [ + "ConnectorSettings", + "DomainToolsFeedsConnector", +] diff --git a/external-import/domaintools-feeds/src/connector/connector.py b/external-import/domaintools-feeds/src/connector/connector.py new file mode 100644 index 00000000000..f33b5f163c0 --- /dev/null +++ b/external-import/domaintools-feeds/src/connector/connector.py @@ -0,0 +1,254 @@ +import json +import sys +from datetime import datetime, timezone + +from client import DomainToolsClient +from connector.converter_to_stix import ConverterToStix +from connector.settings import ConnectorSettings +from pycti import OpenCTIConnectorHelper + + +class DomainToolsFeedsConnector: + """ + Specifications of the external import connector: + + This class encapsulates the main actions, expected to be run by any connector of type `EXTERNAL_IMPORT`. + This type of connector aim to fetch external data to create STIX bundle and send it to OpenCTI. + The STIX bundle in the queue will be processed by OpenCTI workers. + This type of connector uses the basic methods of the helper. + + --- + + Attributes: + config (ConnectorSettings): + Store the connector's configuration. It defines how to connector will behave. + helper (OpenCTIConnectorHelper): + Handle the connection and the requests between the connector, OpenCTI and the workers. + _All connectors MUST use the connector helper with connector's configuration._ + client (TemplateClient): + Provide methods to request the external API. + converter_to_stix (ConnectorConverter): + Provide methods for converting various types of input data into STIX 2.1 objects. + + --- + + Best practices: + - `self.helper.api.work.initiate_work(...)` is used to initiate a new work + - `self.helper.schedule_iso()` is used to schedule connector's runs frequency + - `self.helper.connector_logger.[info/debug/warning/error]` is used when logging a message + - `self.helper.stix2_create_bundle(stix_objects)` is used when creating a bundle + - `self.helper.send_stix2_bundle(stix_objects_bundle)` is used to send the bundle to OpenCTI + - `self.helper.set_state()` is used to store persistent data in connector's state + + """ + + def __init__(self, config: ConnectorSettings, helper: OpenCTIConnectorHelper): + """ + Initialize `DomainToolsFeedsConnector` with its configuration. + + Args: + config (ConnectorSettings): Configuration of the connector + helper (OpenCTIConnectorHelper): Helper to manage connection and requests to OpenCTI + """ + self.config = config + self.helper = helper + + self.params = {} + for k, v in self.config.domaintools: + if not k in ["session_id", "domain", "top"]: + continue + + if v: + if k == "session_id": + self.params["sessionID"] = v + else: + self.params[k] = v + + self.client = DomainToolsClient( + self.helper, + base_url=self.config.domaintools.api_base_url, + api_key=self.config.domaintools.api_key, + feed_type=self.config.domaintools.feed_type, + # Pass any arguments necessary to the client + ) + self.converter_to_stix = ConverterToStix( + self.helper, + tlp_level=self.config.domaintools.tlp_level, + # Pass any arguments necessary to the converter + ) + + def _collect_intelligence(self) -> list: + """ + Collect intelligence from the source and convert into STIX object + :return: List of STIX objects + """ + stix_objects = [] + + # =========================== + # === Add your code below === + # =========================== + + # Get entities from external sources + entities = self.client.get_entities(self.params) + self.helper.connector_logger.info( + f"Successfully downloaded {len(entities)} records." + ) + + if self.config.domaintools.feed_type in ["domainhotlist", "domainrisk"]: + for entity in entities: + _json = json.loads(entity) + domain = _json.get("domain") + timestamp = _json.get("timestamp") + confidence = _json.get("overall_risk", 50) + labels = [self.config.domaintools.feed_type] + for label in [ + "phishing_risk", + "malware_risk", + "spam_risk", + "proximity_risk", + ]: + score = _json.get(label) + if score: + _name = label.replace("_risk", "") + labels.append(f"{_name}={score}") + + entity_to_stix = self.converter_to_stix.create_indicator( + domain, timestamp, confidence, labels + ) + if entity_to_stix: + stix_objects.append(entity_to_stix) + + else: + # Convert into STIX2 object and add it on a list + labels = [self.config.domaintools.feed_type] + for entity in entities: + _json = json.loads(entity) + domain = _json.get("domain") + entity_to_stix = self.converter_to_stix.create_obs(domain, labels) + if entity_to_stix: + stix_objects.append(entity_to_stix) + + # =========================== + # === Add your code above === + # =========================== + + # Ensure consistent bundle by adding the author and TLP marking + if len(stix_objects): + stix_objects.append(self.converter_to_stix.author) + stix_objects.append(self.converter_to_stix.tlp_marking) + + return stix_objects + + def process_message(self) -> None: + """ + Connector main process to collect intelligence + :return: None + """ + self.helper.connector_logger.info( + "[CONNECTOR] Starting connector...", + {"connector_name": self.helper.connect_name}, + ) + + try: + # Get the current state + now = datetime.now() + current_timestamp = int(datetime.timestamp(now)) + current_state = self.helper.get_state() + + if current_state is not None and "last_run" in current_state: + last_run = current_state["last_run"] + + self.helper.connector_logger.info( + "[CONNECTOR] Connector last run", + {"last_run_datetime": last_run}, + ) + else: + self.helper.connector_logger.info( + "[CONNECTOR] Connector has never run..." + ) + + # Friendly name will be displayed on OpenCTI platform + friendly_name = "DomainTools Feeds" + + # Initiate a new work + work_id = self.helper.api.work.initiate_work( + self.helper.connect_id, friendly_name + ) + + self.helper.connector_logger.info( + "[CONNECTOR] Running connector...", + {"connector_name": self.helper.connect_name}, + ) + + # Performing the collection of intelligence + # =========================== + # === Add your code below === + # =========================== + stix_objects = self._collect_intelligence() + + ### May move this into _collect_intelligence method to help with batching + if len(stix_objects): + stix_objects_bundle = self.helper.stix2_create_bundle(stix_objects) + bundles_sent = self.helper.send_stix2_bundle( + stix_objects_bundle, + work_id=work_id, + cleanup_inconsistent_bundle=True, + ) + + self.helper.connector_logger.info( + "Sending STIX objects to OpenCTI...", + {"bundles_sent": str(len(bundles_sent))}, + ) + # =========================== + # === Add your code above === + # =========================== + + # Store the current timestamp as a last run of the connector + self.helper.connector_logger.debug( + "Getting current state and update it with last run of the connector", + {"current_timestamp": current_timestamp}, + ) + current_state = self.helper.get_state() + current_state_datetime = now.strftime("%Y-%m-%d %H:%M:%S") + last_run_datetime = datetime.fromtimestamp( + current_timestamp, tz=timezone.utc + ).strftime("%Y-%m-%d %H:%M:%S") + if current_state: + current_state["last_run"] = current_state_datetime + else: + current_state = {"last_run": current_state_datetime} + self.helper.set_state(current_state) + + message = ( + f"{self.helper.connect_name} connector successfully run, storing last_run as " + + str(last_run_datetime) + ) + + self.helper.api.work.to_processed(work_id, message) + self.helper.connector_logger.info(message) + + except (KeyboardInterrupt, SystemExit): + self.helper.connector_logger.info( + "[CONNECTOR] Connector stopped...", + {"connector_name": self.helper.connect_name}, + ) + sys.exit(0) + except Exception as err: + self.helper.connector_logger.error(str(err)) + + def run(self) -> None: + """ + Start the connector, schedule its runs and trigger the first run. + It allows you to schedule the process to run at a certain interval. + This specific scheduler from the `OpenCTIConnectorHelper` will also check the queue size of a connector. + If `CONNECTOR_QUEUE_THRESHOLD` is set, and if the connector's queue size exceeds the queue threshold, + the connector's main process will not run until the queue is ingested and reduced sufficiently, + allowing it to restart during the next scheduler check. (default is 500MB) + + Example: + - If `CONNECTOR_DURATION_PERIOD=PT5M`, then the connector is running every 5 minutes. + """ + self.helper.schedule_process( + message_callback=self.process_message, + duration_period=self.config.connector.duration_period.total_seconds(), + ) diff --git a/external-import/domaintools-feeds/src/connector/converter_to_stix.py b/external-import/domaintools-feeds/src/connector/converter_to_stix.py new file mode 100644 index 00000000000..5b50422bd63 --- /dev/null +++ b/external-import/domaintools-feeds/src/connector/converter_to_stix.py @@ -0,0 +1,221 @@ +import ipaddress +from typing import Literal + +import stix2 +import validators +from pycti import ( + Identity, + Indicator, + MarkingDefinition, + OpenCTIConnectorHelper, + StixCoreRelationship, +) + + +class ConverterToStix: + """ + Provides methods for converting various types of input data into STIX 2.1 objects. + + REQUIREMENTS: + - `generate_id()` methods from `pycti` library MUST be used to generate the `id` of each entity (except observables), + e.g. `pycti.Identity.generate_id(name="Source Name", identity_class="organization")` for a STIX Identity. + """ + + def __init__( + self, + helper: OpenCTIConnectorHelper, + tlp_level: Literal["clear", "white", "green", "amber", "amber+strict", "red"], + ): + """ + Initialize the converter with necessary configuration. + For log purpose, the connector's helper CAN be injected. + Other arguments CAN be added (e.g. `tlp_level`) if necessary. + + Args: + helper (OpenCTIConnectorHelper): The helper of the connector. Used for logs. + tlp_level (str): The TLP level to add to the created STIX entities. + """ + self.helper = helper + + self.author = self.create_author() + self.tlp_marking = self._create_tlp_marking(level=tlp_level.lower()) + + @staticmethod + def create_author() -> dict: + """ + Create Author + :return: Author in Stix2 object + """ + author = stix2.Identity( + id=Identity.generate_id(name="DomainTools", identity_class="organization"), + name="DomainTools", + identity_class="organization", + description="DomainTools is a leading provider of Whois and other DNS profile data for threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). DomainTools data helps security analysts investigate malicious activity on their networks.", + external_references=[ + stix2.ExternalReference( + source_name="External Source", + url="https://domaintools.com", + description="DomainTools is a leading provider of Whois and other DNS profile data for threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). DomainTools data helps security analysts investigate malicious activity on their networks.", + ) + ], + ) + return author + + @staticmethod + def _create_tlp_marking(level): + mapping = { + "white": stix2.TLP_WHITE, + "clear": stix2.TLP_WHITE, + "green": stix2.TLP_GREEN, + "amber": stix2.TLP_AMBER, + "amber+strict": stix2.MarkingDefinition( + id=MarkingDefinition.generate_id("TLP", "TLP:AMBER+STRICT"), + definition_type="statement", + definition={"statement": "custom"}, + custom_properties={ + "x_opencti_definition_type": "TLP", + "x_opencti_definition": "TLP:AMBER+STRICT", + }, + ), + "red": stix2.TLP_RED, + } + return mapping[level] + + def create_relationship( + self, source_id: str, relationship_type: str, target_id: str + ) -> dict: + """ + Creates Relationship object + :param source_id: ID of source in string + :param relationship_type: Relationship type in string + :param target_id: ID of target in string + :return: Relationship STIX2 object + """ + relationship = stix2.Relationship( + id=StixCoreRelationship.generate_id( + relationship_type, source_id, target_id + ), + relationship_type=relationship_type, + source_ref=source_id, + target_ref=target_id, + created_by_ref=self.author["id"], + ) + return relationship + + # ===========================# + # Other Examples + # ===========================# + + @staticmethod + def _is_ipv6(value: str) -> bool: + """ + Determine whether the provided IP string is IPv6 + :param value: Value in string + :return: A boolean + """ + try: + ipaddress.IPv6Address(value) + return True + except ipaddress.AddressValueError: + return False + + @staticmethod + def _is_ipv4(value: str) -> bool: + """ + Determine whether the provided IP string is IPv4 + :param value: Value in string + :return: A boolean + """ + try: + ipaddress.IPv4Address(value) + return True + except ipaddress.AddressValueError: + return False + + @staticmethod + def _is_domain(value: str) -> bool: + """ + Valid domain name regex including internationalized domain name + :param value: Value in string + :return: A boolean + """ + is_valid_domain = validators.domain(value) + + if is_valid_domain: + return True + else: + return False + + # Feeds + def create_obs(self, value: str, labels: list[str]) -> dict: + """ + Create observable according to value given + :param value: Value in string + :return: Stix object for IPV4, IPV6 or Domain + """ + if self._is_ipv6(value) is True: + stix_ipv6_address = stix2.IPv6Address( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + }, + ) + return stix_ipv6_address + elif self._is_ipv4(value) is True: + stix_ipv4_address = stix2.IPv4Address( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + }, + ) + return stix_ipv4_address + elif self._is_domain(value) is True: + stix_domain_name = stix2.DomainName( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + "x_opencti_labels": labels, + }, + ) + return stix_domain_name + else: + self.helper.connector_logger.error( + "This observable value is not a valid IPv4 or IPv6 address nor DomainName: ", + {"value": value}, + ) + + def create_indicator( + self, value: str, timestamp: str, confidence: int, labels: list[str] + ) -> dict: + """ + Create observable according to value given + :param value: Value in string + :return: Stix object for IPV4, IPV6 or Domain + """ + if self._is_domain(value) is True: + pattern = f"[domain-name:value = '{value}']" + indicator_id = Indicator.generate_id(pattern) + + stix_domain_name = stix2.Indicator( + name=f"Indicator for {value}", + id=indicator_id, + pattern_type="stix", + pattern=pattern, + labels=labels, + valid_from=timestamp, + confidence=confidence, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + "x_opencti_main_observable_type": "Domain-Name", # Required for OpenCTI + }, + ) + return stix_domain_name + else: + self.helper.connector_logger.error( + "This observable value is not a valid IPv4 or IPv6 address nor DomainName: ", + {"value": value}, + ) diff --git a/external-import/domaintools-feeds/src/connector/settings.py b/external-import/domaintools-feeds/src/connector/settings.py new file mode 100644 index 00000000000..7309272ed66 --- /dev/null +++ b/external-import/domaintools-feeds/src/connector/settings.py @@ -0,0 +1,71 @@ +from datetime import timedelta +from typing import Literal + +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseExternalImportConnectorConfig, +) +from pydantic import Field, HttpUrl + + +class ExternalImportConnectorConfig(BaseExternalImportConnectorConfig): + """ + Override the `BaseExternalImportConnectorConfig` to add parameters and/or defaults + to the configuration for connectors of type `EXTERNAL_IMPORT`. + """ + + name: str = Field( + description="The name of the connector.", + default="DomainToolsFeedsConnector", + ) + duration_period: timedelta = Field( + description="The period of time to await between two runs of the connector.", + default=timedelta(hours=1), + ) + + +class DomainToolsConfig(BaseConfigModel): + """ + Define parameters and/or defaults for the configuration specific to the `DomainToolsFeedsConnector`. + """ + + api_base_url: HttpUrl = Field(description="API base URL.") + api_key: str = Field(description="API key for authentication.") + feed_type: Literal[ + "domainhotlist", "domainrisk", "nod", "nad", "noh", "domaindiscovery" + ] = Field(description="Name of the feed.") + session_id: str | None = Field( + description="A unique identifier for the session, used for resuming data retrieval from the last point.", + default="OpenCTI-DomainTools-Feeds", + ) + domain: str | None = Field( + description="Filter for an exact domain or a domain substring by prefixing or suffixing your string with *.", + default=None, + ) + top: int | None = Field( + description="Limits the number of results in the response payload. Primarily intended for testing.", + default=None, + ) + tlp_level: Literal[ + "clear", + "white", + "green", + "amber", + "amber+strict", + "red", + ] = Field( + description="Default TLP level of the imported entities.", + default="clear", + ) + + +class ConnectorSettings(BaseConnectorSettings): + """ + Override `BaseConnectorSettings` to include `ExternalImportConnectorConfig` and `TemplateConfig`. + """ + + connector: ExternalImportConnectorConfig = Field( + default_factory=ExternalImportConnectorConfig + ) + domaintools: DomainToolsConfig = Field(default_factory=DomainToolsConfig) diff --git a/external-import/domaintools-feeds/src/connector/utils.py b/external-import/domaintools-feeds/src/connector/utils.py new file mode 100644 index 00000000000..e66b977a2e6 --- /dev/null +++ b/external-import/domaintools-feeds/src/connector/utils.py @@ -0,0 +1 @@ +# Utilities: helper functions, classes, or modules that provide common, reusable functionality across a codebase diff --git a/external-import/domaintools-feeds/src/main.py b/external-import/domaintools-feeds/src/main.py new file mode 100644 index 00000000000..85f3c14d9c8 --- /dev/null +++ b/external-import/domaintools-feeds/src/main.py @@ -0,0 +1,24 @@ +import traceback + +from connector import ConnectorSettings, DomainToolsFeedsConnector +from pycti import OpenCTIConnectorHelper + +if __name__ == "__main__": + """ + Entry point of the script + + - traceback.print_exc(): This function prints the traceback of the exception to the standard error (stderr). + The traceback includes information about the point in the program where the exception occurred, + which is very useful for debugging purposes. + - exit(1): effective way to terminate a Python program when an error is encountered. + It signals to the operating system and any calling processes that the program did not complete successfully. + """ + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + + connector = DomainToolsFeedsConnector(config=settings, helper=helper) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/external-import/domaintools-feeds/src/requirements.txt b/external-import/domaintools-feeds/src/requirements.txt new file mode 100644 index 00000000000..11bf2d83186 --- /dev/null +++ b/external-import/domaintools-feeds/src/requirements.txt @@ -0,0 +1,5 @@ +pycti==7.260728.0 +pydantic~= 2.11.3 +requests +validators==0.35.0 +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk \ No newline at end of file diff --git a/external-import/domaintools-irisql/.dockerignore b/external-import/domaintools-irisql/.dockerignore new file mode 100644 index 00000000000..c5d360e969a --- /dev/null +++ b/external-import/domaintools-irisql/.dockerignore @@ -0,0 +1,9 @@ +__metadata__ +**/__pycache__ +**/__docs__ +**/.venv +**/venv +**/logs +**/config.yml +**/*.egg-info +**/*.gql diff --git a/external-import/domaintools-irisql/Dockerfile b/external-import/domaintools-irisql/Dockerfile new file mode 100644 index 00000000000..df62cc75a95 --- /dev/null +++ b/external-import/domaintools-irisql/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-alpine +ENV CONNECTOR_TYPE=EXTERNAL_IMPORT + +# Copy the connector +COPY src /opt/opencti-connector-domaintools-irisql + +# Install Python modules +# hadolint ignore=DL3003 +RUN apk update && apk upgrade && \ + apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev + +RUN cd /opt/opencti-connector-domaintools-irisql && \ + pip3 install --no-cache-dir -r requirements.txt && \ + apk del git build-base + +# Expose and entrypoint +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/external-import/domaintools-irisql/README.md b/external-import/domaintools-irisql/README.md new file mode 100644 index 00000000000..79ea1717117 --- /dev/null +++ b/external-import/domaintools-irisql/README.md @@ -0,0 +1,160 @@ +# OpenCTI DomainTools IrisQL Connector + +| Status | Date | Comment | +|------------------|------------|---------| +| Partner Verified | - | - | + +## Table of Contents + +- [OpenCTI DomainTools IrisQL Connector](#opencti-domaintools-irisql-connector) + - [Table of Contents](#table-of-contents) + - [Introduction](#introduction) + - [Installation](#installation) + - [Requirements](#requirements) + - [Configuration variables](#configuration-variables) + - [OpenCTI environment variables](#opencti-environment-variables) + - [Base connector environment variables](#base-connector-environment-variables) + - [DomainTools extra parameters environment variables](#domaintools-extra-parameters-environment-variables) + - [Deployment](#deployment) + - [Docker Deployment](#docker-deployment) + - [Manual Deployment](#manual-deployment) + - [Usage](#usage) + - [Behavior](#behavior) + - [Debugging](#debugging) + - [Additional information](#additional-information) + +## Introduction + +This connector uses IrisQL to fetch data from DomainTools Iris Investigate and import it into OpenCTI. It maps each query result to a STIX 2.1 Observable, enriching it with context like IP addresses, name servers, mail servers, email addresses, etc. To ensure secure data handling, all created objects are assigned a Traffic Light Protocol (TLP) marking for downstream sharing. + +## Installation + +### Requirements + +- OpenCTI Platform version >= 6.x +- DomainTools API credential + +## Configuration variables + +There are a number of configuration options, which are set either in `docker-compose.yml` (for Docker) or in `config.yml` (for manual deployment). + +Below are the parameters you'll need to set for OpenCTI: + +### OpenCTI environment variables + +| Parameter | config.yml `opencti` | Docker environment variable | Default | Mandatory | Description | +|---------------|----------------------|-----------------------------|---------|-----------|------------------------------------------------------| +| OpenCTI URL | `url` | `OPENCTI_URL` | / | Yes | The URL of the OpenCTI platform. | +| OpenCTI Token | `token` | `OPENCTI_TOKEN` | / | Yes | The default admin token set in the OpenCTI platform. | + +### Base connector environment variables + +Below are the parameters you'll need to set for running the connector properly: + +| Parameter | config.yml `connector` | Docker environment variable | Default | Mandatory | Description | +|-----------------|------------------------|-----------------------------|---------|-----------|------------------------------------------------------------------------------------------| +| Connector ID | `id` | `CONNECTOR_ID` | / | Yes | A unique `UUIDv4` identifier for this connector instance. | +| Connector Name | `name` | `CONNECTOR_NAME` | / | Yes | Name of the connector. | +| Connector Scope | `scope` | `CONNECTOR_SCOPE` | stix2 | Yes | The scope or type of data the connector is importing, either a MIME type or Stix Object. | +| Log Level | `log_level` | `CONNECTOR_LOG_LEVEL` | error | No | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`. | +| Duration Period | `duration_period` | `CONNECTOR_DURATION_PERIOD` | PT1H | Yes | The period of time between two connector runs (ISO 8601 duration format). | + +### DomainTools extra parameters environment variables + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|-------------------------|--------------------------------|----------------------------------|---------|-----------|---------------------------------------| +| API base URL | domaintools.api_base_url | `DOMAINTOOLS_API_BASE_URL` | | https://api.domaintools.com/v1/iris-investigate/ | Yes | DomainTools API base URL | +| API key | domaintools.api_key | `DOMAINTOOLS_API_KEY` | | Yes | DomainTools API key | +| IrisQL Query | domaintools.iris_ql | `DOMAINTOOLS_IRIS_QL` | | Yes | IrisQL query to execute | +| STORE IRIS DATA | domaintools.store_iris_data | `DOMAINTOOLS_STORE_IRIS_DATA` | false |No| Store DomainTools Iris data as note object. | +| TLP Level | domaintools.tlp_level | `DOMAINTOOLS_TLP_LEVEL` | clear | No | TLP marking for created STIX objects. | + +## Deployment + +### Docker Deployment + +Before building the Docker container, you need to set the version of pycti in `requirements.txt` equal to whatever version of OpenCTI you're running. Example, `pycti==6.5.1`. If you don't, it will take the latest version, but sometimes the OpenCTI SDK fails to initialize. + +Build a Docker Image using the provided `Dockerfile`. + +Example: + +Register connector in the **main** OpenCTI `docker-compose.yml`: + +```yaml + connector-domaintools: + image: opencti/connector-domaintools:latest + environment: + - OPENCTI_URL=http://opencti:8080 + - OPENCTI_TOKEN=changeme + - CONNECTOR_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + - CONNECTOR_NAME=DomainTools IrisQL + - CONNECTOR_SCOPE=stix2 + - CONNECTOR_LOG_LEVEL=error + - CONNECTOR_DURATION_PERIOD=PT5M + - DOMAINTOOLS_API_BASE_URL=https://api.domaintools.com/v1/iris-investigate/ + - DOMAINTOOLS_API_KEY=changeme + - DOMAINTOOLS_IRIS_QL="# IrisQL-1.0\ndomain contains \"sso\"\nAND\nfirst_seen within \"the last 3 hour\"\nAND\nrisk_score greater_than_or_equal \"90\"" + - DOMAINTOOLS_STORE_IRIS_DATA=true + - DOMAINTOOLS_TLP_LEVEL=clear + restart: always +``` + +Make sure to replace the environment variables in `docker-compose.yml` with the appropriate configurations for your environment. Then, start the docker container with the provided `docker-compose.yml`. + +```shell +docker compose up -d +# -d for detached +``` + +### Manual Deployment + +Create a file `config.yml` based on the provided `config.yml.sample`. + +Replace the configuration variables (especially the "**ChangeMe**" variables) with the appropriate configurations for your environment. + +Install the required python dependencies (preferably in a virtual environment): + +```shell +pip3 install -r requirements.txt +``` + +Then, start the connector from the `src` directory: + +```shell +python3 main.py +``` + +## Usage + +After installation, the connector should require minimal interaction to use, and should update automatically at a regular interval specified in your `docker-compose.yml` or `config.yml` in `duration_period`. + +However, if you would like to force an immediate download of a new batch of data, navigate to: + +**Data management → Ingestion → Connectors** in the OpenCTI platform. + +Find the "DomainTools IrisQL" connector, and click on the refresh button to reset the connector's state and force a new download of data by re-running the connector. + +## Behavior + +- Converts each query result into a STIX 2.1 Observable object +- Bundles and sends the STIX objects to OpenCTI + +## Debugging + +The connector can be debugged by setting the appropriate log level. Note that logging messages can be added using `self.helper.connector_logger.{LOG_LEVEL}("Sample message")`, for example, `self.helper.connector_logger.error("An error message")`. + +Set `CONNECTOR_LOG_LEVEL=debug` for verbose logging. Log output includes: + +- API call details and retry behavior +- Data count fetched per run +- Page-by-page fetch progress +- STIX conversion trace per entry +- Bundle send status + +## Additional information + +- Each Iris Investigate result becomes a STIX 2.1 Domain-Name observable, carrying the DomainTools risk score and per-component risk labels (proximity, malware, phishing, spam). +- Related infrastructure is expanded into its own observables and linked with STIX relationships: resolving IP addresses (`resolves-to`), mail and name servers (`resolves-to`, with their IPs `related-to`), and registrant, SOA, SSL, and additional WHOIS email addresses (`related-to`). +- When `DOMAINTOOLS_STORE_IRIS_DATA` is enabled, the raw Iris record for each domain is attached as a STIX Note. +- Every object is attributed to a DomainTools author identity and tagged with the configured TLP marking. diff --git a/external-import/domaintools-irisql/__metadata__/connector_manifest.json b/external-import/domaintools-irisql/__metadata__/connector_manifest.json new file mode 100644 index 00000000000..c8b0d2a7489 --- /dev/null +++ b/external-import/domaintools-irisql/__metadata__/connector_manifest.json @@ -0,0 +1,22 @@ +{ + "title": "DomainTools IrisQL", + "slug": "domaintools-irisql", + "description": "Import DomainTools IrisQL", + "short_description": "Import DomainTools IrisQL", + "logo": null, + "use_cases": ["Infrastructure & Attack Surface Visibility", "Adversary & Campaign Insights"], + "solution_categories": ["Enrichment & Reputation"], + "contact": null, + "license_type": null, + "verified": true, + "last_verified_date": null, + "playbook_supported": false, + "max_confidence_level": 100, + "support_version": ">=6.8.12", + "subscription_link": null, + "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/external-import/domaintools-irisql", + "manager_supported": false, + "container_version": "rolling", + "container_image": "opencti/connector-domaintools-irisql", + "container_type": "EXTERNAL_IMPORT" +} \ No newline at end of file diff --git a/external-import/domaintools-irisql/__metadata__/logo.png b/external-import/domaintools-irisql/__metadata__/logo.png new file mode 100644 index 00000000000..c94ed391094 Binary files /dev/null and b/external-import/domaintools-irisql/__metadata__/logo.png differ diff --git a/external-import/domaintools-irisql/config.yml.sample b/external-import/domaintools-irisql/config.yml.sample new file mode 100644 index 00000000000..f49745fe5fe --- /dev/null +++ b/external-import/domaintools-irisql/config.yml.sample @@ -0,0 +1,19 @@ +opencti: + url: 'http://localhost' + token: 'ChangeMe' + +connector: + type: 'EXTERNAL_IMPORT' + id: 'ChangeMe' + name: 'DomainTools IrisQL' + scope: 'stix2' + log_level: 'error' # optional (default: 'error') + duration_period: 'PT1H' # # optional, interval in ISO-8601 format between two runs of the connector (default: 'PT1H') + +domaintools: + api_base_url: 'https://api.domaintools.com/v1/iris-investigate/' + api_key: 'ChangeMe' + iris_ql: "# IrisQL-1.0\ndomain contains \"sso\"\nAND\nfirst_seen within \"the last 1 hour\"\nAND\nrisk_score greater_than_or_equal \"90\"" + store_iris_data: true + tlp_level: 'clear' # optional, available values: 'clear', 'white', 'green', 'amber', 'amber+strict', 'red' (default: 'clear') + \ No newline at end of file diff --git a/external-import/domaintools-irisql/docker-compose.yml b/external-import/domaintools-irisql/docker-compose.yml new file mode 100644 index 00000000000..e278e4d54f1 --- /dev/null +++ b/external-import/domaintools-irisql/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3' +services: + opencti-connector-domaintools-irisql: + image: opencti/opencti-connector-domaintools-irisql:latest + environment: + # Generic parameters (connection with OpenCTI) + - OPENCTI_URL=http://localhost + - OPENCTI_TOKEN=CHANGEME + # Common parameters for connectors of type EXTERNAL_IMPORT + - CONNECTOR_ID=CHANGEME + - CONNECTOR_NAME=CHANGEME # optional + - CONNECTOR_SCOPE=CHANGEME + - CONNECTOR_LOG_LEVEL=error # optional (default: 'error') + - CONNECTOR_DURATION_PERIOD=CHANGEME # optional, interval in ISO-8601 format between two runs of the connector (default: 'PT1H') + - DOMAINTOOLS_API_BASE_URL=https://api.domaintools.com/v1/iris-investigate/ + - DOMAINTOOLS_API_KEY=ChangeMe + - DOMAINTOOLS_IRIS_QL=ChangeMe + - DOMAINTOOLS_STORE_IRIS_DATA=true # optional + - DOMAINTOOLS_TLP_LEVEL=clear # optional, available values: 'clear', 'white', 'green', 'amber', 'amber+strict', 'red' (default: 'clear') + restart: always diff --git a/external-import/domaintools-irisql/entrypoint.sh b/external-import/domaintools-irisql/entrypoint.sh new file mode 100644 index 00000000000..1cea6401e33 --- /dev/null +++ b/external-import/domaintools-irisql/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Go to the right directory +cd /opt/opencti-connector-domaintools-irisql + +# Launch the worker +python3 main.py \ No newline at end of file diff --git a/external-import/domaintools-irisql/src/client/__init__.py b/external-import/domaintools-irisql/src/client/__init__.py new file mode 100644 index 00000000000..30c04e41ff4 --- /dev/null +++ b/external-import/domaintools-irisql/src/client/__init__.py @@ -0,0 +1,5 @@ +from client.api_client import DomainToolsClient + +__all__ = [ + "DomainToolsClient", +] diff --git a/external-import/domaintools-irisql/src/client/api_client.py b/external-import/domaintools-irisql/src/client/api_client.py new file mode 100644 index 00000000000..5bb3e51c7f5 --- /dev/null +++ b/external-import/domaintools-irisql/src/client/api_client.py @@ -0,0 +1,89 @@ +import requests +from pycti import OpenCTIConnectorHelper +from pydantic import HttpUrl + + +class DomainToolsClient: + def __init__(self, helper: OpenCTIConnectorHelper, base_url: HttpUrl, api_key: str): + """ + Initialize the client with necessary configuration. + For log purpose, the connector's helper CAN be injected. + Other arguments CAN be added (e.g. `api_key`) if necessary. + + Args: + helper (OpenCTIConnectorHelper): The helper of the connector. Used for logs. + base_url (str): The external API base URL. + api_key (str): The API key to authenticate the connector to the external API. + """ + self.helper = helper + self.base_url = base_url + # Define headers in session and update when needed + headers = {"x-api-key": api_key, "Content-Type": "text/plain"} + self.session = requests.Session() + self.session.headers.update(headers) + + def _request_data(self, api_url: str, params=None, body=None): + """ + Internal method to handle API requests + :return: Response in JSON format + """ + try: + + response = self.session.post(api_url, params=params, data=body) + self.helper.connector_logger.info( + "[API] HTTP POST Request to endpoint", {"url_path": api_url} + ) + + response.raise_for_status() + return response + + except requests.RequestException as err: + error_msg = "[API] Error while fetching data: " + self.helper.connector_logger.error( + error_msg, {"url_path": api_url, "error": str(err)} + ) + return None + + def get_entities(self, body=None) -> list: + """Fetch IrisQL results from the DomainTools Iris Investigate API. + Args: + body: Optional IrisQL query body. + Returns: + A list of result objects from the API. + """ + try: + # =========================== + # === Add your code below === + # =========================== + result_data = [] + params = {} + while True: + response = self._request_data(self.base_url, params=params, body=body) + response.raise_for_status() + if response is None: + raise RuntimeError( + "Failed to fetch IrisQL results from DomainTools API" + ) + + json_response = response.json() + current_results = json_response["response"]["results"] + result_data.extend(current_results) + + if not json_response["response"]["has_more_results"]: + break + + # Update the 'position' field for pagination + params["position"] = json_response["response"]["position"] + + return result_data + + # return response.json() + # =========================== + # === Add your code above === + # =========================== + + # raise NotImplementedError + + except Exception as err: + self.helper.connector_logger.error(err) + return [] diff --git a/external-import/domaintools-irisql/src/connector/__init__.py b/external-import/domaintools-irisql/src/connector/__init__.py new file mode 100644 index 00000000000..6a0d1645f75 --- /dev/null +++ b/external-import/domaintools-irisql/src/connector/__init__.py @@ -0,0 +1,7 @@ +from connector.connector import DomainToolsIrisQLConnector +from connector.settings import ConnectorSettings + +__all__ = [ + "ConnectorSettings", + "DomainToolsIrisQLConnector", +] diff --git a/external-import/domaintools-irisql/src/connector/connector.py b/external-import/domaintools-irisql/src/connector/connector.py new file mode 100644 index 00000000000..287a95a89f6 --- /dev/null +++ b/external-import/domaintools-irisql/src/connector/connector.py @@ -0,0 +1,362 @@ +import json +import sys +from datetime import datetime, timezone + +from client import DomainToolsClient +from connector.converter_to_stix import ConverterToStix +from connector.settings import ConnectorSettings +from pycti import OpenCTIConnectorHelper + + +class DomainToolsIrisQLConnector: + """ + Specifications of the external import connector: + + This class encapsulates the main actions, expected to be run by any connector of type `EXTERNAL_IMPORT`. + This type of connector aim to fetch external data to create STIX bundle and send it to OpenCTI. + The STIX bundle in the queue will be processed by OpenCTI workers. + This type of connector uses the basic methods of the helper. + + --- + + Attributes: + config (ConnectorSettings): + Store the connector's configuration. It defines how to connector will behave. + helper (OpenCTIConnectorHelper): + Handle the connection and the requests between the connector, OpenCTI and the workers. + _All connectors MUST use the connector helper with connector's configuration._ + client (TemplateClient): + Provide methods to request the external API. + converter_to_stix (ConnectorConverter): + Provide methods for converting various types of input data into STIX 2.1 objects. + + --- + + Best practices: + - `self.helper.api.work.initiate_work(...)` is used to initiate a new work + - `self.helper.schedule_iso()` is used to schedule connector's runs frequency + - `self.helper.connector_logger.[info/debug/warning/error]` is used when logging a message + - `self.helper.stix2_create_bundle(stix_objects)` is used when creating a bundle + - `self.helper.send_stix2_bundle(stix_objects_bundle)` is used to send the bundle to OpenCTI + - `self.helper.set_state()` is used to store persistent data in connector's state + + """ + + def __init__(self, config: ConnectorSettings, helper: OpenCTIConnectorHelper): + """ + Initialize `DomainToolsIrisQLConnector` with its configuration. + + Args: + config (ConnectorSettings): Configuration of the connector + helper (OpenCTIConnectorHelper): Helper to manage connection and requests to OpenCTI + """ + self.config = config + self.helper = helper + self.iris_ql = self.config.domaintools.iris_ql + self.store_iris_data = self.config.domaintools.store_iris_data + if not self.iris_ql: + self.helper.log_error("Missing IrisQL data.") + raise ValueError("Missing IrisQL data.") + + self.client = DomainToolsClient( + self.helper, + base_url=self.config.domaintools.api_base_url, + api_key=self.config.domaintools.api_key, + # Pass any arguments necessary to the client + ) + self.converter_to_stix = ConverterToStix( + self.helper, + tlp_level=self.config.domaintools.tlp_level, + # Pass any arguments necessary to the converter + ) + + def _collect_intelligence(self) -> list: + """ + Collect intelligence from the source and convert into STIX object + :return: List of STIX objects + """ + stix_objects = [] + + # =========================== + # === Add your code below === + # =========================== + + # Get entities from external sources + entities = self.client.get_entities(self.iris_ql) + self.helper.connector_logger.info( + f"Successfully downloaded {len(entities)} records." + ) + + for entity in entities: + domain = entity.get("domain") + score = entity.get("domain_risk", {}).get("risk_score") + + labels = ["IrisQL"] + for component in entity.get("domain_risk", {}).get("components", []): + _name = component.get("name") or "" + _score = component.get("risk_score") + if "proximity" in _name: + labels.append(f"proximity:{_score}") + elif "malware" in _name: + labels.append(f"malware:{_score}") + elif "phishing" in _name: + labels.append(f"phishing:{_score}") + elif "spam" in _name: + labels.append(f"spam:{_score}") + else: + continue + + domain_obs = self.converter_to_stix.create_obs(domain, score, labels) + if not domain_obs: + continue + stix_objects.append(domain_obs) + + ###### Note + if self.store_iris_data: + note_content = json.dumps(entity) + note_abstract = "DomainTools Iris Data" + + note_obj_id = self.converter_to_stix.create_note_id( + note_content, note_abstract + ) + + stix_objects.append( + { + "type": "note", + "id": note_obj_id, + "content": note_content, + "abstract": note_abstract, + "object_refs": [domain_obs.id], + } + ) + + ###### IP + stix_objects.extend(self._process_IP(domain_obs, entity.get("ip", []))) + + ###### MX + stix_objects.extend( + self._process_MX_NS(domain_obs, entity.get("mx", []), "MX") + ) + + ###### NS + stix_objects.extend( + self._process_MX_NS(domain_obs, entity.get("name_server", []), "NS") + ) + + ####### EMAIL + # Get all EmailAddress + all_emails = [] + for contact in [ + "admin_contact", + "billing_contact", + "registrant_contact", + "technical_contact", + ]: + all_emails.extend(entity.get(contact, {}).get("email", [])) + + all_emails.extend(entity.get("soa_email", [])) + all_emails.extend(entity.get("ssl_email", [])) + all_emails.extend(entity.get("additional_whois_email", [])) + for ssl in entity.get("ssl_info", []): + all_emails.extend(ssl.get("email", [])) + + # Create Object + unique_emails = list({item["value"]: item for item in all_emails}.values()) + for item in unique_emails: + email = item.get("value") + if ("https://" in email) or ("http://" in email): + continue + + email_obs = self.converter_to_stix.create_obs(email) + + if not email_obs: + continue + + stix_objects.append(email_obs) + + entity_relation = self.converter_to_stix.create_relationship( + domain_obs.id, "related-to", email_obs.id + ) + stix_objects.append(entity_relation) + + # =========================== + # === Add your code above === + # =========================== + + # Ensure consistent bundle by adding the author and TLP marking + if len(stix_objects): + stix_objects.append(self.converter_to_stix.author) + stix_objects.append(self.converter_to_stix.tlp_marking) + + return stix_objects + + def _process_IP(self, _source, data): + all_ip_object = [] + for ip_entity in data: + ip = ip_entity.get("address", {}).get("value") + ip_obs = self.converter_to_stix.create_obs(ip) + + if not ip_obs: + continue + + all_ip_object.append(ip_obs) + + entity_relation = self.converter_to_stix.create_relationship( + _source.id, "resolves-to", ip_obs.id + ) + all_ip_object.append(entity_relation) + + return all_ip_object + + def _process_MX_NS(self, _source, data, data_type): + all_ip_object = [] + for mx_entity in data: + host = mx_entity.get("host", {}).get("value") + + ### Note: some value point to itself + if _source.value == host: + continue + + host_obs = self.converter_to_stix.create_obs(host, labels=[data_type]) + + if not host_obs: + continue + + all_ip_object.append(host_obs) + + entity_relation = self.converter_to_stix.create_relationship( + _source.id, "resolves-to", host_obs.id + ) + all_ip_object.append(entity_relation) + + ### IP + for ip_entity in mx_entity.get("ip", []): + ip = ip_entity.get("value") + ip_obs = self.converter_to_stix.create_obs( + ip, labels=[f"{data_type} IP"] + ) + if not ip_obs: + continue + + all_ip_object.append(ip_obs) + + entity_relation = self.converter_to_stix.create_relationship( + host_obs.id, "related-to", ip_obs.id + ) + all_ip_object.append(entity_relation) + + return all_ip_object + + def process_message(self) -> None: + """ + Connector main process to collect intelligence + :return: None + """ + self.helper.connector_logger.info( + "[CONNECTOR] Starting connector...", + {"connector_name": self.helper.connect_name}, + ) + + try: + # Get the current state + now = datetime.now() + current_timestamp = int(datetime.timestamp(now)) + current_state = self.helper.get_state() + + if current_state is not None and "last_run" in current_state: + last_run = current_state["last_run"] + + self.helper.connector_logger.info( + "[CONNECTOR] Connector last run", + {"last_run_datetime": last_run}, + ) + else: + self.helper.connector_logger.info( + "[CONNECTOR] Connector has never run..." + ) + + # Friendly name will be displayed on OpenCTI platform + friendly_name = "DomainTools IrisQL" + + # Initiate a new work + work_id = self.helper.api.work.initiate_work( + self.helper.connect_id, friendly_name + ) + + self.helper.connector_logger.info( + "[CONNECTOR] Running connector...", + {"connector_name": self.helper.connect_name}, + ) + + # Performing the collection of intelligence + # =========================== + # === Add your code below === + # =========================== + stix_objects = self._collect_intelligence() + + ### May move this into _collect_intelligence method to help with batching + if len(stix_objects): + stix_objects_bundle = self.helper.stix2_create_bundle(stix_objects) + bundles_sent = self.helper.send_stix2_bundle( + stix_objects_bundle, + work_id=work_id, + cleanup_inconsistent_bundle=True, + ) + + self.helper.connector_logger.info( + "Sending STIX objects to OpenCTI...", + {"bundles_sent": str(len(bundles_sent))}, + ) + # =========================== + # === Add your code above === + # =========================== + + # Store the current timestamp as a last run of the connector + self.helper.connector_logger.debug( + "Getting current state and update it with last run of the connector", + {"current_timestamp": current_timestamp}, + ) + current_state = self.helper.get_state() + current_state_datetime = now.strftime("%Y-%m-%d %H:%M:%S") + last_run_datetime = datetime.fromtimestamp( + current_timestamp, tz=timezone.utc + ).strftime("%Y-%m-%d %H:%M:%S") + if current_state: + current_state["last_run"] = current_state_datetime + else: + current_state = {"last_run": current_state_datetime} + self.helper.set_state(current_state) + + message = ( + f"{self.helper.connect_name} connector successfully run, storing last_run as " + + str(last_run_datetime) + ) + + self.helper.api.work.to_processed(work_id, message) + self.helper.connector_logger.info(message) + + except (KeyboardInterrupt, SystemExit): + self.helper.connector_logger.info( + "[CONNECTOR] Connector stopped...", + {"connector_name": self.helper.connect_name}, + ) + sys.exit(0) + except Exception as err: + self.helper.connector_logger.error(str(err)) + + def run(self) -> None: + """ + Start the connector, schedule its runs and trigger the first run. + It allows you to schedule the process to run at a certain interval. + This specific scheduler from the `OpenCTIConnectorHelper` will also check the queue size of a connector. + If `CONNECTOR_QUEUE_THRESHOLD` is set, and if the connector's queue size exceeds the queue threshold, + the connector's main process will not run until the queue is ingested and reduced sufficiently, + allowing it to restart during the next scheduler check. (default is 500MB) + + Example: + - If `CONNECTOR_DURATION_PERIOD=PT5M`, then the connector is running every 5 minutes. + """ + self.helper.schedule_process( + message_callback=self.process_message, + duration_period=self.config.connector.duration_period.total_seconds(), + ) diff --git a/external-import/domaintools-irisql/src/connector/converter_to_stix.py b/external-import/domaintools-irisql/src/connector/converter_to_stix.py new file mode 100644 index 00000000000..4fda3b58cc5 --- /dev/null +++ b/external-import/domaintools-irisql/src/connector/converter_to_stix.py @@ -0,0 +1,211 @@ +import ipaddress +from typing import Literal + +import stix2 +import validators +from pycti import ( + Identity, + MarkingDefinition, + Note, + OpenCTIConnectorHelper, + StixCoreRelationship, +) + + +class ConverterToStix: + """ + Provides methods for converting various types of input data into STIX 2.1 objects. + + REQUIREMENTS: + - `generate_id()` methods from `pycti` library MUST be used to generate the `id` of each entity (except observables), + e.g. `pycti.Identity.generate_id(name="Source Name", identity_class="organization")` for a STIX Identity. + """ + + def __init__( + self, + helper: OpenCTIConnectorHelper, + tlp_level: Literal["clear", "white", "green", "amber", "amber+strict", "red"], + ): + """ + Initialize the converter with necessary configuration. + For log purpose, the connector's helper CAN be injected. + Other arguments CAN be added (e.g. `tlp_level`) if necessary. + + Args: + helper (OpenCTIConnectorHelper): The helper of the connector. Used for logs. + tlp_level (str): The TLP level to add to the created STIX entities. + """ + self.helper = helper + + self.author = self.create_author() + self.tlp_marking = self._create_tlp_marking(level=tlp_level.lower()) + + @staticmethod + def create_author() -> dict: + """ + Create Author + :return: Author in Stix2 object + """ + author = stix2.Identity( + id=Identity.generate_id(name="DomainTools", identity_class="organization"), + name="DomainTools", + identity_class="organization", + description="DomainTools is a leading provider of Whois and other DNS profile data for threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). DomainTools data helps security analysts investigate malicious activity on their networks.", + external_references=[ + stix2.ExternalReference( + source_name="External Source", + url="https://domaintools.com", + description="DomainTools is a leading provider of Whois and other DNS profile data for threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). DomainTools data helps security analysts investigate malicious activity on their networks.", + ) + ], + ) + return author + + @staticmethod + def _create_tlp_marking(level): + mapping = { + "white": stix2.TLP_WHITE, + "clear": stix2.TLP_WHITE, + "green": stix2.TLP_GREEN, + "amber": stix2.TLP_AMBER, + "amber+strict": stix2.MarkingDefinition( + id=MarkingDefinition.generate_id("TLP", "TLP:AMBER+STRICT"), + definition_type="statement", + definition={"statement": "custom"}, + custom_properties={ + "x_opencti_definition_type": "TLP", + "x_opencti_definition": "TLP:AMBER+STRICT", + }, + ), + "red": stix2.TLP_RED, + } + return mapping[level] + + def create_relationship( + self, source_id: str, relationship_type: str, target_id: str + ) -> dict: + """ + Creates Relationship object + :param source_id: ID of source in string + :param relationship_type: Relationship type in string + :param target_id: ID of target in string + :return: Relationship STIX2 object + """ + relationship = stix2.Relationship( + id=StixCoreRelationship.generate_id( + relationship_type, source_id, target_id + ), + relationship_type=relationship_type, + source_ref=source_id, + target_ref=target_id, + created_by_ref=self.author["id"], + ) + return relationship + + # ===========================# + # Other Examples + # ===========================# + + @staticmethod + def _is_ipv6(value: str) -> bool: + """ + Determine whether the provided IP string is IPv6 + :param value: Value in string + :return: A boolean + """ + try: + ipaddress.IPv6Address(value) + return True + except ipaddress.AddressValueError: + return False + + @staticmethod + def _is_ipv4(value: str) -> bool: + """ + Determine whether the provided IP string is IPv4 + :param value: Value in string + :return: A boolean + """ + try: + ipaddress.IPv4Address(value) + return True + except ipaddress.AddressValueError: + return False + + @staticmethod + def _is_domain(value: str) -> bool: + """ + Valid domain name regex including internationalized domain name + :param value: Value in string + :return: A boolean + """ + is_valid_domain = validators.domain(value) + + if is_valid_domain: + return True + else: + return False + + @staticmethod + def _is_email(value: str) -> bool: + if validators.email(value): + return True + else: + return False + + def create_obs( + self, value: str, score: int | None = None, labels: list[str] | None = None + ) -> dict: + """ + Create observable according to value given + :param value: Value in string + :return: Stix object for IPV4, IPV6 or Domain + """ + if self._is_ipv6(value) is True: + stix_ipv6_address = stix2.IPv6Address( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + "x_opencti_labels": labels, + }, + ) + return stix_ipv6_address + elif self._is_ipv4(value) is True: + stix_ipv4_address = stix2.IPv4Address( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + "x_opencti_labels": labels, + }, + ) + return stix_ipv4_address + elif self._is_email(value) is True: + stix_email = stix2.EmailAddress( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + }, + ) + return stix_email + elif self._is_domain(value) is True: + stix_domain_name = stix2.DomainName( + value=value, + object_marking_refs=[self.tlp_marking], + custom_properties={ + "x_opencti_created_by_ref": self.author["id"], + "x_opencti_score": score, + "x_opencti_labels": labels, + }, + ) + return stix_domain_name + else: + self.helper.connector_logger.error( + "This observable value is not a valid IPv4 or IPv6 address nor DomainName: ", + {"value": value}, + ) + + def create_note_id(self, content: str, abstract: str) -> str: + return Note.generate_id(created=None, content=content, abstract=abstract) diff --git a/external-import/domaintools-irisql/src/connector/settings.py b/external-import/domaintools-irisql/src/connector/settings.py new file mode 100644 index 00000000000..49683505e71 --- /dev/null +++ b/external-import/domaintools-irisql/src/connector/settings.py @@ -0,0 +1,60 @@ +from datetime import timedelta +from typing import Literal + +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseExternalImportConnectorConfig, +) +from pydantic import Field, HttpUrl + + +class ExternalImportConnectorConfig(BaseExternalImportConnectorConfig): + """ + Override the `BaseExternalImportConnectorConfig` to add parameters and/or defaults + to the configuration for connectors of type `EXTERNAL_IMPORT`. + """ + + name: str = Field( + description="The name of the connector.", + default="DomainToolsIrisQLConnector", + ) + duration_period: timedelta = Field( + description="The period of time to await between two runs of the connector.", + default=timedelta(hours=1), + ) + + +class DomainToolsConfig(BaseConfigModel): + """ + Define parameters and/or defaults for the configuration specific to the `DomainToolsIrisQLConnector`. + """ + + api_base_url: HttpUrl = Field(description="API base URL.") + api_key: str = Field(description="API key for authentication.") + iris_ql: str = Field(description="The query search string.") + store_iris_data: bool | None = Field( + description="Store DomainTools Iris data as note object.", default=False + ) + tlp_level: Literal[ + "clear", + "white", + "green", + "amber", + "amber+strict", + "red", + ] = Field( + description="Default TLP level of the imported entities.", + default="clear", + ) + + +class ConnectorSettings(BaseConnectorSettings): + """ + Override `BaseConnectorSettings` to include `ExternalImportConnectorConfig` and `TemplateConfig`. + """ + + connector: ExternalImportConnectorConfig = Field( + default_factory=ExternalImportConnectorConfig + ) + domaintools: DomainToolsConfig = Field(default_factory=DomainToolsConfig) diff --git a/external-import/domaintools-irisql/src/connector/utils.py b/external-import/domaintools-irisql/src/connector/utils.py new file mode 100644 index 00000000000..e66b977a2e6 --- /dev/null +++ b/external-import/domaintools-irisql/src/connector/utils.py @@ -0,0 +1 @@ +# Utilities: helper functions, classes, or modules that provide common, reusable functionality across a codebase diff --git a/external-import/domaintools-irisql/src/main.py b/external-import/domaintools-irisql/src/main.py new file mode 100644 index 00000000000..117406e42ec --- /dev/null +++ b/external-import/domaintools-irisql/src/main.py @@ -0,0 +1,24 @@ +import traceback + +from connector import ConnectorSettings, DomainToolsIrisQLConnector +from pycti import OpenCTIConnectorHelper + +if __name__ == "__main__": + """ + Entry point of the script + + - traceback.print_exc(): This function prints the traceback of the exception to the standard error (stderr). + The traceback includes information about the point in the program where the exception occurred, + which is very useful for debugging purposes. + - exit(1): effective way to terminate a Python program when an error is encountered. + It signals to the operating system and any calling processes that the program did not complete successfully. + """ + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + + connector = DomainToolsIrisQLConnector(config=settings, helper=helper) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/external-import/domaintools-irisql/src/requirements.txt b/external-import/domaintools-irisql/src/requirements.txt new file mode 100644 index 00000000000..11bf2d83186 --- /dev/null +++ b/external-import/domaintools-irisql/src/requirements.txt @@ -0,0 +1,5 @@ +pycti==7.260728.0 +pydantic~= 2.11.3 +requests +validators==0.35.0 +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk \ No newline at end of file