feat(domaintools): add DomainTools IrisQL and Feeds connector for OpenCTI (#7128) (#7127)

external-import/domaintools-feeds/.dockerignore

@@ -0,0 +1,9 @@
+__metadata__
+**/__pycache__
+**/__docs__
+**/.venv
+**/venv
+**/logs
+**/config.yml
+**/*.egg-info
+**/*.gql

external-import/domaintools-feeds/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-alpine
+ENV CONNECTOR_TYPE=EXTERNAL_IMPORT
+
+# Copy the connector
+COPY src /opt/opencti-connector-domaintools-feeds
+
+# Install Python modules
+# hadolint ignore=DL3003
+RUN apk update && apk upgrade && \
+    apk --no-cache add git build-base libmagic libffi-dev libxml2-dev libxslt-dev
+
+RUN cd /opt/opencti-connector-domaintools-feeds && \
+    pip3 install --no-cache-dir -r requirements.txt && \
+    apk del git build-base
+
+# Expose and entrypoint
+COPY entrypoint.sh /
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

external-import/domaintools-feeds/README.md

@@ -0,0 +1,161 @@
+# OpenCTI DomainTools Feeds Connector
+
+| Status           | Date       | Comment |
+|------------------|------------|---------|
+| Partner Verified | -          | -       |
+
+## Table of Contents
+
+- [OpenCTI DomainTools Feeds Connector](#opencti-domaintools-feeds-connector)
+  - [Introduction](#introduction)
+  - [Installation](#installation)
+    - [Requirements](#requirements)
+  - [Configuration variables](#configuration-variables)
+    - [OpenCTI environment variables](#opencti-environment-variables)
+    - [Base connector environment variables](#base-connector-environment-variables)
+    - [DomainTools extra parameters environment variables](#domaintools-extra-parameters-environment-variables)
+  - [Deployment](#deployment)
+    - [Docker Deployment](#docker-deployment)
+    - [Manual Deployment](#manual-deployment)
+  - [Usage](#usage)
+  - [Behavior](#behavior)
+  - [Debugging](#debugging)
+  - [Additional information](#additional-information)
+
+## Introduction
+
+This connector fetches data from DomainTools Threat Feeds (via the Real-time Feed API) and imports it into OpenCTI as Observables. Each feed entry is mapped to a Structured Threat Information Expression (STIX) 2.1 Observable object, enriched with metadata such as severity, entity state, platform, audit logs, etc. Created objects carry a Traffic Light Protocol (TLP) marking for downstream sharing.
+
+## Installation
+
+### Requirements
+
+- OpenCTI Platform version >= 6.x
+- DomainTools API credential
+
+## Configuration variables
+
+There are a number of configuration options, which are set either in `docker-compose.yml` (for Docker) or in `config.yml` (for manual deployment).
+
+Below are the parameters you'll need to set for OpenCTI:
+
+### OpenCTI environment variables
+
+| Parameter     | config.yml `opencti` | Docker environment variable | Default | Mandatory | Description                                          |
+|---------------|----------------------|-----------------------------|---------|-----------|------------------------------------------------------|
+| OpenCTI URL   | `url`                | `OPENCTI_URL`               | /       | Yes       | The URL of the OpenCTI platform.                     |
+| OpenCTI Token | `token`              | `OPENCTI_TOKEN`             | /       | Yes       | The default admin token set in the OpenCTI platform. |
+
+### Base connector environment variables
+
+Below are the parameters you'll need to set for running the connector properly:
+
+| Parameter       | config.yml `connector` | Docker environment variable | Default | Mandatory | Description                                                                              |
+|-----------------|------------------------|-----------------------------|---------|-----------|------------------------------------------------------------------------------------------|
+| Connector ID    | `id`                   | `CONNECTOR_ID`              | /       | Yes       | A unique `UUIDv4` identifier for this connector instance.                                |
+| Connector Name  | `name`                 | `CONNECTOR_NAME`            | /       | Yes       | Name of the connector.                                                                   |
+| Connector Scope | `scope`                | `CONNECTOR_SCOPE`           | stix2  | Yes       | The scope or type of data the connector is importing, either a MIME type or Stix Object. |
+| Log Level       | `log_level`            | `CONNECTOR_LOG_LEVEL`       | error   | No        | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`.   |
+| Duration Period | `duration_period`      | `CONNECTOR_DURATION_PERIOD` | PT1H    | Yes       | The period of time between two connector runs (ISO 8601 duration format).                |
+
+### DomainTools extra parameters environment variables
+
+| Parameter               | config.yml                     | Docker environment variable      |  Default | Mandatory | Description                           |
+|-------------------------|--------------------------------|----------------------------------|---------|-----------|---------------------------------------|
+| API base URL            | domaintools.api_base_url            | `DOMAINTOOLS_API_BASE_URL`            |         https://api.domaintools.com        | Yes       | DomainTools API base URL                   |
+| API key                 | domaintools.api_key                 | `DOMAINTOOLS_API_KEY`                 |                   | Yes       | DomainTools API key                        |
+| Feed Type               | domaintools.feed_type         | `DOMAINTOOLS_FEED_TYPE`         |           nod      | Yes       | Type of feed to ingest (domainhotlist, domainrisk, nod, nad, noh, domaindiscovery)      |
+| Session ID              | domaintools.session_id       | `DOMAINTOOLS_SESSION_ID`            |                  | No        | A unique identifier for the session, used for resuming data retrieval from the last point. |
+| TLP Level               | domaintools.tlp_level               | `DOMAINTOOLS_TLP_LEVEL`                         | clear   | No        | TLP marking for created STIX objects. |
+| Domain Filter           | domaintools.domain       | `DOMAINTOOLS_DOMAIN`            |                   | No        | Filter for an exact domain or a domain substring by prefixing or suffixing your string with *. |
+| Top Results             | domaintools.top       | `DOMAINTOOLS_TOP`            |                  | No        | Limits the number of results in the response payload.    |
+
+## Deployment
+
+### Docker Deployment
+
+Before building the Docker container, you need to set the version of pycti in `requirements.txt` equal to whatever version of OpenCTI you're running. Example, `pycti==6.5.1`. If you don't, it will take the latest version, but sometimes the OpenCTI SDK fails to initialize.
+
+Build a Docker Image using the provided `Dockerfile`.
+
+Example:
+
+Register connector in the **main** OpenCTI `docker-compose.yml`:
+
+```yaml
+  connector-domaintools:
+    image: opencti/connector-domaintools:latest
+    environment:
+      - OPENCTI_URL=http://opencti:8080
+      - OPENCTI_TOKEN=changeme
+      - CONNECTOR_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+      - CONNECTOR_NAME=DomainTools Feeds
+      - CONNECTOR_SCOPE=stix2
+      - CONNECTOR_LOG_LEVEL=error
+      - CONNECTOR_DURATION_PERIOD=PT5M
+      - DOMAINTOOLS_API_BASE_URL=https://api.domaintools.com
+      - DOMAINTOOLS_API_KEY=changeme
+      - DOMAINTOOLS_FEED_TYPE=nod
+      - DOMAINTOOLS_SESSION_ID=OpenCTI-NOD
+      - DOMAINTOOLS_TLP_LEVEL=clear
+    restart: always
+```
+
+Make sure to replace the environment variables in `docker-compose.yml` with the appropriate configurations for your environment. Then, start the docker container with the provided `docker-compose.yml`.
+
+```shell
+docker compose up -d
+# -d for detached
+```
+
+### Manual Deployment
+
+Create a file `config.yml` based on the provided `config.yml.sample`.
+
+Replace the configuration variables (especially the "**ChangeMe**" variables) with the appropriate configurations for your environment.
+
+Install the required python dependencies (preferably in a virtual environment):
+
+```shell
+pip3 install -r requirements.txt
+```
+
+Then, start the connector from the `src` directory:
+
+```shell
+python3 main.py
+```
+
+## Usage
+
+After installation, the connector should require minimal interaction to use, and should update automatically at a regular interval specified in your `docker-compose.yml` or `config.yml` in `duration_period`.
+
+However, if you would like to force an immediate download of a new batch of data, navigate to:
+
+**Data management → Ingestion → Connectors** in the OpenCTI platform.
+
+Find the "DomainTools Feeds" connector, and click on the refresh button to reset the connector's state and force a new download of data by re-running the connector.
+
+## Behavior
+
+- Fetches data from DomainTools Feeds paginated by `session_id`
+- Converts each feed entry into a STIX 2.1 Observable object
+- Bundles and sends the STIX objects to OpenCTI
+
+## Debugging
+
+The connector can be debugged by setting the appropriate log level. Note that logging messages can be added using `self.helper.connector_logger.{LOG_LEVEL}("Sample message")`, for example, `self.helper.connector_logger.error("An error message")`.
+
+Set `CONNECTOR_LOG_LEVEL=debug` for verbose logging. Log output includes:
+
+- API call details and retry behavior
+- Data count fetched per run
+- Page-by-page fetch progress
+- STIX conversion trace per entry
+- Bundle send status
+
+## Additional information
+
+- For the risk feeds (`domainhotlist`, `domainrisk`), each entry becomes a STIX 2.1 Indicator with a domain-name pattern, a confidence derived from the overall risk score, and per-category risk labels (phishing, malware, spam, proximity).
+- For the observation feeds (`nod`, `nad`, `noh`, `domaindiscovery`), each entry becomes a STIX 2.1 Domain-Name observable labeled with its feed type.
+- Every object is attributed to a DomainTools author identity and tagged with the configured TLP marking.

external-import/domaintools-feeds/__metadata__/connector_manifest.json

@@ -0,0 +1,22 @@
+{
+  "title": "DomainTools Feeds",
+  "slug": "domaintools-feeds",
+  "description": "Import DomainTools Feeds",
+  "short_description": "Import DomainTools Feeds",
+  "logo": null,
+  "use_cases": ["Infrastructure & Attack Surface Visibility", "Detection & Response Enablement"],
+  "solution_categories": ["Threat Intelligence Feed"],
+  "contact": null,
+  "license_type": null,
+  "verified": true,
+  "last_verified_date": null,
+  "playbook_supported": false,
+  "max_confidence_level": 100,
+  "support_version": ">=6.8.12",
+  "subscription_link": null,
+  "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/external-import/domaintools-feeds",
+  "manager_supported": false,
+  "container_version": "rolling",
+  "container_image": "opencti/connector-domaintools-feeds",
+  "container_type": "EXTERNAL_IMPORT"
+}

external-import/domaintools-feeds/config.yml.sample

@@ -0,0 +1,20 @@
+opencti:
+  url: 'http://localhost'
+  token: 'ChangeMe'
+
+connector:
+  type: 'EXTERNAL_IMPORT'
+  id: 'ChangeMe'
+  name: 'DomainTools Feeds' # optional
+  scope: 'stix2'
+  log_level: 'error' # optional (default: 'error')
+  duration_period: 'PT5M' #  # optional, interval in ISO-8601 format between two runs of the connector (default: 'PT1H')
+
+domaintools:
+  api_base_url: 'https://api.domaintools.com'
+  api_key: 'ChangeMe'
+  feed_type: 'ChangeMe'  # domainhotlist, domainrisk, nod, nad, noh, domaindiscovery
+  session_id: 'ChangeMe'     #
+  # domain: *bank*              # optional
+  # top: 10                     # optional
+  tlp_level: 'clear' # optional, available values: 'clear', 'white', 'green', 'amber', 'amber+strict', 'red' (default: 'clear')

external-import/domaintools-feeds/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  opencti-connector-domaintools-feeds:
+    image: opencti/opencti-connector-domaintools-feeds:latest
+    environment:
+      # Generic parameters (connection with OpenCTI)
+      - OPENCTI_URL=http://localhost
+      - OPENCTI_TOKEN=CHANGEME
+      # Common parameters for connectors of type EXTERNAL_IMPORT
+      - CONNECTOR_ID=CHANGEME
+      - CONNECTOR_NAME=CHANGEME # optional
+      - CONNECTOR_SCOPE=CHANGEME
+      - CONNECTOR_LOG_LEVEL=error # optional (default: 'error')
+      - CONNECTOR_DURATION_PERIOD=CHANGEME # optional, interval in ISO-8601 format between two runs of the connector (default: 'PT1H')
+      - DOMAINTOOLS_API_BASE_URL=https://api.domaintools.com
+      - DOMAINTOOLS_API_KEY=ChangeMe
+      - DOMAINTOOLS_FEED_TYPE=ChangeMe  # domainhotlist, domainrisk, nod, nad, noh, domaindiscovery
+      - DOMAINTOOLS_SESSION_ID=ChangeMe     #
+      # - DOMAINTOOLS_DOMAIN=*bank*              # optional
+      # - DOMAINTOOLS_TOP=10                     # optional
+      - DOMAINTOOLS_TLP_LEVEL=clear
+    restart: always

external-import/domaintools-feeds/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Go to the right directory
+cd /opt/opencti-connector-domaintools-feeds
+
+# Launch the worker
+python3 main.py

external-import/domaintools-feeds/src/client/__init__.py

@@ -0,0 +1,5 @@
+from client.api_client import DomainToolsClient
+
+__all__ = [
+    "DomainToolsClient",
+]

external-import/domaintools-feeds/src/client/api_client.py

@@ -0,0 +1,80 @@
+import requests
+from pycti import OpenCTIConnectorHelper
+from pydantic import HttpUrl
+
+
+class DomainToolsClient:
+    def __init__(
+        self,
+        helper: OpenCTIConnectorHelper,
+        base_url: HttpUrl,
+        api_key: str,
+        feed_type: str,
+    ):
+        """
+        Initialize the client with necessary configuration.
+        For log purpose, the connector's helper CAN be injected.
+        Other arguments CAN be added (e.g. `api_key`) if necessary.
+
+        Args:
+            helper (OpenCTIConnectorHelper): The helper of the connector. Used for logs.
+            base_url (str): The external API base URL.
+            api_key (str): The API key to authenticate the connector to the external API.
+        """
+        self.helper = helper
+
+        self.base_url = f"{base_url}/v1/feed/{feed_type}/"
+        self.feed_type = feed_type
+        # Define headers in session and update when needed
+        headers = {"x-api-key": api_key}
+        self.session = requests.Session()
+        self.session.headers.update(headers)
+
+    def _request_data(self, api_url: str, params=None):
+        """
+        Internal method to handle API requests
+        :return: Response in JSON format
+        """
+        try:
+            response = self.session.get(api_url, params=params)
+
+            self.helper.connector_logger.info(
+                "[API] HTTP Get Request to endpoint", {"url_path": api_url}
+            )
+
+            response.raise_for_status()
+            return response
+
+        except requests.RequestException as err:
+            error_msg = "[API] Error while fetching data: "
+            self.helper.connector_logger.error(
+                error_msg, {"url_path": api_url, "error": str(err)}
+            )
+            return None
+
+    def get_entities(self, params=None) -> list:
+        """Fetch feed entries from the DomainTools Threat Feeds API.
+         Args:
+             params: Optional query parameters (e.g., sessionID, domain filter, top).
+         Returns:
+             A list of feed entries (one per line) as strings.
+        """
+        try:
+            # ===========================
+            # === Add your code below ===
+            # ===========================
+
+            response = self._request_data(self.base_url, params=params)
+            if response is None:
+                 raise RuntimeError("Failed to fetch DomainTools feed entries")
+            return response.text.splitlines()
+
+            # return response.json()
+            # ===========================
+            # === Add your code above ===
+            # ===========================
+
+            raise NotImplementedError
+
+        except Exception as err:
+            self.helper.connector_logger.error(err)

external-import/domaintools-feeds/src/connector/__init__.py

@@ -0,0 +1,7 @@
+from connector.connector import DomainToolsFeedsConnector
+from connector.settings import ConnectorSettings
+
+__all__ = [
+    "ConnectorSettings",
+    "DomainToolsFeedsConnector",
+]