diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..805eeccb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +tests +.venv +.local +.github +.vscode +.gitignore + +docs/ +Makefile +Changelog.md + +*.md \ No newline at end of file diff --git a/.github/instructions/snyk_rules.instructions.md b/.github/instructions/snyk_rules.instructions.md new file mode 100644 index 00000000..eef06341 --- /dev/null +++ b/.github/instructions/snyk_rules.instructions.md @@ -0,0 +1,14 @@ +--- +alwaysApply: true +always_on: true +trigger: always_on +applyTo: "**" +description: Snyk Security At Inception +--- + +# Project security best practices + +- Always run snyk_code_scan tool for new first party code that is generated in a Snyk-supported language. +- If any security issues are found based on newly introduced or modified code or dependencies, attempt to fix the issues using the results context from Snyk. +- Rescan the code after fixing the issues to ensure that the issues were fixed and that there are no newly introduced issues. +- Repeat this process until no new issues are found. diff --git a/.gitignore b/.gitignore index 87d9689b..a118d1b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,35 +1,54 @@ -# IDE -.vscode/ - -# Flask logging server logs -flask_logging_server/logs/simplified/* -flask_logging_server/logs/dicom_raw_logs/* -flask_logging_server/logs/exceptions/* - -# DICOM server logs -dicom_server/exceptions/*.log - -# API logs -API/logs/*.log - -# Python cache files -dicom_server/**/__pycache__/ -*.pyc - -# Environment files -API/.env +# IDE +.vscode + +# Byte-compiled / optimized / DLL files +__pycache__ +__pycache__/ +*.py[cod] +*$py.class + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +*.hypothesis/ +.pytest_cache/ + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Environments .env - -# Python specific generated files -__pycache__/ -*.pyo -*.pyd -*.so - -# Node.js -node_modules/ - -data/ - - - +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# snyk things +.github/instructions/ \ No newline at end of file diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..45481802 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,8 @@ +cff-version: 1.2.0 +message: "DICOMHawk citation" +authors: +- family-names: "Vasilomanolakis" + given-names: "Emmanouil" + orcid: "https://orcid.org/0000-0001-5068-9158" +title: "DICOMHawk" +url: "https://github.com/honeynet/DICOMHawk" \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e69de29b diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index 9cb881e1..00000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,28 +0,0 @@ -# Contributors - -## Project: DicomHawk Honeypot - -### Creator & Initial Developer -- **Georgios Theodoridis** (GitHub: [@gtheodoridis](https://github.com/gtheodoridis)) - - Developed the initial version of DicomHawk - - Designed and implemented core honeypot functionalities - - Conducted initial research and testing - -### Lead Developers (Version 2.0 and Beyond) -- **[Alexandra Babanuta]** (GitHub: [@Alexandra Babanuta](https://github.com/albab19)) - - Led development of **DicomHawk v2.0** - - Improved honeypot detection and response mechanisms - - Refactored and optimized core components - -- **[Nawras Mouala]** (GitHub: [@Nawras Mouala](https://github.com/Nawras67))) - - Led development of **DicomHawk v2.0** - - Improved honeypot detection and response mechanisms - - Developed additional logging and analysis tools - -### Additional Contributors -(If others contribute in the future, they can be listed here.) - -### Acknowledgments -- Special thanks to Dr. Emmanouil Vasilomanolakis for guidance and support. -- The Honeynet Project for hosting and maintaining the repository. - diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 00000000..06125c0a --- /dev/null +++ b/Changelog.md @@ -0,0 +1,3 @@ +Dicomhawk 0.1.0 (resease 2026-01-30) + * Refactored application + * First release. \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 128eba42..00000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Georgios Theodoridis - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..c59dee79 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +.PHONY: docker +build-docker: + docker build -t dicomhawk:latest -f build/docker/Dockerfile.dicomhawk + +run-docker: + docker run --rm -it -p 104:104 -p 11112:11112 --network=bridge --name dicomhawk dicomhawk:latest + +format: + black . \ No newline at end of file diff --git a/README.md b/README.md index 61e786a4..486e73b9 100644 --- a/README.md +++ b/README.md @@ -1,542 +1,18 @@ -# DICOMHawk +

-[![DICOMHawk Logo](cover_images/dicomhawk_logo.png)](cover_images/dicomhawk_logo.png) +![Header](./docs/figures/logo.png) -> **A powerful and efficient honeypot for DICOM servers, designed to attract and log unauthorized access attempts and interactions in healthcare environments.** +

-DICOMHawk is a sophisticated cybersecurity tool built using Flask and pynetdicom that offers a streamlined web interface for monitoring and managing DICOM interactions in real-time. It serves as an advanced deception technology specifically designed for medical imaging environments, helping security teams detect, analyze, and respond to potential threats targeting DICOM infrastructure. +
-## 🚀 Key Features +[Quick Start](docs/quick_start.md) | +[Installation](docs/installation.md) | +[Configuration](docs/configuration.md) | +[Commands](docs/commands.md) | +[Features](docs/features.md) | +[FAQ](docs/faq.md) | +[Contact](docs/contact.md) | -### Core DICOM Functionality -- **Full DICOM Protocol Support**: Enables potential attackers to perform DICOM operations on both standard DICOM information models (STUDYROOT and PATIENTROOT) through its DICOM port -- **REST API Service**: Provides an API service enabling attackers to interact with the DICOM server content. Using the API endpoints, an attacker can search and download studies, series, patient and images data. Moreover, they can upload files to the Web API server. -- **Real Medical Data Integration**: Stores real DICOM files that are updated periodically through "The Cancer Imaging Archive (TCIA)" API, which metadata as PHI is modified to resemble real patient data of Danish -citizens in the Danish settings. +
-### Advanced Security Features -- **Comprehensive Honeytoken System**: Multiple types of honeytokens including: - - Encapsulated PDF canary tokens - - HoneyURLs (fake URLs seeded into DICOM datasets) - - Credential honeytokens - - Hidden endpoints and credentials in source code -- **Threat Intelligence Integration**: Automatic reputation checks on each unique IP address interacting with the honeypot -- **Kernel-Level Protection**: Optional Blackhole service for blocking known mass-scanner services at the kernel level - -### Data Localization -- **Multi-Locale Patient Data**: Offers multi-locale patient data generation with configurable regional setting that support different locales for realistic patient name generation. - -### Monitoring & Management -- **Centralized Security Monitoring**: Elastic Stack integration with Logstash for comprehensive attacker activity tracking -- **Automated Log Management**: Daily rotation and compression with intelligent disk space management -- **Interactive Configuration**: Command-line setup wizard guiding users through essential configurations - -## Table of Contents - -- [Quick Start](#quick-start) -- [Deploying DICOMHawk Using Docker Compose](#deploying-dicomhawk-using-docker-compose) -- [Running DICOMHawk Locally](#running-dicomhawk-locally) -- [Configuration](#configuration) -- [Advanced Configuration](#advanced-configuration) -- [Usage Examples](#usage-examples) -- [DICOMHawk Monitoring System](#dicomhawk-monitoring-system) -- [Honeytokens](#honeytokens) -- [Log Management](#log-management) - -## Quick Start - -Get DICOMHawk running quickly with simple installation process. - -### Prerequisites - -Before installing DICOMHawk, ensure you have: - -- **Docker** installed on your system (with Docker Hub access) -- **Docker Compose** for managing multiple containers -- **TCIA account** - free credentials from [The Cancer Imaging Archive](https://www.cancerimagingarchive.net/access-data/) -- **Port availability** - ensure required ports are available (see [Port Requirements](#port-requirements)) - -### Docker Daemon Check - -Before proceeding with installation, verify that the Docker daemon is running: - -**Linux/macOS:** -```bash -sudo systemctl status docker -# or -docker info -``` - -**Windows:** -```powershell -Get-Service docker -# or -docker info -``` - -**macOS (Docker Desktop):** -```bash -docker info -``` - -If Docker is not running, start it: - -**Linux:** -```bash -sudo systemctl start docker -``` - -**Windows:** -```powershell -Start-Service docker -``` - -**macOS:** -Open Docker Desktop application or run: -```bash -open -a Docker -``` - -### Quick Installation - -1. **Clone the Repository** - ```bash - git clone https://github.com/honeynet/DICOMHawk.git - cd DICOMHawk - ``` - -2. **Run the Automated Setup** - ```bash - ./install.sh - ``` - -3. **Follow the Interactive Wizard** - - The script guides you through configuration - - Accept defaults for most settings - - Focus on essential TCIA credentials(Username & Password) - - For detailed configuration options, see the [Configuration](#configuration) section - -### Access Your DICOMHawk Instance - -Once deployed, access your honeypot through: - -- **Web Dashboard**: http://localhost:5000 -- **API Service**: http://localhost:3702 -- **DICOM Server**: localhost:11112 - -## Deploying DICOMHawk Using Docker Compose - -> **💡 Alternative Deployment Method**: This method can be used as an alternative to the Quick Start installation. However, you'll need to manually create your `.env` file with all required configuration settings before deployment. - -### Docker Daemon Check - -Before proceeding with deployment, ensure the Docker daemon is running: - -**Linux/macOS:** -```bash -sudo systemctl status docker -# or -docker info -``` - -**Windows:** -```powershell -Get-Service docker -# or -docker info -``` - -**macOS (Docker Desktop):** -```bash -docker info -``` - -If Docker is not running, start it: - -**Linux:** -```bash -sudo systemctl start docker -``` - -**Windows:** -```powershell -Start-Service docker -``` - -**macOS:** -Open Docker Desktop application or run: -```bash -open -a Docker -``` - -### Port Requirements - -Before deployment, ensure these ports are available: - -| Service | Port | Purpose | -|-------------------|------------|----------------------------------------| -| **Web Dashboard** | 5000 | Main web interface for monitoring | -| **API Service** | 3702 | REST API for programmatic access | -| **DICOM Server** | 11112 | Medical imaging protocol server | -| **Redis** | 6379 | Fast data storage (internal) | -| **Elasticsearch** | 9200, 9300 | Search and analytics (monitoring mode) | -| **Kibana** | 5601 | Data visualization (monitoring mode) | - -### Port Availability Check - -**Linux:** -```bash -netstat -tuln | grep -E '11112|5601|3702|5000|6379' -``` - -**Windows:** -```powershell -Get-NetTCPConnection | Where-Object { $_.LocalPort -eq 11112 -or $_.LocalPort -eq 3702 -or $_.LocalPort -eq 5000 -or $_.LocalPort -eq 6379 } | Format-Table -``` - -### Deployment Architecture - -![DICOMHawk Deployment Architecture](cover_images/deployment.png) - -### Service Profiles - -DICOMHawk uses Docker Compose profiles for flexible deployment: - -| Profile | Services Included | Use Case | -|------------|-------------------------------------|----------------------------| -| **main** | DICOM server, API, Redis, log server | Core honeypot functionality | -| **monitoring** | Elasticsearch, Kibana, Logstash | Advanced monitoring stack | - -### Available Interfaces - -| Interface | URL | Purpose | -|-----------|-----|---------| -| **Kibana Dashboard** | http://localhost:5601/app/dashboards | Advanced monitoring and visualization | -| **Simplified Logging Server** | http://localhost:5000 | Basic log viewing and management | -| **Web API User Interface** | http://localhost:3000 | API interaction and testing | - - -### Default Credentials of API Interface -- **Username**: `test` -- **Password**: `test` - -> **Note**: These are honey credentials designed to detect unauthorized access attempts. -### Deployment Commands - -```bash -# Clone repository (may take time due to large DICOM files) -git clone https://github.com/honeynet/DICOMHawk.git -cd ./DICOMHawk - -# Start core services only -docker compose --profile main up -d - -# Start monitoring services only -docker compose --profile monitoring up -d - -# Start all services -docker compose --profile main --profile monitoring up -d -``` - -## Running DICOMHawk Locally - -> **💡 Development/Testing Deployment Method**: This method is ideal for development, testing, or when you need full control over individual services. It requires manual setup of each component and configuration management. - -### Prerequisites - -**Port availability** - ensure required ports are available (see [Port Requirements](#port-requirements)) - -#### Redis Service -**Ensure Redis is running on port 6379. You can start Redis using the following command if you have Redis installed:** - -```bash -redis-server --port 6379 -``` - -**If you do not have Redis installed, you can easily run a Redis instance using Docker with the following command:** -```bash -docker run -p 6379:6379 --name redis-db -d redis -``` - -#### Installing Packages -```bash -cd ./dicom_server -pip install -r requirements.txt -``` - -### Service Startup - -#### DICOM Server -```bash -cd ./dicom_server -python main.py # Use python3 main.py if your environment defaults to Python 3 -``` - -#### Run the API using Node.js: -```bash -cd ./API -node app.js -``` - -#### Flask Logging Server -```bash -cd ./flask_logging_server -python logserver.py # Use python3 logserver.py if your environment defaults to Python 3 -``` - -### Monitoring Stack - -To deploy the monitoring stack, navigate to the root directory and run the Docker Compose file which contains the monitoring stack's configurations. -```bash -cd monitoring_stack/ -docker compose --profile main up -d -``` - -## Configuration - -### Essential Configuration (Required) - -#### [1] TCIA Credentials -**Required for downloading real medical images** - -DICOMHawk integrates with The Cancer Imaging Archive (TCIA) to provide authentic medical imaging data: - -- **TCIA Account Setup**: Create a free account at [TCIA](https://www.cancerimagingarchive.net/access-data/) following the [account creation guide](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=23691309) -- **Automatic Updates**: Files are retrieved from publicly available repositories with licenses saved in: - ``` - dicom_server/dicom_storage/tcia_data/modality/[StudyInstanceUID]/SeriesInstanceUID/LICENSE - ``` - -**Configuration Parameters:** -- `TCIA_USER_NAME`: Username for TCIA API authentication -- `TCIA_PASSWORD`: Password for TCIA API authentication -- `TCIA_ACTIVATED`: Boolean (`yes`/`no`) to activate/deactivate TCIA service -- `TCIA_PERIOD_UNIT`: Time unit (`day`, `week`, `hour`, `minutes`) for update frequency -- `TCIA_PERIOD`: Numerical frequency value (e.g., `2` weeks = updates twice weekly) -- `MODALITIES`: Array of modalities to retrieve (e.g., `["CT", "MR", "US", "DX"]`) -- `MINIMUM_TCIA_FILES_IN_SERIE`: Minimum files per series -- `MAXIMUM_TCIA_FILES_IN_SERIE`: Maximum files per series - -### Optional Configuration - -#### [2] Security Settings -**Auto-generated for production use** - -JWT and session management secrets for secure authentication: - -- **Access Token Secret**: JWT authentication signing -- **Refresh Token Secret**: Session refresh token signing -- **Admin Secret**: Admin authentication signing -- **Admin Refresh Token Secret**: Admin session refresh -- **Session Secret**: User session management - -#### [3] API Settings -**Web API configuration** - -- **API Port**: REST API service port (default: 3702) - -> **⚠️ Important**: If changing the API port, update both `docker-compose.yml` and `API/Dockerfile` accordingly. - -**Example**: If you change API port to 8080, update: - -- [`docker-compose.yml`](docker-compose.yml#L42) → `api` service: `"3702:3702"` → `"8080:8080"` -- [`API/Dockerfile`](API/Dockerfile#L8): `EXPOSE 3702` → `EXPOSE 8080` - -#### [4] Threat Intelligence APIs -**Enhanced security detection (optional)** - -Integrate with external threat intelligence services: - -- **[AbuseIPDB](https://www.abuseipdb.com/)**: IP reputation checking -- **[IPQualityScore](https://www.ipqualityscore.com/)**: Enhanced IP analysis -- **[VirusTotal](https://www.virustotal.com/gui/home/upload)**: Malware detection - -#### [5] DICOM Settings -**Multi-port DICOM server configuration** - -- **DICOM Ports**: Server listening ports (default: 11112) -- **DICOM_IMPLEMENTATION_NAME**: Server identification (default: ORTHANC) -- **DICOM_IMPLEMENTATION_UID**: Unique server identifier - -> **⚠️ Important**: Port changes require updates to `docker-compose.yml` and `dicom_server/Dockerfile`. - -**Example**: If you change DICOM port to 104, update: - -- [`docker-compose.yml`](docker-compose.yml#L109) → `dicom_server` service: `"11112:11112"` → `"104:104"` -- [`dicom_server/Dockerfile`](dicom_server/Dockerfile#L37): `EXPOSE 11112` → `EXPOSE 104` - -#### [6] Regional Settings -**Patient data localization** - -- **Faker Locale**: Language for patient names (default: en_US) -- **OSM Enabled**: Location services (default: true) -- **OSM Country Code**: Country for location data (default: DK) -- **OSM City**: Specific city (optional) - -#### [7] Honeypot Settings -**Decoy configuration for intrusion detection** - -- **Honey URL**: Fake URL that triggers alerts when accessed - -## Advanced Configuration - -### DICOMHawk Configuration File - -`config.py` contains the main configuration constants and is located in the project root. These settings can be overridden via environment variables in docker compose file. - -### Key Configurable Parameters - -#### General Configuration -- **PROD**: Environment mode (`yes`/`no`) - - `yes`: Production mode with optimized settings - - `no`: Development mode with debug details and system information - -#### Logging Configuration -- **FLASK_ACTIVATED**: Flask server logging (`yes`/`no`) - -#### Integrity Checks -- **INTEGRITY_CHECK**: Periodic DICOM file integrity verification (`yes`/`no`) - -#### DICOM Server and Blackhole Configuration -- **DICOM_SERVER_HOST**: DICOM server IP address or hostname -- **BLOCK_SCANNERS**: Mass scanner blocking (`yes`/`no`) - -## Usage Examples - -### DICOM Protocol Interaction - -Users can interact with the DICOM server using standard DCMTK tools: - -#### Connection Verification -```bash -echoscu localhost 11112 -``` - -#### Patient Queries -```bash -findscu -v -S -k QueryRetrieveLevel=PATIENT localhost 11112 -``` - -#### Study Queries -```bash -findscu -v -S -k QueryRetrieveLevel=STUDY localhost 11112 -``` - -#### File Storage -```bash -storescu -v -d localhost 11112 [Path to your DICOM file] -``` - -### DICOM Client Applications - -DICOMHawk is compatible with various DICOM client applications: - -- **Sante DICOM Viewer**: [Download here](https://santesoft.com/win/sante-dicom-viewer-lite/sante-dicom-viewer-lite.html) -- **Other DICOM viewers**: Any DICOM-compliant client application - -## DICOMHawk Monitoring System - -### Overview - -DICOMHawk implements a centralized security monitoring infrastructure designed to track and analyze attacker behavior in healthcare environments. This system enables cybersecurity teams to: - -- **Quick Detection**: Rapidly identify security incidents -- **Pattern Analysis**: Understand attacker techniques and interaction patterns -- **Forensic Capabilities**: Maintain detailed logs for comprehensive analysis -- **Impact Assessment**: Trace the source and impact of each interaction - -![DICOMHawk Monitoring System](cover_images/kibana.png) - -### Monitoring Components - -The monitoring system provides: - -- **Real-time Metrics**: Summary statistics and detailed analysis -- **Multi-format Visualizations**: Numbers, tables, pie charts, and timelines -- **Threat Scoring**: Immediate malicious and abuse scoring for each interaction -- **Comprehensive Logging**: Detailed tracking of DICOM sessions and API requests - -### Architecture - -The monitoring system utilizes the Elastic Stack: - -1. **Logstash**: Collects data from log files integrated with the honeypot -2. **Elasticsearch**: Indexes and stores security events for analysis -3. **Kibana**: Provides powerful data visualization and dashboard capabilities - -## Honeytokens - -Honeytokens (canary PDFs and honeyURLs) are security measures used to detect and alert on unauthorized access or potential breaches. - -### DICOM Server Honeytokens - -The DICOM server in DICOMHawk is designed to automatically update its DICOM file repository periodically, pulling new files from The Cancer Imaging Archive (TCIA). During this update process, the system injects selected DICOM files with honeytokens, specifically canary PDFs and honeyURLs, as part of its enhanced security measures. - -When the DICOM server periodically removes old DICOM files and retrieves new ones from TCIA, the updated canary PDF and honeyURL are automatically injected into some of these new files. This ensures that the security features are consistently refreshed and tailored to current monitoring and security needs. - -#### Canary PDFs -Canary PDF files serve as monitored tokens within DICOM files. - -- **Location**: `dicom_server/storage/can.pdf` (maps to `/opt/dicomhawk/storage/can.pdf` in container) -- The server uses this file as a template for generating canary PDFs injected into new DICOM files retrieved from TCIA. Make sure the updated PDF is named can.pdf to ensure it is properly recognized and utilized by the system. - -#### HoneyURLs -HoneyURLs are URLs embedded within DICOM data. When accessed, they indicate potential unauthorized interactions. - -```bash -HONEY_URL="https://[YOURHONEYURL]" -``` -- Replace `[YOURHONEYURL]` with your desired honey URL - -- This change in the environment variable ensures that any new DICOM files automatically fetched and updated by the server will include the new honeyURL. - -### Web API Honeytokens - -The Web API has also employed four honeytoken types to detect different attack vectors. - -#### robots.txt and Hidden Endpoints -- Allows an attacker to be misguided and mislead to, for example, the endpoints called: "/admin", "/admin-config", "/secure" and "/ensurance_data". -- The purpose of this file is to make the attackers curious to explore the Web API and think of ways to get access to those protected resources. In this way, more meaningful information on attackers' actions can be collected. -- If someone accesses the "robots.txt" file, the interaction is immediately logged and visualized within the visualization dashboard which helps identifying potential crawling or scraping activities. -- When for example, the "/admin" endpoint is accessed a fake admin access token is generated, which is not differing in size from the original access token. This is meant to provide inspiration for the potential adversaries. - -#### Honey Credentials - -- Fake credentials appear to be "leaked" in the login page of the Web API. They are to be found in the raw html source. If these credentials are used by a potential adversary, they are taken to an "Under development" screen. -- Moreover, in order to access the Web API from the very start, the potential adversary has to login into the system. -- Honey credentials "test" - "test" are used. -- The login page is continuously monitored for login attempts and therefore guessing, credential stuffing and brute force attacks can be identified. - -## Log Management - -DICOMHawk implements comprehensive automated log management through the `dicomhawkinit` service. - -### Features - -- **Daily Log Rotation**: Automatic log file rotation -- **Compression**: Efficient storage using pigz compression -- **Cleanup**: Automatic removal of logs older than 30 days (configurable) -- **Organization**: Structured log storage and management - -DICOMHawk captures detailed information about: - -All logs are stored under `/data/dicomhawk/logs/`: - -| Directory/File | Content | Purpose | -|----------------|---------|---------| -| `dicom_raw_logs/` | Raw DICOM protocol messages, association requests/releases, C-FIND/C-GET/C-STORE operations, detailed packet-level communication | Deep protocol analysis and debugging | -| `simplified/` | Clean DICOM transaction summaries, patient queries, study retrievals, association events with timestamps and IP addresses | Quick event review and monitoring | -| `exceptions/` | Python exceptions, service errors, configuration issues, startup failures, runtime problems | Troubleshooting and system health monitoring | -| `api_logs.log` | REST API requests/responses, authentication attempts, file uploads/downloads, user sessions, endpoint access | API usage monitoring and security analysis | -| `reputation.log` | IP reputation scores, threat intelligence results, abuse scores, geographic data, proxy/VPN detection | Security analysis and threat assessment | -| `scanned_ips.log` | IP scanning patterns, port scans, connection attempts, attack signatures, frequency analysis | Attack detection and pattern recognition | - -### Configuration - -Customize log retention through environment variables: - -```yaml -dicomhawkinit: - environment: - - PERSISTENCE_CYCLES=30 # Days to retain logs -``` diff --git a/customizer/Dockerfile b/customizer/Dockerfile deleted file mode 100644 index c0cef02d..00000000 --- a/customizer/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.9-slim - -ARG APP_UID=2000 -ARG APP_GID=2000 - -WORKDIR /customizer - -# Copy the customizer script -COPY customizer.py /customizer/ -COPY entrypoint.sh /customizer/ - -# Make scripts executable -RUN chmod +x /customizer/customizer.py -RUN chmod +x /customizer/entrypoint.sh - -# Install required packages -RUN pip install --no-cache-dir colorama - -RUN groupadd -g ${APP_GID} dicom \ - && useradd -m -u ${APP_UID} -g ${APP_GID} -s /usr/sbin/nologin dicom \ - && chown -R ${APP_UID}:${APP_GID} /customizer - -USER ${APP_UID}:${APP_GID} - -ENTRYPOINT ["/customizer/entrypoint.sh"] \ No newline at end of file diff --git a/customizer/customizer.py b/customizer/customizer.py deleted file mode 100644 index f87095e6..00000000 --- a/customizer/customizer.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python3 -""" -DICOMHawk Configuration Customizer -This script prompts users for necessary configurations and generates a .env file -for the DICOMHawk honeypot system. -""" - -import os -import sys -import secrets -import string -from pathlib import Path - -class Colors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - -def print_header(): - """Print the DICOMHawk header""" - print(f"{Colors.HEADER}{Colors.BOLD}") - print("=" * 60) - print(" DICOMHawk Configuration Customizer") - print("=" * 60) - print(f"{Colors.ENDC}") - print("This script will help you configure DICOMHawk for your environment.") - print("Press Enter to use default values (shown in brackets).") - print() - -def generate_secret(length=32): - """Generate a random secret string""" - # Use only alphanumeric characters to avoid Docker Compose variable interpretation issues - alphabet = string.ascii_letters + string.digits - return ''.join(secrets.choice(alphabet) for _ in range(length)) - -def get_input(prompt, default="", required=False, secret=False): - """Get user input with validation""" - while True: - if secret: - import getpass - value = getpass.getpass(f"{prompt} [{default}]: ") - else: - value = input(f"{prompt} [{default}]: ").strip() - - if not value: - if required and not default: - print(f"{Colors.FAIL}This field is required. Please enter a value.{Colors.ENDC}") - continue - value = default - - if required and not value: - print(f"{Colors.FAIL}This field is required. Please enter a value.{Colors.ENDC}") - continue - - return value - -def check_first_run(): - """Check if this is the first run""" - env_file = Path(".env") - if env_file.exists(): - print(f"{Colors.WARNING}Configuration file .env already exists.{Colors.ENDC}") - response = input("Do you want to overwrite it? (y/n): ").lower().strip() - if response not in ['y', 'yes']: - print("Configuration cancelled.") - sys.exit(0) - return True - -def main(): - """Main configuration function""" - print_header() - - if not check_first_run(): - return - - print(f"{Colors.OKBLUE}Step 1: Security Configuration{Colors.ENDC}") - print("-" * 40) - - # Generate default secrets - access_token_secret = generate_secret(32) - refresh_token_secret = generate_secret(32) - admin_secret = generate_secret(32) - admin_refresh_token_secret = generate_secret(32) - session_secret = generate_secret(32) - - # Security tokens - ACCESS_TOKEN_SECRET = get_input( - "Access Token Secret (for JWT authentication)", - access_token_secret, - required=True, - secret=True - ) - - REFRESH_TOKEN_SECRET = get_input( - "Refresh Token Secret (for JWT refresh)", - refresh_token_secret, - required=True, - secret=True - ) - - ADMIN_SECRET = get_input( - "Admin Secret (for admin authentication)", - admin_secret, - required=True, - secret=True - ) - - ADMIN_REFRESH_TOKEN_SECRET = get_input( - "Admin Refresh Token Secret", - admin_refresh_token_secret, - required=True, - secret=True - ) - - SESSION_SECRET = get_input( - "Session Secret (for session management)", - session_secret, - required=True, - secret=True - ) - - print(f"\n{Colors.OKBLUE}Step 2: API Configuration{Colors.ENDC}") - print("-" * 40) - - API_PORT = get_input("API Port", "3702") - - print(f"\n{Colors.OKBLUE}Step 3: TCIA Configuration{Colors.ENDC}") - print("-" * 40) - print("TCIA credentials are optional. Leave blank to use sample DICOM files (fallback is automatic).") - print("Get free credentials: https://www.cancerimagingarchive.net/") - - TCIA_USER_NAME = get_input("TCIA Username (optional)") - TCIA_PASSWORD = get_input("TCIA Password (optional)", secret=True) - TCIA_PERIOD_UNIT = get_input("TCIA Period Unit (minutes/hours/days)", "minutes") - TCIA_PERIOD = get_input("TCIA Period (frequency of downloads)", "1") - # Always enable fallback mode; it will automatically be used when credentials are missing or TCIA is disabled - TCIA_FALLBACK_MODE = "true" - - print(f"\n{Colors.OKBLUE}Step 4: Threat Intelligence APIs (Optional){Colors.ENDC}") - print("-" * 40) - print("These APIs are used for IP reputation checking. Leave empty if you don't have keys.") - - ABUSE_IP_API_KEY = get_input("AbuseIPDB API Key", "", secret=True) - IP_QUALITY_SCORE_API_KEY = get_input("IPQualityScore API Key", "", secret=True) - VIRUS_TOTAL_API_KEY = get_input("VirusTotal API Key", "", secret=True) - - print(f"\n{Colors.OKBLUE}Step 5: DICOM Configuration{Colors.ENDC}") - print("-" * 40) - - DICOM_PORTS = get_input("DICOM Ports (comma-separated)", "11112") - DICOM_IMPLEMENTATION_NAME = get_input("DICOM Implementation Name", "ORTHANC") - DICOM_IMPLEMENTATION_UID = get_input("DICOM Implementation UID", "1.2.826.0.1.3680043.9.3811.2.0.1") - - print(f"\n{Colors.OKBLUE}Step 6: Regional Configuration{Colors.ENDC}") - print("-" * 40) - - FAKER_LOCALE = get_input("Faker Locale (for generating patient names)", "en_US") - OSM_ENABLED = get_input("Enable OpenStreetMap integration (true/false)", "true") - OSM_COUNTRY = get_input("OSM Country Code (ISO 3166-1 alpha-2)", "DK") - OSM_CITY = get_input("OSM City (optional)", "") - - print(f"\n{Colors.OKBLUE}Step 7: Honeypot Configuration{Colors.ENDC}") - print("-" * 40) - - HONEY_URL = get_input("Honey URL (for honeytoken injection)", "https://example.com/honey") - - # Generate .env file - env_content = f"""# DICOMHawk Configuration File -# Generated by customizer.py - -# Security Tokens -ACCESS_TOKEN_SECRET={ACCESS_TOKEN_SECRET} -REFRESH_TOKEN_SECRET={REFRESH_TOKEN_SECRET} -ADMIN_SECRET={ADMIN_SECRET} -ADMIN_REFRESH_TOKEN_SECRET={ADMIN_REFRESH_TOKEN_SECRET} -SESSION_SECRET={SESSION_SECRET} - -# API Configuration -API_PORT={API_PORT} - -# TCIA Configuration -TCIA_USER_NAME={TCIA_USER_NAME} -TCIA_PASSWORD={TCIA_PASSWORD} -TCIA_PERIOD_UNIT={TCIA_PERIOD_UNIT} -TCIA_PERIOD={TCIA_PERIOD} -TCIA_FALLBACK_MODE={TCIA_FALLBACK_MODE} - -# Threat Intelligence APIs -ABUSE_IP_API_KEY={ABUSE_IP_API_KEY} -IP_QUALITY_SCORE_API_KEY={IP_QUALITY_SCORE_API_KEY} -VIRUS_TOTAL_API_KEY={VIRUS_TOTAL_API_KEY} - -# DICOM Configuration -DICOM_PORTS={DICOM_PORTS} -DICOM_IMPLEMENTATION_NAME={DICOM_IMPLEMENTATION_NAME} -DICOM_IMPLEMENTATION_UID={DICOM_IMPLEMENTATION_UID} - -# Regional Configuration -FAKER_LOCALE={FAKER_LOCALE} -OSM_ENABLED={OSM_ENABLED} -OSM_COUNTRY={OSM_COUNTRY} -OSM_CITY={OSM_CITY} - -# Honeypot Configuration -HONEY_URL={HONEY_URL} -""" - - # Write .env file - with open(".env", "w") as f: - f.write(env_content) - - print(f"\n{Colors.OKGREEN}Configuration completed successfully!{Colors.ENDC}") - print(f"{Colors.BOLD}Generated .env file with your configuration.{Colors.ENDC}") - print() - print("Next steps:") - print("1. Review the generated .env file") - print("2. Run: docker-compose --profile main up -d") - print("3. Access the web interface at: http://localhost:5000") - print("4. Access the API at: http://localhost:3702") - print() - print(f"{Colors.WARNING}Important:{Colors.ENDC}") - print("- TCIA credentials are optional - the system will use sample files if not provided") - print("- Update your TCIA credentials if they expire") - print("- The fallback system ensures the honeypot always has realistic DICOM data") - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print(f"\n{Colors.FAIL}Configuration cancelled by user.{Colors.ENDC}") - sys.exit(1) - except Exception as e: - print(f"\n{Colors.FAIL}Error: {e}{Colors.ENDC}") - sys.exit(1) \ No newline at end of file diff --git a/customizer/entrypoint.sh b/customizer/entrypoint.sh deleted file mode 100644 index 74a811b0..00000000 --- a/customizer/entrypoint.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# DICOMHawk Customizer Entrypoint -# This script runs the customizer and ensures it only runs once - -set -e - -CONFIG_FLAG="/customizer/.configured" - -echo "DICOMHawk Customizer Starting..." - -# Check if already configured -if [ -f "$CONFIG_FLAG" ]; then - echo "Configuration already completed. Skipping customizer." - exit 0 -fi - -# Check if .env file exists -if [ -f "/workspace/.env" ]; then - echo "Configuration file .env already exists." - echo "If you want to reconfigure, remove the .env file and restart." - exit 0 -fi - -# Run the customizer -echo "Starting configuration process..." -python3 /customizer/customizer.py - -# Check if configuration was successful -if [ -f "/workspace/.env" ]; then - echo "Configuration completed successfully!" - touch "$CONFIG_FLAG" - exit 0 -else - echo "Configuration failed. Please check the output above." - exit 1 -fi \ No newline at end of file diff --git a/dicom_server/Dockerfile b/dicom_server/Dockerfile deleted file mode 100644 index 1662cb3e..00000000 --- a/dicom_server/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM python:3.9-slim -USER root - -RUN mkdir -p /var/log/dicomhawk/dicom_raw_logs \ - /var/log/dicomhawk/simplified \ - /var/log/dicomhawk/exceptions \ - /var/log/dicomhawk/api_logs \ - /var/log/dicomhawk/reputation \ - /var/log/dicomhawk/scanned_ips \ - /opt/dicomhawk/storage/dicom_storage \ - /opt/dicomhawk/storage/c_store_files \ - /opt/dicomhawk/tcia/data \ - /opt/dicomhawk/tcia/stagger - -RUN touch /var/log/dicomhawk/dicom_raw_logs/dicom_raw.log \ - /var/log/dicomhawk/simplified/simplified_logger.log \ - /var/log/dicomhawk/exceptions/exceptions.log \ - /var/log/dicomhawk/api_logs/api_logs.log \ - /var/log/dicomhawk/reputation/reputation.log \ - /var/log/dicomhawk/scanned_ips/scanned_ips.log - -COPY . dicom_server/ -WORKDIR /dicom_server/. - -RUN pip install --upgrade pip \ - && pip install -r requirements.txt \ - && apt-get update \ - && apt-get -qq install -y iptables ipset - -COPY ./pydicom_and_pynetdicom_libs/service_class.py /usr/local/lib/python3.9/site-packages/pynetdicom/service_class.py -COPY ./pydicom_and_pynetdicom_libs/db.py /usr/local/lib/python3.9/site-packages/pynetdicom/apps/qrscp/db.py -COPY ./pydicom_and_pynetdicom_libs/valuerep.py /usr/local/lib/python3.9/site-packages/pydicom/valuerep.py - -ARG APP_UID=2000 -ARG APP_GID=2000 -RUN groupadd -g ${APP_GID} dicom \ - && useradd -m -u ${APP_UID} -g ${APP_GID} -s /usr/sbin/nologin dicom \ - && chown -R ${APP_UID}:${APP_GID} \ - /var/log/dicomhawk \ - /opt/dicomhawk - -EXPOSE 11112 -USER ${APP_UID}:${APP_GID} -CMD ["python","-u","main.py"] diff --git a/dicom_server/config.py b/dicom_server/config.py deleted file mode 100644 index db0aa2a5..00000000 --- a/dicom_server/config.py +++ /dev/null @@ -1,211 +0,0 @@ -"""This module defines configuration constants and paths for the server service. -Many values can be overridden via environment variables - -TCIA Serviceconstants: ---------------------- -* TCIA_USER_NAME -* TCIA_ACTIVATED -* TCIA_PASSWORD -* TCIA_PERIOD_UNIT -* TCIA_PERIOD -* TCIA_FILES_DIRECTORY -* MODALITIES -* MINIMUM_TCIA_FILES_IN_SERIE -* MAXIMUM_TCIA_FILES_IN_SERIE -* TCIA_STUDIES_PER_MODALITY -* FAKER_LOCALE - -OpenStreetMap Integration: ---------------------- -* OSM_ENABLED -* OSM_COUNTRY -* OSM_CITY -* OSM_CACHE_DURATION -* OSM_MAX_INSTITUTIONS -* OSM_TIMEOUT -* OSM_FALLBACK_INSTITUTIONS - -Logging Server: ---------------------- -* FLASK_ACTIVATED (Important to avoid logging on test environment) -* MAIN_LOG_DIRECTORY -* SIMPLIFIED_LOG_DIRECTORY -* EXCEPTIONS_LOG_DIRECTORY - -Integrity Check: ---------------------- -* INTEGRITY_CHECK -* HASH_STORAGE_PATH - -Threat Intelligence: ---------------------- -* ABUSE_IP_API_KEY -* IP_QUALITY_SCORE_API_KEY -* VIRUS_TOTAL_API_KEY - -Blackhole: ---------------------- -* BLOCK_SCANNERS -* BLACKHOLE_FILE_PATH - -DICOM server: ---------------------- -* PROD (environment will be production if this constant is true and development if it is false) -* DICOM_STORAGE_DIR -* C_STORE_STORAGE -* DICOM_PORTS -* DICOM_SERVER_HOST -* REDIS_HOST -* DICOM_DATABASE -* CANARY_PDF_PATH - -""" - -import os, json - -TRUE_LIST = ["true", "1", "t", "yes"] -"""Envirnoment""" -PROD = os.getenv("PROD", "False").lower() in TRUE_LIST - -DOCKER = os.getenv("DOCKER", "False").lower() in TRUE_LIST -"""Flask server status""" -FLASK_ACTIVATED = os.getenv("FLASK_ACTIVATED", "True").lower() in TRUE_LIST - -"""Null routing the incomming requests if belong a known mass scanner """ -BLOCK_SCANNERS = os.getenv("BLOCK_SCANNERS", "False").lower() in TRUE_LIST - -"""Blackhole list file""" -BLACKHOLE_FILE_PATH = os.getenv("BLACKHOLE_FILE_PATH", "/opt/dicomhawk/storage/blackhole_list.txt" if DOCKER else "./storage/blackhole_list.txt") - -"""DICOM files storage""" - -DICOM_STORAGE_DIR = "/opt/dicomhawk/storage/dicom_storage" if DOCKER else "./storage/dicom_storage" - -"""DICOM files recieved through the server storage""" - -C_STORE_STORAGE = "/opt/dicomhawk/storage/c_store_files" if DOCKER else "./storage/c_store_files" - -"""DICOM server port configuration""" -try: - DICOM_PORTS = json.loads(os.getenv("DICOM_PORTS", "[11112]")) -except json.JSONDecodeError: - DICOM_PORTS = [11112] - -# DICOM Implementation Name -IMPLEMENTATION_NAME = os.getenv("DICOM_IMPLEMENTATION_NAME", "ORTHANC") -IMPLEMENTATION_UID = os.getenv("DICOM_IMPLEMENTATION_UID", "1.2.826.0.1.3680043.9.3811.2.0.1") - -"""DICOM server host ip configuration""" -DICOM_SERVER_HOST = "172.29.0.3" if DOCKER else "0.0.0.0" - - -"""Redis host configuration""" -REDIS_HOST = os.getenv("REDIS_HOST", "172.29.0.4") if DOCKER else "localhost" - -"""Logs directories""" -MAIN_LOG_DIRECTORY, SIMPLIFIED_LOG_DIRECTORY, EXCEPTIONS_LOG_DIRECTORY = ( - ("/var/log/dicomhawk/dicom_raw_logs", "/var/log/dicomhawk/simplified", "/var/log/dicomhawk/exceptions") - if DOCKER - else ( - "../flask_logging_server/logs/dicom_raw_logs", - "../flask_logging_server/logs/simplified", - "./exceptions", - ) -) - -"""The sqlite file path""" -DICOM_DATABASE = "/opt/dicomhawk/storage/db.db" if DOCKER else "./storage/db.db" - -"""TCIA username and password to use it in API calls""" -TCIA_USER_NAME = os.getenv("TCIA_USER_NAME", "user") -TCIA_PASSWORD = os.getenv("TCIA_PASSWORD", "pass") - -"""Time unit to schedule tcia files retrieval""" -TCIA_PERIOD_UNIT = os.getenv("TCIA_PERIOD_UNIT", "week") - - -"""Default update dicom files from tcia API each 1 week""" -TCIA_PERIOD = int(os.getenv("TCIA_PERIOD", 1)) - -"""The path where TCIA dicom files save on retrieval""" -TCIA_FILES_DIRECTORY = "/opt/dicomhawk/tcia/data" if DOCKER else "./storage/tcia_data" -"""Files stagger directory""" -TCIA_FILES_STAGGER_DIRECTORY = "/opt/dicomhawk/tcia/stagger" if DOCKER else "./storage/stagger" -""" API key Abuseipdb """ -ABUSE_IP_API_KEY = os.getenv( - "ABUSE_IP__KEY", - "apikey", -) - -""" API key Abuseipdb """ -IP_QUALITY_SCORE_API_KEY = os.getenv( - "IP_QUALITY_SCORE_API_KEY", - "apikey", -) - -""" API key Virus Total """ -VIRUS_TOTAL_API_KEY = os.getenv( - "VIRUS_TOTAL_API_KEY", - "apikey", -) - -"""Canary pdf path""" -CANARY_PDF_PATH = "/opt/dicomhawk/storage/can.pdf" if DOCKER else "./storage/can.pdf" - - -"""TCIA activated""" -TCIA_ACTIVATED = os.getenv("TCIA_ACTIVATED", "True").lower() in TRUE_LIST - -"""TCIA fallback mode - use sample files when TCIA is unavailable""" -TCIA_FALLBACK_MODE = os.getenv("TCIA_FALLBACK_MODE", "True").lower() in TRUE_LIST - - -""" Modalities of the studies should be retrieved from TCIA """ -MODALITIES = json.loads(os.getenv("MODALITIES", '["CT", "MR", "US", "DX"]')) - -"""Minimum number of files in each serie retrieved from The Cancer Imaging Archeive API""" -MINIMUM_TCIA_FILES_IN_SERIE = int(os.getenv("MINIMUM_TCIA_FILES_IN_SERIE", 1)) - -"""Maximum number of files in each serie retrieved from The Cancer Imaging Archeive API""" - -MAXIMUM_TCIA_FILES_IN_SERIE = int(os.getenv("MAXIMUM_TCIA_FILES_IN_SERIE", 3)) - -"""Number of studies for each modality from TCIA""" -TCIA_STUDIES_PER_MODALITY = int(os.getenv("TCIA_STUDIES_PER_MODALITY", 10)) - -"""Honeytoken URL""" -HONEY_URL = os.getenv("HONEY_URL","VALUE") - -"""Activate DICOM files integrity checks every 6 hours""" -INTEGRITY_CHECK = os.getenv("INTEGRITY_CHECK", "True").lower() in TRUE_LIST - -"""Integrity checker file storage path""" -HASH_STORAGE_PATH = "/opt/dicomhawk/storage/hash_store.json" if DOCKER else "./storage/hash_store.json" - -"""Faker locale for generating patient names and data""" -FAKER_LOCALE = os.getenv("FAKER_LOCALE", "en_US") - -"""OpenStreetMap Integration Configuration""" -OSM_ENABLED = os.getenv("OSM_ENABLED", "True").lower() in TRUE_LIST - -"""Country to search for medical institutions (ISO 3166-1 alpha-2 code)""" -OSM_COUNTRY = os.getenv("OSM_COUNTRY", "DK") - -"""City to search for medical institutions (optional, searches entire country if not specified)""" -OSM_CITY = os.getenv("OSM_CITY", "") - -"""Cache duration for OSM data in hours""" -OSM_CACHE_DURATION = int(os.getenv("OSM_CACHE_DURATION", 24)) - -"""Maximum number of institutions to fetch from OSM""" -OSM_MAX_INSTITUTIONS = int(os.getenv("OSM_MAX_INSTITUTIONS", 50)) - -"""Timeout for OSM API requests in seconds""" -OSM_TIMEOUT = int(os.getenv("OSM_TIMEOUT", 30)) - -"""Fallback institutions when OSM is disabled or fails""" -OSM_FALLBACK_INSTITUTIONS = json.loads(os.getenv("OSM_FALLBACK_INSTITUTIONS", - '["Københavns Sundhedscenter", "Aarhus Kliniken", "Odense Patienthus", "Nordjylland Med Institut"]')) - -"""OSM cache file path""" -OSM_CACHE_FILE = "/opt/dicomhawk/storage/osm_institutions_cache.json" if DOCKER else "./storage/osm_institutions_cache.json" \ No newline at end of file diff --git a/dicom_server/core/app_container.py b/dicom_server/core/app_container.py deleted file mode 100644 index f9cdbcc6..00000000 --- a/dicom_server/core/app_container.py +++ /dev/null @@ -1,222 +0,0 @@ -""" -This module defines the ApplicationContext class using the Dependency Injector library to manage dependencies throughout the start of the application. - -The ApplicationContext class acts as a central configuration hub for all services on the server - -The setup includes adding the following providers: - - Loggers provider - ------------------ - - data-access agents: - ------------------ - - * dicom_db: Manages the DICOM database service, responsible for storing/retrieving dicom files information maintained at the storage block. - - * redis_handler: Implements a redis service responsible for storing of dicom sessions information, provides rapid access for external analysis/visualization middleware. - - TCIA providers: - ------------------ - - * tcia_api: Handles communication between the application and The Cancer Imaging Archive API. - - * tcia_manager: Handles file exchange by removing old files, organizing new ones, injectting new files with honeytokens and initializing the database. - - * tcia_scheduler: Schedules the retrieval of new DICOM files on a periodic basis. - - Threat Intelligence: - ------------------ - - * threat_intelligence: provides IP information from three Threat Intelligence services (AbuseIPdatabase, IPQualityScore, and VirusTotal). - - Files Integrity Checker: - ------------------ - * files_checker: Verifies the integrity of the dicom files stored at the dicom storage every seven hours, detecting any unauthorized access attemp. - - Network Management: - ------------------ - - * blackhole: Null-route the requests with IP addresses that belong to one of the known mass-scanners mitigitating the potential of the heneypot been identified as a honeypot in the future - - * session_collector: Collects and manages data from dicom sessions in order to ensure consistency in data analysis and visualization - - DICOM Services: - ------------------ - - * dicom_handler: implements the standard dicom operations, acting as a SCP (Service Provider) on C-Echo and C-Store first part of C-Get, C-Find and C-Move (receiving and responding to requests) operations - and as a SCU (Serive User) for the second part of C-Get, C-Find and C-Move (initiating requests) operations - - * dicom_application: handles the configuration, initialization of the DICOM application entity and starts the DICOM server. -""" - -import sys, os - -from dependency_injector import containers, providers -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -import redis -import config -from loggers import Loggers -from dicomdb import DicomDatabase -from redis_handler import RedisClient -from dicom_session_manager import SessionCollector -from dicom_handlers import DICOMHandlers -from dicom_application import DicomStarter -from threat_intelligence_handler import ThreatIntelligence -import logging -from custom_units.integrity_checker import FilesChecker -from custom_units.tcia_management import TCIAAPI -from custom_units.tcia_management import TCIAManager -from custom_units.tcia_management import TCIAScheduler -from custom_units.network_manager import Blackhole -from custom_units.osm_institutions import OSMInstitutionsService - - - -class ApplicationContainer(containers.DeclarativeContainer): - - # Loggers service - loggers = providers.Singleton( - Loggers, - config.PROD, - config.MAIN_LOG_DIRECTORY, - config.SIMPLIFIED_LOG_DIRECTORY, - config.EXCEPTIONS_LOG_DIRECTORY, - ) - - if config.FLASK_ACTIVATED: - loggers() - app_logger = providers.Singleton(logging.getLogger, "app_logger") - exceptions_logger = providers.Singleton(logging.getLogger, "exceptions") - simplified_logger = providers.Singleton(logging.getLogger, "simplified_logger") - - # DICOM database session configuration - engine = providers.Singleton(create_engine, f"sqlite:///{config.DICOM_DATABASE}") - session_factory = providers.Singleton(sessionmaker, bind=engine) - session = providers.Singleton(lambda sf: sf(), session_factory) - - # DICOM_database provider - dicom_db = providers.Singleton( - DicomDatabase, app_logger, exceptions_logger, config.DICOM_STORAGE_DIR, session - ) - - # Redis provider - redis_client = providers.Singleton(redis.Redis, config.REDIS_HOST, 6379) - try: - redis_client().client_list() - except ConnectionError: - print( - f"No Redis database is running on {config.REDIS_HOST}, port 6379. Please start the Redis service.", - ) - sys.exit(1) - - redis_handler = providers.Singleton( - RedisClient, app_logger, exceptions_logger, redis_client - ) - - # OSM provider - osm_service = providers.Singleton( - OSMInstitutionsService, - app_logger, - exceptions_logger, - ) - - # TCIA providers - - tcia_api = providers.Factory( - TCIAAPI, - app_logger, - exceptions_logger, - config.TCIA_USER_NAME, - config.TCIA_PASSWORD, - config.MINIMUM_TCIA_FILES_IN_SERIE, - config.MAXIMUM_TCIA_FILES_IN_SERIE, - config.MODALITIES, - config.TCIA_STUDIES_PER_MODALITY, - ) - tcia_manager = providers.Singleton( - TCIAManager, - app_logger, - exceptions_logger, - config.HONEY_URL, - config.DICOM_STORAGE_DIR, - config.TCIA_FILES_DIRECTORY, - config.TCIA_FILES_STAGGER_DIRECTORY, - config.CANARY_PDF_PATH, - dicom_db, - redis_handler, - tcia_api, - osm_service, - ) - - tcia_scheduler = providers.Singleton( - TCIAScheduler, - app_logger, - exceptions_logger, - config.TCIA_PERIOD, - config.TCIA_PERIOD_UNIT, - tcia_manager, - ) - - # IP Threat Intelligence provider - threat_intelligence = providers.Singleton( - ThreatIntelligence, - app_logger, - exceptions_logger, - config.ABUSE_IP_API_KEY, - config.IP_QUALITY_SCORE_API_KEY, - config.VIRUS_TOTAL_API_KEY, - ) - - # DICOM files integrity cheacker provider - files_checker = providers.Singleton( - FilesChecker, - app_logger, - exceptions_logger, - config.DICOM_STORAGE_DIR, - config.HASH_STORAGE_PATH, - redis_handler, - ) - - # Blackholee service provider - blackhole = providers.Singleton( - Blackhole, - app_logger, - exceptions_logger, - config.BLOCK_SCANNERS, - config.BLACKHOLE_FILE_PATH, - ) - - # A DICOM connection comprises multiple DICOM requests. A collector service provider is added to manage session information before logging. - session_collector = providers.Singleton( - SessionCollector, - app_logger, - simplified_logger, - exceptions_logger, - redis_handler, - threat_intelligence, - ) - - # DICOM handlers provider - dicom_handlers = providers.Singleton( - DICOMHandlers, app_logger, exceptions_logger, session_collector, dicom_db - ) - - # The DICOM application handles the application entity configuration - dicom_application = providers.Singleton( - DicomStarter, - app_logger, - exceptions_logger, - config.DICOM_PORTS, - config.DICOM_SERVER_HOST, - dicom_handlers, - ) - - if config.TCIA_ACTIVATED: - tcia_scheduler() - - if config.INTEGRITY_CHECK: - files_checker() - - if config.BLOCK_SCANNERS: - blackhole() diff --git a/dicom_server/core/dicom_application.py b/dicom_server/core/dicom_application.py deleted file mode 100644 index fda2b427..00000000 --- a/dicom_server/core/dicom_application.py +++ /dev/null @@ -1,152 +0,0 @@ -""" - -A `DicomStarter` class to initialize and launch the DICOM server. -It sets up the Application Entity (AE), registers event handlers, -and verifies port availability before starting the server - -""" - -import socket -import time -from pynetdicom import evt -from pynetdicom.sop_class import ( - PatientRootQueryRetrieveInformationModelFind, - Verification, - StudyRootQueryRetrieveInformationModelMove, - PatientRootQueryRetrieveInformationModelGet, - StudyRootQueryRetrieveInformationModelFind, - StudyRootQueryRetrieveInformationModelGet, - PatientRootQueryRetrieveInformationModelMove, -) -from pynetdicom import ( - AE, - AllStoragePresentationContexts, - StoragePresentationContexts, -) - -from config import IMPLEMENTATION_NAME, IMPLEMENTATION_UID - -class DicomStarter: - - def __init__(self, app_logger, exceptions_logger, ports, ip, handlers): - """ - - Constructor for DicomStarter. - Parameters: - ---------- - exceptions_logger : Logger - A previously sat logger to handle exceptions. - ports : list or int - DICOM server ports. - ip : str - DICOM host IP address. - handlers : object - Assoc, C-FIND, C-GET, C-MOVE, Release, Abort handler methods. - - """ - self.logger = app_logger - self.ports = [ports] if isinstance(ports, int) else ports - self.ip = ip - self.handlers = handlers - self.exceptions_logger = exceptions_logger - - def register_dicom_handlers(self): - """ - List of event-handler tuples for the DICOM server. - - """ - try: - handlers = [ - (evt.EVT_ACSE_RECV, self.handlers.handle_assoc), - (evt.EVT_RELEASED, self.handlers.handle_release), - (evt.EVT_C_FIND, self.handlers.handle_find), - (evt.EVT_C_STORE, self.handlers.handle_store), - (evt.EVT_C_ECHO, self.handlers.handle_echo), - (evt.EVT_C_MOVE, self.handlers.handle_move), - (evt.EVT_C_GET, self.handlers.handle_get), - (evt.EVT_ABORTED, self.handlers.handle_abort), - ] - - return handlers - - except Exception as e: - self.exceptions_logger.exception( - "Unexpected error while registering the DICOM handlers" - ) - - def start_the_application(self): - """ - Start the DICOM server. - Sets up the Application Entity and registers event handlers if the ports are not already used. - - """ - try: - handlers = self.register_dicom_handlers() - - # Start server on each port - for port in self.ports: - if not self.is_port_in_use(port): - # Create a new AE instance for each port - ae = self.initialize_application_entity() - if ae: - self.logger.info(f"Starting DICOM Server on port {port}") - ae.start_server( - (self.ip, port), - evt_handlers=handlers, - block=False - ) - self.logger.info(f"Successfully started DICOM Server on port {port}") - else: - self.logger.warning(f"Port {port} is already in use") - - - self.logger.info("All DICOM servers started successfully") - - while True: - time.sleep(1) - except Exception: - self.exceptions_logger.exception( - "Unexpected error starting the application" - ) - - def initialize_application_entity(self): - """ - Create and configure the Application Entity (AE). - Registers supported and requested presentation contexts - for storage, query/retrieve, and verification services. - """ - try: - ae = AE() - ae.implementation_version_name = IMPLEMENTATION_NAME - ae.implementation_class_uid = IMPLEMENTATION_UID - ae.supported_contexts = AllStoragePresentationContexts - ae.requested_contexts = StoragePresentationContexts - ae.add_supported_context(PatientRootQueryRetrieveInformationModelFind) - ae.add_supported_context(PatientRootQueryRetrieveInformationModelGet) - ae.add_supported_context(StudyRootQueryRetrieveInformationModelGet) - ae.add_supported_context(StudyRootQueryRetrieveInformationModelFind) - ae.add_supported_context(StudyRootQueryRetrieveInformationModelMove) - ae.add_supported_context(PatientRootQueryRetrieveInformationModelMove) - ae.add_supported_context(Verification) - self.initialize_storage_contexts(StoragePresentationContexts) - self.logger.debug("Application entity initialized") - return ae - except Exception: - self.exceptions_logger.exception( - "Unexpected error while initializing the application entity object" - ) - - def initialize_storage_contexts(self, StoragePresentationContexts): - """ - Configure the roles (SCP/SCU) for each Storage Presentation Context. - - """ - for context in StoragePresentationContexts: - context._as_scp = True - context._as_scu = True - context.scp_role = True - context.scu_role = True - - def is_port_in_use(self, port): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - return s.connect_ex((self.ip, port)) == 0 diff --git a/dicom_server/core/dicom_handlers.py b/dicom_server/core/dicom_handlers.py deleted file mode 100644 index 2aa4f25d..00000000 --- a/dicom_server/core/dicom_handlers.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Implementation of the DICOM handlers""" - -import traceback, utilities.dicom_util as dicom_util -from dependency_injector.wiring import inject -from services.dicom_session_service import ISessionCollector -from services.dicom_database_service import IDicomDatabase -from pydicom.dataset import Dataset -import traceback -from pydicom.pixel_data_handlers.util import apply_modality_lut -from typing import Generator, Tuple, Optional -from enums.dicom_session_keys import Sessionkeys as session_keys - - -class DICOMHandlers: - @inject - def __init__( - self, - app_logger, - exceptions_logger=None, - event_collector: ISessionCollector = None, - dicomdb: IDicomDatabase = None, - ): - self.event_collector = event_collector or ISessionCollector() - self.dicomdb = dicomdb or IDicomDatabase() - self.exceptions_logger = exceptions_logger - self.logger = app_logger - dicomdb.initialize_database() - - def handle_assoc(self, event): - try: - version_name = ( - str(event.assoc.requestor.implementation_version_name) - if event.assoc.requestor.implementation_version_name - else "N/A" - ) - ip = str(event.assoc.requestor.address) - port = event.assoc.requestor.port - local_port = event.assoc.acceptor.port - self.event_collector.collect_session_info( - { - session_keys.LOCAL_PORT.key: local_port, - }, - ) - self.event_collector.session_started(ip, port, version_name) - except Exception as e: - self.exceptions_logger.exception( - "Unexpected error while handling association" - ) - - def handle_echo(self, event): - try: - local_port = event.assoc.acceptor.port - self.event_collector.collect_session_info( - { - session_keys.LOG_LEVEL.key: "Info", - session_keys.REQUEST_TYPE.key: "C_ECHO", - session_keys.SESSION_MAIN_OPERATION.key: "C_ECHO", - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - return 0x0000 - except Exception: - self.exceptions_logger.exception( - "Unexpected error while handling ECHO operation" - ) - return 0xC000 - - def handle_find( - self, event - ) -> Generator[Tuple[int, Optional[Dataset]], None, None]: - matches = [] - - try: - local_port = event.assoc.acceptor.port - self.event_collector.collect_session_info( - { - session_keys.LOG_LEVEL.key: "Info", - session_keys.REQUEST_TYPE.key: "C_FIND", - session_keys.SESSION_MAIN_OPERATION.key: "C_FIND", - session_keys.LOCAL_PORT.key: local_port, - } - ) - sop_class_uid = event.request.AffectedSOPClassUID - identifier = event.identifier - - # Validate the SOPClassUID - if dicom_util.is_sopclassuid_valid(sop_class_uid): - self.logger.info("Request dataset has invaild model") - yield (0xA900, None) # Identifier does not match SOP Class - return - - # Validate Identifier - if dicom_util.identifier_invalid(identifier): - self.logger.info("Request dataset has invaild identifier") - yield (0xC006, None) # Invalid attribute value - return - - dicom_util.filter_identifier_tags(identifier) - query_level = dicom_util.get_query_level(identifier) - self.event_collector.collect_session_info( - {session_keys.QUERY_LEVEL.key: query_level} - ) - - # Determine if it's a "all" request - if dicom_util.all_requested(identifier): - if query_level == "STUDY": - matches = self.dicomdb.query_all_studies() - elif query_level == "SERIES": - - # The pynetdicom ORM lacks query/retrieve model on SERIES level, that is why we filter here based on studies and not on series. - # This add limitition in quering all series otherwise, we used the studyinstanceuid (STUDY model) as an identifier tag to then filter specific serie based on the SeriesInstanceUID tag - - matches = self.dicomdb.query_all_studies() - elif query_level == "PATIENT": - matches = self.dicomdb.query_all_patients() - self.event_collector.collect_session_info( - { - session_keys.SESSION_PARAMETERS.key: "ALL " + query_level, - session_keys.MATCHES.key: len(matches), - }, - True, - ) - else: - query_parameters = dicom_util.get_query_parameters(identifier) - self.event_collector.collect_session_info( - {session_keys.SESSION_PARAMETERS.key: query_parameters} - ) - # Hierarchical query level determination - if dicom_util.is_patient_level(identifier): - matches = self.dicomdb.query_patient_level(identifier) - elif dicom_util.is_study_level(identifier): - matches = self.dicomdb.query_study_level(identifier) - elif dicom_util.is_series_level(identifier): - matches = self.dicomdb.query_series_level(identifier) - - self.event_collector.collect_session_info( - {session_keys.MATCHES.key: len(matches)}, True - ) - - # Send Matching Results - for instance in matches: - response_dataset = Dataset() - try: - self.dicomdb.get_response_data( - identifier, instance, response_dataset - ) - yield (0xFF00, response_dataset) # Pending response - except Exception: - self.exceptions_logger.exception( - "Exception in building response set" - ) - yield (0xC001, None) - - # Final success response - yield (0x0000, None) - - except Exception: - self.exceptions_logger.exception("Exception in handling C-FIND operation") - yield (0xC001, None) # Unable to process - - def handle_get(self, event) -> Generator[Tuple[int, Optional[Dataset]], None, None]: - try: - local_port = event.assoc.acceptor.port - assoc = event.assoc - identifier = event.identifier - - if dicom_util.identifier_invalid(identifier): - yield 0xC000, None - return - instances = dicom_util.get_instances() - matching = [] - query_level = dicom_util.get_query_level(identifier) - matching = self.get_matching_instances(event, instances) - self.event_collector.collect_session_info( - { - session_keys.QUERY_LEVEL.key: query_level, - session_keys.LOG_LEVEL.key: "Info", - session_keys.SESSION_MAIN_OPERATION.key: "C_GET", - session_keys.REQUEST_TYPE.key: "C_GET", - session_keys.MATCHES.key: len(matching), - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - yield len(matching) - for instance in matching: - if event.is_cancelled: - yield 0xFE00, None - # Ensure the accepted contexts act as a SCP - dicom_util.assign_runtime_contexts_support(assoc) - if dicom_util.file_compressed(instance): - instance.decompress() - apply_modality_lut(instance.pixel_array, instance) - yield 0xFF00, instance - - yield 0x0000, None - except Exception: - self.exceptions_logger.exception( - "Unexpected error while handling C-GET operation" - ) - yield (0xC001, None) - - def handle_store(self, event): - local_port = event.assoc.acceptor.port - self.event_collector.collect_session_info( - { - session_keys.LOG_LEVEL.key: "Info", - session_keys.REQUEST_TYPE.key: "C_STORE", - session_keys.SESSION_MAIN_OPERATION.key: "C_STORE", - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - dicom_util.store_received_file(event) - return 0x0000 - - def handle_move( - self, event - ) -> Generator[Tuple[int, Optional[Dataset]], None, None]: - - local_port = event.assoc.acceptor.port - addr = assoc.requestor.address - port = assoc.requestor.port - yield (str(addr), port) - assoc = event.assoc - identifier = event.identifier - - if dicom_util.identifier_invalid(identifier): - yield 0xC000, None - return - instances = dicom_util.get_instances() - matching = [] - query_level = dicom_util.get_query_level(identifier) - matching = self.get_matching_instances(event, instances) - self.event_collector.collect_session_info( - { - session_keys.QUERY_LEVEL.key: query_level, - session_keys.LOG_LEVEL.key: "Info", - session_keys.REQUEST_TYPE.key: "C_MOVE", - session_keys.MATCHES.key: len(matching), - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - yield len(matching) - for instance in matching: - if event.is_cancelled: - yield 0xFE00, None - dicom_util.assign_runtime_contexts_support(assoc) - if dicom_util.file_compressed(instance): - instance.decompress() - apply_modality_lut(instance.pixel_array, instance) - yield 0xFF00, instance - - yield 0x0000, None - - def handle_release(self, event): - local_port = event.assoc.acceptor.port - self.event_collector.collect_session_info( - { - session_keys.LOG_LEVEL.key: "Warning", - session_keys.REQUEST_TYPE.key: "Association Released", - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - self.event_collector.session_ended() - - def handle_abort(self, event): - local_port = event.assoc.acceptor.port - if event.assoc.requestor.implementation_version_name: - self.event_collector.collect_session_info( - { - session_keys.LOG_LEVEL.key: "Warning", - session_keys.REQUEST_TYPE.key: "Association Aborted", - session_keys.VERSION.key: str( - event.assoc.requestor.implementation_version_name - ), - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - else: - self.event_collector.collect_session_info( - { - session_keys.LOG_LEVEL.key: "Warning", - session_keys.REQUEST_TYPE.key: "Association Aborted", - session_keys.LOCAL_PORT.key: local_port, - }, - True, - ) - - self.event_collector.session_ended() - - def get_matching_instances(self, event, instances): - matching = [] - - if dicom_util.is_study_level(event.identifier): - if hasattr(event.identifier, "StudyInstanceUID"): - study_uid = event.identifier.StudyInstanceUID - self.event_collector.collect_session_info( - { - session_keys.SESSION_PARAMETERS.key: "StudyInstanceUID: " - + str(study_uid) - } - ) - matching = [ - instance - for instance in instances - if instance.StudyInstanceUID == study_uid - ] - - elif dicom_util.is_series_level(event.identifier): - if hasattr(event.identifier, "SeriesInstanceUID"): - series_uid = event.identifier.SeriesInstanceUID - self.event_collector.collect_session_info( - { - session_keys.SESSION_PARAMETERS.key: "SeriesInstanceUID: " - + str(series_uid) - } - ) - - matching = [ - instance - for instance in instances - if instance.SeriesInstanceUID == series_uid - ] - - return matching diff --git a/dicom_server/core/dicom_session_manager.py b/dicom_server/core/dicom_session_manager.py deleted file mode 100644 index a8bb9fbe..00000000 --- a/dicom_server/core/dicom_session_manager.py +++ /dev/null @@ -1,179 +0,0 @@ -from datetime import datetime -from dependency_injector.wiring import inject -from services.redis_service import IRedisService -from services.dicom_session_service import ISessionCollector -from services.threat_intelligence_service import IThreatIntelligence -import time, pytz -import utilities.dicom_util as dicom_util -from enums.dicom_session_keys import Sessionkeys as sk - - -class SessionCollector(ISessionCollector): - @inject - def __init__( - self, - app_logger, - simp_logger, - exceptions_logger, - redis_handler: IRedisService = None, - threat_intelligence: IThreatIntelligence = None, - ): - - self.session_info = {key.key: key.default for key in sk} - self.redis_data = {} - self.redis_handler = redis_handler or IRedisService() - self.simp_logger = simp_logger - self.exceptions_logger = exceptions_logger - self.threat_intelligence = threat_intelligence - self.logger = app_logger - self.timezone = pytz.timezone("Europe/Copenhagen") - - def session_started(self, ip, port, v_name): - """ - Starting a new DICOM session - whenever an association request recieved - - """ - try: - - if not self.session_locked(): - self.logger.warning( - f"\033[93mDICOM session started at {datetime.now(self.timezone).strftime('%Y-%m-%dT%H:%M:%S')}\n\033[0mInitializing session info for host: \033[92m{ip}\033[0m port: \033[92m{port}\033[0m" - ) - self.initialize_session_info(ip, port) - self.get_session_requestor_reputation(ip) - self.set_session_lock(1) - else: - self.session_info[sk.VERSION.key] = v_name - self.logger.info( - f"Requester DICOM client version: \033[92m{v_name}\033[0m" - ) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while starting a DICOM session" - ) - - def get_session_requestor_reputation(self, ip): - """ - Using the data-access provider "redis_handler" to populate data to redis - - """ - try: - rep_dat = {} - ip_scanned = self.redis_handler.is_ip_scanned(ip) - if not ip_scanned: - rep_dat = self.threat_intelligence.get_reputation_data(ip) - self.redis_handler.add_reputation_data(rep_dat) - self.redis_handler.add_scanned_ip(ip) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while getting session requestor reputation data" - ) - - def initialize_session_info(self, ip, port): - """ - Initializing session information by setting session keys values - - """ - try: - current_time = time.time() - known_scanner = dicom_util.is_known_scanner(ip) - self.session_info[sk.KNOWN_SCANNER.key] = known_scanner - self.session_info[sk.SESSION_MAIN_OPERATION.key] = "Association Requested" - self.session_info[sk.REQUEST_TYPE.key] = "Association Requested" - self.session_info[sk.IP.key] = str(ip) - self.session_info[sk.PORT.key] = port - self.session_info[sk.LOG_LEVEL.key] = "Warning" - self.set_session_id(str(int(current_time * 1000))) - self.collect_session_info({}, True) - self.logger.info( - f'Host identified as a \033[92m"Known_scanner"\033' - if known_scanner - else 'Host identified as a\033[91m"Non_known_scanner"\033[0m\nHost reputation data can be found on the Redis server or the visualization dashbord' - ) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while initializing DICOM session" - ) - - def collect_session_info(self, params, sub_process_finished=False): - """ - Collecting session information through the dicom_handlers provider - If sub_process_finished is True means a dicom request in this session is elapsed then we log a simplified message - - """ - try: - for key, value in params.items(): - self.session_info[key] = value - if sub_process_finished: - - current_time = datetime.now(self.timezone).strftime( - "%Y-%m-%dT%H:%M:%S.%f" - ) - self.session_info[sk.TIMESTAMP.key] = str(current_time) - self.simp_logger.info(self.session_info) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while collecting DICOM session information" - ) - - def session_ended(self): - """ - push session data to Redis and reset the session information object - - """ - try: - - self.redis_handler.add_request_data(self.build_redis_object()) - self.logger.info( - f'Main operation:\033[92m"{self.session_info[sk.SESSION_MAIN_OPERATION.key]}" \033[0m Status:\033[92m"{self.session_info[sk.STATUS.key]}"\033[0m' - ) - self.logger.warning( - f"\033[93mDICOM session ended at {datetime.now(self.timezone).strftime('%Y-%m-%dT%H:%M:%S')}\n\033[0m" - ) - self.reset_session() - except Exception: - self.exceptions_logger.exception( - "Unexpected error while ending a DICOM session" - ) - - def reset_session(self): - self.session_info = {key.key: key.default for key in sk} - - def build_redis_object(self): - """ - build a Redis object and filtering the not log-stash relevant keys - - """ - - try: - self.session_info[sk.STATUS.key] = ( - "Finished" - if self.session_info[sk.REQUEST_TYPE.key] == "Association Released" - else "Aborted" - ) - redis_object = self.session_info.copy() - redis_object[sk.REQUEST_TYPE.key] = self.session_info[ - sk.SESSION_MAIN_OPERATION.key - ] - keys_to_remove = {sk.LOCK.key, sk.SESSION_MAIN_OPERATION.key} - for key in keys_to_remove: - redis_object.pop(key, None) - js_redis = dicom_util.format_log_entry(str(redis_object)) - self.logger.debug(f"Redis object formatted: {js_redis}") - return js_redis - except Exception: - self.exceptions_logger.exception( - "Unexpected error while building Redis object" - ) - - def set_session_lock(self, value): - self.session_info[sk.LOCK.key] = value - - def session_locked(self): - - return self.session_info[sk.LOCK.key] == 1 - - def set_session_id(self, s_id): - self.session_info[sk.SESSION_ID.key] = s_id - \ No newline at end of file diff --git a/dicom_server/core/dicomdb.py b/dicom_server/core/dicomdb.py deleted file mode 100644 index 988f2917..00000000 --- a/dicom_server/core/dicomdb.py +++ /dev/null @@ -1,294 +0,0 @@ -import os, sys - -sys.path.append(os.path.abspath("pydicom_and_pynetdicom_libs")) -from pydicom import dcmread -from sqlalchemy import String, delete -import db -from sqlalchemy import cast, String -from contextlib import contextmanager -from services.dicom_database_service import IDicomDatabase - - -class DicomDatabase(IDicomDatabase): - - def __init__(self, app_logger, exceptions_logger, storagedirectory, session): - try: - self.session = session - self.storagedirectory = storagedirectory - self.exceptions_logger = exceptions_logger - self.logger = app_logger - except Exception: - self.exceptions_logger.exception( - "Unexpected error while building DICOMDatabase instance." - ) - - @contextmanager - def session_scope(self): - try: - yield self.session - self.session.commit() - except Exception: - self.session.rollback() - self.logger.debug(f"Session rollback!") - - def initialize_database(self): - try: - self.delete_database() - self.fill_database_tables_from_dicom_files() - self.logger.info("Database initialized from DICOM storage") - except Exception: - self.exceptions_logger.exception( - "Unexpected error while initializing the database from dicom files" - ) - - def fill_database_tables_from_dicom_files(self): - with self.session_scope() as session: - try: - for path in os.listdir(self.storagedirectory): - instance = dcmread(os.path.join(self.storagedirectory, path)) - db.add_instance(instance, session, path) - session.commit() - except Exception: - session.rollback() - self.exceptions_logger.exception("Exception filling database") - - def delete_database(self): - with self.session_scope() as session: - try: - delete_statement = delete(db.Instance) - session.execute(delete_statement) - session.commit() - self.logger.info("Database cleared") - except Exception: - session.rollback() - self.exceptions_logger.exception("Exception clearing database") - - def query_all_studies(self): - - all_studies = [] - try: - with self.session_scope() as session: - studyQuery = session.query(db.Study) - all_studies = studyQuery.all() - return all_studies - except Exception: - self.exceptions_logger.exception("Exception querying all studies") - - def query_all_series(self): - all_studies = [] - try: - with self.session_scope() as session: - studyQuery = session.query(db.Series) - all_studies = studyQuery.all() - return all_studies - except Exception: - self.exceptions_logger.exception("Exception querying all series") - - def query_all_patients(self): - all_patients = [] - try: - with self.session_scope() as session: - studyQuery = session.query(db.Patient) - all_patients = studyQuery.all() - return all_patients - except Exception: - self.exceptions_logger.exception("Exception querying all patients") - - def query_study_level(self, identifier): - matches = [] - with self.session_scope() as session: - try: - matchedInstances = db.search( - "1.2.840.10008.5.1.4.1.2.2.1", identifier, session - ) - - unique_studies = self.get_unique_studies(matchedInstances) - studyQuery = session.query(db.Study) - studyQuery = studyQuery.filter( - db.Study.study_instance_uid.in_(unique_studies) - ) - self.logger.debug("Querying studies from the database") - matches = studyQuery.all() - return matches - except Exception: - self.exceptions_logger.exception("Exception in STUDY level query") - - def query_series_level(self, identifier): - - matched_studies = [] - with self.session_scope() as session: - try: - matchedInstances = db.search( - "1.2.840.10008.5.1.4.1.2.2.1", identifier, session - ) - uniqueSeries = self.get_uniqueSeries(matchedInstances, identifier) - seriesQuery = session.query(db.Series) - seriesQuery = seriesQuery.filter( - db.Series.series_instance_uid.in_(uniqueSeries) - ) - matched_studies = seriesQuery.all() - return matched_studies - except Exception: - self.exceptions_logger.exception("Exception in SERIES level query") - - def query_patient_level(self, identifier): - matches = [] - with self.session_scope() as session: - try: - matchedInstances = db.search( - "1.2.840.10008.5.1.4.1.2.1.1", identifier, session - ) - uniquePatients = self.get_unique_patients(matchedInstances) - patientQuery = session.query(db.Patient) - patientQuery = patientQuery.filter( - db.Patient.patient_id.in_(uniquePatients) - ) - matches = patientQuery.all() - return matches - except Exception: - self.exceptions_logger.exception("Exception in PATIENT level query") - - def get_response_data(self, identifier, instance, response_dataset): - - if identifier.QueryRetrieveLevel == "STUDY": - self.get_studyRoot_dataset(instance, response_dataset) - elif identifier.QueryRetrieveLevel == "SERIES": - self.get_seriesRoot_dataset(identifier, instance, response_dataset) - elif identifier.QueryRetrieveLevel == "PATIENT": - self.get_patientRoot_dataset(identifier, instance, response_dataset) - - def get_patientRoot_dataset(self, identifier, instance, response_dataset): - - response_dataset.PatientID = getattr(instance, "patient_id") - response_dataset.PatientName = getattr(instance, "patient_name") - - def get_seriesRoot_dataset(self, identifier, instance, response_dataset): - try: - if len(identifier) == 1: - - response_dataset.Modality = self.get_other_levels_tags( - "STUDY", "modality", getattr(instance, "study_instance_uid") - ) - response_dataset.SeriesInstanceUID = self.get_other_levels_tags( - "STUDY", - "series_instance_uid", - getattr(instance, "study_instance_uid"), - ) - response_dataset.SeriesNumber = self.get_other_levels_tags( - "STUDY", "series_number", getattr(instance, "study_instance_uid") - ) - else: - response_dataset.Modality = getattr(instance, "modality") - response_dataset.SeriesInstanceUID = getattr( - instance, "series_instance_uid" - ) - response_dataset.SeriesNumber = getattr(instance, "series_number") - response_dataset.PatientName = self.get_other_levels_tags( - "SERIES", "patient_name", getattr(instance, "series_instance_uid") - ) - response_dataset.PatientID = self.get_other_levels_tags( - "SERIES", "patient_id", getattr(instance, "series_instance_uid") - ) - response_dataset.NumberOfSeriesRelatedInstances = ( - self.get_other_levels_tags( - "SERIES", - "NumberOfSeriesRelatedInstances", - getattr(instance, "series_instance_uid"), - ) - ) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while getting series data sets" - ) - - def get_studyRoot_dataset(self, instance, response_dataset): - - direct_attributes = { - "StudyInstanceUID": "study_instance_uid", - "StudyDate": "study_date", - "StudyTime": "study_time", - "AccessionNumber": "accession_number", - "StudyID": "study_id", - } - - other_level_attributes = { - "InstitutionName": "institution_name", - "PatientBirthDate": "birth_date", - "PatientSex": "patient_sex", - "PatientName": "patient_name", - "PatientID": "patient_id", - "NumberOfStudyRelatedInstances": "NumberOfStudyRelatedInstances", - "ModalitiesInStudy": "modality", - } - - for response_attr, instance_attr in direct_attributes.items(): - setattr(response_dataset, response_attr, getattr(instance, instance_attr)) - - study_instance_uid = getattr(instance, "study_instance_uid") - for response_attr, tag in other_level_attributes.items(): - value = self.get_other_levels_tags("STUDY", tag, study_instance_uid) - setattr(response_dataset, response_attr, value) - - def get_other_levels_tags(self, level, required_tag, query_identifier): - - with self.session_scope() as session: - query = session.query(db.Instance) - - if level == "STUDY": - - query = query.filter( - db.Instance.study_instance_uid == cast(query_identifier, String) - ) - - if required_tag == "NumberOfStudyRelatedInstances": - return query.count() - - else: - result = query.first() - if result: - return getattr(result, required_tag) - elif level == "SERIES": - query = query.filter( - db.Instance.series_instance_uid == cast(query_identifier, String) - ) - if required_tag == "NumberOfSeriesRelatedInstances": - return query.count() - result = query.first() - if result: - return getattr(result, required_tag) - elif level == "PATIENT": - query = query.filter( - db.Instance.study_instance_uid == cast(query_identifier, String) - ) - result = query.first() - if result: - return getattr(result, required_tag) - return None - - def get_unique_studies(self, li): - unique_st = [] - for a in li: - sUID = getattr(a, "study_instance_uid") - if sUID not in unique_st: - unique_st.append(sUID) - return unique_st - - def get_uniqueSeries(self, li, identifier): - unique_st = [] - for a in li: - serieUID = getattr(a, "series_instance_uid") - studyUID = getattr(a, "study_instance_uid") - if serieUID not in unique_st and studyUID == identifier.StudyInstanceUID: - unique_st.append(serieUID) - - return unique_st - - def get_unique_patients(self, li): - unique_st = [] - for a in li: - sUID = getattr(a, "patient_id") - if sUID not in unique_st: - unique_st.append(sUID) - # print("MatchedInstancesPatient",unique_st) - - return unique_st diff --git a/dicom_server/core/loggers.py b/dicom_server/core/loggers.py deleted file mode 100644 index bdb87a9e..00000000 --- a/dicom_server/core/loggers.py +++ /dev/null @@ -1,156 +0,0 @@ -import os, logging, json, sys -from datetime import datetime -from services.loggers_service import ILoggers -import traceback - - -class SimplifiedLogsFormatter(logging.Formatter): - def __init__(self, is_production): - super().__init__(style="%") - self.is_production = is_production - - def format(self, record): - msg = str(record.msg) - - log_object = json.loads( - msg.replace("'", '"').replace("False", "false").replace("True", "true") - ) - - keys_to_remove = {"lock", "main_operation", "known_scanner", "status"} - for key in keys_to_remove: - log_object.pop(key, None) - if ( - log_object["request_type"] == "Association Aborted" - or log_object["request_type"] == "Association Released" - ): - # Remove none subprocess info - log_object["session_parameters"] = "N/A" - log_object["query_level"] = "N/A" - log_object["matches"] = "N/A" - - return json.dumps(log_object, default=str) - - -class ExceptionFormatter(logging.Formatter): - def __init__(self, is_production, app_logger): - super().__init__(style="%") - self.is_production = is_production - self.logger = app_logger - - def format(self, record): - - excep = ( - "Exception at " - + datetime.now().strftime("%d-%m-%Y : %H-%M-%S") - + "\n.............................................\n" - ) - trace = "" - - if record.exc_info: - trace = "".join(traceback.format_exception(*record.exc_info)) - - tb = record.exc_info[2] - while tb.tb_next: - tb = tb.tb_next - frame = tb.tb_frame - excep += "In " + frame.f_globals["__name__"] + "\n" - - self.logger.error( - "Exception in " - + frame.f_globals["__name__"] - + "\n" - + "Message: " - + record.msg - + "\nSee\033[91m exceptions.log \033[0m file for more details\n.......................................\n" - ) - - excep += ( - "Message: " - + str(record.msg) - + "\n" - + "Traceback :\n ............\n" - + trace - ) - return excep - - -class Loggers(ILoggers): - - def __init__( - self, - is_production, - main_logger_directory, - simplified_log_directory, - exception_log_directory, - ): - - self.is_production = is_production - self.app_logger = self.setup_logger("app_logger", None) - self.main_logger = self.setup_logger( - "pynetdicom", main_logger_directory, logging.DEBUG, None - ) - self.simplified_logger = self.setup_logger( - "simplified_logger", - simplified_log_directory, - logging.INFO, - SimplifiedLogsFormatter(is_production), - ) - self.exceptions_logger = self.setup_logger( - "exceptions", - exception_log_directory, - logging.ERROR, - ExceptionFormatter(is_production, self.app_logger), - ) - - def setup_logger( - self, - name, - log_directory, - level=logging.INFO, - formatter=None, - ): - if name != "app_logger": - if log_directory: - os.makedirs(log_directory, exist_ok=True) - - file_basename = "dicom_raw" if name == "pynetdicom" else name - handler = logging.FileHandler( - os.path.join(log_directory, file_basename + ".log") - ) - handler.setFormatter(formatter) - stream_handler = logging.StreamHandler(stream=sys.stdout) - logger = logging.getLogger(name) - logger.setLevel(level) - if not self.is_production: - logger.addHandler(stream_handler) - logger.addHandler(handler) - self.app_logger.debug(f'Logger: {name} initialized at "{log_directory}"') - return logger - else: - logger = logging.getLogger("app_logger") - handler = logging.StreamHandler(sys.stdout) - if self.is_production: - logger.setLevel(logging.INFO) - handler.setLevel(logging.INFO) - else: - logger.setLevel(logging.DEBUG) - handler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(levelname)s - %(message)s") - handler.setFormatter(formatter) - handler.addFilter(self.add_color) - logger.addHandler(handler) - return logger - - def add_color(self, record): - colors = { - "DEBUG": "\033[94m", # Blue - "INFO": "\033[92m", # Green - "WARNING": "\033[93m", # Yellow - "ERROR": "\033[91m", # Red - "CRITICAL": "\033[1;91m", # Bold Red - } - reset = "\033[0m" - levelname = record.levelname - if levelname in colors: - record.levelname = colors[levelname] + levelname + reset - return True diff --git a/dicom_server/core/redis_handler.py b/dicom_server/core/redis_handler.py deleted file mode 100644 index fcad7900..00000000 --- a/dicom_server/core/redis_handler.py +++ /dev/null @@ -1,90 +0,0 @@ -import json -from services.redis_service import IRedisService - - -class RedisClient(IRedisService): - - def __init__(self, app_logger, exceptions_logger, redis_client): - - self.redis_client = redis_client - self.exceptions_logger = exceptions_logger - self.logger = app_logger - - def is_ip_scanned(self, ip): - try: - return ip.encode() in self.redis_client.lrange("scannedIPs", 0, -1) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while retrieving IPs list from Redis" - ) - - def add_scanned_ip(self, ip): - try: - self.redis_client.rpush("scannedIPs", ip) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while adding a scanned IP to the scanned list" - ) - - def add_reputation_data(self, rep_dat): - try: - self.redis_client.rpush("reputation", json.dumps(rep_dat)) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while pushing repution object" - ) - - def add_request_data(self, redis_log_data): - try: - self.redis_client.rpush("requests", redis_log_data) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while adding a request information to Redis" - ) - - def get_TCI_existing_studies( - self, - ): - try: - return set(self.redis_client.lrange("TCIA_studies", 0, -1)) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while checking TCIA studies" - ) - - def add_TCI_study(self, study_uid): - try: - self.redis_client.rpush("TCIA_studies", study_uid) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while adding a TCIA studyInstanceUID" - ) - - def add_injected_file(self, patient_name, modality): - try: - self.redis_client.rpush( - "injected_files", - str({"patient_name": patient_name, "modality": modality}), - ) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while adding injected file identifiers to redis" - ) - - def get_honey_url( - self, - ): - try: - self.redis_client.get("webhook") - except Exception: - self.exceptions_logger.exception( - "Unexpected error while getting webhook key" - ) - - def update_files_integrity_state(self, changed_files): - try: - self.redis_client.rpush("fileChange", json.dumps(changed_files)) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while adding integrity check identifier" - ) diff --git a/dicom_server/core/threat_intelligence_handler.py b/dicom_server/core/threat_intelligence_handler.py deleted file mode 100644 index d29cbae8..00000000 --- a/dicom_server/core/threat_intelligence_handler.py +++ /dev/null @@ -1,120 +0,0 @@ -import requests -from datetime import datetime -from services.threat_intelligence_service import IThreatIntelligence - - -class ThreatIntelligence(IThreatIntelligence): - def __init__( - self, - app_logger, - exceptions_logger, - abuse_ip_api_key, - ip_quality_score_api_key, - virus_total_api_key, - ): - self.abuse_ip_api_key = abuse_ip_api_key - self.ip_quality_score_api_key = ip_quality_score_api_key - self.virus_total_api_key = virus_total_api_key - self.exceptions_logger = exceptions_logger - self.logger = app_logger - - # Get IP security score from ABUSEIPDB - - def getIPSecurityScore(self, ip): - - api_key = self.abuse_ip_api_key - url = "https://api.abuseipdb.com/api/v2/check" - headers = {"Accept": "application/json", "Key": api_key} - params = {"ipAddress": ip, "maxAgeInDays": 90} - - try: - response = requests.get(url, headers=headers, params=params) - if response.status_code == 200: - data = response.json()["data"] - country = data.get("countryCode", "N/A") - isp = data.get("isp", "N/A") - abuseIPConfidenceScore = data["abuseConfidenceScore"] - return [isp, abuseIPConfidenceScore, country] - else: - self.exceptions_logger.exception( - f" {response.status_code} - {response.json().get('errors', [{'detail': 'Unknown error'}])[0]['detail']}" - ) - except Exception: - self.exceptions_logger.exception( - f'Unexpected error while getting IP security score from "abuseipdb.com"' - ) - - # Get IP security score from IPQUALITYSCORE - - def getIpqualityScore(self, ip): - try: - api_key = self.ip_quality_score_api_key - url = f"https://ipqualityscore.com/api/json/ip/{api_key}/{ip}" - response = requests.get(url) - if response.status_code == 200: - data = response.json() - return [ - data.get("fraud_score", "N/A"), - data.get("proxy", "N/A"), - data.get("city", "N/A"), - data.get("bot_status", "N/A"), - data.get("vpn", "N/A"), - data.get("latitude", "N/A"), - data.get("longitude", "N/A"), - ] - return {"service": "IPQualityScore", "error": response.text} - except Exception: - self.exceptions_logger.exception( - f'Unexpected error while getting IP quality score from "ipqualityscore.com"' - ) - - # Get IP security score from VIRUSTOTAL - - def getVirusTotalScore(self, ip): - try: - url = f"https://www.virustotal.com/api/v3/ip_addresses/{ip}" - headers = {"x-apikey": self.virus_total_api_key} - response = requests.get(url, headers=headers) - if response.status_code == 200: - data = response.json()["data"] - # print(data) - result_counts = {} - - for analysis in data["attributes"]["last_analysis_results"].values(): - result = analysis["result"] - result_counts[result] = result_counts.get(result, 0) + 1 - return result_counts - except Exception: - self.exceptions_logger.exception( - f'Unexpected error while getting IP information from "virustotal.com"' - ) - - # Build reputation object - - def get_reputation_data(self, ip): - - current_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") - abusedb = self.getIPSecurityScore(ip) - abusedb_object = abusedb if abusedb else ["", "", ""] - ip_quality = self.getIpqualityScore(ip) - ip_quality_score = ip_quality if ip_quality else ["", "", "", "", "", "", ""] - - vt = self.getVirusTotalScore(ip) - virus_total = vt if vt else {} - try: - rep_dat = {} - rep_dat["timestamp"] = str(current_time) - rep_dat["virus_total_results"] = virus_total - rep_dat["ip"] = ip - rep_dat["ip_quality_score"] = ip_quality_score[0] - rep_dat["proxy"] = ip_quality_score[1] - rep_dat["region"] = ip_quality_score[2] - rep_dat["vpn"] = ip_quality_score[4] - rep_dat["country"] = abusedb_object[2] - rep_dat["ISP"] = abusedb_object[0] - rep_dat["AbuseDBScore"] = abusedb_object[1] - return rep_dat - except Exception: - self.exceptions_logger.exception( - "Unexpected error while building IP reputation object" - ) diff --git a/dicom_server/custom_units/integrity_checker.py b/dicom_server/custom_units/integrity_checker.py deleted file mode 100644 index 86c38511..00000000 --- a/dicom_server/custom_units/integrity_checker.py +++ /dev/null @@ -1,93 +0,0 @@ -import os, hashlib, threading, json, schedule, time, logging -from dependency_injector.wiring import inject -from services.redis_service import IRedisService -from services.integrity_checker_service import IIntegrityChecker - - -class FilesChecker(threading.Thread, IIntegrityChecker): - @inject - def __init__( - self, - app_logger, - exceptions_logger, - storage_directory, - hash_store_path, - redis_handler: IRedisService = None, - ): - - super().__init__(daemon=True) - self.storage_directory = storage_directory - self.hash_store_path = hash_store_path - # self.event= event - self.redis_handler = redis_handler or IRedisService() - self.exceptions_logger = exceptions_logger - self.logger = app_logger - self.start() - - def run(self): - try: - schedule.every(6).hours.do(self.check_hashes) - self.logger.info("Checking files integrity each 6 hours") - except Exception: - self.exceptions_logger.exception( - "Unable to schedule files integrity cheack" - ) - while True: - schedule.run_pending() - time.sleep(10300) - - def hash_file(self, filename): - hash_sha256 = hashlib.sha256() - try: - with open(filename, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_sha256.update(chunk) - except Exception: - self.exceptions_logger.exception( - f"Unexpected error while generating hash value for file {filename}" - ) - return None - return hash_sha256.hexdigest() - - def check_hashes(self): - - self.logger.debug("Checking files integrity") - new_hashes = {} - changed_files = [] - try: - with open(self.hash_store_path, "r") as f: - old_hashes = json.load(f) - except FileNotFoundError: - self.exceptions_logger.exception( - "Unexpected error while retrieving hashes from the hash storage file " - ) - old_hashes = {} - - for path in os.listdir(self.storage_directory): - full_path = os.path.join(self.storage_directory, path) - try: - file_hash = self.hash_file(full_path) - if file_hash: - new_hashes[path] = file_hash - if path in old_hashes and old_hashes[path] != file_hash: - changed_files.append(path) - - except Exception: - self.exceptions_logger.exception( - f"Unexpected error while proccessing hashes in {full_path}" - ) - - with open(self.hash_store_path, "w") as f: - json.dump(new_hashes, f) - - if changed_files: - try: - self.redis_handler.update_files_integrity_state(changed_files) - except Exception: - self.exceptions_logger.exception( - "Unexpected error while populating file integrity checks to Redis" - ) - self.logger.debug("Changed files:", changed_files) - else: - - self.logger.debug("No changes detected.") diff --git a/dicom_server/custom_units/network_manager.py b/dicom_server/custom_units/network_manager.py deleted file mode 100644 index 4cf8c081..00000000 --- a/dicom_server/custom_units/network_manager.py +++ /dev/null @@ -1,122 +0,0 @@ -import subprocess -from services.blackhole_service import IBlackhole - - -class Blackhole(IBlackhole): - - def __init__( - self, app_logger, exceptions_logger, mass_scanners_blocked, blackhole_file_path - ): - self.blackhole_file_path = blackhole_file_path - try: - self.logger = app_logger - if mass_scanners_blocked: - self.block_scanners(self.blackhole_file_path, "known_scanners") - else: - if self.is_scanners_blocked("known_scanners"): - self.allow_scanners("known_scanners") - self.exceptions_logger = exceptions_logger - except Exception as e: - pass - - def is_scanners_blocked(self, known_scanners): - """ - - Check if the ipset known_scanners exists - - """ - - try: - result = subprocess.run( - ["ipset", "list", known_scanners], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - return result.returncode == 0 - except FileNotFoundError: - self.exceptions_logger.exception( - "ipset command not found. Make sure ipset is installed." - ) - return False - - def get_known_scanners(self, scanners_file): - """ - - Get the known scanners list - - """ - knownScanners = [] - try: - with open(scanners_file, "r") as file: - for line in file: - line = line.strip() - if line and not line.startswith("#"): - ip_address = line.split("#")[0].strip() - knownScanners.append(ip_address) - except Exception as e: - self.exceptions_logger.exception(f"Error reading file: {e}") - return knownScanners - - def create_ipset(self, ipset_name): - """ - - Create an ipset on the kernal - - """ - try: - subprocess.run( - f" ipset create {ipset_name} hash:ip", shell=True, check=True - ) - self.logger.debug(f"IP set '{ipset_name}' created.") - except subprocess.CalledProcessError as e: - self.exceptions_logger.exception(f"Failed to create IP set: {e}") - - def add_ip_to_ipset(self, ipset_name, ip_address): - try: - subprocess.run( - f" ipset add {ipset_name} {ip_address}", shell=True, check=True - ) - self.logger.debug( - f"IP address {ip_address} added to IP set '{ipset_name}'." - ) - except subprocess.CalledProcessError as e: - self.exceptions_logger.exception( - f"Failed to add IP {ip_address} to IP set: {e}" - ) - - def setup_iptables_rule(self, ipset_name): - try: - subprocess.run( - f" iptables -I INPUT -m set --match-set {ipset_name} src -j DROP", - shell=True, - check=True, - ) - self.logger.debug(f"iptables rule added for IP set '{ipset_name}'.") - except subprocess.CalledProcessError as e: - self.exceptions_logger.exception(f"Failed to add iptables rule: {e}") - - """Null-route the known scanners IPs""" - - def block_scanners(self, source_list, set_name): - try: - self.create_ipset(set_name) - for ip in self.get_known_scanners(source_list): - self.add_ip_to_ipset(set_name, ip) - - self.setup_iptables_rule(set_name) - except Exception as e: - self.exceptions_logger.exception( - "Unexpected error while blocking mass scanners", e - ) - - """Allow known scanners to interact with the server""" - - def allow_scanners(self, set_name): - subprocess.run( - f" iptables -D INPUT -m set --match-set {set_name} src -j DROP", - shell=True, - check=True, - ) - - subprocess.run(" ipset destroy {set_name}", shell=True, check=True) diff --git a/dicom_server/custom_units/osm_institutions.py b/dicom_server/custom_units/osm_institutions.py deleted file mode 100644 index e5ec2fb9..00000000 --- a/dicom_server/custom_units/osm_institutions.py +++ /dev/null @@ -1,219 +0,0 @@ -""" -This module provides functionality to fetch real medical institutions from OpenStreetMap -and cache them for optimal performance. -""" - -import json -import os -import random -from datetime import datetime, timedelta -from typing import List, Dict, Optional -import overpy -from services.osm_service import IOSMService -import config - - -class OSMInstitutionsService(IOSMService): - """ - Service to fetch and manage medical institutions from OpenStreetMap. - - """ - - def __init__(self, app_logger, exceptions_logger): - self.logger = app_logger - self.exceptions_logger = exceptions_logger - self.api = overpy.Overpass() - self.cache_file = config.OSM_CACHE_FILE - self.cache_duration_hours = config.OSM_CACHE_DURATION - self.max_institutions = config.OSM_MAX_INSTITUTIONS - self.timeout = config.OSM_TIMEOUT - self.country = config.OSM_COUNTRY - self.city = config.OSM_CITY - self.fallback_institutions = config.OSM_FALLBACK_INSTITUTIONS - self.enabled = config.OSM_ENABLED - - # Medical facility tags for OSM queries - self.medical_tags = ['amenity=hospital', 'healthcare=hospital'] - - # Ensure cache directory exists - os.makedirs(os.path.dirname(self.cache_file), exist_ok=True) - - def get_medical_institutions(self) -> List[str]: - try: - if not self.enabled: - self.logger.info("OSM is disabled, using fallback institutions") - return self.fallback_institutions - - # Try to get from cache first - if self.is_cache_valid(): - institutions = self._load_from_cache() - if institutions: - self.logger.debug(f"Loaded {len(institutions)} institutions from cache") - return institutions - - self.logger.info("Cache invalid or empty, fetching institutions from OSM") - institutions = self._fetch_from_osm() - - if institutions: - self._save_to_cache(institutions) - self.logger.info(f"Successfully fetched {len(institutions)} institutions from OSM") - return institutions - else: - self.logger.warning("No institutions found from OSM, using fallback") - self._save_to_cache(self.fallback_institutions) - return self.fallback_institutions - - except Exception as e: - self.exceptions_logger.exception("Error getting medical institutions") - self.logger.warning("Failed to get institutions from OSM, using fallback") - self._save_to_cache(self.fallback_institutions) - return self.fallback_institutions - - def refresh_cache(self) -> bool: - try: - self.logger.info("Force refreshing OSM institutions cache") - institutions = self._fetch_from_osm() - - if institutions: - self._save_to_cache(institutions) - self.logger.info(f"Cache refreshed with {len(institutions)} institutions") - return True - else: - self.logger.warning("Failed to refresh cache - no institutions found") - return False - - except Exception as e: - self.exceptions_logger.exception("Error refreshing OSM cache") - return False - - def is_cache_valid(self) -> bool: - """ - Check if the current cache is still valid. - """ - try: - if not os.path.exists(self.cache_file): - return False - - with open(self.cache_file, 'r', encoding='utf-8') as f: - cache_data = json.load(f) - - cache_time = datetime.fromisoformat(cache_data.get('timestamp', '')) - expiry_time = cache_time + timedelta(hours=self.cache_duration_hours) - - return datetime.now() < expiry_time - - except Exception as e: - self.exceptions_logger.exception("Error checking cache validity") - return False - - def _fetch_from_osm(self) -> List[str]: - try: - query = self._build_osm_query() - self.logger.debug(f"OSM Query: {query}") - - result = self.api.query(query) - - institutions = [] - - # Process both nodes and ways - for element in list(result.nodes) + list(result.ways): - name = self._extract_institution_name(element.tags) - if name: - institutions.append(name) - - # Remove duplicates and limit results - institutions = list(set(institutions))[:self.max_institutions] - - self.logger.info(f"Found {len(institutions)} medical institutions from OSM") - return institutions - - except Exception as e: - self.exceptions_logger.exception("Error fetching from OSM") - raise - - def _build_osm_query(self) -> str: - medical_queries = [] - for tag in self.medical_tags: - key, value = tag.split('=') - medical_queries.extend([ - f'node[{key}={value}](area);', - f'way[{key}={value}](area);' - ]) - - medical_query_str = '\n '.join(medical_queries) - - # Build area query based on configuration - if self.city and self.country: - area_query = f"""( - area["name"="{self.city}"]["place"~"^(city|town)$"]; - area["name:en"="{self.city}"]["admin_level"~"^(4|5|6|7|8)$"]; -) -> .area;""" - elif self.city: - area_query = f"""( - area["name"="{self.city}"]["place"~"^(city|town)$"]; - area["name:en"="{self.city}"]["admin_level"]; -) -> .area;""" - else: - area_query = f'area["ISO3166-1"="{self.country}"]["admin_level"="2"] -> .area;' - - return f"""[out:json][timeout:{self.timeout}]; -{area_query} -( - {medical_query_str} -); -out tags;""" - - def _extract_institution_name(self, tags: Dict) -> Optional[str]: - name_keys = ['name', 'name:en', 'official_name', 'alt_name', 'brand'] - - for key in name_keys: - if key in tags and tags[key].strip(): - name = tags[key].strip() - - if self._is_valid_institution_name(name): - return name.title() - - return None - - def _is_valid_institution_name(self, name: str) -> bool: - if not name or len(name) < 3 or len(name) > 100: - return False - - skip_terms = ['pharmacy', 'apotek', 'dentist', 'tandlæge', 'veterinary', 'dyrlæge'] - name_lower = name.lower() - return not any(term in name_lower for term in skip_terms) - - def _load_from_cache(self) -> List[str]: - try: - with open(self.cache_file, 'r', encoding='utf-8') as f: - cache_data = json.load(f) - return cache_data.get('institutions', []) - except Exception as e: - self.exceptions_logger.exception("Error loading from cache") - return [] - - def _save_to_cache(self, institutions: List[str]) -> None: - try: - cache_data = { - 'timestamp': datetime.now().isoformat(), - 'institutions': institutions - } - - with open(self.cache_file, 'w', encoding='utf-8') as f: - json.dump(cache_data, f, indent=2, ensure_ascii=False) - - self.logger.debug(f"Saved {len(institutions)} institutions to cache") - - except Exception as e: - self.exceptions_logger.exception("Error saving to cache") - - -def get_random_institution(osm_service: Optional[OSMInstitutionsService] = None) -> str: - if osm_service and config.OSM_ENABLED: - try: - institutions = osm_service.get_medical_institutions() - return random.choice(institutions) - except Exception: - pass - - return random.choice(config.OSM_FALLBACK_INSTITUTIONS) \ No newline at end of file diff --git a/dicom_server/custom_units/tcia_management.py b/dicom_server/custom_units/tcia_management.py deleted file mode 100644 index 714be57c..00000000 --- a/dicom_server/custom_units/tcia_management.py +++ /dev/null @@ -1,404 +0,0 @@ -import threading -from pydicom import uid -import schedule, requests -import json -import utilities.tcia_util as tcia_util -import utilities.tcia_fallback as tcia_fallback -from services.redis_service import IRedisService -from services.dicom_database_service import IDicomDatabase -from services.tci_services import ITCIAAPI -from services.tci_services import ITCIAScheduler -from services.tci_services import ITCIAManager -from services.osm_service import IOSMService -from dependency_injector.wiring import inject -import logging -import time - - -class TCIAScheduler(threading.Thread, ITCIAScheduler): - - @inject - def __init__( - self, - app_logger, - exceptions_logger, - period, - period_unit, - tcia_manager: ITCIAManager = None, - ): - super().__init__(daemon=True) - self.period = period - self.period_unit = period_unit - self.exceptions_logger = exceptions_logger - self.tcia_manager = tcia_manager or ITCIAManager() - self.logger = app_logger - self.start() - - def run(self): - self.schedule_files_retrieval() - self.logger.info( - f"TCIA retrieving schedule is started, DICOM files storage and database will be directly updated from The Cancer Imaging Archive each {self.period} {self.period_unit}" - ) - - while True: - schedule.run_pending() - time.sleep(10) - - def schedule_files_retrieval(self): - try: - schedule_unit = getattr(schedule.every(self.period), str(self.period_unit)) - schedule_unit.do(self.tcia_manager.change_dicom_files) - - except Exception: - self.exceptions_logger.exception( - "Unexpected error while running TCIA files retrieve scheduler" - ) - - -class TCIAManager(ITCIAManager): - def __init__( - self, - app_logger, - exceptions_logger, - honeytoken_url, - storage_directory, - tcia_dir, - stagger_dir, - pdf_canary_path, - dicomdb: IDicomDatabase = None, - redis_handler: IRedisService = None, - tcia_api: ITCIAAPI = None, - osm_service: IOSMService = None, - ): - self.honeytoken_url = honeytoken_url - self.tcia_dir = tcia_dir - self.stagger_dir = stagger_dir - self.storage_directory = storage_directory - self.dicomdb = dicomdb or IDicomDatabase() - self.redis_handler = redis_handler or IRedisService() - self.osm_service = osm_service - self.exceptions_logger = exceptions_logger - self.logger = app_logger - - self.tcia_api = tcia_api - self.pdf_canary_path = pdf_canary_path - self.change_dicom_files_called = False - - def change_dicom_files(self): - # to be used in unit testing - self.change_dicom_files_called = True - - self.logger.info("Scheduled change of dicom files started") - - tcia_username = getattr(self.tcia_api, 'username', 'user') if self.tcia_api else 'user' - tcia_password = getattr(self.tcia_api, 'password', 'pass') if self.tcia_api else 'pass' - - try: - # Ensure storage directory exists before staging - tcia_util.initialize_dicom_directory_if_not_exist(self.storage_directory) - - self.logger.debug("Stagging existing files") - tcia_util.stage_old_files( - self.storage_directory, self.tcia_dir, self.stagger_dir - ) - - if tcia_fallback.should_use_fallback(tcia_username, tcia_password): - self.logger.info("Using fallback mode - copying sample TCIA files") - files_copied = tcia_fallback.copy_sample_files_to_tcia_directory(self.tcia_dir) - if files_copied == 0: - raise Exception("Failed to copy sample files") - else: - self.logger.debug("Getting access token from TCIA") - self.tcia_api.get_access_token() - - self.logger.debug("New files retrieval") - exist_studies = self.redis_handler.get_TCI_existing_studies() - - self.tcia_api.get_new_files(exist_studies, self.tcia_dir) - - self.logger.debug("Organizing downloaded files") - self.organize_downloaded_files() - - self.logger.debug("Cleaning up staged files") - tcia_util.delete_staged_files(self.stagger_dir) - - self.logger.debug("Updating DICOM database") - self.dicomdb.initialize_database() - - except Exception as e: - self.exceptions_logger.exception( - "Changing Dicom files failed: Rolling back changes" - ) - # Only rollback if not in fallback mode to avoid rollback errors - if not tcia_fallback.should_use_fallback(tcia_username, tcia_password): - self.roll_back_changes() - else: - self.logger.warning("Skipping rollback in fallback mode due to error: " + str(e)) - - def roll_back_changes(self): - self.logger.debug("Roll-back: Deleting downloaded files") - tcia_util.delete_downloded_files_if_exist(self.tcia_dir) - self.logger.debug("Roll-back: restorring old files") - tcia_util.restore_old_files( - self.storage_directory, self.tcia_dir, self.stagger_dir - ) - - def organize_downloaded_files(self): - self.logger.info("Organizing TCIA retrieved files") - directory = tcia_util.initialize_dicom_directory_if_not_exist( - self.storage_directory - ) - series_files_counter = 0 - self.logger.debug("Parsing files based on modalities") - self.logger.debug("Adding StudyInstanceUIDs to Redis") - for modality in tcia_util.get_downloaded_modalitis(self.tcia_dir): - for study_uid in tcia_util.get_studies_from_modality( - modality, self.tcia_dir - ): - try: - - self.redis_handler.add_TCI_study(study_uid) - except Exception: - self.exceptions_logger.exception("Study addition failed:") - - study_files_counter = 0 - ( - patient_name, - patient_id, - patient_sex, - birth_date, - study_id, - study_date, - accession_number, - ) = tcia_util.generate_patient_info() - institution = tcia_util.get_random_institution(self.osm_service) - for se_uid in tcia_util.get_downloaded_series_per_study( - modality, study_uid, self.tcia_dir - ): - for file in tcia_util.get_files_per_serie( - modality, study_uid, se_uid, self.tcia_dir - ): - study_files_counter += 1 - series_files_counter += 1 - self.process_serie_file( - directory, - series_files_counter, - modality, - study_uid, - study_files_counter, - patient_name, - patient_id, - patient_sex, - birth_date, - study_id, - study_date, - accession_number, - institution, - se_uid, - file, - ) - - def process_serie_file( - self, - directory, - series_files_counter, - modality, - study_uid, - study_files_counter, - patient_name, - patient_id, - patient_sex, - birth_date, - study_id, - study_date, - accession_number, - institution, - se_uid, - file, - ): - if not tcia_util.is_licience_file(file): - - dataset = tcia_util.build_file_dataset( - modality, - study_uid, - patient_name, - patient_id, - patient_sex, - birth_date, - study_id, - study_date, - accession_number, - institution, - se_uid, - file, - self.tcia_dir, - ) - if ( - series_files_counter == 4 - or series_files_counter == 12 - or series_files_counter == 18 - or series_files_counter == 24 - ): - # Injecting 4 retrieved dicom file with canary token and honeyURL token - self.inject_honey_url(dataset) - - self.inject_pdf_canary_token(dataset) - - self.redis_handler.add_injected_file(patient_name, modality) - - tcia_util.store_retrieved_file( - directory, - modality, - study_files_counter, - patient_name, - dataset, - ) - - def inject_pdf_canary_token(self, dataset): - try: - self.logger.debug( - f"DICOM file for Patient: {dataset.PatientName}, modality: {dataset.Modality} injected with pdf canary token" - ) - dataset.SOPClassUID = uid.EncapsulatedPDFStorage - dataset.MIMETypeOfEncapsulatedDocument = "application/pdf" - dataset.EncapsulatedDocument = self.get_canary_token() - except Exception: - self.exceptions_logger.exception( - "Unexpected error while injecting canary token" - ) - - def inject_honey_url(self, dataset): - try: - dataset.RetrieveURL = ( - f"{str(self.honeytoken_url)}/{dataset.StudyInstanceUID}" - ) - self.logger.debug( - f"DICOM file for Patient {dataset.PatientName}, modality: {dataset.Modality} injected with honeyURL" - ) - except Exception: - self.exceptions_logger.exception( - f"Unexpected error while injecting HoneyURL for Patient {dataset.PatientName}, modality: {dataset.Modality}" - ) - - def get_canary_token(self): - pdf_path = self.pdf_canary_path - try: - with open(pdf_path, "rb") as pdf_file: - pdf_data = pdf_file.read() - return pdf_data - except Exception: - self.exceptions_logger.exception("Canary token retrieval failed:") - - -class TCIAAPI(ITCIAAPI): - - def __init__( - self, - app_logger, - exceptions_logger, - username, - password, - min_series, - max_series, - modalities, - studies_per_mod, - ): - self.username = username - self.password = password - self.exceptions_logger = exceptions_logger - self.minimum_files_in_each_retrieved_serie = min_series - self.maximum_files_in_each_retrieved_serie = max_series - self.number_of_studies_in_each_retrieved_modality = studies_per_mod - self.modalities = modalities - self.logger = app_logger - - def get_new_files(self, existing_studies, tcia_dir): - self.logger.debug("Calling TCIA API") - try: - - _json = {} - for mod in self.modalities: - metadata = {} - _json = self.get_studies_based_on_modalities(mod) - if _json: - study_counter = 0 - metadata = tcia_util.filter_retrieved_studies( - existing_studies, - _json, - study_counter, - self.number_of_studies_in_each_retrieved_modality, - self.minimum_files_in_each_retrieved_serie, - self.maximum_files_in_each_retrieved_serie, - ) - - for st_uid, se_uids in metadata.items(): - for se_uid_dict in se_uids: - se_uid = se_uid_dict["se_uid"] - mod = se_uid_dict["modality"] - response = requests.get( - f"https://services.cancerimagingarchive.net/nbia-api/services/v2/getImage?SeriesInstanceUID={se_uid}", - headers={ - "Authorization": f"Bearer {self.access_token}" - }, - ) - response.raise_for_status() - - tcia_util.extract_and_save_zip_data( - mod, st_uid, se_uid, response, tcia_dir - ) - metadata = {} - except Exception: - self.exceptions_logger.exception("New files retrieval failed:") - - def get_access_token(self): - session = requests.Session() - adapter = requests.adapters.HTTPAdapter(max_retries=3) - session.mount("https://", adapter) - try: - res = session.post( - "https://services.cancerimagingarchive.net/nbia-api/oauth/token", - data={ - "username": self.username, - "password": self.password, - "client_id": "NBIA", - "grant_type": "password", - }, - timeout=10, - ) - json_object = json.loads(res.content) - self.access_token = json_object["access_token"] - self.refresh_token = json_object["refresh_token"] - - except Exception: - self.exceptions_logger.exception("Access token retrieval failed:") - - def refresh_access_token(self): - session = requests.Session() - adapter = requests.adapters.HTTPAdapter(max_retries=3) - session.mount("https://", adapter) - try: - session.post( - "https://services.cancerimagingarchive.net/nbia-api/oauth/token", - data={ - "refresh_token": self.refresh_token, - "client_id": "nbia", - "grant_type": "refresh_token", - }, - timeout=10, - ) - except Exception: - self.exceptions_logger.exception("Token refresh failed:") - - def get_studies_based_on_modalities(self, mod): - session = requests.Session() - adapter = requests.adapters.HTTPAdapter(max_retries=3) - session.mount("https://", adapter) - try: - res = session.get( - "https://services.cancerimagingarchive.net/nbia-api/services/v1/getSeries", - params={f"Modality": {mod}}, - ) - _json = json.loads(res.content) - except Exception: - self.exceptions_logger.exception("Studies retrieval failed:") - return _json diff --git a/dicom_server/enums/dicom_session_keys.py b/dicom_server/enums/dicom_session_keys.py deleted file mode 100644 index d2ebcc7f..00000000 --- a/dicom_server/enums/dicom_session_keys.py +++ /dev/null @@ -1,31 +0,0 @@ -from aenum import Enum - - -class Sessionkeys(Enum): - _init_ = "key default" - - LOCK = "lock", 0 - SESSION_ID = "session_id", 0 - REQUEST_TYPE = "request_type", "N/A" - SESSION_MAIN_OPERATION = "main_operation", "N/A" - QUERY_LEVEL = "query_level", "N/A" - SESSION_PARAMETERS = "session_parameters", "N/A" - LOG_LEVEL = "log_level", "N/A" - VERSION = "version", "N/A" - IP = "ip", "N/A" - PORT = "port", "N/A" - LOCAL_PORT = "local_port", "N/A" - KNOWN_SCANNER = "known_scanner", "N/A" - MATCHES = "matches", "N/A" - STATUS = "status", "N/A" - TIMESTAMP = "timestamp", "N/A" - - -class RequestType(Enum): - ASSO_RQ = "Association Requested", 0 - ASSO_RE = "Association Released", 0 - C_ECHO = "C_ECHO", 0 - C_FIND = "C_FIND", 0 - C_GET = "C_GET", 0 - C_STORE = "C_STORE", 0 - C_MOVE = "C_MOVE" diff --git a/dicom_server/logo.txt b/dicom_server/logo.txt deleted file mode 100644 index c5a11e61..00000000 --- a/dicom_server/logo.txt +++ /dev/null @@ -1,37 +0,0 @@ -............................................................=@@@@@@@@@@@@@@............................................................. -.........................................................*@@@@@@@*....%@@@@@@@.......................................................... -......................................................%@@@@@@@...........-@@@@@@@....................................................... -................+*.................................%@@@@@@*.....:@@.........:@@@@@@@-.................................*-................ -................+@@@@@#.........................@@@@@@@=........................@@@@@@@+........................:%@@@@@=................ -................=@@**#%@@@@#:...............-@@@@@@@-.........+%@@@@%#-............%@@@@@@*................=%@@@@%#**@@:................ -..........@#.....@@@@@@#**#%@@@@%-.......=@@@@@@@:........-@@@=:.:::::-%@@+...........*@@@@@@#........+@@@@@%#**#%@@@@@....:#@.......... -..........=@@@@@@#-..=%@@@@@#**#%@@@@@#@@@@@@#...........@@%+++#@@@@+@@@@@@@-............=@@@@@@@%@@@@@#***%@@@@@%=..-*@@@@@@:.......... -...........+@@@#*#%@@@@@%=..+%@@@%#**#%@@@@=....#@%.....@@*+++++=*@*+%@#.:+-:*@+...=@##+....@@@@%#***#%@@@%=..=%@@@@@%#*#@@@-........... -..............*%@@@@@@@@@@@@@@@%=::*%@@%*%@@=...+@*....@@*#@@#=:......:--+----:@=....-=....%@@**@@@%+::+@@@@@@@@@@@@@@@@%+.............. -.............@@@@*====**+.##@*..%@@@@=*@%*%@@*........=@@@*........-*#%@@@%#%@-##.........@@@**@@.@@@@@...@##--##++===#@@@@............. -..............#@@+-=@@...:@@@@.:@@@@@@@=*#*%@@*.......@@*..........+@@@@@=....@@....:....@@@**%-@@@@@@@#.=@%@+...@@%=-*@@*.............. -.............@@@@@@@@.@@@@%.@@@%%@@@@#*@@@%*#@@*.....#@=@-......+@%@@@@@+.....:#........@@@**@@@@*%@@@@#@@@#:@@@@%.@@@@@@@@............. -..............#@@@@@@@@@@@@@@%%%%%%%%%%%%%%#+#@@#....@@@..=......+@@=@@@@+............:@@@**%%%%%%%%%%%%%%%@@@@@@@@@@@@@@*.............. -.................@@@@@@@@@@@@@@@@@@@@@@@@@@@@**@@@...*@@@.......%@.+@@@@@@@@#........@@@%*@@@@@@@@@@@@@@@@@@@@@@@@@@@@%................. -................@@@@@....................+++@@@*%@@@.@@%.......:..@@@==+@@@@@@@%...@@@@*#@@+++-....................@@@@%................ -.................=@@@+.%@@@@*-@@@@@@@@@@@+#@@=::**#@@@@##........@@++%**.+++++@@@@@@%*#-::@@@+%@@@@@@@@@@-%@@@@@%.%@@@-................. -....................+@@@@@@@::*@@@@@@@@@@@@@@@@@@@@@@@@@........@@%:#@+.==++++++@@@@@@@@@@@@@@@%@@@@@@@@+::@@@@@@@@-.................... -.......................-@@@@@@@@@@@-.@@@@@@@@@@@@@@=@@@.........@@%:*@==+..=+*##%@@@@@@@@@@@@@@@@@+.%@@@@@@@@@@@........................ -.......................-@@@@@@......:@@...@@#..@@@*+@@#.........@@@:-@@+++++++*###@@@@@#.:@@@=..@@*:.....:@@@@@@........................ -.......................-@@@@:..@@@-----=@@@@*..@@@*#@@..........@@@#:*@#+#++++*#@%#%@@@+..+..-@#-----+@@@..+@@@@........................ -.......................-@@@@..@@.%@@@@@@#:=..@@@@-@@@@.@=.......*:@@*:#@*+@++*##%@%%@@@@@@=..*:@@@@@@@+-@@..@@@@........................ -.......................-@@@@...%@@=..@@@@@+.-@@@@-#@@@.@@.........=@@#-#@#%@%####@@@%@@@@@*..@@@@@=..*@@=:..@@@@........................ -.......................-@@@@*%@@@-*@@@*--------@@++@@@@@@@..+......:@@@-+@@@@%#%%%@@@@@@@-------=@@@@-*@@@*#@@@@........................ -.......................-@@@@@@@:.%@@*-@@@@@@@@@@@@=#@@@@:@@:+@..-..-+@@@++@@@@%%@@@@@@@@@@@@@@@@@@.@@@+.*@@@@@@@........................ -.......................-@@@@@@@@@@+=@@:::::::::-@@@+%@@@.-@@%@@..**.#@@@@%+*@@@@%@@@@@@@@@%::::::*@@.%@@@@@@@@@@........................ -.......................-@@@@@@..:+@@:%@@@@@@@@@@@@@@+#@@@#=@@@@@..%@=@@@@@@#+*@@@@@@@@@@@@@@@@@@@@-+@@.:.=@@@@@@........................ -.......................=@@@@@@*:@@:%@@-...........@@@#+@@@@@@@@@@.=@@@@@@@@@@%+*@@@@@@@@@@@@@....%@@=-@#:%@@@@@@........................ -................=@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@:................ -................+@@@@:::::-#@@@@-:+@@@@=..:#@@@@@#:...#@@@@=::@@@@@::=@@+:-@@@@::%@@@@#::=@@@-:=@@@@@@%::%@-:*@@*::@@@@-................ -................+@@@@...+=...=@@..:@@...:+...@@@...=*...+@@....+@+....@@-..@@@%..#@@@@....%@@*..@@@@@@:..@@..=@...@@@@@-................ -................+@@@@...@@@...@@..:@@..+@@@@@@@:..@@@@...@@...-...=...@@-..@@@%..#@@@:..+..@@@...@*:@%..*@@.....-@@@@@@-................ -................+@@@@...@@@...@@..:@@..+@@@@@@@:..@@@@...@@...@*.*@...@@-........#@@#..@@..-@@#..@...-..@@@.....@@@@@@@-................ -................+@@@@...@@@...@@..:@@..+@@@@@@@:..@@@@...@@...@@@@@:..@@-..@@@%..#@@........#@@........+@@@...=..@@@@@@-................ -................+@@@@...@@:..*@@:.-@@:.......@@@........@@@:.:@@@@@:.:@@-..@@@%..#@=..@@@@..:@@%...@..:@@@@..=@+..@@@@@-................ -................+@@@@......+@@@@##%@@@@%-.=@@@@@@@+..+@@@@@%#%@@@@@%#%@@%##@@@@##@@##%@@@@@**@@@*+@@%+%@@@@..=@@*..#@@@-................ -................+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*#@@@@@@@@@@@@@@@@@@@:+@@@@@@@@@@@@@@@@@@@=@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@-................ \ No newline at end of file diff --git a/dicom_server/main.py b/dicom_server/main.py deleted file mode 100644 index 4cd4d556..00000000 --- a/dicom_server/main.py +++ /dev/null @@ -1,10 +0,0 @@ -import sys, os - -with open("logo.txt", "r") as f: - print("\033[92m", f.read()) -sys.path.append(os.path.abspath("./core/")) -from app_container import ApplicationContainer - - -app = ApplicationContainer() -app.dicom_application().start_the_application() diff --git a/dicom_server/pydicom_and_pynetdicom_libs/db.py b/dicom_server/pydicom_and_pynetdicom_libs/db.py deleted file mode 100644 index 935e1651..00000000 --- a/dicom_server/pydicom_and_pynetdicom_libs/db.py +++ /dev/null @@ -1,838 +0,0 @@ -""" -The original implementation of this module can be found at "https://github.com/pydicom/pynetdicom/blob/c22be4b79a20eea0f176340629b37c6e30dd10b2/pynetdicom/apps/qrscp/db.py#L4" -""" - -"""Database interface for the qrscp application. - -Unique Keys ------------ -* At each level one attribute is unique -* A unique key shall uniquely identify a single instance at a given level -* Unique keys **may** be in a C-FIND request's Identifier -* Unique keys **shall** be in a C-MOVE or C-GET request's Identifier -* C-FIND, C-GET and C-MOVE shall support existence and matching of all -* unique keys. All instances managed shall have specific non-zero length - unique key values - -Required Keys -------------- -* Multiple instances may have the same value for required keys. -* Required keys may be in a C-FIND request's Identifier -* Required keys shall not be in a C-GET or C-MOVE request's Identifier - -Honeypot Adjustments --------------------- -* Added `PatientSex`, `InstitutionName`, and `PatientBirthDate` to the attribute - list at the STUDY level, allowing queries to filter by these fields. -* Changed `InstitutionName` VR from `LO` to `DA`. -* Changed `InstanceNumber` VR from `IS` to `UI`. - -""" - -from collections import OrderedDict -import sys - -try: - from sqlalchemy import create_engine, Column, ForeignKey, Integer, String -except ImportError: - sys.exit("qrscp requires the sqlalchemy package") - -from sqlalchemy.orm import declarative_base - -from pydicom.dataset import Dataset - -from pynetdicom import build_context -from pynetdicom.sop_class import ( - PatientRootQueryRetrieveInformationModelFind, - PatientRootQueryRetrieveInformationModelMove, - PatientRootQueryRetrieveInformationModelGet, - StudyRootQueryRetrieveInformationModelFind, - StudyRootQueryRetrieveInformationModelMove, - StudyRootQueryRetrieveInformationModelGet, -) - - -class InvalidIdentifier(Exception): - pass - - -# C.2.2.2: The total length of the attribute may be larger than given in Part 5 -# C.2.2.2: The VM may be larger than the VM from Part 6, depending -# on the matching type - -# VRs for supported elements - Part 5 -# CS - 16 bytes maximum - str -# DA - 8 bytes fixed, format YYYYMMDD - str -# IS - 12 bytes maximum, range - int -# LO - 64 characters maximum - str -# PN - 64 characters maximum per component group (5 components per group) - str -# SH - 16 characters maximum - str -# TM - 14 bytes maximum, format HHMMSS.FFFFFF - str -# UI - 64 bytes maximum - str - - -# Translate from the element keyword to the db attribute -_TRANSLATION = { - "PatientID": "patient_id", # PATIENT | Unique | VM 1 | LO - "PatientSex": "patient_sex", - "InstitutionName": "institution_name", - "PatientBirthDate": "birth_date", - "PatientName": "patient_name", # PATIENT | Required | VM 1 | PN - "StudyInstanceUID": "study_instance_uid", # STUDY | Unique | VM 1 | UI - "StudyDate": "study_date", # STUDY | Required | VM 1 | DA - "StudyTime": "study_time", # STUDY | Required | VM 1 | TM - "AccessionNumber": "accession_number", # STUDY | Required | VM 1 | SH - "StudyID": "study_id", # STUDY | Required | VM 1 | SH - "SeriesInstanceUID": "series_instance_uid", # SERIES | Unique | VM 1 | UI - "Modality": "modality", # SERIES | Required | VM 1 | CS - "SeriesNumber": "series_number", # SERIES | Required | VM 1 | IS - "SOPInstanceUID": "sop_instance_uid", # IMAGE | Unique | VM 1 | UI - "InstanceNumber": "instance_number", # IMAGE | Required | VM 1 | IS -} - -# Unique and required keys and their level, VR and VM for Patient Root -# Study Root is the same but includes the PATIENT attributes -_ATTRIBUTES = { - "PatientID": ("PATIENT", "U", "LO", 1), - "PatientName": ("PATIENT", "R", "PN", 1), - "InstitutionName": ("STUDY", "R", "DA", 1), - "PatientBirthDate": ("STUDY", "R", "DA", 1), - "StudyInstanceUID": ("STUDY", "U", "UI", 1), - "PatientSex": ("PATIENT", "R", "PN", 1), - "StudyDate": ("STUDY", "R", "DA", 1), - "StudyTime": ("STUDY", "R", "TM", 1), - "AccessionNumber": ("STUDY", "R", "SH", 1), - "StudyID": ("STUDY", "R", "SH", 1), - "SeriesInstanceUID": ("SERIES", "U", "UI", 1), - "Modality": ("SERIES", "R", "VS", 1), - "SeriesNumber": ("SERIES", "R", "IS", 1), - "SOPInstanceUID": ("IMAGE", "U", "UI", 1), - "InstanceNumber": ("IMAGE", "R", "UI", 1), -} -_PATIENT_ROOT_ATTRIBUTES = OrderedDict( - { - "PATIENT": ["PatientID", "PatientName"], - "STUDY": [ - "StudyInstanceUID", - "StudyDate", - "StudyTime", - "AccessionNumber", - "StudyID", - ], - "SERIES": ["SeriesInstanceUID", "Modality", "SeriesNumber"], - "IMAGE": ["SOPInstanceUID", "InstanceNumber"], - } -) -_STUDY_ROOT_ATTRIBUTES = OrderedDict( - { - "STUDY": [ - "StudyInstanceUID", - "StudyDate", - "StudyTime", - "AccessionNumber", - "StudyID", - "PatientID", - "PatientName", - "PatientSex", - "PatientBirthDate", - "InstitutionName", - ], - "SERIES": ["SeriesInstanceUID", "Modality", "SeriesNumber"], - "IMAGE": ["SOPInstanceUID", "InstanceNumber"], - } -) - -# Supported Information Models -_C_FIND = [ - PatientRootQueryRetrieveInformationModelFind, - StudyRootQueryRetrieveInformationModelFind, -] -_C_GET = [ - PatientRootQueryRetrieveInformationModelGet, - StudyRootQueryRetrieveInformationModelGet, -] -_C_MOVE = [ - PatientRootQueryRetrieveInformationModelMove, - StudyRootQueryRetrieveInformationModelMove, -] - -_PATIENT_ROOT = { - PatientRootQueryRetrieveInformationModelFind: _PATIENT_ROOT_ATTRIBUTES, - PatientRootQueryRetrieveInformationModelGet: _PATIENT_ROOT_ATTRIBUTES, - PatientRootQueryRetrieveInformationModelMove: _PATIENT_ROOT_ATTRIBUTES, -} -_STUDY_ROOT = { - StudyRootQueryRetrieveInformationModelFind: _STUDY_ROOT_ATTRIBUTES, - StudyRootQueryRetrieveInformationModelGet: _STUDY_ROOT_ATTRIBUTES, - StudyRootQueryRetrieveInformationModelMove: _STUDY_ROOT_ATTRIBUTES, -} - - -def add_instance(ds, session, fpath=None): - """Add a SOP Instance to the database or update existing instance. - - Parameters - ---------- - ds : pydicom.dataset.Dataset - The SOP Instance to be added to the database. - session : sqlalchemy.orm.session.Session - The session we are using to query the database. - fpath : str, optional - The path to where the SOP Instance is stored, taken relative - to the database file. - """ - # Check if instance is already in the database - result = ( - session.query(Instance) - .filter(Instance.sop_instance_uid == ds.SOPInstanceUID) - .all() - ) - if result: - instance = result[0] - else: - instance = Instance() - - # Unique or Required attributes - required = [ - # (Instance attribute, DICOM keyword, max length, req'd) - ("patient_id", "PatientID", 64, True), - ("patient_name", "PatientName", 64, False), - ("study_instance_uid", "StudyInstanceUID", 64, True), - ("study_date", "StudyDate", 8, False), - ("study_time", "StudyTime", 14, False), - ("accession_number", "AccessionNumber", 16, False), - ("study_id", "StudyID", 16, False), - ("series_instance_uid", "SeriesInstanceUID", 64, True), - ("modality", "Modality", 16, False), - ("series_number", "SeriesNumber", None, False), - ("sop_instance_uid", "SOPInstanceUID", 64, True), - ("instance_number", "InstanceNumber", None, False), - ("patient_sex", "PatientSex", 64, False), - ("institution_name", "InstitutionName", 64, False), - ("birth_date", "PatientBirthDate", 64, False), - ] - - # Unique and Required attributes - for attr, keyword, max_len, unique in required: - if not unique and keyword not in ds: - value = None - else: - elem = ds[keyword] - value = elem.value - - if value is not None: - # All supported attributes have VM 1 - # assert elem.VM == 1 - if max_len: - if elem.VR == "PN": - value = str(value) - - assert len(value) <= max_len - else: - assert -(2**31) <= value <= 2**31 - 1 - - setattr(instance, attr, value) - - instance.filename = fpath - - # Transfer Syntax UID - try: - tsyntax = ds.file_meta.TransferSyntaxUID - if tsyntax: - assert len(tsyntax) < 64 - instance.transfer_syntax_uid = tsyntax - except (AttributeError, AssertionError): - pass - - # SOP Class UID - try: - uid = ds.SOPClassUID - if uid: - assert len(uid) < 64 - instance.sop_class_uid = uid - except (AttributeError, AssertionError): - pass - - session.add(instance) - session.commit() - - -def build_query(identifier, session, query=None): - """Perform a query against the database. - - Parameters - ---------- - identifier : pydicom.dataset.Dataset - The request's *Identifier* dataset containing the query attributes. - session : sqlalchemy.orm.session.Session - The session we are using to query the database. - query : sqlalchemy.orm.query.Query, optional - If not used then start a new query, otherwise extend the existing - `query`. - - Returns - ------- - sqlalchemy.orm.query.Query - The resulting query. - """ - # VRs for Single Value Matching and Wild Card Matching - _text_vr = ["AE", "CS", "LO", "LT", "PN", "SH", "ST", "UC", "UR", "UT"] - for elem in [e for e in identifier if e.keyword in _ATTRIBUTES]: - vr = elem.VR - val = elem.value - # Convert PersonName3 to str - if vr == "PN" and val: - val = str(val) - - # Part 4, C.2.2.2.1 Single Value Matching - if vr != "SQ" and val is not None: - if vr in _text_vr and ("*" in val or "?" in val): - pass - elif vr in ["DA", "TM", "DT"] and "-" in val: - pass - else: - # print('Performing single value matching...') - query = _search_single_value(elem, session, query) - continue - - # Part 4, C.2.2.2.3 Universal Matching - if val is None: - # print('Performing universal matching...') - query = _search_universal(elem, session, query) - continue - - # Part 4, C.2.2.2.2 List of UID Matching - if vr == "UI": - # print('Performing list of UID matching...') - query = _search_uid_list(elem, session, query) - continue - - # Part 4, C.2.2.2.4 Wild Card Matching - if vr in _text_vr and ("*" in val or "?" in val): - # print('Performing wildcard matching...') - query = _search_wildcard(elem, session, query) - continue - - # Part 4, C.2.2.2.5 Range Matching - if vr in ["DT", "TM", "DA"] and "-" in val: - query = _search_range(elem, session, query) - continue - - # Part 4, C.2.2.2.6 Sequence Matching - # No supported attributes are sequences - - return query - - -def _check_identifier(identifier, model): - """Check that the C-FIND, C-GET or C-MOVE `identifier` is valid. - - Parameters - ---------- - identifier : pydicom.dataset.Dataset - The *Identifier* dataset to check. - model : pydicom.uid.UID - The Query/Retrieve Information Model. - - Raises - ------ - InvalidIdentifier - If the Identifier is invalid. - """ - # Part 4, C.4.1.1.3.1, C.4.2.1.4 and C.4.3.1.3.1: - # (0008,0052) Query Retrieve Level is required in the Identifier - if "QueryRetrieveLevel" not in identifier: - raise InvalidIdentifier( - "The Identifier contains no Query Retrieve Level element" - ) - - if model in _PATIENT_ROOT: - attr = _PATIENT_ROOT[model] - else: - attr = _STUDY_ROOT[model] - - levels = list(attr.keys()) - if identifier.QueryRetrieveLevel not in levels: - raise InvalidIdentifier( - "The Identifier's Query Retrieve Level value is invalid" - ) - - if len(identifier) == 1: - raise InvalidIdentifier("The Identifier contains no keys") - - for ii, level in enumerate(levels): - if level == identifier.QueryRetrieveLevel: - # Check if identifier has elements below current level - for sublevel in levels[ii + 1 :]: - if any([kw in identifier for kw in attr[sublevel]]): - raise InvalidIdentifier( - "The Identifier contains keys below the level " - "specified by the Query Retrieve Level" - ) - - # The level is the same as that in the identifier so we're OK - return - - # The level is above that in the identifier so make sure the unique - # keyword is present - if attr[level][0] not in identifier: - raise InvalidIdentifier( - f"The Identifier is missing a unique key for " f"the '{level}' level" - ) - - -def clear(session): - """Delete all entries from the database. - - Parameters - ---------- - session : sqlalchemy.orm.session.Session - The session we are using to clear the database. - """ - for instance in session.query(Instance).all(): - session.delete(instance) - - session.commit() - - -def create(db_location, echo=False): - """Create a new database at `db_location` if one doesn't already exist. - - Parameters - ---------- - db_location : str - The location of the database. - echo : bool, optional - Turn the sqlalchemy logging on (default ``False``). - """ - engine = create_engine(db_location, echo=echo) - - # Create the tables (won't recreate tables already present) - Base.metadata.create_all(engine) - - return engine - - -def remove_instance(instance_uid, session): - """Remove a SOP Instance from the database. - - Parameters - ---------- - instance_uid : pydicom.uid.UID - The (0008,0018) *SOP Instance UID* of the SOP Instance to be removed - from the database. - session : sqlalchemy.orm.session.Session - The session to use when querying the database for the instance. - """ - matches = ( - session.query(Instance).filter(Instance.sop_instance_uid == instance_uid).all() - ) - if matches: - session.delete(matches[0]) - session.commit() - - -def search(model, identifier, session): - """Search the database. - - Optional keys are not supported. - - Parameters - ---------- - model : pydicom.uid.UID - The Query/Retrieve Information Model. Supported models are: - - - *Patient Root Query Retrieve Information Model* for C-FIND, C-GET - and C-MOVE - - *Study Root Query Retrieve Information Model* for C-FIND, C-GET and - C-MOVE - identifier : pydicom.dataset.Dataset - The Query/Retrieve request's *Identifier* dataset. - session : sqlalchemy.orm.session.Session - The session we are using to query the database. - - Returns - ------- - list of Instance - The matching database Instances. - - Raises - ------ - ValueError - If the `identifier` is invalid. - """ - if model not in _STUDY_ROOT and model not in _PATIENT_ROOT: - raise ValueError(f"Unknown information model '{model.name}'") - - # Remove all optional keys, after this only unique/required will remain - for elem in identifier: - kw = elem.keyword - if kw != "QueryRetrieveLevel" and kw not in _ATTRIBUTES: - delattr(identifier, kw) - - if model in _C_GET or model in _C_MOVE: - # Part 4, C.2.2.1.2: remove required keys from C-GET/C-MOVE - for kw, value in _ATTRIBUTES.items(): - if value[1] == "R" and kw in identifier: - delattr(identifier, kw) - - return _search_qr(model, identifier, session) - - -def _search_qr(model, identifier, session): - """Search the database using a Query/Retrieve *Identifier* query. - - Parameters - ---------- - model : pydicom.uid.UID - Either *Patient Root Query Retrieve Information Model* or *Study Root - Query Retrieve Information Model* for C-FIND, C-GET or C-MOVE. - identifier : pydicom.dataset.Dataset - The request's *Identifier* dataset. - session : sqlalchemy.orm.session.Session - The session we are using to query the database. - - Returns - ------- - list of db.Instance - The Instances that match the query. - """ - # Will raise InvalidIdentifier if check failed - _check_identifier(identifier, model) - - if model in _PATIENT_ROOT: - attr = _PATIENT_ROOT[model] - else: - attr = _STUDY_ROOT[model] - - # Hierarchical search method: C.4.1.3.1.1 - query = None - for level, keywords in attr.items(): - # Keywords at current level that are in the identifier - keywords = [kw for kw in keywords if kw in identifier] - # Create query dataset for only the current level and run it - ds = Dataset() - [setattr(ds, kw, getattr(identifier, kw)) for kw in keywords] - query = build_query(ds, session, query) - - if level == identifier.QueryRetrieveLevel: - break - - return query.all() - - -def _search_range(elem, session, query=None): - """Perform a range search for DA, DT and TM elements with '-' in them. - - Parameters - ---------- - elem : pydicom.dataelem.DataElement - The attribute to perform the search with. - session : sqlalchemy.orm.session.Session - The session we are using to query the database. - query : sqlalchemy.orm.query.Query, optional - An existing query within which this search should be performed. If - not used then all the Instances in the database will be searched - (default). - - Returns - ------- - sqlalchemy.orm.query.Query - The resulting query. - """ - # range matching - # - : matches any date within the range, inclusive - # - : match all dates prior to and including - # -: match all dates after and including - #