diff --git a/infra/docker/docker-compose.agent.yml b/infra/docker/docker-compose.agent.yml new file mode 100644 index 0000000..0efaf5d --- /dev/null +++ b/infra/docker/docker-compose.agent.yml @@ -0,0 +1,35 @@ +# Persys Compute Agent Docker Compose + +services: + compute-agent: + build: + context: ../.. + dockerfile: compute-agent/Dockerfile + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-jaeger:4318} + - OTEL_SERVICE_NAME=${COMPUTE_AGENT_OTEL_SERVICE_NAME:-compute-agent} + - PERSYS_VAULT_ENABLED=true + - PERSYS_VAULT_ADDR=${COMPUTE_AGENT_VAULT_ADDR:-http://vault:8200} + - PERSYS_VAULT_AUTH_METHOD=${COMPUTE_AGENT_VAULT_AUTH_METHOD:-approle} + - PERSYS_VAULT_SERVICE_NAME=${COMPUTE_AGENT_VAULT_SERVICE_NAME:-compute-agent} + - PERSYS_TLS_ENABLED=${PERSYS_TLS_ENABLED:-true} + - PERSYS_GRPC_PORT=${PERSYS_GRPC_PORT:-50051} + - PERSYS_LOG_LEVEL=${PERSYS_LOG_LEVEL:-info} + - PERSYS_RECONCILE_ENABLED=${PERSYS_RECONCILE_ENABLED:-true} + - PERSYS_VM_ENABLED=${PERSYS_VM_ENABLED:-false} + - PERSYS_SCHEDULER_INSECURE=${PERSYS_SCHEDULER_INSECURE:-false} + - PERSYS_SCHEDULER_ADDR=${PERSYS_SCHEDULER_ADDR:-persys-scheduler:8085} + user: "0:0" + volumes: + - /etc/localtime:/etc/localtime:ro + - ${DOCKER_SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock + ports: + - "50051:50051" + - "8089:8089" + networks: + - persys-cloud-net + +# --- Shared overlay network for all services --- +networks: + persys-cloud-net: + driver: bridge diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml index c7aa3c7..3b58434 100644 --- a/infra/docker/docker-compose.yml +++ b/infra/docker/docker-compose.yml @@ -14,20 +14,16 @@ services: ports: - 8551:8551 depends_on: - - vault-manager - - mongodb - - vault - coredns environment: - OTEL_EXPORTER_JAEGER_ENDPOINT=${OTEL_EXPORTER_JAEGER_ENDPOINT:-jaeger:4318} - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-jaeger:4318} - PERSYS_VAULT_ADDR=${PERSYS_GATEWAY_VAULT_ADDR:-http://vault:8200} - PERSYS_VAULT_AUTH_METHOD=${PERSYS_GATEWAY_VAULT_AUTH_METHOD:-approle} - - PERSYS_VAULT_TOKEN=${PERSYS_GATEWAY_VAULT_TOKEN:-} - PERSYS_GATEWAY_VAULT_ADDR=${PERSYS_GATEWAY_VAULT_ADDR:-http://vault:8200} - - PERSYS_GATEWAY_VAULT_ROLE_ID=${PERSYS_GATEWAY_VAULT_ROLE_ID:-} - - PERSYS_GATEWAY_VAULT_SECRET_ID=${PERSYS_GATEWAY_VAULT_SECRET_ID:-} - - PERSYS_GATEWAY_MONGO_URI=${PERSYS_GATEWAY_MONGO_URI:-mongodb://admin:admin@mongodb:27017/persys-gateway?authSource=admin} + - PERSYS_GATEWAY_DEPLOYMENT_MODE=${PERSYS_GATEWAY_DEPLOYMENT_MODE:-self-hosted} + - PERSYS_GATEWAY_JWT_SECRET=${PERSYS_GATEWAY_JWT_SECRET} + - PERSYS_GATEWAY_POSTGRES_DSN=${PERSYS_GATEWAY_POSTGRES_DSN} - PERSYS_GATEWAY_GITHUB_WEBHOOK_SECRET=${PERSYS_GATEWAY_GITHUB_WEBHOOK_SECRET:-} - PERSYS_GATEWAY_GITHUB_WEBHOOK_SECRET_FILE=${PERSYS_GATEWAY_GITHUB_WEBHOOK_SECRET_FILE:-} - PERSYS_GATEWAY_GITHUB_CLIENT_ID=${PERSYS_GATEWAY_GITHUB_CLIENT_ID:-} @@ -40,7 +36,7 @@ services: - /etc/localtime:/etc/localtime:ro networks: - persys-cloud-net - + # --- Persys Scheduler (Workload scheduler and reconciler) --- persys-scheduler: build: @@ -51,7 +47,6 @@ services: - 8085:8085 # gRPC + mTLS API depends_on: - etcd - - vault-manager - coredns - redis environment: @@ -65,9 +60,6 @@ services: - PERSYS_EXTERNAL_IP=${PERSYS_EXTERNAL_IP} - PERSYS_VAULT_ADDR=${PERSYS_SCHEDULER_VAULT_ADDR:-http://vault:8200} - PERSYS_VAULT_AUTH_METHOD=${PERSYS_SCHEDULER_VAULT_AUTH_METHOD:-approle} - - PERSYS_VAULT_TOKEN=${PERSYS_SCHEDULER_VAULT_TOKEN} - - PERSYS_VAULT_APPROLE_ROLE_ID=${PERSYS_SCHEDULER_VAULT_ROLE_ID} - - PERSYS_VAULT_APPROLE_SECRET_ID=${PERSYS_SCHEDULER_VAULT_SECRET_ID} - PERSYS_VAULT_PKI_MOUNT=${PERSYS_VAULT_PKI_MOUNT:-pki} - REDIS_ADDR=redis:6379 - REDIS_PASSWORD= @@ -85,7 +77,6 @@ services: ports: - 8087:8087 depends_on: - - vault-manager - mysql - redis environment: @@ -98,10 +89,6 @@ services: - PERSYS_FORGERY_PIPELINE_STATUS_KEY=${PERSYS_FORGERY_PIPELINE_STATUS_KEY:-pipeline_status} - PERSYS_FORGERY_VAULT_ADDR=${PERSYS_FORGERY_VAULT_ADDR:-http://vault:8200} - PERSYS_FORGERY_VAULT_AUTH_METHOD=${PERSYS_FORGERY_VAULT_AUTH_METHOD:-approle} - - PERSYS_FORGERY_VAULT_TOKEN=${PERSYS_FORGERY_VAULT_TOKEN:-} - - PERSYS_FORGERY_VAULT_TOKEN_FILE=${PERSYS_FORGERY_VAULT_TOKEN_FILE:-} - - PERSYS_FORGERY_VAULT_ROLE_ID=${PERSYS_FORGERY_VAULT_ROLE_ID:-} - - PERSYS_FORGERY_VAULT_SECRET_ID=${PERSYS_FORGERY_VAULT_SECRET_ID:-} - PERSYS_FORGERY_GITHUB_WEBHOOK_SECRET=${PERSYS_FORGERY_GITHUB_WEBHOOK_SECRET:-} - PERSYS_FORGERY_GITHUB_WEBHOOK_SECRET_FILE=${PERSYS_FORGERY_GITHUB_WEBHOOK_SECRET_FILE:-} volumes: @@ -139,9 +126,6 @@ services: - AUTOMATION_VAULT_ENABLED=${AUTOMATION_VAULT_ENABLED:-false} - AUTOMATION_VAULT_ADDR=${AUTOMATION_VAULT_ADDR:-http://vault:8200} - AUTOMATION_VAULT_AUTH_METHOD=${AUTOMATION_VAULT_AUTH_METHOD:-approle} - - AUTOMATION_VAULT_TOKEN=${AUTOMATION_VAULT_TOKEN:-} - - AUTOMATION_VAULT_APPROLE_ROLE_ID=${AUTOMATION_VAULT_APPROLE_ROLE_ID:-} - - AUTOMATION_VAULT_APPROLE_SECRET_ID=${AUTOMATION_VAULT_APPROLE_SECRET_ID:-} - AUTOMATION_VAULT_PKI_MOUNT=${AUTOMATION_VAULT_PKI_MOUNT:-pki} - AUTOMATION_VAULT_PKI_ROLE=${AUTOMATION_VAULT_PKI_ROLE:-persys-automation} - AUTOMATION_VAULT_CERT_TTL=${AUTOMATION_VAULT_CERT_TTL:-24h} @@ -154,7 +138,7 @@ services: - AUTOMATION_FORGERY_REDIS_DB=${AUTOMATION_FORGERY_REDIS_DB:-0} - AUTOMATION_FORGERY_PIPELINE_KEY=${AUTOMATION_FORGERY_PIPELINE_KEY:-pipeline_status} - AUTOMATION_STORE_BACKEND=${AUTOMATION_STORE_BACKEND:-postgres} - - AUTOMATION_POSTGRES_DSN=${AUTOMATION_POSTGRES_DSN:-postgres://automation:automation@automation-postgres:5432/persys_automation?sslmode=disable} + - AUTOMATION_POSTGRES_DSN=${AUTOMATION_POSTGRES_DSN:-postgres://automation:automation@postgres:5432/persys_automation?sslmode=disable} - AUTOMATION_LEADER_ELECTION_ENABLED=${AUTOMATION_LEADER_ELECTION_ENABLED:-true} - AUTOMATION_LEADER_ELECTION_LOCK_ID=${AUTOMATION_LEADER_ELECTION_LOCK_ID:-771001} - AUTOMATION_LEADER_ELECTION_POLL_INTERVAL=${AUTOMATION_LEADER_ELECTION_POLL_INTERVAL:-5s} @@ -192,9 +176,6 @@ services: - INTELLIGENCE_VAULT_ENABLED=${INTELLIGENCE_VAULT_ENABLED:-false} - INTELLIGENCE_VAULT_ADDR=${INTELLIGENCE_VAULT_ADDR:-http://vault:8200} - INTELLIGENCE_VAULT_AUTH_METHOD=${INTELLIGENCE_VAULT_AUTH_METHOD:-approle} - - INTELLIGENCE_VAULT_TOKEN=${INTELLIGENCE_VAULT_TOKEN:-} - - INTELLIGENCE_VAULT_APPROLE_ROLE_ID=${INTELLIGENCE_VAULT_APPROLE_ROLE_ID:-} - - INTELLIGENCE_VAULT_APPROLE_SECRET_ID=${INTELLIGENCE_VAULT_APPROLE_SECRET_ID:-} - INTELLIGENCE_VAULT_PKI_MOUNT=${INTELLIGENCE_VAULT_PKI_MOUNT:-pki} - INTELLIGENCE_VAULT_PKI_ROLE=${INTELLIGENCE_VAULT_PKI_ROLE:-persys-intelligence} - INTELLIGENCE_VAULT_CERT_TTL=${INTELLIGENCE_VAULT_CERT_TTL:-24h} @@ -208,40 +189,6 @@ services: - /etc/localtime:/etc/localtime:ro networks: - persys-cloud-net - - compute-agent: - build: - context: ../.. - dockerfile: compute-agent/Dockerfile - environment: - - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-jaeger:4318} - - OTEL_SERVICE_NAME=${COMPUTE_AGENT_OTEL_SERVICE_NAME:-compute-agent} - - PERSYS_VAULT_ENABLED=true - - PERSYS_VAULT_ADDR=${COMPUTE_AGENT_VAULT_ADDR:-http://vault:8200} - - PERSYS_VAULT_AUTH_METHOD=${COMPUTE_AGENT_VAULT_AUTH_METHOD:-approle} - - PERSYS_VAULT_TOKEN=${COMPUTE_AGENT_VAULT_TOKEN:-} - - PERSYS_VAULT_APPROLE_ROLE_ID=${COMPUTE_AGENT_VAULT_ROLE_ID:-} - - PERSYS_VAULT_APPROLE_SECRET_ID=${COMPUTE_AGENT_VAULT_SECRET_ID:-} - - PERSYS_VAULT_SERVICE_NAME=${COMPUTE_AGENT_VAULT_SERVICE_NAME:-compute-agent} - - PERSYS_TLS_ENABLED=${PERSYS_TLS_ENABLED:-true} - - PERSYS_GRPC_PORT=${PERSYS_GRPC_PORT:-50051} - - PERSYS_LOG_LEVEL=${PERSYS_LOG_LEVEL:-info} - - PERSYS_RECONCILE_ENABLED=${PERSYS_RECONCILE_ENABLED:-true} - - PERSYS_VM_ENABLED=${PERSYS_VM_ENABLED:-false} - - PERSYS_SCHEDULER_INSECURE=${PERSYS_SCHEDULER_INSECURE:-false} - - PERSYS_SCHEDULER_ADDR=${PERSYS_SCHEDULER_ADDR:-persys-scheduler:8085} - user: "0:0" - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DOCKER_SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock - depends_on: - - vault-manager - - persys-scheduler - ports: - - "50051:50051" - - "8089:8089" - networks: - - persys-cloud-net # --- Certificate Authority / Secret Manager --- vault: @@ -325,7 +272,7 @@ services: networks: - persys-cloud-net -# --- Postgres (Automation policy and audit store) --- +# --- Postgres (Automation policy and audit store/ Gateway Auth Store) --- postgres: image: postgres:17-alpine restart: unless-stopped @@ -337,25 +284,10 @@ services: - "5432:5432" volumes: - /etc/localtime:/etc/localtime:ro - - automation_postgres_data:/var/lib/postgresql/data + - postgres_data:/var/lib/postgresql/data networks: - persys-cloud-net -# --- MongoDB (Database for Persys Gateway) --- - mongodb: - image: mongo:latest - restart: always - environment: - MONGO_INITDB_ROOT_USERNAME: "${MONGO_INITDB_ROOT_USERNAME:-admin}" - MONGO_INITDB_ROOT_PASSWORD: "${MONGO_INITDB_ROOT_PASSWORD:-admin}" - MONGODB_DATABASE: "${MONGODB_DATABASE:-persys-gateway}" - ports: - - 27017:27017 - volumes: - - /etc/localtime:/etc/localtime:ro - - mongodb_data_container:/data/db - networks: - - persys-cloud-net # --- etcd (Key-value store for Persys Scheduler and CoreDNS) --- etcd: @@ -374,7 +306,7 @@ services: # --- Volumes for persistent data --- volumes: - mongodb_data_container: + postgres_data: mysql_data: redis_data: automation_postgres_data: diff --git a/infra/docker/grafana/provisioning/datasources/prometheus-datasource.yml b/infra/docker/grafana/provisioning/datasources/prometheus-datasource.yml index 0779b7d..9a8cf1f 100644 --- a/infra/docker/grafana/provisioning/datasources/prometheus-datasource.yml +++ b/infra/docker/grafana/provisioning/datasources/prometheus-datasource.yml @@ -6,4 +6,11 @@ datasources: access: proxy url: http://prometheus:9090 isDefault: true + editable: true + + - name: pyroscope + type: pyroscope + access: proxy + url: http://pyroscope:4040 + isDefault: true editable: true \ No newline at end of file diff --git a/infra/docker/sample.env b/infra/docker/sample.env index 4a6fcf1..08fe377 100644 --- a/infra/docker/sample.env +++ b/infra/docker/sample.env @@ -13,10 +13,9 @@ VAULT_ROOT_TOKEN= # Vault auth per service PERSYS_GATEWAY_VAULT_ADDR=http://vault:8200 PERSYS_GATEWAY_VAULT_AUTH_METHOD=approle -PERSYS_GATEWAY_VAULT_TOKEN= -PERSYS_GATEWAY_VAULT_ROLE_ID= -PERSYS_GATEWAY_VAULT_SECRET_ID= -PERSYS_GATEWAY_MONGO_URI=mongodb://admin:admin@mongodb:27017/persys-gateway?authSource=admin +PERSYS_GATEWAY_JWT_SECRET= +PERSYS_GATEWAY_POSTGRES_DSN= +PERSYS_GATEWAY_DEPLOYMENT_MODE= # self-hosted | managed PERSYS_GATEWAY_GITHUB_WEBHOOK_SECRET= PERSYS_GATEWAY_GITHUB_WEBHOOK_SECRET_FILE= PERSYS_GATEWAY_GITHUB_CLIENT_ID= @@ -33,10 +32,6 @@ PERSYS_FORGERY_PIPELINE_STATUS_KEY=forge:pipeline_status PERSYS_FORGERY_OTEL_SERVICE_NAME=persys-forgery PERSYS_FORGERY_VAULT_ADDR=http://vault:8200 PERSYS_FORGERY_VAULT_AUTH_METHOD=approle -PERSYS_FORGERY_VAULT_TOKEN= -PERSYS_FORGERY_VAULT_TOKEN_FILE= -PERSYS_FORGERY_VAULT_ROLE_ID= -PERSYS_FORGERY_VAULT_SECRET_ID= PERSYS_FORGERY_GITHUB_WEBHOOK_SECRET= PERSYS_FORGERY_GITHUB_WEBHOOK_SECRET_FILE= @@ -69,9 +64,6 @@ AUTOMATION_SERVER_TLS_KEY=/etc/persys/certs/persys_automation/persys_automation- AUTOMATION_VAULT_ENABLED=true AUTOMATION_VAULT_ADDR=http://vault:8200 AUTOMATION_VAULT_AUTH_METHOD=approle -AUTOMATION_VAULT_TOKEN= -AUTOMATION_VAULT_APPROLE_ROLE_ID= -AUTOMATION_VAULT_APPROLE_SECRET_ID= AUTOMATION_VAULT_PKI_MOUNT=pki AUTOMATION_VAULT_PKI_ROLE=persys-automation AUTOMATION_VAULT_CERT_TTL=24h @@ -86,15 +78,9 @@ AUTOMATION_FORGERY_PIPELINE_KEY=pipeline_status PERSYS_SCHEDULER_VAULT_ADDR=http://vault:8200 PERSYS_SCHEDULER_VAULT_AUTH_METHOD=approle -PERSYS_SCHEDULER_VAULT_TOKEN= -PERSYS_SCHEDULER_VAULT_ROLE_ID= -PERSYS_SCHEDULER_VAULT_SECRET_ID= COMPUTE_AGENT_VAULT_ADDR=http://vault:8200 COMPUTE_AGENT_VAULT_AUTH_METHOD=approle -COMPUTE_AGENT_VAULT_TOKEN= -COMPUTE_AGENT_VAULT_ROLE_ID= -COMPUTE_AGENT_VAULT_SECRET_ID= # Scheduler ETCD_ENDPOINTS=etcd:2379 @@ -116,9 +102,4 @@ DOCKER_SOCK_PATH=/var/run/docker.sock # Grafana GF_SECURITY_ADMIN_USER=admin GF_SECURITY_ADMIN_PASSWORD=admin -GF_USERS_ALLOW_SIGN_UP=false - -# MongoDB -MONGO_INITDB_ROOT_USERNAME=admin -MONGO_INITDB_ROOT_PASSWORD=admin -MONGODB_DATABASE=persys-gateway +GF_USERS_ALLOW_SIGN_UP=false \ No newline at end of file