.env.example

# Environment Variables
# Copy this file to .env and fill in your API keys
#
# Quick start:
#   cp .env.example .env
#   # Fill in all required sections below, then:
#   python main.py --debug   # runs 1 record to verify your setup
#
# Tip: To verify just the LLM configuration before setting up audio,
# you can run the text-only flow (no ElevenLabs/STT/TTS needed):
#   python scripts/run_text_only.py --record-id 1.1.2

# ==============================================
# Required: API Keys
# ==============================================

# --- ElevenLabs (user simulator) ---
ELEVENLABS_API_KEY=your_elevenlabs_api_key_here

# ElevenLabs Conversational AI agent IDs for user simulation.
# Create a Conversational AI agent at https://elevenlabs.io/conversational-ai and copy its agent ID.
# You need two agents: one with a female voice (persona 1) and one with a male voice (persona 2).
# These are used to simulate different caller personas during benchmark conversations.
ELEVENLABS_USER_AGENT_ID_USER_PERSONA_1=your_elevenlabs_agent_id_for_persona_1
ELEVENLABS_USER_AGENT_ID_USER_PERSONA_2=your_elevenlabs_agent_id_for_persona_2

# --- LLM (assistant + text judge metrics) ---
OPENAI_API_KEY=your_openai_api_key_here

# --- STT/TTS (voice pipeline) ---
# The API key and model for your chosen provider must be passed via the *_PARAMS JSON.

# STT provider: assemblyai | cartesia | deepgram | deepgram-flux | elevenlabs | nvidia | nvidia-baseten | openai
EVA_MODEL__STT=cartesia
# Must include "api_key" and "model" for your chosen provider:
EVA_MODEL__STT_PARAMS='{"api_key": "your_cartesia_api_key", "model": "ink-whisper"}'

# TTS provider: cartesia | chatterbox | elevenlabs | gemini | kokoro | nvidia-baseten | openai | xtts
EVA_MODEL__TTS=cartesia
# Must include "api_key" and "model" for your chosen provider:
EVA_MODEL__TTS_PARAMS='{"api_key": "your_cartesia_api_key", "model": "sonic"}'
# For round-robin load balancing, use "urls" instead of "url":
#   EVA_MODEL__TTS_PARAMS='{"api_key": "...", "model": "sonic", "urls": ["http://server1/v1", "http://server2/v1"]}'

# --- Metrics judge models ---
# Google credentials (audio judge metrics default to Gemini)
GOOGLE_APPLICATION_CREDENTIALS=path/to/your/service-account-credentials.json

# AWS credentials (faithfulness metric defaults to Claude via Bedrock)
AWS_ACCESS_KEY_ID=your_aws_access_key_id_here
AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key_here

# If you only have an OpenAI key, you can skip the AWS credentials above and
# override all text judge models (including faithfulness) to use OpenAI instead
# (results may be less accurate):
#   JUDGE_MODEL=gpt-5.2
# Audio judge metrics (agent_speech_fidelity, user_speech_fidelity) still require
# Gemini. To skip them, run only text-based metrics, e.g.:
#   EVA_METRICS=task_completion,faithfulness,conciseness,turn_taking

# ==============================================
# Required: Model Deployments
# ==============================================
#
# EVA_MODEL_LIST: JSON array of LiteLLM Router deployments.
#   - model_name: alias your code uses (e.g., "gpt-5.2")
#   - litellm_params.model: provider-specific identifier (e.g., "openai/gpt-4o")
#   - Use "os.environ/VAR_NAME" syntax to reference other env vars
#
# EVA needs at minimum:
#   1. An LLM for the assistant (matches EVA_MODEL__LLM below)
#   2. Gemini for audio judge metrics
#   3. Claude (Bedrock) for the faithfulness metric
#
# See docs/llm_configuration.md for more provider examples and load balancing.

EVA_MODEL_LIST='[
  {
    "model_name": "gpt-5.2",
    "litellm_params": {
      "model": "openai/gpt-5.2",
      "api_key": "os.environ/OPENAI_API_KEY",
      "max_parallel_requests": 5
    },
    "model_info": {"base_model": "gpt-5.2"}
  },
  {
    "model_name": "gemini-3.1-pro-preview",
    "litellm_params": {
      "model": "vertex_ai/gemini-3.1-pro-preview",
      "vertex_project": "your-gcp-project-id",
      "vertex_location": "global",
      "vertex_credentials": "os.environ/GOOGLE_APPLICATION_CREDENTIALS",
      "max_parallel_requests": 5
    }
  },
  {
    "model_name": "us.anthropic.claude-opus-4-6",
    "litellm_params": {
      "model": "bedrock/us.anthropic.claude-opus-4-6-v1",
      "aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID",
      "aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY",
      "max_parallel_requests": 5
    }
  }
]'

# --- Optional: additional model deployments ---
# Uncomment and add to EVA_MODEL_LIST above as needed.
#
# Azure OpenAI (alternative to direct OpenAI):
#   {
#     "model_name": "gpt-5.2",
#     "litellm_params": {
#       "model": "azure/gpt-5.2",
#       "api_key": "os.environ/AZURE_OPENAI_API_KEY",
#       "api_base": "https://your-resource.openai.azure.com",
#       "max_parallel_requests": 5
#     },
#     "model_info": {"base_model": "gpt-5.2"}
#   }
#
# Self-hosted model (e.g., vLLM, NVIDIA NIM):
#   {
#     "model_name": "my-model",
#     "litellm_params": {
#       "model": "openai/my-model-name",
#       "api_key": "os.environ/MY_MODEL_KEY",
#       "api_base": "http://my-server:8000/v1",
#       "max_parallel_requests": 5
#     }
#   }
#
# Load balancing (multiple endpoints for the same model):
#   {
#     "model_name": "my-model",
#     "litellm_params": {"model": "openai/my-model", "api_base": "http://server1:8000/v1", ...}
#   },
#   {
#     "model_name": "my-model",
#     "litellm_params": {"model": "openai/my-model", "api_base": "http://server2:8000/v1", ...}
#   }

# ==============================================
# Required: Framework Configuration
# ==============================================

# Domain name — determines dataset, agent config, and scenario paths:
#   data/{domain}_dataset.jsonl
#   configs/agents/{domain}_agent.yaml
#   data/{domain}_scenarios/
# The included sample domain is "airline".
EVA_DOMAIN=airline

# LLM model name — must match a model_name in EVA_MODEL_LIST above.
EVA_MODEL__LLM=gpt-5.2

# ==============================================
# Optional: Alternative LLM Provider Keys
# ==============================================

# Azure OpenAI (alternative to direct OpenAI)
# AZURE_OPENAI_API_KEY=your_azure_openai_api_key_here
# AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/

# Google API key (alternative to service account credentials for Gemini)
# GOOGLE_API_KEY=your_google_api_key_here

# ==============================================
# Optional: Speech-to-Speech / Audio-LLM Configuration
# ==============================================
# Only needed if benchmarking speech-to-speech models.

# EVA_MODEL__S2S=openai
# EVA_MODEL__S2S_PARAMS='{"model": "gpt-realtime-mini", "api_key": ""}'

# EVA_MODEL__AUDIO_LLM=
# EVA_MODEL__AUDIO_LLM_PARAMS='{"url": "", "api_key": ""}'

# ==============================================
# Optional: Execution Settings
# ==============================================

# Maximum number of concurrent conversations (1-100)
EVA_MAX_CONCURRENT_CONVERSATIONS=1

# Conversation timeout in seconds (30-10000)
EVA_CONVERSATION_TIMEOUT_SECONDS=360

# Maximum number of rerun attempts for failed records (1-10)
EVA_MAX_RERUN_ATTEMPTS=3

# Output directory for results (default: output)
EVA_OUTPUT_DIR=output

# Starting port for WebSocket servers (1024-65000)
EVA_BASE_PORT=10000

# Number of ports in the pool (10-500)
EVA_PORT_POOL_SIZE=150

# Comma-separated list of metrics to run, use 'all' to run all metrics
EVA_METRICS=all

# Debug mode: run only 1 record regardless of dataset size (true | false)
EVA_DEBUG=false

# Comma-separated list of specific record IDs to run (empty = run all)
# Example: EVA_RECORD_IDS=1.2.1,1.2.2,1.3.1
EVA_RECORD_IDS=

# Logging level (DEBUG | INFO | WARNING | ERROR | CRITICAL)
EVA_LOG_LEVEL=INFO

# ==============================================
# Optional: Turn Detection & VAD Configuration
# ==============================================
# Fine-tune user turn detection and voice activity detection.
# Leave commented to use smart defaults.

# User turn start strategy: vad | transcription | external
# - vad: Start turn when VAD detects speech (default)
# - transcription: Start turn when STT produces transcription
# - external: Delegate to external service (e.g., Deepgram Flux)
# EVA_MODEL__TURN_START_STRATEGY=vad

# User turn start strategy parameters (JSON)
# EVA_MODEL__TURN_START_STRATEGY_PARAMS='{}'

# User turn stop strategy: turn_analyzer | speech_timeout | external
# - turn_analyzer: Use smart turn analyzer to detect natural turn end (default)
# - speech_timeout: Stop after fixed silence duration
# - external: Delegate to external service
# EVA_MODEL__TURN_STOP_STRATEGY=turn_analyzer

# User turn stop strategy parameters (JSON)
# For speech_timeout: {"user_speech_timeout": 0.8}
# For turn_analyzer: automatically uses smart turn detection
# EVA_MODEL__TURN_STOP_STRATEGY_PARAMS='{}'

# Note: For services with built-in turn detection (e.g., Deepgram Flux), set both to 'external':
#   EVA_MODEL__TURN_START_STRATEGY=external
#   EVA_MODEL__TURN_STOP_STRATEGY=external

# VAD (Voice Activity Detection) analyzer: silero
# EVA_MODEL__VAD=silero

# VAD parameters (JSON)
# - confidence: Minimum confidence threshold (0.0-1.0, default: 0.7)
# - start_secs: Duration to wait before confirming voice start (default: 0.2)
# - stop_secs: Duration to wait before confirming voice stop (default: 0.2)
# - min_volume: Minimum audio volume threshold (0.0-1.0, default: 0.6)
# EVA_MODEL__VAD_PARAMS='{"start_secs": 0.1, "stop_secs": 0.8, "min_volume": 0.6, "confidence": 0.7}'