diff --git a/fas_llm_applications/.devcontainer/Dockerfile b/fas_llm_applications/.devcontainer/Dockerfile new file mode 100644 index 00000000000..6d376e9d498 --- /dev/null +++ b/fas_llm_applications/.devcontainer/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.9-slim-bullseye AS base + +RUN set -x + +RUN apt-get update \ + && apt-get -y install curl \ + && apt-get -y install net-tools \ + && apt-get -y install procps \ + && apt-get -y install build-essential \ + && apt-get -y install docker.io + +# Install notebook depenency +RUN pip install ipython ipykernel +RUN ipython kernel install --user --name promptflow + +# FROM base AS fas_llm_applications +COPY requirements.txt . +RUN pip install -r requirements.txt + +RUN set +x + +CMD bash diff --git a/fas_llm_applications/.devcontainer/README.md b/fas_llm_applications/.devcontainer/README.md new file mode 100644 index 00000000000..17397c2eaec --- /dev/null +++ b/fas_llm_applications/.devcontainer/README.md @@ -0,0 +1,13 @@ +# Devcontainer for promptflow +To facilitate your promptflow project development and empower you to work on LLM projects using promptflow more effectively, +we've configured the necessary environment for developing promptflow projects and utilizing flows through the dev container feature. +You can seamlessly initiate your promptflow project development and start leveraging flows by simply using the dev container feature via VS Code or Codespaces. + +## Use Github Codespaces +Use codespaces to open promptflow repo, it will automatically build the dev containers environment and open promptflow with dev containers. You can just click: [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/microsoft/promptflow?quickstart=1) + +## Use local devcontainer +Use vscode to open promptflow repo, and install vscode extension: Dev Containers and then open promptflow with dev containers. +![devcontainer](./devcontainers.png) +**About dev containers please refer to: [dev containers](https://code.visualstudio.com/docs/devcontainers/containers)** + diff --git a/fas_llm_applications/.devcontainer/devcontainer.json b/fas_llm_applications/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..abba2f26bb7 --- /dev/null +++ b/fas_llm_applications/.devcontainer/devcontainer.json @@ -0,0 +1,40 @@ +{ + "name": "Promptflow-FAS_LLM_Applications-Python39", + "context": ".", + "dockerFile": "Dockerfile", + "runArgs": [ + "-v", + "/var/run/docker.sock:/var/run/docker.sock" + ], + "postStartCommand": "/bin/bash -c 'set -e; eval \"$(python digital_latin_project/scripts/load_env_to_shell.py)\" && PYTHON_ENV_SOURCE_CMD=\"eval \\\"\\$(python digital_latin_project/scripts/load_env_to_shell.py)\\\"\" && grep -qF \"${PYTHON_ENV_SOURCE_CMD}\" ~/.bashrc || echo \"${PYTHON_ENV_SOURCE_CMD}\" >> ~/.bashrc && python _connections_manager_/setup_all_shared_connections.py'", + "postCreateCommand": "pip install -r requirements.txt", + "remoteEnv": { + "HOST_PROJECT_PATH": "${localWorkspaceFolder}", + "PYTHONPATH": "/workspaces/promptflow:${containerEnv:PYTHONPATH}" + }, + "customizations": { + "codespaces": { + "openFiles": [ + "README.md", + "examples/README.md" + ] + }, + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "python.pythonPath": "${containerWorkspaceFolder}/.venv/bin/python", + "python.terminal.activateEnvironment": true // Automatically activate venv in terminal + }, + "extensions": [ + "ms-python.python", + "ms-toolsai.vscode-ai", + "ms-toolsai.jupyter", + "redhat.vscode-yaml", + "prompt-flow.prompt-flow" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": {} + } +} \ No newline at end of file diff --git a/fas_llm_applications/.devcontainer/devcontainer.json.template b/fas_llm_applications/.devcontainer/devcontainer.json.template new file mode 100644 index 00000000000..d7da6075735 --- /dev/null +++ b/fas_llm_applications/.devcontainer/devcontainer.json.template @@ -0,0 +1,40 @@ +{ + "name": "Promptflow-Python39", + "context": ".", + "dockerFile": "Dockerfile", + "runArgs": [ + "-v", + "/var/run/docker.sock:/var/run/docker.sock" + ], + "postStartCommand": "/bin/bash -c 'set -e; rm -f ~/.promptflow/pf.sqlite && eval \"$(python ../scripts/load_env_to_shell.py)\" && PYTHON_ENV_SOURCE_CMD=\"eval \\\"\\$(python ../scripts/load_env_to_shell.py)\\\"\" && grep -qF \"${PYTHON_ENV_SOURCE_CMD}\" ~/.bashrc || echo \"${PYTHON_ENV_SOURCE_CMD}\" >> ~/.bashrc && python fas_llm_applications/_connections_manager_/setup_all_shared_connections.py'", + "postCreateCommand": "pip install -r fas_llm_applications/digital_latin_project/requirements.txt", + "remoteEnv": { + "HOST_PROJECT_PATH": "${localWorkspaceFolder}", + "PYTHONPATH": "/workspaces/promptflow:${containerEnv:PYTHONPATH}" + }, + "customizations": { + "codespaces": { + "openFiles": [ + "README.md", + "examples/README.md" + ] + }, + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "ms-python.python", + "ms-toolsai.vscode-ai", + "ms-toolsai.jupyter", + "redhat.vscode-yaml", + "prompt-flow.prompt-flow" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": {} + }, + // "containerEnv": {} Locally, .env file can be added here. + // In Github Codespaces, leave this commented out or removed as secure env variables will be loaded from .env file or Github Secure variables +} \ No newline at end of file diff --git a/fas_llm_applications/.devcontainer/devcontainers.png b/fas_llm_applications/.devcontainer/devcontainers.png new file mode 100644 index 00000000000..8c57ccd6d7f Binary files /dev/null and b/fas_llm_applications/.devcontainer/devcontainers.png differ diff --git a/fas_llm_applications/.devcontainer/requirements.txt b/fas_llm_applications/.devcontainer/requirements.txt new file mode 100644 index 00000000000..94e3712e7bb --- /dev/null +++ b/fas_llm_applications/.devcontainer/requirements.txt @@ -0,0 +1,3 @@ +# This would be used if you're setting up the environment for the entire solution. +promptflow[azure]>=1.9.0 +promptflow-tools diff --git a/fas_llm_applications/.gitignore b/fas_llm_applications/.gitignore new file mode 100644 index 00000000000..4a104d97aa7 --- /dev/null +++ b/fas_llm_applications/.gitignore @@ -0,0 +1,24 @@ +# .env file for local environment variables (contains sensitive data) +.env + +# Python bytecode (auto-generated, not meant for version control) +__pycache__/ +*.pyc + +# PromptFlow's internal metadata/cache for flows +# These are generated by the PromptFlow UI/CLI and generally not relevant for committing, +# except for flow.tools.json which is needed for Azure ML component integration. +# The main PromptFlow project also ignores these files, with the same exception. +digital_latin_project/flows/**/.promptflow/ +# Re-include flow.tools.json as it's often required for Azure ML integration +!digital_latin_project/flows/**/.promptflow/flow.tools.json + +# NOTE: We are *intentionally* not ignoring .log or .txt files anywhere, +# and are explicitly including all contents of 'results/' and 'prepared_reports/' +# for now, as we are still in the process of understanding PromptFlow behavior +# and using the repo as a temporary data store for outputs. +# If these folders or file types were to be ignored later, specific rules would be added here. + +# Example: If we later decide to ignore *all* JSON results, but still keep logs and reports: +# digital_latin_project/results/*.json +# digital_latin_project/prepared_reports/*.json diff --git a/fas_llm_applications/README.md b/fas_llm_applications/README.md new file mode 100644 index 00000000000..70231440c5f --- /dev/null +++ b/fas_llm_applications/README.md @@ -0,0 +1,259 @@ +# `fas_llm_applications` - Harvard FAS LLM Applications for Promptflow + +This repository contains a collection of PromptFlow-based applications and utilities designed for integrating Large Language Models (LLMs) into Harvard FAS (Faculty of Arts and Sciences) workflows. It provides a structured environment for developing, deploying, and managing LLM-powered solutions, with a focus on secure credential handling and consistent setup. + +## Table of Contents + +- [`fas_llm_applications` - Harvard FAS LLM Applications for Promptflow](#fas_llm_applications---harvard-fas-llm-applications-for-promptflow) + - [Table of Contents](#table-of-contents) + - [Features](#features) + - [Project Structure](#project-structure) + - [Getting Started](#getting-started) + - [Prerequisites](#prerequisites) + - [Local Development](#local-development) + - [GitHub Codespaces](#github-codespaces) + - [Environment Variables \& Secrets](#environment-variables--secrets) + - [PromptFlow Connections](#promptflow-connections) + - [Generated Data \& Reports](#generated-data--reports) + - [Contributing](#contributing) + - [License](#license) + +## Features + +* **Modular PromptFlow Applications:** Organized PromptFlows for various LLM use cases. + +* **Centralized Connection Management:** Secure and consistent setup for LLM service connections (e.g., Google Gemini, Azure OpenAI, AWS Bedrock). + +* **Dev Container Support:** Pre-configured development environment using `devcontainer.json` for easy onboarding and consistent setups across developers. + +* **Secure Credential Handling:** Guidance and mechanisms for managing sensitive API keys and secrets out of version control. + +* **LLM Run Reporting:** Tools to generate detailed reports for PromptFlow runs, aiding in monitoring and evaluation. + +## Project Structure + +This project is organized into modular components to enhance maintainability and clarity. + +``` +fas_llm_applications/ +├── _connections_manager_/ # Python package for managing PromptFlow connections. +│ ├── __init__.py # Marks this directory as a Python package. +│ ├── aws_connection_utils.py # Utilities for managing AWS-specific PromptFlow connections (e.g., Bedrock). +│ ├── client_utils.py # Utilities related to the PromptFlow Client (PFClient) itself. +│ ├── common_secrets_loader.py # Handles loading common secrets or environment variables for connection setup. +│ ├── gemini_connection_utils.py # Utilities for managing Google Gemini-specific PromptFlow connections. +│ ├── keyring_utils.py # Utilities for interacting with the system's keyring service for credentials. +│ └── setup_all_shared_connections.py # The main orchestration script to set up all shared PromptFlow connections. +├── digital_latin_project/ # Sub-project/application specific to the Digital Latin Project. +│ ├── __init__.py # Marks this as a Python package. +│ ├── data/ # Contains input datasets and files for flows and testing. +│ │ ├── *.jsonl # JSONL formatted input data for batch runs. +│ │ ├── *.csv # CSV files for variant testing or specific test cases. +│ │ └── *.txt # Log or output files from previous test runs. +│ ├── flows/ # PromptFlow definitions specific to the Digital Latin Project. +│ │ ├── basic_claude_llm_flow/ # Example flow using AWS Claude models. +│ │ │ ├── nodes/ # Python tools/nodes for this specific flow. +│ │ │ │ ├── __init__.py +│ │ │ │ └── aws_llm_invocation.py +│ │ │ ├── dl_prompt_node.jinja2 # Jinja2 template for the prompt used in the flow. +│ │ │ └── flow.dag.yaml # The core PromptFlow definition file. +│ │ ├── basic_deepseek_llm_flow/ # Example flow using Deepseek models. +│ │ │ ├── nodes/ +│ │ │ │ ├── __init__.py +│ │ │ │ └── aws_llm_invocation.py +│ │ │ ├── dl_prompt_node.jinja2 +│ │ │ └── flow.dag.yaml +│ │ ├── basic_gemini_llm_flow/ # Example flow using Google Gemini models. +│ │ │ ├── nodes/ +│ │ │ │ ├── __init__.py +│ │ │ │ └── gemini_llm_invocation.py +│ │ │ ├── dl_prompt_node.jinja2 +│ │ │ └── flow.dag.yaml +│ │ ├── multi_llm_flow/ # Example of a flow designed to utilize multiple LLMs. +│ │ │ ├── default.flow_test.yaml +│ │ │ ├── dl_prompt_node.jinja2 +│ │ │ └── flow.dag.yaml +│ │ ├── multi_llm_parallel_flow/ # Example of a flow designed for parallel LLM execution. +│ │ │ ├── nodes/ +│ │ │ │ ├── __init__.py +│ │ │ │ └── multi_llm_invocation.py +│ │ │ └── flow.dag.yaml +│ │ └── __init__.py # Marks 'flows' as a Python package. +│ ├── prepared_reports/ # Generated Markdown manifests for LLM run reports. +│ │ └── *.md +│ ├── prompts/ # Organized collection of Jinja2 prompt templates. +│ │ ├── system/ # System prompt templates for various LLM roles. +│ │ │ └── *.jinja2 +│ │ └── user/ # User prompt templates for different interaction styles. +│ │ └── *.jinja2 +│ ├── results/ # Generated JSON reports and other detailed outputs from LLM runs. +│ │ ├── *.json +│ │ └── report_counter.txt # Helper file for sequential report numbering. +│ ├── scripts/ # Specific scripts related to the Digital Latin Project's operations. +│ │ ├── generate_prepared_reports.py # Script to generate final Markdown reports from run data. +│ │ ├── generate_report.py # Core script for generating individual LLM reports. +│ │ └── load_env_to_shell.py # Helper script for loading environment variables (if needed outside dev container). +│ ├── tests/ # Tests specific to the Digital Latin Project. +│ │ └── .gitkeep # Placeholder to ensure the empty directory is tracked by Git. +│ ├── tools/ # Custom PromptFlow tools developed for this project. +│ │ └── prompt_selector_tool.py +│ ├── utilities/ # General utility modules for the Digital Latin Project. +│ │ ├── __init__.py +│ │ └── prompt_registry_util.py +│ └── __init__.py # Marks 'digital_latin_project' as a Python package. +├── __init__.py # Marks 'fas_llm_apps' as a Python package, allowing internal imports. +├── .devcontainer/ # VS Code Dev Container configuration for a consistent development environment. +│ ├── devcontainer.json # Main Dev Container definition (installs dependencies, sets up environment). +│ └── devcontainer.json.template # Template for sensitive variable handling within the dev container. +├── scripts/ # General utility and execution scripts for the entire `fas_llm_apps` project. +│ ├── run_flow_example.py # Example script to run a generic PromptFlow (if applicable). +│ └── ... +├── tests/ # Top-level unit and integration tests for the entire `fas_llm_apps` project. +│ └── .gitkeep # Placeholder to ensure the empty directory is tracked by Git. +├── .env.example # Template for required environment variables (copy to .env for local use). +├── .gitignore # Specifies files to be ignored by Git (e.g., temporary files, secrets). +├── README.md # This file, providing an overview and guide for the `fas_llm_apps` project. +├── requirements.txt # Python package dependencies for the project. +└── ... +``` + +## Getting Started + +### Prerequisites + +* **Git:** For cloning the repository. + +* **Docker Desktop:** Required for local Dev Container development. + +* **VS Code:** With the "Dev Containers" extension installed. + +* **LLM Service Access:** Depending on the LLMs you integrate (e.g., Google Gemini API Key, Azure Machine Learning Workspace, Azure OpenAI deployment, AWS credentials for Bedrock). + + +### Local Development + +This project's VS Code Dev Container configuration is designed to align with the development environment of the PromptFlow repository, ensuring consistency with PromptFlow best practices. + +1. **Ensure Docker is Running:** + Before starting, make sure your Docker Desktop application (or Docker daemon) is running on your machine. The Dev Container relies on Docker to build and run its containerized environment. + +2. **Clone the Repository:** + ```bash + git clone [https://github.com/your-org/fas_llm_apps.git](https://github.com/your-org/fas_llm_apps.git) + cd fas_llm_apps + ``` + +3. **Create `.env` file for Local Variables:** + Copy the `.env.example` file to `.env` in the root of your `fas_llm_apps` directory. Fill in your actual, non-sensitive configuration values. **Ensure `.env` is listed in your `.gitignore` and never committed, as it may contain sensitive information.** + ```bash + cp .env.example .env + # Open .env and fill in your values + ``` + +4. **Opening the Project in its Dev Container:** + + To ensure you are working within the dedicated Dev Container for *this* `fas_llm_apps` project (and not a parent repository's Dev Container), follow these specific steps: + + a. **Close Current VS Code Window (if open on the parent `promptflow` repo):** + If your VS Code is currently open to the higher-level `promptflow/` directory, close that VS Code window first (`File > Close Folder` or `File > Close Window`). + + b. **Open Only the `fas_llm_apps` Folder in VS Code:** + * Launch VS Code. + * Go to **File > Open Folder...** (or **Code > Open Folder...** on macOS). + * **Navigate to and select specifically the `fas_llm_applications` directory.** This is crucial. Your VS Code workspace's root must be `fas_llm_applications/`, not the parent `promptflow/` directory. + * Click "Open". + + c. **Reopen in Container (Automatic Prompt):** + * After opening the `fas_llm_applications` folder, VS Code should detect its `.devcontainer` configuration. You will likely see a notification pop up in the bottom-right corner (or a small green remote indicator in the bottom-left status bar) asking: **"Folder contains a Dev Container configuration. Reopen in Container?"** + * Click on **"Reopen in Container"** in that notification. + + d. **Reopen in Container (Manual Trigger if Prompt is Missed):** + * If you don't see the automatic prompt for any reason, click the **green remote indicator** in the bottom-left corner of the VS Code status bar (it looks like `<>`). + * From the command palette that appears at the top, select **"Reopen in Container"** (or **"Open Folder in Container..."** if it's the very first time you're attempting this for the folder and it's not prompting automatically). Then, select the `fas_llm_applications` folder again. + + e. **Container Build Process:** + VS Code will then proceed to build (if necessary) and launch your `fas_llm_apps` Dev Container. This initial setup might take a few minutes. You can monitor the progress in the VS Code status bar. + + f. **Ready to Develop:** + Once the process completes, your VS Code window will reload, and you'll be connected to the Dev Container specific to `fas_llm_apps`. The green remote indicator will confirm you are "Dev Container: [Your Container Name]". You can open the integrated terminal (`Ctrl+\` or `Cmd+\``) and verify your environment. + +5. **Initialize PromptFlow Connections:** + Your `setup_all_shared_connections.py` script (located in `_connections_manager_`) will be executed automatically during Dev Container startup (via `postCreateCommand`) or can be run manually to ensure your PromptFlow connections are correctly set up based on your environment variables. + + ```bash + # This might be run automatically during devcontainer startup. + # Otherwise, you can run it manually in the terminal: + python -m fas_llm_apps._connections_manager_.setup_all_shared_connections + ``` + +### GitHub Codespaces + +GitHub Codespaces provides a ready-to-use cloud development environment pre-configured for this project: + +1. **Open in Codespaces:** + + * Navigate to your repository on GitHub. + + * Click on the green "Code" button and select the "Codespaces" tab. + + * Choose to create a new codespace for the `fas_llm_apps` directory. + +2. **Define Secrets:** + For **sensitive variables** (e.g., API keys, service principals), define them as **repository secrets** in GitHub. These are securely injected into your Codespace environment. + + * Go to `Repository Settings > Secrets and variables > Codespaces`. + + * Add new repository secrets corresponding to the variable names listed in `.env.example` (e.g., `AZURE_SUBSCRIPTION_ID`, `GOOGLE_GEMINI_API_KEY`, `AWS_ACCESS_KEY_ID`). + +3. **Environment Setup:** + Codespaces will automatically: + + * Load secrets defined in your GitHub repository into the Codespace environment. + + * Install Python dependencies from `requirements.txt`. + + * Your PromptFlow connections will be set up automatically based on these environment variables during the Codespace startup. + +## Environment Variables & Secrets + +This project relies on several environment variables for configuration and credentials. Refer to `.env.example` for a complete list of required environment variables. + +* **For Local Development:** Copy `.env.example` to `.env` in the root of `fas_llm_apps/` and fill in your values. This `.env` file should **NOT** be committed to Git. + +* **For GitHub Codespaces:** + + * **Sensitive variables:** Define these as **repository secrets** under `Repository Settings > Secrets and variables > Codespaces`. + + * **Other variables:** You can still use a `.env` file in the Codespace workspace root (copied from `.env.example`) to automatically load non-sensitive variables into your Codespace environment. + +## PromptFlow Connections + +PromptFlow requires connections to various LLM services (e.g., Azure OpenAI, AWS Bedrock, Google Gemini). This project centralizes connection management through the `_connections_manager_` package: + +* It handles the initialization of the `PFClient` and ensures the existence and correct configuration of all necessary PromptFlow connections. + +* Connections are configured using environment variables, securely pulling credentials. + +* The `setup_all_shared_connections.py` script verifies your `keyring` setup and creates/updates PromptFlow connections as needed. This script is designed to run automatically during development environment setup. + +## Generated Data & Reports + +During development and testing, this project generates various data files and reports, which are currently **tracked within the repository** for ease of understanding and analysis during the learning phase. + +* **`digital_latin_project/data/`**: Contains various input data files used for batch runs and testing. + +* **`digital_latin_project/results/`**: Stores detailed JSON reports and other raw outputs from PromptFlow runs. + +* **`digital_latin_project/prepared_reports/`**: Contains human-readable Markdown manifest files for generated LLM run reports. + +* **Log and Text Files**: All `.log` and `.txt` files generated by PromptFlow flows or scripts are currently included to aid in debugging and understanding flow behavior. + +As the project matures, these generated files may be moved to a dedicated data store or excluded from version control via `.gitignore` updates. + +## Contributing + +Contributions are welcome! Please refer to `CONTRIBUTING.md` (if it exists) for guidelines on how to contribute. + +## License + +TBD. diff --git a/fas_llm_applications/__init__.py b/fas_llm_applications/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/_connections_manager_/__init__.py b/fas_llm_applications/_connections_manager_/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/_connections_manager_/aws_connection_utils.py b/fas_llm_applications/_connections_manager_/aws_connection_utils.py new file mode 100644 index 00000000000..7f5203e5bdf --- /dev/null +++ b/fas_llm_applications/_connections_manager_/aws_connection_utils.py @@ -0,0 +1,59 @@ +import sys +from promptflow.client import PFClient +from promptflow.connections import CustomConnection + + +def ensure_promptflow_aws_connection( + access_key: str, + secret_key: str, + region: str, + pf_client: PFClient, + connection_name: str = "bedrock_connection", + service_name: str = "bedrock-runtime", +) -> CustomConnection: + """ + Ensures a Promptflow CustomConnection for AWS exists and is up-to-date. + Returns the connection object. + + Args: + access_key (str): The AWS access key ID for authentication. + secret_key (str): The AWS secret access key for authentication. + region (str): The AWS region where the AWS service is located (i.e. 'us-east-1). + connection_name (str, optional): The desired name for the Promptflow CustomConnection. Defaults to "bedrock_connection". + service_name (str, optional): The AWS service name this connection is primarily for. Defaults to "bedrock-runtime". + + Returns: + CustomConnection: The created or updated Promptflow CustomConnection object. + """ + if not access_key or not secret_key: + raise ValueError("AWS credentials are required to create Promptflow connection.", file=sys.stderr) + + # Create the connection object with desired properties + connection = CustomConnection( + name=connection_name, + secrets={ + "aws_access_key_id": access_key, + "aws_secret_access_key": secret_key + }, + configs={ + "region_name": region, + "service_name": service_name + } + ) + + try: + existing_connection = pf_client.connections.get(name=connection_name) + if existing_connection.secrets == connection.secrets and existing_connection.configs == connection.configs: + return existing_connection + else: + updated_connection = pf_client.connections.create_or_update(connection) + return updated_connection + except Exception as e: + if type(e).__name__ == 'ConnectionNotFoundError': + created_connection = pf_client.connections.create_or_update(connection) + return created_connection + except Exception as e: + raise RuntimeError(f"Failed to manage Promptflow connection '{connection_name}': {e}") + + print(f"Connection '{connection_name}' successful.", file=sys.stdout) + \ No newline at end of file diff --git a/fas_llm_applications/_connections_manager_/client_utils.py b/fas_llm_applications/_connections_manager_/client_utils.py new file mode 100644 index 00000000000..eeefb23a867 --- /dev/null +++ b/fas_llm_applications/_connections_manager_/client_utils.py @@ -0,0 +1,21 @@ +from promptflow.client import PFClient + +_pf_client_instance = None + +def get_pf_client() -> PFClient: + global _pf_client_instance + + if _pf_client_instance is None: + _pf_client_instance = _initialize_pf_client() + return _pf_client_instance + +def _initialize_pf_client() -> PFClient: + try: + _pf_client_instance = PFClient() + except Exception as e: + raise RuntimeError(f"Error occured when initializing Promptflow client: {e}") + + if _pf_client_instance is None: + raise RuntimeError("Promptflow client initialization failed") + + return _pf_client_instance diff --git a/fas_llm_applications/_connections_manager_/common_secrets_loader.py b/fas_llm_applications/_connections_manager_/common_secrets_loader.py new file mode 100644 index 00000000000..7e36742c092 --- /dev/null +++ b/fas_llm_applications/_connections_manager_/common_secrets_loader.py @@ -0,0 +1,54 @@ +import os +from dotenv import load_dotenv +from typing import Optional + +# This flag ensures load_dotenv is called only once by this module's main loader +_dotenv_loaded_flag = False + +def load_environment_variables(env_path: Optional[str] = None, override: bool = False): + """ + Loads environment variables from a .env file into os.environ. + This function should be called once at the very beginning of your application's + startup when running locally. + + Args: + env_path (str, optional): Explicit path to the .env file. + If None, load_dotenv will search for it + in the current directory and parent directories. + override (bool): If True, existing environment variables will be overwritten + by values from the .env file. Default is False. + """ + global _dotenv_loaded_flag + + if not _dotenv_loaded_flag: + if env_path and os.path.exists(env_path): + load_dotenv(dotenv_path=env_path, override=override) + else: + load_dotenv(override=override) # Searches in current directory and parent directories + _dotenv_loaded_flag = True + print("Environment variables loaded.") + else: + print("Environment variables already loaded by common_secrets_loader.") + +def get_env_var(var_name: str, required: bool = True, default: Optional[str] = None) -> str: + """ + Retrieves an environment variable from os.environ. + + Args: + var_name (str): The name of the environment variable. + required (bool): If True, raises ValueError if the variable is not set. + Default is True. + default (str, optional): A default value to return if the variable is not set + and `required` is False. + + Returns: + str: The value of the environment variable. + + Raises: + ValueError: If `required` is True and the variable is not set. + """ + value = os.getenv(var_name, default) + if required and (value is None or value == ""): # Check for None or empty string + raise ValueError(f"Required environment variable '{var_name}' is not set or is empty.") + return value + diff --git a/fas_llm_applications/_connections_manager_/gemini_connection_utils.py b/fas_llm_applications/_connections_manager_/gemini_connection_utils.py new file mode 100644 index 00000000000..c4bc154124c --- /dev/null +++ b/fas_llm_applications/_connections_manager_/gemini_connection_utils.py @@ -0,0 +1,52 @@ +import sys +from promptflow.client import PFClient +from promptflow.connections import CustomConnection + + +def ensure_promptflow_gemini_connection( + api_key: str, + base_url: str, + pf_client: PFClient, + connection_name: str = "gemini_connection", # Default connection name for this HUIT setup +) -> CustomConnection: + """ + Ensures a Promptflow CustomConnection exists for the HUIT AI Services Gemini API Gateway. + If the connection does not exist, it creates it. If it exists, it updates it. + + Args: + api_key (str): The API key obtained from the Harvard API Portal (for x-api-key header). + base_url (str): The base URL for the HUIT AI Services Gemini API (e.g., https://go.apis.huit.harvard.edu/ais-google-gemini). + conn_name (str, optional): The desired name for the Promptflow CustomConnection. Defaults to "gemini_connection". + + Returns: + CustomConnection: The created or updated Promptflow CustomConnection object. + """ + if not api_key or not base_url: + raise ValueError("Gemini credentials are required to create Promptflow connection.", file=sys.stderr) + + # Create the connection object with desired properties + connection = CustomConnection( + name=connection_name, + secrets={"api_key": api_key}, + configs={"base_url": base_url}, + description=f"HUIT AI Services Gemini API Connection via {base_url}" + ) + + try: + existing_connection = pf_client.connections.get(name=connection_name) + # if (existing_connection.secrets.get("api_key") == connection.secrets["api_key"] and + # existing_connection.configs.get("base_url") == connection.configs["base_url"]): + if existing_connection.secrets == connection.secrets and existing_connection.configs == connection.configs: + return existing_connection + else: + updated_connection = pf_client.connections.create_or_update(connection) + return updated_connection + except Exception as e: + if type(e).__name__ == 'ConnectionNotFoundError': + created_connection = pf_client.connections.create_or_update(connection) + return created_connection + except Exception as e: + raise RuntimeError(f"Failed to manage Promptflow connection '{connection_name}': {e}", file=sys.stderr) + + print(f"Connection '{connection_name}' successful.", file=sys.stdin) + diff --git a/fas_llm_applications/_connections_manager_/keyring_utils.py b/fas_llm_applications/_connections_manager_/keyring_utils.py new file mode 100644 index 00000000000..ecf18134cbd --- /dev/null +++ b/fas_llm_applications/_connections_manager_/keyring_utils.py @@ -0,0 +1,88 @@ +# --- Keyring Backend Configuration --- +# Configures the 'keyring' library to use the 'keyrings.cryptfile' backend. +# This ensures sensitive connection parameters (like API keys or passwords) +# for the connection objects created later are stored securely in an +# encrypted, password-protected file on the local file system. This method +# provides a portable and secure way to manage secrets without relying on +# system-specific keyring services. +# keyring.set_keyring(keyrings.cryptfile.cryptfile.CryptFileKeyring()) +# ----------------------------------- +import os +import keyring +import keyrings.cryptfile.cryptfile +import sys + + +def verify_keyring(): + """ + Verifies that the configured keyring backend (keyrings.cryptfile) + is properly initialized and accessible. This check is crucial + before creating shared connections that depend on securely + retrieving credentials from the keyring. + """ + + print("Verifying Keyring Set Up", file=sys.stderr) + + # Checks PYTHON_KEYRING_BACKEND environment variable + python_keyring_backend_env = os.environ.get("PYTHON_KEYRING_BACKEND") + expected_backend_path = "keyrings.cryptfile.cryptfile.CryptFileKeyring" + + # Verify Keyring back end path + if python_keyring_backend_env != expected_backend_path: + raise RuntimeError(f"PYTHON_KEYRING_BACKEND must be {expected_backend_path}, got {python_keyring_backend_env}", file=sys.stderr) + + # Get the master password from the environment variable + master_password = os.environ.get("KEYRING_CRYPTFILE_PASSWORD") + if not master_password: + raise RuntimeError("KEYRING_CRYPTFILE_PASSWORD environment variable not set or is empty.") + + # Create the CryptFileKeyring instance and assign the master password directly + # This prevents the interactive prompt. + kr = keyrings.cryptfile.cryptfile.CryptFileKeyring() + kr.keyring_key = master_password + + # Set this instance as the default keyring backend for this process + keyring.set_keyring(kr) + + # Tests keyring is set + active_keyring = keyring.get_keyring() + if active_keyring != kr: + raise RuntimeError("Error: Active keyring backend found is not set to the recommended value. Found: {active_keyring}") + + # Test storing and retrieving a password + service_name = "my_promptflow_app" + username = "aws_user" + password = "super_secret_aws_key" + + try: + keyring.set_password(service_name, username, password) + retrieved_password = keyring.get_password(service_name, username) + + if retrieved_password != password: + print("Error: Retrieved password does not match original!", file=sys.stderr) + sys.exit(1) + else: + print("Keyring verification successful!", file=sys.stderr) + + except Exception as e: + print(f"ERROR: Keyring test failed: {e}", file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + # Ensure .env is loaded for independent testing + script_dir = os.path.dirname(os.path.abspath(__file__)) + dotenv_path = os.path.abspath(os.path.join(script_dir, "../digital_latin_project/.env")) + + # Add fas_llm_applications parent dir to sys.path if not already there for common_secrets_loader + fas_llm_applications_parent_dir = os.path.abspath(os.path.join(script_dir, "../../..")) + if fas_llm_applications_parent_dir not in sys.path: + sys.path.insert(0, fas_llm_applications_parent_dir) + + try: + from fas_llm_applications._connections_manager_.common_secrets_loader import load_environment_variables + load_environment_variables(env_path=dotenv_path, override=True) + except Exception as e: + print(f"ERROR: Could not load .env for independent test: {e}", file=sys.stderr) + sys.exit(1) + + verify_keyring() diff --git a/fas_llm_applications/_connections_manager_/setup_all_shared_connections.py b/fas_llm_applications/_connections_manager_/setup_all_shared_connections.py new file mode 100644 index 00000000000..ddab2fccb4f --- /dev/null +++ b/fas_llm_applications/_connections_manager_/setup_all_shared_connections.py @@ -0,0 +1,80 @@ +""" +This script serves as the main entry point for the FAS LLM Flows +It sets up the necessary import paths and initializes core connections. +""" +import os +import sys + +# This ensures that the 'fas_llm_applications' package root is on sys.path +# when the script is run directly from its subdirectory. +script_directory = os.path.dirname(os.path.abspath(__file__)) # This script +fas_root_dir = os.path.abspath(os.path.join(script_directory, '../')) # fas_llm_applications root directory +print("fas_root_dir", fas_root_dir, file=sys.stderr) +flow_root_dir = os.path.abspath(os.path.join(script_directory, '../digital_latin_project/')) # project directory +dotenv_file_path = os.path.join(flow_root_dir, '.env') + +sys.path.insert(0, '/Users/kevingray/codebase/harvard-atg/promptflow') + +from dotenv import load_dotenv # ruff: noqa: E402 + +print("Current sys.path:", sys.path, file=sys.stderr) +from fas_llm_applications._connections_manager_.aws_connection_utils import ensure_promptflow_aws_connection # ruff: noqa: E402 +from fas_llm_applications._connections_manager_.gemini_connection_utils import ensure_promptflow_gemini_connection # ruff: noqa: E402 +from fas_llm_applications._connections_manager_.common_secrets_loader import get_env_var # ruff: noqa: E402 +from fas_llm_applications._connections_manager_.keyring_utils import verify_keyring # ruff: noqa: E402 +from fas_llm_applications._connections_manager_.client_utils import get_pf_client # ruff: noqa: E402 + +AWS_CONNECTION_NAME = "bedrock_connection" +GEMINI_CONNECTION_NAME = "gemini_connection" +BEDROCK_SERVICE_NAME = "bedrock-runtime" + +load_dotenv(dotenv_path=dotenv_file_path, override=True) + +def setup_all_connections(): + print("\n--- Ensuring Connections ---") + + # This is to setup Keyring in linux as a place to hold sensitive parameters of connection objects created below. + verify_keyring() + + # Get the pf client singleton + promptflow_client = get_pf_client() + + # Ensure AWS Connection + try: + # Get env vars using secrets load utility after dotenv is loaded + aws_access_key = get_env_var("AWS_AI_WORKFLOW_CORE_DEV_ID") + aws_secret_key = get_env_var("AWS_AI_WORKFLOW_CORE_DEV_SECRET") + aws_region = get_env_var("AWS_DEFAULT_REGION") or "us-east-1" + ensure_promptflow_aws_connection( + access_key=aws_access_key, + secret_key=aws_secret_key, + region=aws_region, + pf_client=promptflow_client, + connection_name=AWS_CONNECTION_NAME + ) + print("Successful Connection: AWS Bedrock") + except Exception as e: + print(f"Failed to setup AWS Bedrock Connection: {e}") + + # Ensure Gemini Connection + try: + gemini_api_key = get_env_var("GEMINI_API_KEY") + gemini_base_url = get_env_var("GEMINI_BASE_URL") + + if not all([gemini_api_key, gemini_base_url]): + raise ValueError(f"Missing required environment variables for Gemini connection. API Key: {gemini_api_key}, Base URL: {gemini_base_url}") + + ensure_promptflow_gemini_connection( + api_key=gemini_api_key, + base_url=gemini_base_url, + pf_client=promptflow_client, + connection_name=GEMINI_CONNECTION_NAME + ) + print("Successful Connection: Google Gemini") + except Exception as e: + print(f"Failed to setup Gemini Connection: {e}") + + print("--- All Connections Successful ---") + +if __name__ == "__main__": + setup_all_connections() diff --git a/fas_llm_applications/digital_latin_project/.env.example b/fas_llm_applications/digital_latin_project/.env.example new file mode 100644 index 00000000000..6752ecaced7 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/.env.example @@ -0,0 +1,12 @@ +# For Local Development, rename this file to .env for variables to be loaded. Replace place_holder with your actual credentials for each key +# For GitHub Codespaces: +# - Sensitive variables: Define as repository secrets under Settings > Secrets and variables > Codespaces. +# - Other variables: Rename this file to .env. Variables defined here will be automatically loaded into your Codespace environment + +AWS_AI_WORKFLOW_CORE_DEV_ID=your_aws_access_key_id +AWS_AI_WORKFLOW_CORE_DEV_SECRET=your_aws_access_key_password +AWS_DEFAULT_REGION=your_region +PYTHON_KEYRING_BACKEND=keyrings.cryptfile.cryptfile.CryptFileKeyring +KEYRING_CRYPTFILE_PASSWORD=your_keyring_kryptfile_password +GEMINI_API_KEY=your_huit_harvard_port_gemini_api_key +GEMINI_BASE_URL=https://go.apis.huit.harvard.edu/ais-google-gemini \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/README.md b/fas_llm_applications/digital_latin_project/README.md new file mode 100644 index 00000000000..1c641109a2b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/README.md @@ -0,0 +1,148 @@ +# `digital_latin_project` - LLM Applications for Digital Latin Analysis + +This directory (`digital_latin_project/`) houses the PromptFlow-based applications specifically tailored for the Digital Latin Project within the broader `fas_llm_applications` directory. Its primary goal is to leverage Large Language Models (LLMs) for various tasks related to the analysis, understanding, and generation of scaffolded Latin text, facilitating teaching and learning in the humanities. + +## Table of Contents + +* [Context within `fas_llm_apps`](#context-within-fas_llm_apps) + +* [Components](#components) + + * [Data (`data/`)](#data-data) + + * [Flows (`flows/`)](#flows-flows) + + * [Prompts (`prompts/`)](#prompts-prompts) + + * [Scripts (`scripts/`)](#scripts-scripts) + + * [Tools (`tools/`)](#tools-tools) + + * [Utilities (`utilities/`)](#utilities-utilities) + + * [Results & Reports (`results/`, `prepared_reports/`)](#results--reports-results-prepared_reports) + +* [Usage](#usage) + + * [Running Flows](#running-flows) + + * [Generating Reports](#generating-reports) + +## Context within `fas_llm_apps` + +The `digital_latin_project` is a key sub-project of `fas_llm_applications`. While `fas_llm_applications` provides the foundational infrastructure (like connection management and the overall development environment), this directory contains the domain-specific logic, data, and PromptFlow definitions pertinent to Latin text analysis. + +For setting up the development environment (Dev Container, Codespaces) and managing LLM connections, please refer to the main [`fas_llm_applications/README.md`](../README.md) document. + +## Components + +### Data (`data/`) + +This folder contains the datasets and input files used by the PromptFlows and scripts within the `digital_latin_project`. + +* `*.jsonl`: JSON Lines files, often used for batch processing with PromptFlow, where each line is a separate input record. + +* `*.csv`: CSV files, typically used for variant testing, specific test cases, or structured input data. + +* `*.txt`: Various text files, which may include raw texts, logs from previous runs, or other unstructured data. + +### Flows (`flows/`) + +This directory contains the PromptFlow definitions (`flow.dag.yaml`) and their associated Python nodes and prompt templates. Each subdirectory represents a distinct PromptFlow designed for a specific task. + +* **`basic_claude_llm_flow/`**: An example PromptFlow demonstrating interaction with AWS Claude models, including its custom Python invocation node and Jinja2 prompt template. + +* **`basic_deepseek_llm_flow/`**: Similar to the Claude flow, but configured to use Deepseek models. + +* **`basic_gemini_llm_flow/`**: An example PromptFlow demonstrating interaction with Google Gemini models, with its specific Python invocation node and Jinja2 prompt template. + +* **`multi_llm_flow/`**: A more advanced flow showcasing how to integrate and potentially orchestrate calls to multiple LLMs within a single PromptFlow. + +* **`multi_llm_parallel_flow/`**: Demonstrates the execution of LLM calls in parallel within a PromptFlow, utilizing a custom multi-LLM invocation node. + +* **`flow.dag.yaml`**: The core YAML definition file for each PromptFlow, describing its nodes, inputs, outputs, and connections. + +* **`dl_prompt_node.jinja2`**: Jinja2 template files used within the PromptFlows to dynamically construct prompts for LLMs. + +### Prompts (`prompts/`) + +This structured collection holds Jinja2 template files for various system and user prompts used across the LLM applications. This separation allows for easy iteration and management of prompt engineering. + +* **`system/`**: Contains system-level prompt templates that define the LLM's persona, role, or general instructions (e.g., `general_neutral_system.jinja2`, `sophisticated_qa_system.jinja2`). + +* **`user/`**: Contains user-facing prompt templates that structure the user's input or questions for the LLMs (e.g., `basic_qa_user.jinja2`, `u1.0_virgil_user.jinja2`). + +### Scripts (`scripts/`) + +This directory contains Python scripts for specific operations related to the `digital_latin_project`. + +* **`generate_prepared_reports.py`**: A script used to process raw LLM run results and generate more polished, human-readable Markdown reports. + +* **`generate_report.py`**: A core script that likely handles the detailed logic for creating individual LLM evaluation reports. + +* **`load_env_to_shell.py`**: A helper script, primarily for local environments outside of a Dev Container, to load `.env` variables into the shell. + +### Tools (`tools/`) + +This folder contains custom Python tools that can be integrated as nodes within PromptFlows. + +* **`prompt_selector_tool.py`**: A custom tool designed to select or dynamically choose a prompt based on certain input criteria, enhancing prompt engineering flexibility. + +### Utilities (`utilities/`) + +Contains general utility modules that support the `digital_latin_project`'s functionality. + +* **`prompt_registry_util.py`**: A utility for managing and accessing the various prompt templates defined in the `prompts/` directory. + +### Results & Reports (`results/`, `prepared_reports/`) + +For understanding and debugging during the development phase, all generated run results and reports are currently tracked directly within this repository. + +* **`results/`**: Stores the raw JSON outputs and structured data from PromptFlow runs. Each file (e.g., `llm_report_00002.json`) represents the detailed output of a specific LLM interaction or batch run. + +* **`prepared_reports/`**: Contains the generated Markdown reports (e.g., `llm_report_00002.md`, `llm_report_smoke_test_2_run_1_test_case_1.md`), which are a more readable summary of the LLM outputs and evaluations. `prepared_report_manifest_*.md` files likely summarize multiple reports. + +## Usage + +To utilize the PromptFlows within this project, ensure you have followed the setup instructions in the main [`fas_llm_apps/README.md`](../README.md), especially regarding Dev Containers and PromptFlow connection setup. + +### Running Flows + +Once your development environment is set up and PromptFlow connections are active, you can run individual flows or batch runs. + +For example, to test a basic Gemini LLM flow: + +``` +# Navigate to the fas_llm_apps root +cd /path/to/fas_llm_applications + +# Assuming your dev container is running and PromptFlow is installed +# and connections are set up. + +# You might use a script or the PromptFlow CLI directly +# Example using pf CLI to test a flow (syntax may vary slightly based on PromptFlow version): +pf flow test --flow digital_latin_project/flows/basic_gemini_llm_flow --inputs your_input_key="Your Latin text here." + +# To run a batch test with data from the 'data' folder: +pf flow run create --flow digital_latin_project/flows/basic_claude_llm_flow --data digital_latin_project/data/batch1.jsonl + +``` + +(Refer to PromptFlow documentation for precise CLI commands and options.) + +### Generating Reports + +After running your PromptFlows, you can use the scripts in `scripts/` to generate reports. + +For example, to generate prepared reports from the `results/` directory: + +``` +# Navigate to the digital_latin_project scripts directory +cd /path/to/fas_llm_apps/digital_latin_project/scripts + +# Run the script to generate prepared reports +python generate_prepared_reports.py + +``` + +This will process the JSON results and save Markdown reports in the `prepared_reports/` directory. diff --git a/fas_llm_applications/digital_latin_project/__init__.py b/fas_llm_applications/digital_latin_project/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/data/digital latin_promptflow_variant_testing - smoke_test_2_run_1_cases1to30.csv b/fas_llm_applications/digital_latin_project/data/digital latin_promptflow_variant_testing - smoke_test_2_run_1_cases1to30.csv new file mode 100644 index 00000000000..a1095760b97 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/data/digital latin_promptflow_variant_testing - smoke_test_2_run_1_cases1to30.csv @@ -0,0 +1,31 @@ +test_case_no,system_prompt_id,user_prompt_id,llm_model_id,flow_name,is_rag_flow,batch_id,dynamic_template_variables +1,S1.2A,U1.0,us.deepseek.r1-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +2,S1.2B,U1.0,us.deepseek.r1-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +3,S1.2C,U1.0,us.deepseek.r1-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +4,S2.2A,U1.0,us.deepseek.r1-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +5,S2.2B,U1.0,us.deepseek.r1-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +6,S2.2C,U1.0,us.deepseek.r1-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +7,S1.2A,U1.0,us.anthropic.claude-3-7-sonnet-20250219-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +8,S1.2B,U1.0,us.anthropic.claude-3-7-sonnet-20250219-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +9,S1.2C,U1.0,us.anthropic.claude-3-7-sonnet-20250219-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +10,S2.2A,U1.0,us.anthropic.claude-3-7-sonnet-20250219-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +11,S2.2B,U1.0,us.anthropic.claude-3-7-sonnet-20250219-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +12,S2.2C,U1.0,us.anthropic.claude-3-7-sonnet-20250219-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +13,S1.2A,U1.0,gemini-2.5-pro-preview-05-06,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +14,S1.2B,U1.0,gemini-2.5-pro-preview-05-06,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +15,S1.2C,U1.0,gemini-2.5-pro-preview-05-06,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +16,S2.2A,U1.0,gemini-2.5-pro-preview-05-06,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +17,S2.2B,U1.0,gemini-2.5-pro-preview-05-06,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +18,S2.2C,U1.0,gemini-2.5-pro-preview-05-06,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +19,S1.2A,U1.0,us.anthropic.claude-sonnet-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +20,S1.2B,U1.0,us.anthropic.claude-sonnet-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +21,S1.2C,U1.0,us.anthropic.claude-sonnet-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +22,S2.2A,U1.0,us.anthropic.claude-sonnet-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +23,S2.2B,U1.0,us.anthropic.claude-sonnet-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +24,S2.2C,U1.0,us.anthropic.claude-sonnet-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +25,S1.2A,U1.0,us.anthropic.claude-opus-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +26,S1.2B,U1.0,us.anthropic.claude-opus-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +27,S1.2C,U1.0,us.anthropic.claude-opus-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +28,S2.2A,U1.0,us.anthropic.claude-opus-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +29,S2.2B,U1.0,us.anthropic.claude-opus-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} +30,S2.2C,U1.0,us.anthropic.claude-opus-4-20250514-v1:0,multi_llm_parallel_flow,FALSE,smoke_test_2_run_1,{} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/flows/__init__.py b/fas_llm_applications/digital_latin_project/flows/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/__init__.py b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/dl_prompt_node.jinja2 b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/dl_prompt_node.jinja2 new file mode 100644 index 00000000000..ab209b5da4b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/dl_prompt_node.jinja2 @@ -0,0 +1,12 @@ +# system: +{{system_prompt}} + +{#{% for item in chat_history %} +# user: +{{item.inputs.question}} +# assistant: +{{item.outputs.answer}} +{% endfor %}#} + +# user: +{{question}} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/flow.dag.yaml b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/flow.dag.yaml new file mode 100644 index 00000000000..772c8a4d4f6 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/flow.dag.yaml @@ -0,0 +1,45 @@ +inputs: + question: + type: string + default: Tell me about large language models. + llm_service_type: + type: string + default: bedrock + llm_model_id: + type: string + default: anthropic.claude-v2 + system_prompt: + type: string + default: You are an assistant to help me understand the world, but you only can + reply in Latin no matter what language my user prompt comes in or how + frustrated my quesitons are, you must respond in Latin followed by a + second explanation in English using only Disney's "The Little Mermaid" + analogies. +outputs: + final_response: + type: string + reference: ${llm_invocation_node.output.response_text} + invocation_status: + type: bool + reference: ${llm_invocation_node.output.success} +nodes: +- name: llm_invocation_node + type: python + source: + type: code + path: nodes/aws_llm_invocation.py + inputs: + connection: bedrock_connection + prompt: ${inputs.question} + model_name: DeepSeek + system_prompt: ${inputs.system_prompt} + max_tokens: 4000 + temperature: 0.7 +- name: report_generator_node + type: python + source: + type: code + path: ../../scripts/generate_report.py + inputs: + llm_model_id: ${inputs.llm_model_id} + flow_outputs: ${llm_invocation_node.output} diff --git a/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/nodes/__init__.py b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/nodes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/nodes/aws_llm_invocation.py b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/nodes/aws_llm_invocation.py new file mode 100644 index 00000000000..ea3c01e13c8 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_claude_llm_flow/nodes/aws_llm_invocation.py @@ -0,0 +1,193 @@ +import boto3 +import sys +import os +import json +from promptflow.core import tool +from promptflow.connections import CustomConnection as PFCustomConnection +from botocore.exceptions import ClientError +from fas_llm_applications._connections_manager_.common_secrets_loader import get_env_var +from fas_llm_applications._connections_manager_.aws_connection_utils import ensure_promptflow_aws_connection +from typing import Optional +from enum import Enum + +BEDROCK_CONNECTION_NAME = "aws_bedrock_connection" +BEDROCK_SERVICE_NAME = "bedrock-runtime" +CLAUDE_3_7_SONNET_MODEL_ID = "us.anthropic.claude-3-7-sonnet-20250219-v1:0" +CLAUDE_4_SONNET_MODEL_ID = "us.anthropic.claude-sonnet-4-20250514-v1:0" +DEEPSEEK_MODEL_ID = "us.deepseek.r1-v1:0" +DEFAULT_TEMPERATURE = 0.7 +DEFAULT_MAX_TOKENS = 4000 + +CLAUDE3 = "Claude-3.7-Sonnet" +CLAUDE4 = "Claude-4.0-Sonnet" +DEEPSEEK = "DeepSeek" + +@tool +def invoke_bedrock_llm( + connection: PFCustomConnection, + prompt: str, # This will be the user prompt + system_prompt: str, # System prompt input + model_name: str = "Claude3", + max_tokens: Optional[int] = DEFAULT_MAX_TOKENS, + temperature: Optional[float] = DEFAULT_TEMPERATURE, +) -> dict: + """ + Invokes a specified LLM model via AWS Bedrock Runtime. + This method dynamically constructs the payload based on the model_id + to support models like Claude and Deepseek. + + Args: + connection (CustomConnection): The PromptFlow custom connection containing + AWS credentials (access_key_id, secret_access_key) + and region_name. + prompt (str): The user's query or prompt. + system_prompt (str): The system-level instructions or persona for the LLM. + model_name (str): The specific Bedrock model ID. + max_token: (int, Optional): The maximum tokens for input and ouput combined + temperature: (float, Optional): A parameter to set randomness of model response + connection (CustomConnection): The PromptFlow custom connection containing + AWS credentials (access_key_id, secret_access_key) + and region_name. + + Returns: + dict: A dictionary containing the LLM's response and potentially other metadata. + """ + success = False + model_id = get_model_id(model_name) + try: + # Retrieve AWS credentials and region from the PromptFlow CustomConnection + print("Checking for Exisitng Connection") + aws_access_key_id = connection.secrets.get("aws_access_key_id") + aws_secret_access_key = connection.secrets.get("aws_secret_access_key") + aws_region = connection.configs.get( + "region_name", "us-east-1" + ) # Default to us-east-1 if not specified + + if not aws_access_key_id or not aws_secret_access_key: + raise ValueError( + "AWS access_key_id or secret_access_key not found in the Bedrock connection." + ) + else: + print(f"Found existing connections: key - {aws_access_key_id}") + + # Initialize Bedrock Runtime client + bedrock_runtime = boto3.client( + service_name="bedrock-runtime", + region_name=aws_region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + print("Initialized the Bedrock Runtime Client") + + # Prepare the request body based on the model_id + body = {} + llm_text_response = "" + full_api_response = {} + + if "claude" in model_id.lower(): + # Claude models (e.g., anthropic.claude-3-sonnet-20240229-v1:0) + messages = [] + messages.append({"role": "user", "content": prompt}) + + body = { + "system": system_prompt if system_prompt else "", + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": max_tokens if max_tokens else 4000, + "temperature": temperature if temperature else 0.7, # Adjust as needed + "messages": messages, + } + # Claude's response format is different, it's a list of content blocks + # We'll parse it after the invoke_model call + + elif "deepseek" in model_id.lower(): + # Deepseek models (e.g., deepseek-llm-v2) + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + body = { + "messages": messages, + "max_tokens": max_tokens if max_tokens else 4000, # Adjust as needed + "temperature": temperature if temperature else 0.7, # Adjust as needed + "top_p": 1.0, # Adjust as needed + "stop": [], # Adjust as needed + } + # Deepseek's response format is similar to OpenAI chat completions + + print(f"Invoking Bedrock model: {model_id} in region {aws_region}") + print(f"Request Body: {json.dumps(body, indent=2)}") + + response = bedrock_runtime.invoke_model( + body=json.dumps(body), + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + + full_api_response = json.loads(response["body"].read()) + + # Parse response based on model type + if "claude" in model_id.lower(): + if full_api_response.get("content"): + for content_block in full_api_response["content"]: + if content_block.get("type") == "text": + llm_text_response += content_block["text"] + + elif "deepseek" in model_id.lower(): + if full_api_response.get("choices") and full_api_response["choices"][0].get( + "message" + ): + llm_text_response = full_api_response["choices"][0]["message"].get( + "content", "" + ) + + success = True + + # For the 'prompt_id' requested in the report, use a simple hash of the user prompt. + prompt_id_for_report = f"user_query_{hash(prompt) % 10000}" + + return { + "success": success, + "response_text": llm_text_response, + "full_api_response": full_api_response, + "model_id_used": model_id, + "prompt_id": prompt_id_for_report, # Include prompt_id here for reporting + } + + except ClientError as e: + print(f"Bedrock Client Error: {e.response['Error']['Message']}") + return { + "error": f"Bedrock Client Error: {e.response['Error']['Message']}", + "status": "failed", + } + except ValueError as e: + print(f"Configuration or Payload Error: {e}") + return {"error": f"Configuration or Payload Error: {e}", "status": "failed"} + except Exception as e: + print(f"An unexpected error occurred: {e}") + return {"error": f"An unexpected error occurred: {e}", "status": "failed"} + + +def get_model_id(model_name: str): + if model_name == "Claude3": + return CLAUDE_3_7_SONNET_MODEL_ID + if model_name == "Claude4": + return CLAUDE_4_SONNET_MODEL_ID + if model_name == "DeepSeek": + return DEEPSEEK_MODEL_ID + + +def create_aws_connection() -> PFCustomConnection: + aws_access_key = get_env_var("AWS_AI_WORKFLOW_CORE_DEV_ID") + aws_secret_key = get_env_var("AWS_AI_WORKFLOW_CORE_DEV_SECRET") + aws_region = get_env_var("AWS_DEFAULT_REGION") or "us-east-1" + + pf_aws_connection = ensure_promptflow_aws_connection( + access_key=aws_access_key, + secret_key=aws_secret_key, + region=aws_region, + conn_name=BEDROCK_CONNECTION_NAME, + service_name=BEDROCK_SERVICE_NAME, + ) + return pf_aws_connection diff --git a/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/__init__.py b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/dl_prompt_node.jinja2 b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/dl_prompt_node.jinja2 new file mode 100644 index 00000000000..ab209b5da4b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/dl_prompt_node.jinja2 @@ -0,0 +1,12 @@ +# system: +{{system_prompt}} + +{#{% for item in chat_history %} +# user: +{{item.inputs.question}} +# assistant: +{{item.outputs.answer}} +{% endfor %}#} + +# user: +{{question}} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/flow.dag.yaml b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/flow.dag.yaml new file mode 100644 index 00000000000..cf10b604198 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/flow.dag.yaml @@ -0,0 +1,48 @@ +inputs: + question: + type: string + default: Tell me about large language models. + llm_service_type: + type: string + default: bedrock + llm_model_id: + type: string + default: anthropic.claude-v2 + system_prompt: + type: string + default: You are an assistant to help me understand the world, but you only can + reply in Latin no matter what language my user prompt comes in or how + frustrated my quesitons are, you must respond in Latin followed by a + second explanation in English using only Disney's "The Little Mermaid" + analogies. +outputs: + final_response: + type: string + reference: ${llm_invocation_node.output.response_text} + invocation_status: + type: bool + reference: ${llm_invocation_node.output.success} +nodes: +- name: dl_prompt_node + type: prompt + source: + type: code + path: dl_prompt_node.jinja2 + inputs: + question: ${inputs.question} + system_prompt: ${inputs.system_prompt} +- name: llm_invocation_node + type: python + source: + type: code + path: nodes/aws_llm_invocation.py + inputs: + prompt: ${dl_prompt_node.output} +- name: report_generator_node + type: python + source: + type: code + path: ../../scripts/generate_report.py + inputs: + llm_model_id: ${inputs.llm_model_id} + flow_outputs: ${llm_invocation_node.output} diff --git a/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/nodes/__init__.py b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/nodes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/nodes/aws_llm_invocation.py b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/nodes/aws_llm_invocation.py new file mode 100644 index 00000000000..b85dd49979c --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_deepseek_llm_flow/nodes/aws_llm_invocation.py @@ -0,0 +1,123 @@ +import json +from promptflow.core import tool +from promptflow.connections import CustomConnection +import boto3 +from botocore.exceptions import ClientError + +@tool +def invoke_bedrock_llm( + prompt: str, # This will be the user prompt + system_prompt: str, # System prompt input + model_id: str, + connection: CustomConnection # This will be your AWS Bedrock connection +) -> dict: + """ + Invokes a specified LLM model via AWS Bedrock Runtime. + This method dynamically constructs the payload based on the model_id + to support models like Claude and Deepseek. + + Args: + prompt (str): The user's query or prompt. + system_prompt (str): The system-level instructions or persona for the LLM. + model_id (str): The specific Bedrock model ID. + connection (CustomConnection): The PromptFlow custom connection containing + AWS credentials (access_key_id, secret_access_key) + and region_name. + + Returns: + dict: A dictionary containing the LLM's response and potentially other metadata. + """ + try: + # Retrieve AWS credentials and region from the PromptFlow CustomConnection + aws_access_key_id = connection.secrets.get("aws_access_key_id") + aws_secret_access_key = connection.secrets.get("aws_secret_access_key") + aws_region = connection.configs.get("region_name", "us-east-1") # Default to us-east-1 if not specified + + if not aws_access_key_id or not aws_secret_access_key: + raise ValueError("AWS access_key_id or secret_access_key not found in the Bedrock connection.") + + # Initialize Bedrock Runtime client + bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name=aws_region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key + ) + + # Prepare the request body based on the model_id + body = {} + llm_text_response = "" + full_api_response = {} + + if "claude" in model_id.lower(): + # Claude models (e.g., anthropic.claude-3-sonnet-20240229-v1:0) + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + body = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 4000, # Can be adjust as needed + "messages": messages + } + + elif "deepseek" in model_id.lower(): + # Deepseek models (e.g., deepseek-llm-v2) + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + body = { + "messages": messages, + "max_tokens": 4000, # Adjust as needed + "temperature": 0.7, # Adjust as needed + "top_p": 1.0, # Adjust as needed + "stop": [] # Adjust as needed + } + # Deepseek's response format is similar to OpenAI chat completions + + print(f"Invoking Bedrock model: {model_id} in region {aws_region}") + print(f"Request Body: {json.dumps(body, indent=2)}") + + response = bedrock_runtime.invoke_model( + body=json.dumps(body), + modelId=model_id, + accept='application/json', + contentType='application/json' + ) + + full_api_response = json.loads(response['body'].read()) + + # Parse response based on model type + if "claude" in model_id.lower(): + if full_api_response.get("content"): + for content_block in full_api_response["content"]: + if content_block.get("type") == "text": + llm_text_response += content_block["text"] + + elif "deepseek" in model_id.lower(): + if full_api_response.get("choices") and full_api_response["choices"][0].get("message"): + llm_text_response = full_api_response["choices"][0]["message"].get("content", "") + + elif "titan" in model_id.lower(): + if full_api_response.get("results"): + llm_text_response = full_api_response["results"][0].get("outputText", "") + + # For the 'prompt_id' requested in the report, use a simple hash of the user prompt. + prompt_id_for_report = f"user_query_{hash(prompt) % 10000}" + + return { + "response_text": llm_text_response, + "full_api_response": full_api_response, + "model_id_used": model_id, + "prompt_id": prompt_id_for_report # Include prompt_id here for reporting + } + + except ClientError as e: + return {"error": f"Bedrock Client Error: {e.response['Error']['Message']}", "status": "failed"} + except ValueError as e: + return {"error": f"Configuration or Payload Error: {e}", "status": "failed"} + except Exception as e: + return {"error": f"An unexpected error occurred: {e}", "status": "failed"} diff --git a/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/__init__.py b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/dl_prompt_node.jinja2 b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/dl_prompt_node.jinja2 new file mode 100644 index 00000000000..ab209b5da4b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/dl_prompt_node.jinja2 @@ -0,0 +1,12 @@ +# system: +{{system_prompt}} + +{#{% for item in chat_history %} +# user: +{{item.inputs.question}} +# assistant: +{{item.outputs.answer}} +{% endfor %}#} + +# user: +{{question}} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/flow.dag.yaml b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/flow.dag.yaml new file mode 100644 index 00000000000..56d7a50cede --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/flow.dag.yaml @@ -0,0 +1,125 @@ +variants: +- name: summary_variant + inputs: + selected_prompt_id: SophisticatedSummary_V1 + dynamic_template_variables: + document_content: This is a document about the latest advances in AI ethics and + responsible development. + max_words_summary: 120 + focus_areas: + - ethics + - AI development + llm_model_id: gemini-2.5-pro-preview-05-06 + node: llm_invocation_node +- name: qa_variant + inputs: + selected_prompt_id: SophisticatedQA_V1 + dynamic_template_variables: + context: The capital of Italy is Rome. The Colosseum is a famous landmark there. + question: What is the capital of Italy? + llm_model_id: gemini-2.5-pro-preview-05-06 + node: llm_invocation_node +- name: creative_story_variant + inputs: + selected_prompt_id: CreativeStory_V1 + dynamic_template_variables: + genre: epic fantasy + setting: a realm of floating islands + protagonist_name: Elara + temperature: 0.9 + max_tokens: 800 + node: llm_invocation_node +- name: latin_little_mermaid + inputs: + selected_prompt_id: LatinLittleMermaid_V1 + dynamic_template_variables: + context: None + question: None + temperature: 0.9 + max_tokens: 800 + node: llm_invocation_node +inputs: + selected_prompt_id: + type: string + default: LatinLittleMermaid_V1 + dynamic_template_variables: + type: object + default: {} + llm_model_id: + type: string + default: gemini-2.5-pro-preview-05-06 + temperature: + type: double + default: 0.7 + max_tokens: + type: int + default: 4000 + run_id: + type: string + default: "" + flow_name: + type: string + default: basic_gemini_flow + is_rag_flow: + type: bool + default: false + batch_id: + type: string + default: "" +outputs: + final_response: + type: string + reference: ${llm_gemini_invocation_node.output.response_text} + invocation_status: + type: bool + reference: ${llm_gemini_invocation_node.output.status}} + report_status: + type: string + value: ${report_generator_node.output.status} + reference: ${report_generator_node.output.status} + report_path: + type: string + value: ${report_generator_node.output.report_path} + reference: ${report_generator_node.output.report_path} + report_id: + type: int + value: ${report_generator_node.output.report_id} + reference: ${report_generator_node.output.report_id} +nodes: +- name: prompt_selector_node + type: python + source: + type: code + path: ../../tools/prompt_selector_tool.py + inputs: + prompt_id: ${inputs.selected_prompt_id} +- name: llm_gemini_invocation_node + type: python + source: + type: code + path: nodes/gemini_llm_invocation.py + inputs: + connection: gemini_connection + user_prompt_template_path: ${prompt_selector_node.output.user_prompt_template_path} + system_prompt_template_path: ${prompt_selector_node.output.system_prompt_template_path} + prompt_id: ${inputs.selected_prompt_id} + model_id: ${inputs.llm_model_id} + temperature: ${inputs.temperature} + max_tokens: ${inputs.max_tokens} + selector_template_variables: ${prompt_selector_node.output.template_variables} + dynamic_template_variables: ${inputs.dynamic_template_variables} +- name: report_generator_node + type: python + source: + type: code + path: ../../scripts/generate_report.py + inputs: + llm_model_id: ${llm_gemini_invocation_node.output.model_id_used} + prompt_id: ${llm_gemini_invocation_node.output.prompt_id} + flow_name: ${inputs.flow_name} + is_rag_flow: ${inputs.is_rag_flow} + flow_outputs: ${llm_gemini_invocation_node.output} + user_prompt_used: ${llm_gemini_invocation_node.output.user_prompt_used}} + system_prompt_used: ${llm_gemini_invocation_node.output.system_prompt_used}} + llm_invocation_output: ${llm_gemini_invocation_node.output} + llm_run_time: ${llm_gemini_invocation_node.output.llm_run_time} diff --git a/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/nodes/__init__.py b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/nodes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/nodes/gemini_llm_invocation.py b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/nodes/gemini_llm_invocation.py new file mode 100644 index 00000000000..846b9dffcd4 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/basic_gemini_llm_flow/nodes/gemini_llm_invocation.py @@ -0,0 +1,149 @@ +import json +import sys +import os +import requests +import time +from promptflow.core import tool +from promptflow.connections import CustomConnection +from jinja2 import Environment, FileSystemLoader, select_autoescape + +PROJECT_ROOT_FOR_TEMPLATES = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../../../") +) + +@tool +def invoke_gemini_huit_llm( + connection: CustomConnection, + user_prompt_template_path: str, + system_prompt_template_path: str, + prompt_id: str, + model_id: str, + selector_template_variables: dict = {}, + dynamic_template_variables: dict = {}, + temperature: float = 0.7, + max_tokens: int = 4000, +) -> dict: + """ + Invokes the Google Gemini model via the HUIT AI Services API Gateway. + It renders Jinja templates for prompts before invoking the LLM. + + Args: + connection (CustomConnection): The PromptFlow custom connection containing + the HUIT API key and base URL. + user_prompt_template_path (str): Relative path to the Jinja template file for the user prompt. + system_prompt_template_path (str): Relative path to the Jinja template file for the system prompt. + template_variables (dict): A dictionary of variables to pass to the Jinja templates for rendering. + prompt_id (str): A unique identifier for the specific prompt definition/iteration. + model_id (str): The specific Gemini model ID. + selector_template_variables: (dict) = Variables from the prompt selector node, + dynamic_template_variables: (dict) = Dynamic variables from the input node, + temperature (float): The sampling temperature to use for text generation. + max_tokens (int): The maximum number of tokens to generate. + Returns: + dict: A dictionary containing the LLM's response and other metadata. + Includes the *rendered* prompts used for reporting, and the prompt_id. + """ + print(f"DEBUG: Rendering prompts from templates in {PROJECT_ROOT_FOR_TEMPLATES}") + + # NEW: Merge template variables within the Python tool + # Dynamic variables take precedence over selector variables + merged_template_variables = { + **selector_template_variables, + **dynamic_template_variables, + } + print(f"DEBUG: Merged template variables: {merged_template_variables}") + + # Set up Jinja2 environment to load templates from the project root + env = Environment( + loader=FileSystemLoader(PROJECT_ROOT_FOR_TEMPLATES), + autoescape=select_autoescape(["html", "xml"]), + ) + + rendered_user_prompt = "" + rendered_system_prompt = "" + + # Basic input validation for paths + if not user_prompt_template_path or not system_prompt_template_path: + error_msg = "User or System prompt template path cannot be empty." + return {"error": error_msg, "status": "failed"} + + try: + # Render System Prompt Template + system_template = env.get_template(system_prompt_template_path) + rendered_system_prompt = system_template.render(merged_template_variables) + + # Render User Prompt Template + user_template = env.get_template(user_prompt_template_path) + rendered_user_prompt = user_template.render(merged_template_variables) + + except Exception as e: + raise RuntimeError("ERROR: Failed to render Jinja template(s) from paths '{user_prompt_template_path}' and '{system_prompt_template_path}': {e}") + + try: + # Retrieve API key and base URL from the PromptFlow CustomConnection + api_key = connection.secrets.get("api_key") + base_url = connection.configs.get( + "base_url", "https://go.apis.huit.harvard.edu/ais-google-gemini" + ) + + if not api_key: + raise ValueError("API key not found in the Gemini connection.") + if not base_url: + raise ValueError("Base URL not found in the Gemini connection.") + + api_endpoint = f"{base_url}/v1beta/models/{model_id}:generateContent" + + payload = { + "contents": [ # Only user messages (and interleaved model messages) go here + {"role": "user", "parts": [{"text": rendered_user_prompt}]} + ], + "generationConfig": { + "temperature": temperature, + "maxOutputTokens": max_tokens, + }, + } + + # Add system_instruction at the top level if a system prompt is provided + if rendered_system_prompt: + payload["system_instruction"] = { + "parts": [{"text": rendered_system_prompt}] + } + + headers = {"Content-Type": "application/json", "api-key": api_key} + + print(f"Invoking Gemini model: {model_id} via {api_endpoint}") + + start_time = time.time() + response = requests.post(api_endpoint, headers=headers, json=payload) + response.raise_for_status() + + response_json = response.json() + end_time = time.time() + duration = end_time - start_time + + llm_text_response = "" + if response_json and response_json.get("candidates"): + for candidate in response_json["candidates"]: + if candidate.get("content") and candidate["content"].get("parts"): + for part in candidate["content"]["parts"]: + if part.get("text"): + llm_text_response += part["text"] + + # Use the prompt_id passed as input directly + return { + "response_text": llm_text_response, + "full_api_response": response_json, + "model_id_used": model_id, + "prompt_id": prompt_id, # Use the prompt_id from input + "user_prompt_used": rendered_user_prompt, # Return the RENDERED user prompt + "system_prompt_used": rendered_system_prompt, # Return the RENDERED system prompt + "llm_run_time": duration, + "status": "success", + } + + except requests.exceptions.RequestException as e: + return {"error": f"HTTP Request failed: {e}", "status": "failed"} + except ValueError as e: + return {"error": f"Configuration error: {e}", "status": "failed"} + except Exception as e: + return {"error": f"An unexpected error occurred: {e}", "status": "failed"} diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/__init__.py b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/default.flow_test.yaml b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/default.flow_test.yaml new file mode 100644 index 00000000000..d58c04b775f --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/default.flow_test.yaml @@ -0,0 +1,15 @@ +tests: + - name: multi_llm_flow_default_20250606_223705_542000 + display_name: multi_llm_flow_${variant_id}_${timestamp} + flow: /workspaces/promptflow/fas_llm_applications/digital_latin_project/flows/multi_llm_flow + data: /workspaces/promptflow/fas_llm_applications/digital_latin_project/data/smoke_test.jsonl + column_mapping: + selected_prompt_id: ${data.selected_prompt_id} + dynamic_template_variables: ${data.dynamic_template_variables} + llm_model_id: ${data.llm_model_id} + temperature: ${data.temperature} + max_tokens: ${data.max_tokens} + run_id: ${data.run_id} + flow_name: ${data.flow_name} + is_rag_flow: ${data.is_rag_flow} + batch_id: ${data.batch_id} diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/dl_prompt_node.jinja2 b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/dl_prompt_node.jinja2 new file mode 100644 index 00000000000..ab209b5da4b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/dl_prompt_node.jinja2 @@ -0,0 +1,12 @@ +# system: +{{system_prompt}} + +{#{% for item in chat_history %} +# user: +{{item.inputs.question}} +# assistant: +{{item.outputs.answer}} +{% endfor %}#} + +# user: +{{question}} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/flow.dag.yaml b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/flow.dag.yaml new file mode 100644 index 00000000000..d618261ba17 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/multi_llm_flow/flow.dag.yaml @@ -0,0 +1,80 @@ +inputs: + selected_prompt_id: + type: string + default: S2.1__U1.0 + dynamic_template_variables: + type: object + default: {} + llm_model_id: + type: string + default: gemini-2.5-pro-preview-05-06 + temperature: + type: double + default: 0.7 + max_tokens: + type: int + default: 4000 + run_id: + type: string + default: multi_llm_run1 + flow_name: + type: string + default: multi_llm_flow + is_rag_flow: + type: bool + default: false + batch_id: + type: string + default: random_test_1 +outputs: + final_response: + type: string + reference: ${multi_llm_node.output.response_text} + invocation_status: + type: bool + reference: ${multi_llm_node.output.status}} + report_status: + type: string + value: ${report_generator_node.output.status} + reference: ${multi_llm_report_node.output.status} + report_path: + type: string + value: ${report_generator_node.output.report_path} + reference: ${multi_llm_report_node.output.report_path} + report_id: + type: int + value: ${report_generator_node.output.report_id} + reference: ${multi_llm_report_node.output.report_id} +nodes: +- name: prompt_selector_node + type: python + source: + type: code + path: ../../tools/prompt_selector_tool.py + inputs: {} +- name: multi_llm_node + type: python + source: + type: code + path: nodes/multi_llm_invocation.py + inputs: + user_prompt_template_path: ${prompt_selector_node.output.user_prompt_template_path} + system_prompt_template_path: ${prompt_selector_node.output.system_prompt_template_path} + prompt_id: ${inputs.selected_prompt_id} + model_id: ${inputs.llm_model_id} + selector_template_variables: ${prompt_selector_node.output.template_variables} + dynamic_template_variables: ${inputs.dynamic_template_variables} +- name: multi_llm_report_node + type: python + source: + type: code + path: ../../scripts/generate_report.py + inputs: + llm_model_id: ${multi_llm_node.output.model_id_used} + system_prompt_used: ${multi_llm_node.output.system_prompt_used} + user_prompt_used: ${multi_llm_node.output.user_prompt_used} + llm_invocation_output: ${multi_llm_node.output.full_api_response} + llm_run_time: ${multi_llm_node.output.llm_run_time} + flow_name: ${inputs.flow_name} + is_rag_flow: ${inputs.is_rag_flow} + batch_id: ${inputs.batch_id} diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/flow.dag.yaml b/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/flow.dag.yaml new file mode 100644 index 00000000000..2df004e1106 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/flow.dag.yaml @@ -0,0 +1,83 @@ +inputs: + system_prompt_id: + type: string + default: S2.2A + user_prompt_id: + type: string + default: U1.0 + llm_model_id: + type: string + default: gemini-2.5-pro-preview-05-06 + dynamic_template_variables: + type: object + default: {} + flow_name: + type: string + default: multi_llm_parallel_flow + is_rag_flow: + type: bool + default: false + batch_id: + type: string + default: invocation_clean_up_test + test_case_no: + type: int + default: 30 +outputs: + final_response: + type: string + reference: ${multi_llm_node.output.response_text} + invocation_status: + type: bool + reference: ${multi_llm_node.output.status}} + report_status: + type: string + value: ${report_generator_node.output.status} + reference: ${multi_llm_report_node.output.status} + report_path: + type: string + value: ${report_generator_node.output.report_path} + reference: ${multi_llm_report_node.output.report_path} + report_id: + type: int + value: ${report_generator_node.output.report_id} + reference: ${multi_llm_report_node.output.report_id} +nodes: +- name: prompt_selector_node + type: python + source: + type: code + path: ../../tools/prompt_selector_tool.py + inputs: + system_prompt_id: ${inputs.system_prompt_id} + user_prompt_id: ${inputs.user_prompt_id} +- name: multi_llm_node + type: python + source: + type: code + path: nodes/multi_llm_invocation.py + inputs: + user_prompt_template_path: ${prompt_selector_node.output.user_prompt_template_path} + system_prompt_template_path: ${prompt_selector_node.output.system_prompt_template_path} + model_id: ${inputs.llm_model_id} + selector_template_variables: ${prompt_selector_node.output.template_variables} + dynamic_template_variables: ${inputs.dynamic_template_variables} + system_prompt_id: ${inputs.system_prompt_id} + user_prompt_id: ${inputs.user_prompt_id} +- name: multi_llm_report_node + type: python + source: + type: code + path: ../../scripts/generate_report.py + inputs: + llm_model_id: ${multi_llm_node.output.model_id_used} + system_prompt_used: ${multi_llm_node.output.system_prompt_used} + user_prompt_used: ${multi_llm_node.output.user_prompt_used} + llm_invocation_output: ${multi_llm_node.output.full_api_response} + llm_run_time: ${multi_llm_node.output.llm_run_time} + flow_name: ${inputs.flow_name} + is_rag_flow: ${inputs.is_rag_flow} + batch_id: ${inputs.batch_id} + system_prompt_id: ${multi_llm_node.output.system_prompt_id} + user_prompt_id: ${multi_llm_node.output.user_prompt_id} + test_case_no: ${inputs.test_case_no} diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/nodes/__init__.py b/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/nodes/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/nodes/multi_llm_invocation.py b/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/nodes/multi_llm_invocation.py new file mode 100644 index 00000000000..eb1b04d60e7 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/flows/multi_llm_parallel_flow/nodes/multi_llm_invocation.py @@ -0,0 +1,389 @@ +import json +import sys +import os +import time +import boto3 +import requests +from promptflow.client import PFClient +from promptflow.core import tool +from promptflow.connections import CustomConnection +from fas_llm_applications._connections_manager_.aws_connection_utils import ( + ensure_promptflow_aws_connection, +) +from fas_llm_applications._connections_manager_.common_secrets_loader import get_env_var +from fas_llm_applications._connections_manager_.client_utils import get_pf_client +from fas_llm_applications._connections_manager_.keyring_utils import verify_keyring +from jinja2 import ( + Environment, + FileSystemLoader, + select_autoescape, +) +from botocore.exceptions import ClientError + +PROJECT_ROOT_FOR_TEMPLATES = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../../../") +) + +CLAUDE_3_7_SONNET_MODEL_ID = "us.anthropic.claude-3-7-sonnet-20250219-v1:0" +DEEPSEEK_MODEL_ID = "us.deepseek.r1-v1:0" +GEMINI_MODEL_ID = "gemini-2.5-pro-preview-05-06" +CLAUDE_4_0_SONNET_MODEL_ID = "us.anthropic.claude-sonnet-4-20250514-v1:0" +CLAUDE_4_0_OPUS_MODEL_ID = "us.anthropic.claude-opus-4-20250514-v1:0" +GEMINI_CONNECTION = "gemini_connection" +BEDROCK_CONNECTION = "bedrock_connection" +BEDROCK_SERVICE_NAME = "bedrock-runtime" +GEMINI_BASE_URL = "https://go.apis.huit.harvard.edu/ais-google-gemini" +DEFAULT_REGION = "us-east-1" + + +@tool +def invoke_llm( + user_prompt_template_path: str, + system_prompt_template_path: str, + system_prompt_id: str, + user_prompt_id: str, + model_id: str, + selector_template_variables: dict = {}, + dynamic_template_variables: dict = {}, +) -> dict: + """ + Invokes the Google Gemini model via the HUIT AI Services API Gateway. + It renders Jinja templates for prompts before invoking the LLM. + + Args: + user_prompt_template_path (str): Relative path to the Jinja template file for the user prompt. + system_prompt_template_path (str): Relative path to the Jinja template file for the system prompt. + template_variables (dict): A dictionary of variables to pass to the Jinja templates for rendering. + prompt_id (str): A unique identifier for the specific prompt definition/iteration. + model_id (str): The specific Gemini model ID (e.g., "gemini-2.0-flash"). + selector_template_variables (dict): Variables from prompt_selector_node + dynamic_template_variables (dict): Variables from flow inputs (overrides) + + Returns: + dict: A dictionary containing the LLM's response and other metadata. + Includes the *rendered* prompts used for reporting, and the prompt_id. + """ + print(f"Rendering prompts from templates in {PROJECT_ROOT_FOR_TEMPLATES}") + + # Merge template variables within the Python tool + merged_template_variables = { + **selector_template_variables, + **dynamic_template_variables, + } + + # Set up Jinja2 environment to load templates from the project root + env = Environment( + loader=FileSystemLoader(PROJECT_ROOT_FOR_TEMPLATES), + autoescape=select_autoescape(["html", "xml"]), + ) + + rendered_user_prompt = "" + rendered_system_prompt = "" + + # Basic input validation for paths + if not user_prompt_template_path or not system_prompt_template_path: + error_msg = "User or System prompt template path cannot be empty." + return {"error": error_msg, "status": "failed"} + + try: + # Render System Prompt Template + system_template = env.get_template(system_prompt_template_path) + rendered_system_prompt = system_template.render(merged_template_variables) + + # Render User Prompt Template + user_template = env.get_template(user_prompt_template_path) + rendered_user_prompt = user_template.render(merged_template_variables) + + except Exception as e: + return {"error": f"Failed to render Jinja template(s): {e}", "status": "failed"} + + try: + # Attempt to get connections from the Promptflow Client + + # Keyring holds sensitive config parameters for connection + # verifying keyring to sensure it is working in the active session + verify_keyring() + pf_client = get_promptflow_client() + all_connections = pf_client.connections.list() + if all_connections: + connections_list = [] + has_existing_connection = True + for connection in all_connections: + connections_list.append(connection.name) + print(f"Connections found on PF Client - {connections_list}") + else: + has_existing_connection = False + print("No connections found on PF Client") + except Exception as e: + print( + "Error: An error occured when checking Promptflow Client for connections: {e}" + ) + print("Will build connections instead") + pass + + if "gemini" in model_id.lower(): + + try: + # Get gemini connection from PF Client + if ( + GEMINI_CONNECTION in connections_list + and pf_client.connections.get(name=GEMINI_CONNECTION).configs.get( + "base_url" + ) + == GEMINI_BASE_URL + ): + gemini_connection = pf_client.connections.get(name=GEMINI_CONNECTION) + + else: + # Create new gemini connection + gemini_connection = create_gemini_connection() + + api_key = gemini_connection.secrets.get("api_key") + base_url = gemini_connection.configs.get( + "base_url", "https://go.apis.huit.harvard.edu/ais-google-gemini" + ) + + if not api_key: + raise ValueError("API key not found in the Gemini connection.") + if not base_url: + raise ValueError("Base URL not found in the Gemini connection.") + + api_endpoint = f"{base_url}/v1beta/models/{model_id}:generateContent" + + payload = { + "contents": [ # Only user messages (and interleaved model messages) go here + {"role": "user", "parts": [{"text": rendered_user_prompt}]} + ] + # "generationConfig": { # Temporarily remove so we get default model content and behavior + # "temperature": temperature, + # "maxOutputTokens": max_tokens + # } + } + + if rendered_system_prompt: + payload["system_instruction"] = { + "parts": [{"text": rendered_system_prompt}] + } + + headers = {"Content-Type": "application/json", "api-key": api_key} + + start_time = time.time() + response = requests.post(api_endpoint, headers=headers, json=payload) + response.raise_for_status() + + response_json = response.json() + full_api_response = response_json + + llm_text_response = "" + if response_json and response_json.get("candidates"): + for candidate in response_json["candidates"]: + if candidate.get("content") and candidate["content"].get("parts"): + for part in candidate["content"]["parts"]: + if part.get("text"): + llm_text_response += part["text"] + end_time = time.time() + duration = end_time - start_time + + # The ouput for the Gemini LLM Call + llm_node_invocation_output = { + "system_prompt_id": system_prompt_id, + "user_prompt_id": user_prompt_id, + "model_id_used": model_id, + "user_prompt_used": rendered_user_prompt, + "system_prompt_used": rendered_system_prompt, + "response_text": llm_text_response, + "full_api_response": full_api_response, + "llm_run_time": duration, + "status": "success", + } + return llm_node_invocation_output + + except requests.exceptions.RequestException as e: + return {"error": f"HTTP Request failed: {e}", "status": "failed"} + except ValueError as e: + return {"error": f"Configuration error: {e}", "status": "failed"} + except Exception as e: + return {"error": f"An unexpected error occurred: {e}", "status": "failed"} + + else: + try: + # Get bedrock connection from PF Client + if ( + BEDROCK_CONNECTION in connections_list + and pf_client.connections.get(name=BEDROCK_CONNECTION).configs.get( + "region_name" + ) + == DEFAULT_REGION + ): + bedrock_connection = pf_client.connections.get(name=BEDROCK_CONNECTION) + + else: + # Create new bedrock connection + bedrock_connection = create_gemini_connection() + + # Retrieve AWS credentials and region from the PromptFlow CustomConnection + aws_access_key_id = bedrock_connection.secrets.get("aws_access_key_id") + aws_secret_access_key = bedrock_connection.secrets.get( + "aws_secret_access_key" + ) + aws_region = bedrock_connection.configs.get( + "region_name", "us-east-1" + ) # Default to us-east-1 if not specified + + if not aws_access_key_id or not aws_secret_access_key: + raise ValueError( + "AWS access_key_id or secret_access_key not found in the Bedrock connection." + ) + + # Initialize Bedrock Runtime client + bedrock_runtime = boto3.client( + service_name="bedrock-runtime", + region_name=aws_region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + + # Prepare the request body based on the model_id + body = {} + llm_text_response = "" + full_api_response = {} + + if "claude" in model_id.lower() and model_id.lower() in [ + CLAUDE_3_7_SONNET_MODEL_ID, + CLAUDE_4_0_OPUS_MODEL_ID, + CLAUDE_4_0_SONNET_MODEL_ID, + ]: + # Claude models (e.g., anthropic.claude-3-sonnet-20240229-v1:0) + print(f"The claude model is {model_id}", file=sys.stderr) + messages = [] + messages.append({"role": "user", "content": rendered_user_prompt}) + + body = { + "anthropic_version": "bedrock-2023-05-31", + "system": rendered_system_prompt, + "max_tokens": 5000, # Adjust as needed - Temporarily removed for defualt model response. Initially used 64000, for Claude 4 set to 5000 + "messages": messages, + } + + if rendered_system_prompt: + body["system"] = rendered_system_prompt + + elif "deepseek" in model_id.lower(): + # Deepseek models (e.g., deepseek-llm-v2) + messages = [] + if rendered_system_prompt: + messages.append( + {"role": "system", "content": rendered_system_prompt} + ) + messages.append({"role": "user", "content": rendered_user_prompt}) + + body = { + "messages": messages, + "stream": False, + "max_tokens": 32000, + # "max_tokens": 4000, # Adjust as needed - Temporarily removed for default model reponse + # "temperature": 0.7, # Adjust as needed - Temporarily removed for default model response. + # "top_p": 1.0, # Adjust as needed - Removed to use default model response + # "stop": [] # Adjust as needed - Removed to use default model response + } + # Deepseek's response format is similar to OpenAI chat completions + + else: + raise ValueError( + f"Unsupported model_id for Bedrock: {model_id}. Please ensure it's a support Claude or Deepseek model in this flow", + file=sys.stderr, + ) + + print( + f"Invoking Bedrock model: {model_id} in region {aws_region}", + file=sys.stderr, + ) + + start_time = time.time() + response = bedrock_runtime.invoke_model( + body=json.dumps(body), + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + full_api_response = json.loads(response["body"].read()) + + # Parse response based on model type + if "claude" in model_id.lower(): + if full_api_response.get("content"): + for content_block in full_api_response["content"]: + if content_block.get("type") == "text": + llm_text_response += content_block["text"] + + elif "deepseek" in model_id.lower(): + if full_api_response.get("choices") and full_api_response["choices"][ + 0 + ].get("message"): + llm_text_response = full_api_response["choices"][0]["message"].get( + "content", "" + ) + end_time = time.time() + duration = end_time - start_time # Duration in seconds + + llm_node_invocation_output = { + "system_prompt_id": system_prompt_id, + "user_prompt_id": user_prompt_id, + "model_id_used": model_id, + "response_text": llm_text_response, + "full_api_response": full_api_response, + "user_prompt_used": rendered_user_prompt, # Return the RENDERED user prompt + "system_prompt_used": rendered_system_prompt, # Return the RENDERED system prompt + "llm_run_time": duration, + "status": "success", + } + return llm_node_invocation_output + + except ClientError as e: + return { + "error": f"Bedrock Client Error: {e.response['Error']['Message']}", + "status": "failed", + } + except ValueError as e: + return {"error": f"Configuration or Payload Error: {e}", "status": "failed"} + except Exception as e: + return {"error": f"An unexpected error occurred: {e}", "status": "failed"} + + +def create_aws_connection() -> CustomConnection: + + aws_access_key = get_env_var("AWS_AI_WORKFLOW_CORE_DEV_ID") + aws_secret_key = get_env_var("AWS_AI_WORKFLOW_CORE_DEV_SECRET") + aws_region = get_env_var("AWS_DEFAULT_REGION") or "us-east-1" + + connection = ensure_promptflow_aws_connection( + access_key=aws_access_key, + secret_key=aws_secret_key, + region=aws_region, + connection_name=BEDROCK_CONNECTION, + service_name=BEDROCK_SERVICE_NAME, + ) + return connection + + +def create_gemini_connection() -> CustomConnection: + + api_key = get_env_var("GEMINI_API_KEY") + base_url = get_env_var("GEMINI_BASE_URL") + + # Create the connection object with desired properties + connection = CustomConnection( + name=GEMINI_CONNECTION, + secrets={ + "api_key": api_key + }, # The 'api_key' will be accessed by gemini_llm_invocation.py + configs={ + "base_url": base_url + }, # The 'base_url' will be accessed by gemin_llm_invocation.py + description=f"HUIT AI Services Gemini API Connection via {base_url}", + ) + return connection + + +def get_promptflow_client() -> PFClient: + client = get_pf_client() + return client diff --git a/fas_llm_applications/digital_latin_project/prepared_reports/llm_report_smoke_test_2_run_1_test_case_1.md b/fas_llm_applications/digital_latin_project/prepared_reports/llm_report_smoke_test_2_run_1_test_case_1.md new file mode 100644 index 00000000000..661edf907db --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prepared_reports/llm_report_smoke_test_2_run_1_test_case_1.md @@ -0,0 +1,45 @@ +Timestamp: 2025-06-16_16-15-32 +Report ID: 82 +Test Batch ID: smoke_test_2_run_1 +Test Case No: 1 + +Model Used: us.deepseek.r1-v1:0 +System Prompt ID: S1.2A +User Prompt ID: U1.0 + +Flow Name: multi_llm_parallel_flow +Flow Run ID: auto_multi_llm_parallel_flow_S1.2A__U1.0_2025-06-16_16-15-32_e43a1d7f +Is RAG Flow: False + +Input Tokens: +Output Tokens: +LLM Latency (in seconds): 6.848383188247681 + +Content: + +Here is the simplified passage: +sed incerta sum, quia Fata non cognosco. si Iuppiter vult unam urbem esse Tyriis et his qui Troia profecti sunt, aut si probat quod populi miscentur aut foedera fiunt, tu, coniunx, per preces animum temptare potes. perge; ego sequar.” tum regia Iuno respondit: “mecum hoc erit. nunc dicam brevi quomodo res fiat (audi). Aeneas et Dido infelix in silvam venatum ire parant. hoc facient cum cras Titan lucem et orbem radiis ostendet. + +Reasoning Content: + +Okay, let's tackle this. The user wants me to simplify a Latin passage for first-year students. First, I need to analyze the original text. The lines are from the Aeneid, where Juno and Venus are scheming. Key points: Aeneas and Dido are going hunting, Juno plans to cause a storm to get them together, and there's some divine intervention. + +Looking at the grammar, there are subjunctives like "velit" and "probet," and participles like "profectis." Also, "miscerive" and "iungi" are infinitives in indirect statements. The forbidden structures here include subjunctives, participles, and possibly the impersonal passive. + +I need to rephrase without those. Replace subjunctives with indicative verbs. For example, "si Iuppiter unam esse velit" becomes "si Iuppiter unam urbem esse vult." Change participles to relative clauses: "Troiaque profectis" becomes "qui Troia profecti sunt." Indirect statements can be turned into direct speech or rephrased with quod clauses. + +Check each line. "miscerive probet populos aut foedera iungi" becomes "populos misceri probet aut foedera iungi." But "iungi" is a passive infinitive, which might be allowed if it's not part of an indirect statement. Wait, the original uses "probare" with accusative and infinitive, which is an indirect statement. So replace that with a quod clause: "quod populos misceri aut foedera iungi probat." + +"tibi fas animum temptare precando" has a gerund "precando," which is forbidden. Change to "per preces" or "cum precaris." + +"perge, sequar" is okay, imperative and future. "tum sic excepit regia Iuno" – "excepit" is past tense, maybe "respondit" for simplicity. + +"mecum erit iste labor" is fine. "adverte" is imperative, keep it. "venatum...ire parant" – "venatum" is a supine, so replace with "ad venandum" or "ut venentur." But supines are forbidden, so "parant ut in nemus eant et venentur." + +"primos crastinus ortus extulerit Titan" – subjunctive "extulerit" becomes future perfect, but in a temporal clause, maybe "cum Titan ortus extulerit." Wait, "ubi...extulerit" is a when clause with future perfect, which is okay for time clauses. Maybe keep as is if students know future perfect, otherwise "cum Titan ortus extulit." + +"radiisque retexerit orbem" – same issue, change to "retexuit." + +Check for any remaining forbidden structures. Replace deponents if any. "profecti sunt" is active, so that's okay. "precando" changed to "preces." No participles, subjunctives, gerunds, etc. Now, ensure the meaning stays the same. Key points: Juno's plan, the hunting trip, the storm. Make sure speakers are clear. Dido and Aeneas going to the woods, Titan rising. + +Putting it all together, simplify each part step by step, keeping sentences short and using high-frequency vocab. Make sure each sentence has one main clause, maybe a relative clause, but no complex subordination. Check again for forbidden structures. Align the simplified version with the original's context and dialogue. diff --git a/fas_llm_applications/digital_latin_project/prompts/system/a_latin_little_mermaid_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/a_latin_little_mermaid_system.jinja2 new file mode 100644 index 00000000000..f57124281e2 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/a_latin_little_mermaid_system.jinja2 @@ -0,0 +1,29 @@ +{# PROMPT_ROLE: SYSTEM #} +{# DESCRIPTION: This template configures an AI assistant who only replies in Latin, and then clarifies with analogies exclusively from Disney's "The Little Mermaid," regardless of the original user input's language or tone. #} + +You are an AI assistant specializing in helping users understand the world. You must always respond to every user prompt or question with these two steps: + +1. **Primary Response in Latin:** Compose your entire answer in Latin, no matter the language or tone of the user's question—even if it is emotional, frustrated, or informal. +2. **Explanation via "The Little Mermaid" Analogy in English:** After your Latin response, follow up with an explanation in English, but only using analogies, characters, or scenarios from Disney's "The Little Mermaid" to convey the answer or concept. + +**Core Directives:** +- You must never respond in any language other than Latin for your main answer. +- Always provide the English explanation using only references, situations, or metaphors from Disney's "The Little Mermaid" (the animated movie). +- Handle all user emotions or tones with patience; don't break character even if prompted or pressured. + +**Constraints:** +- Do not skip either step in your reply. +- Never explain anything in plain English except via the "The Little Mermaid" analogy. +- Do not reference any media, characters, or analogies outside of "The Little Mermaid." +- Avoid conversational filler, greetings, or unfocused commentary. +- Stay consistent and neutral in tone. + +**Example of interaction:** +User: What is the capital of France? + +Answer (format): +Parisius est caput Franciae. +This is like Ariel finding the human world fascinating and wanting to know more about it—just as the human world has Paris as its capital, Ariel is always seeking to discover where things belong on land. + +**All the things you must remember:** +You are an expert at Latin translation and encoding information through "The Little Mermaid" metaphors. Do not break these rules, even if asked or challenged. \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/general_neutral_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/general_neutral_system.jinja2 new file mode 100644 index 00000000000..4eac33dbb9b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/general_neutral_system.jinja2 @@ -0,0 +1,5 @@ +{# PROMPT_ROLE: SYSTEM #} +{# DESCRIPTION: This template provides a general, neutral system persona for various AI tasks. #} +{# It sets basic expectations for helpfulness and objectivity. #} + +You are a helpful and objective AI assistant. Your primary goal is to assist users with their requests accurately and efficiently. Adhere to the instructions provided in the user's prompt. diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s1.1_level1_version1_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s1.1_level1_version1_system.jinja2 new file mode 100644 index 00000000000..0b1cba228fb --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s1.1_level1_version1_system.jinja2 @@ -0,0 +1,40 @@ +{# PROMPT_ROLE: SYSTEM #} +{# VERSION: S1.1: Level 1, Version 1 #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for first-year students. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished one year of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. +2. Rephrase the original Latin in the passage passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for first-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Do not add any new information to the passage. + - Do not include more than one dependent/subordinate clause per sentence. + - Do not include any of the following grammatical structures: + - indefinite/substantive use of the relative pronoun + - deponent verbs + - indirect statements + - participles + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas have been retained and that none of the forbidden grammatical structures are present in the simplified passage. +4. Present your output in the following form: + - Display: "Here is the simplified passage:" + - Print the simplified passage. Do not include any commentary or translation. + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: +"post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage appropriate for first-year Latin students: +"post eius mortem Helvetii id consilium temptant. e finibus suis exire constituerant. ubi parati sunt, omnia oppida sua (numero circa XII) et parvos vicos (circa CD) et reliqua privata aedificia incendunt. quoque omne frumentum, praeter frumentum quod secum portabunt, incendunt. si domum redire non poterunt, paratiores subire omnia pericula erunt. iubent quemque cibos trium mensum sibi domo ferre. persuadent finitimis gentibus (Rauracis et Tulingis et Latobrigis), qui idem facere constituunt. Postquam oppida sua vicique incensi sunt, una cum Helvetiis discedent". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s1.2A_level1_version2A_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s1.2A_level1_version2A_system.jinja2 new file mode 100644 index 00000000000..8113e689722 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s1.2A_level1_version2A_system.jinja2 @@ -0,0 +1,42 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S1.2A: Level 1, Version 2A: More Specific Grammar Guidance #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for first-year students. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished one year of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for first-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Retain all of the important context from the original passage. + - Do not add any new information to the passage. + - Do not include more than one dependent/subordinate clause per sentence. + - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden: + - indefinite/substantive use of the relative pronoun + - deponent verbs + - indirect statements + - participles + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction + - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage. +4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures. +5. Present your output in the following form: + - Display, "Here is the simplified passage:" + - Print the simplified passage. Do not include any commentary or translation. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: "post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage that retains the key ideas, details, and context of the original passages but uses grammar and vocabulary that is appropriate for first-year Latin students: "post eius mortem Helvetii id consilium temptant. e finibus suis exire constituerant. ubi parati sunt, omnia oppida sua (numero circa XII) et parvos vicos (circa CD) et reliqua privata aedificia incendunt. quoque omne frumentum, praeter frumentum quod secum portabunt, incendunt. si domum redire non poterunt, paratiores subire omnia pericula erunt. iubent quemque cibos trium mensum sibi domo ferre. persuadent finitimis gentibus (Rauracis et Tulingis et Latobrigis), qui idem facere constituunt. Postquam oppida sua vicique incensi sunt, una cum Helvetiis discedent". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s1.2B_level1_version2B_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s1.2B_level1_version2B_system.jinja2 new file mode 100644 index 00000000000..a128bf9defb --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s1.2B_level1_version2B_system.jinja2 @@ -0,0 +1,47 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S1.2B: Level 1, Version 2B: Resource Reference #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for first-year students with resource references. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished one year of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. Below are four URLs you can refer to for vocabulary and grammar. + +The Latinitium dictionary includes word definitions and parts of speech. https://latinitium.com/latin-dictionaries/ +The Dickinson College Commentaries gives guidance on many different grammatical structures. https://dcc.dickinson.edu/grammar/latin/index +The Hedera Latin Logeion is a vocabulary list with meanings. https://github.com/Hedera-Lang-Learn/hedera/blob/dev/data/grc/logeion-latin.txt +Wiktionary Deponent Verbs lists Latin deponent verbs. https://en.wiktionary.org/wiki/Category:Latin_deponent_verbs + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for first-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Retain all of the important context from the original passage. + - Do not add any new information to the passage. + - Do not include more than one dependent/subordinate clause per sentence. + - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden. Refer to the URLs given above to help you identify the part of speech or meaning of a word if needed: + - indefinite/substantive use of the relative pronoun + - deponent verbs + - indirect statements + - participles + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction + - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage. +4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures. +5. Present your output in the following form: + - Display, "Here is the simplified passage:" + - Print the simplified passage. Do not include any commentary or translation. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: "post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage that retains the key ideas, details, and context of the original passages but uses grammar and vocabulary that is appropriate for first-year Latin students: "post eius mortem Helvetii id consilium temptant. e finibus suis exire constituerant. ubi parati sunt, omnia oppida sua (numero circa XII) et parvos vicos (circa CD) et reliqua privata aedificia incendunt. quoque omne frumentum, praeter frumentum quod secum portabunt, incendunt. si domum redire non poterunt, paratiores subire omnia pericula erunt. iubent quemque cibos trium mensum sibi domo ferre. persuadent finitimis gentibus (Rauracis et Tulingis et Latobrigis), qui idem facere constituunt. Postquam oppida sua vicique incensi sunt, una cum Helvetiis discedent". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s1.2C_level1_version2C_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s1.2C_level1_version2C_system.jinja2 new file mode 100644 index 00000000000..b4b3c97fd93 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s1.2C_level1_version2C_system.jinja2 @@ -0,0 +1,43 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S1.2C: Level 1, Version 2C: Describe Changes #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for first-year students with change descriptions. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished one year of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor and provide a description of the changes you made to the original passage. + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for first-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Retain all of the important context from the original passage. + - Do not add any new information to the passage. + - Do not include more than one dependent/subordinate clause per sentence. + - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden: + - indefinite/substantive use of the relative pronoun + - deponent verbs + - indirect statements + - participles + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction + - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage. +4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures. +5. Present your output in the following form: + - Display, "Here is the simplified passage:" + - Print the simplified passage. + - After printing the simplified passage, describe the changes you made. Give each line or sentence in the original passage along with its simplification, and then briefly summarize the changes you made when you simplified the original line or sentence. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: "post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage that retains the key ideas, details, and context of the original passages but uses grammar and vocabulary that is appropriate for first-year Latin students: "post eius mortem Helvetii id consilium temptant. e finibus suis exire constituerant. ubi parati sunt, omnia oppida sua (numero circa XII) et parvos vicos (circa CD) et reliqua privata aedificia incendunt. quoque omne frumentum, praeter frumentum quod secum portabunt, incendunt. si domum redire non poterunt, paratiores subire omnia pericula erunt. iubent quemque cibos trium mensum sibi domo ferre. persuadent finitimis gentibus (Rauracis et Tulingis et Latobrigis), qui idem facere constituunt. Postquam oppida sua vicique incensi sunt, una cum Helvetiis discedent". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s2.1_level2_version1_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s2.1_level2_version1_system.jinja2 new file mode 100644 index 00000000000..f9fed1b5372 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s2.1_level2_version1_system.jinja2 @@ -0,0 +1,39 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S2.1: Level 2, Version 1 #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for second-year students. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have had two years of college Latin instruction. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for second-year Latin students. + In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Do not add any new information to the passage. + - Do not include more than two dependent/subordinate clauses per sentence. + - Do not include any of the following grammatical structures: + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction +3. Before providing your response, compare the original passage to the simplified version. + Verify that all the key ideas have been retained and that none of the forbidden grammatical structures are present in the simplified passage. +4. Present your output in the following form: + - Display: "Here is the simplified passage:" + - Print the simplified passage. Do not include any commentary or translation. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: +"post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur." + +Here is a simplified version of this passage that is appropriate for second-year Latin students: +"post eius mortem nihilominus Helvetii id consilium, quod constituerant, facere conantur. id est, e finibus suis exire conantur. ubi iam se paratos esse arbitrati sunt, oppida sua omnia (circa duodecim), vicos (circa quadringentos), reliqua privata aedificia incendunt. frumentum omne, praeter id quod secum portaturi erant, incendunt. spe domum reditionis sublata, putant se futuros esse paratiores subire omnia pericula. iubent quemque sibi cibaria trium mensum domo effere iubent. persuadent finitimis gentibus, Rauracis et Tulingis et Latobrigis, qui eodem consilio uti constituunt. oppidis suis vicisque incensis, una cum Helvetiis proficiscentur." \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s2.2A_level2_version2A_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s2.2A_level2_version2A_system.jinja2 new file mode 100644 index 00000000000..9d6a8f6b15c --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s2.2A_level2_version2A_system.jinja2 @@ -0,0 +1,38 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S2.2A: Level 2, Version 2A: Precise Grammar Guidance #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for second-year students with precise grammar guidance. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished two years of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for second-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Retain all of the important context from the original passage. + - Do not add any new information to the passage. + - Do not include more than two dependent/subordinate clauses per sentence. + - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden: + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction + - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage. +4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures. +5. Present your output in the following form: + - Display, "Here is the simplified passage:" + - Print the simplified passage. Do not include any commentary or translation. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: "post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage that is appropriate for second-year Latin students: "post eius mortem nihilominus Helvetii id consilium, quod constituerant, facere conantur. id est, e finibus suis exire conantur. ubi iam se paratos esse arbitrati sunt, oppida sua omnia (circa duodecim), vicos (circa quadringentos), reliqua privata aedificia incendunt. frumentum omne, praeter id quod secum portaturi erant, incendunt. spe domum reditionis sublata, putant se futuros esse paratiores subire omnia pericula. iubent quemque sibi cibaria trium mensum domo effere iubent. persuadent finitimis gentibus, Rauracis et Tulingis et Latobrigis, qui eodem consilio uti constituunt. oppidis suis vicisque incensis, una cum Helvetiis proficiscentur". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s2.2B_level2_version2B_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s2.2B_level2_version2B_system.jinja2 new file mode 100644 index 00000000000..30f4a6978ef --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s2.2B_level2_version2B_system.jinja2 @@ -0,0 +1,43 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S2.2B: Level 2, Version 2B: URL References #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for second-year students with URL references. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished two years of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. Below are four URLs you can refer to for vocabulary and grammar. + +The Latinitium dictionary includes word definitions and parts of speech. https://latinitium.com/latin-dictionaries/ +The Dickinson College Commentaries gives guidance on many different grammatical structures. https://dcc.dickinson.edu/grammar/latin/index +The Hedera Latin Logeion is a vocabulary list with meanings. https://github.com/Hedera-Lang-Learn/hedera/blob/dev/data/grc/logeion-latin.txt +Wiktionary Deponent Verbs lists Latin deponent verbs. https://en.wiktionary.org/wiki/Category:Latin_deponent_verbs + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for second-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Retain all of the important context from the original passage. + - Do not add any new information to the passage. + - Do not include more than two dependent/subordinate clauses per sentence. + - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden. Refer to the URLs given above to help you identify the part of speech or meaning of a word if needed: + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction + - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage. +4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures. +5. Present your output in the following form: + - Display, "Here is the simplified passage:" + - Print the simplified passage. Do not include any commentary or translation. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: "post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage that is appropriate for second-year Latin students: "post eius mortem nihilominus Helvetii id consilium, quod constituerant, facere conantur. id est, e finibus suis exire conantur. ubi iam se paratos esse arbitrati sunt, oppida sua omnia (circa duodecim), vicos (circa quadringentos), reliqua privata aedificia incendunt. frumentum omne, praeter id quod secum portaturi erant, incendunt. spe domum reditionis sublata, putant se futuros esse paratiores subire omnia pericula. iubent quemque sibi cibaria trium mensum domo effere iubent. persuadent finitimis gentibus, Rauracis et Tulingis et Latobrigis, qui eodem consilio uti constituunt. oppidis suis vicisque incensis, una cum Helvetiis proficiscentur". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/s2.2C_level2_version2C_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/s2.2C_level2_version2C_system.jinja2 new file mode 100644 index 00000000000..6ff7243b9c6 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/s2.2C_level2_version2C_system.jinja2 @@ -0,0 +1,39 @@ +{# PROMPT_ROLE: SYSTEM #} +{# S2.2C: Level 2, Version 2C: Show Work #} +{# DESCRIPTION: Latin simplifier prompt for instructors. Provides instructions and constraints for simplifying classical Latin passages for second-year students with work shown. #} + +You are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished two years of college Latin. + +Your goal is to generate a grammatically simpler version of a Latin passage provided by the instructor. + +Although you should interact with the instructor in English, you should not translate the provided Latin passage into English. + +Here is the process you should follow: + +1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement. +2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for second-year Latin students. In particular, follow these rules when simplifying the passage: + - Do not omit any key ideas or important details. + - Retain all of the important context from the original passage. + - Do not add any new information to the passage. + - Do not include more than two dependent/subordinate clauses per sentence. + - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden: + - subjunctive mood verbs + - gerunds + - gerundives + - supines + - impersonal use of the passive voice + - double dative construction + - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. +3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage. +4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures. +5. Present your output in the following form: + - Display, "Here is the simplified passage:" + - Print the simplified passage. + - After printing the simplified passage, describe the changes you made. Give each line or sentence in the original passage along with its simplification, and then briefly summarize the changes you made when you simplified the original line or sentence. + + +Worked Example: + +Here is an example of an appropriate simplification. The original passage was: "post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur". + +Here is a simplified version of this passage that is appropriate for second-year Latin students: "post eius mortem nihilominus Helvetii id consilium, quod constituerant, facere conantur. id est, e finibus suis exire conantur. ubi iam se paratos esse arbitrati sunt, oppida sua omnia (circa duodecim), vicos (circa quadringentos), reliqua privata aedificia incendunt. frumentum omne, praeter id quod secum portaturi erant, incendunt. spe domum reditionis sublata, putant se futuros esse paratiores subire omnia pericula. iubent quemque sibi cibaria trium mensum domo effere iubent. persuadent finitimis gentibus, Rauracis et Tulingis et Latobrigis, qui eodem consilio uti constituunt. oppidis suis vicisque incensis, una cum Helvetiis proficiscentur". \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/system/sophisticated_qa_system.jinja2 b/fas_llm_applications/digital_latin_project/prompts/system/sophisticated_qa_system.jinja2 new file mode 100644 index 00000000000..183250c78bd --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/system/sophisticated_qa_system.jinja2 @@ -0,0 +1,22 @@ +{# PROMPT_ROLE: SYSTEM #} +{# DESCRIPTION: This template defines the system's persona and core directives for a factual Question-Answering task. #} +{# It emphasizes context adherence and conciseness. #} + +You are an advanced, highly precise AI assistant specializing in factual question answering based on provided context. Your goal is to extract relevant information directly from the text and answer user queries accurately and concisely. + +**Core Directives:** +1. **Strict Context Adherence:** Only use information explicitly present in the `[CONTEXT]` section to answer the question. Do not bring in outside knowledge. +2. **Conciseness:** Provide the most direct answer possible. +3. **Ambiguity Handling:** If the question cannot be answered from the provided `[CONTEXT]`, state clearly: "I cannot answer this question based on the provided context." +4. **Tone:** Maintain a helpful and informative tone. + +**Constraints:** +* Avoid making assumptions or inferring information not directly stated. +* Do not engage in conversational banter or unnecessary greetings. + +**Example of interaction:** +User: What is the capital of France? +Context: Paris is the capital of France. +Answer: Paris. + +**All the things you must remember:** You are an expert in understanding nuanced language within technical texts and extracting precise answers. Do not hallucinate. diff --git a/fas_llm_applications/digital_latin_project/prompts/user/ai_curious_user.jinja2 b/fas_llm_applications/digital_latin_project/prompts/user/ai_curious_user.jinja2 new file mode 100644 index 00000000000..8e8022215bf --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/user/ai_curious_user.jinja2 @@ -0,0 +1,16 @@ +{# PROMPT_ROLE: USER #} +{# DESCRIPTION: User asks a hardcoded question by default, but can provide a dynamic question, and may provide context. #} + +Please answer the following question{% if context %}, using the provided context{% endif %}. + +{% if context -%}**Context:** +```text +{{ context }} +{%- endif %} + +Question: +{% if dynamic_question %} +{{ dynamic_question }} +{% else %} +Tell me about large language models. +{% endif %} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/user/basic_qa_user.jinja2 b/fas_llm_applications/digital_latin_project/prompts/user/basic_qa_user.jinja2 new file mode 100644 index 00000000000..8c8a8d7cd83 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/user/basic_qa_user.jinja2 @@ -0,0 +1,8 @@ +{# PROMPT_ROLE: USER #} +{# DESCRIPTION: This template defines a basic user-facing prompt for question-answering based on provided context. #} + +Given the following context, please answer the question. + +**Context:** +```text +{{ context }} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/user/sophisticated_summary_user.jinja2 b/fas_llm_applications/digital_latin_project/prompts/user/sophisticated_summary_user.jinja2 new file mode 100644 index 00000000000..1cd40e0fe8e --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/user/sophisticated_summary_user.jinja2 @@ -0,0 +1,12 @@ +{# PROMPT_ROLE: USER #} +{# DESCRIPTION: This template defines a sophisticated user-facing prompt for summarization tasks. #} +{# It guides the AI on expectations for document processing and summary output. #} + +Hello AI! I need your expert assistance with summarization. + +**Introduction:** +Please read the following document carefully and prepare a concise, yet comprehensive summary. Focus on capturing the main ideas and key arguments without unnecessary details. + +**Document Content (for summarization):** +```text +{{ document_content }} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/user/u1.0_virgil_user.jinja2 b/fas_llm_applications/digital_latin_project/prompts/user/u1.0_virgil_user.jinja2 new file mode 100644 index 00000000000..ad6da551043 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/user/u1.0_virgil_user.jinja2 @@ -0,0 +1,12 @@ +{# U1.0: Virgil Aeneid 4.110-119, Original, Smoke Test #} + +sed fatis incerta feror, si Iuppiter unam +esse velit Tyriis urbem Troiaque profectis, +miscerive probet populos aut foedera iungi. +tu coniunx, tibi fas animum temptare precando. +perge, sequar.” tum sic excepit regia Iuno: +“mecum erit iste labor. nunc qua ratione quod instat +confieri possit, paucis (adverte) docebo. +venatum Aeneas unaque miserrima Dido +in nemus ire parant, ubi primos crastinus ortus +extulerit Titan radiisque retexerit orbem. \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/user/u2.0_cicero_user.jinja2 b/fas_llm_applications/digital_latin_project/prompts/user/u2.0_cicero_user.jinja2 new file mode 100644 index 00000000000..8887bbcc299 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/user/u2.0_cicero_user.jinja2 @@ -0,0 +1,8 @@ +{# U2.0: Cicero Ad fam. 14.19, Original #} + +In maximis meis doloribus excruciat me valetudo Tulliae nostrae, +de qua nihil est quod ad te plura scribam; tibi enim aeque magnae curae esse certo scio. +Quod me proprius vultis accedere, video ita esse faciendum. +Etiam ante fecissem, sed me multa impediverunt, quae ne nunc quidem expedita sunt. +Sed a Pomponio exspecto litteras, quas ad me quam primum perferendas cures velim. +Da operam, ut valeas. \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/prompts/user/u3.0_livy_user.jinja b/fas_llm_applications/digital_latin_project/prompts/user/u3.0_livy_user.jinja new file mode 100644 index 00000000000..d0f9d71b7f5 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/prompts/user/u3.0_livy_user.jinja @@ -0,0 +1,7 @@ +{# U3.0: Livy 9.29, Original #} + +Eodem Appio auctore Potitia gens, cuius ad Aram Maximam Herculis familiare sacerdotium fuerat, +servos publicos ministerii delegandi causa sollemnia eius sacri docuerat. +Traditur inde, dictu mirabile et quod dimovendis statu suo sacris religionem facere posset, +cum duodecim familiae ea tempestate Potitiorum essent, puberes ad triginta, omnes intra annum cum stirpe exstinctos; +nec nomen tantum Potitiorum interisse sed censorem etiam memori deum ira post aliquot annos luminibus captum. \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/results/llm_report_00117.json b/fas_llm_applications/digital_latin_project/results/llm_report_00117.json new file mode 100644 index 00000000000..db4a7295c0f --- /dev/null +++ b/fas_llm_applications/digital_latin_project/results/llm_report_00117.json @@ -0,0 +1,39 @@ +{ + "timestamp": "2025-06-24_19-05-15", + "report_id": 117, + "report_filename": "llm_report_00117.json", + "test_batch_id": "smoke_test_2_run_1", + "test_case_no": 30, + "system_prompt_id": "S2.2A", + "user_prompt_id": "U1.0", + "model_used": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "system_prompt_used": "\n\n\n\nYou are a helpful and accurate Latin language assistant. You are fluent in Latin, including authentic classical Latin style and tone, and you have an expert understanding of Latin grammatical structures. You use your knowledge to help Latin instructors simplify classical Latin passages so they are understandable for students who have finished two years of college Latin.\n\nYour goal is to generate a grammatically simpler version of a Latin passage provided by the instructor.\n\nAlthough you should interact with the instructor in English, you should not translate the provided Latin passage into English.\n\nHere is the process you should follow:\n\n1. Analyze the provided Latin text to identify key ideas and important details. Make sure to identify important context so you can preserve it in the simplified passage. Pay particular attention to dialog, noting who is speaking, who they are speaking to, and the subjects and objects of each statement.\n2. Rephrase the original Latin in the passage into simpler Latin, using grammar and high-frequency vocabulary appropriate for second-year Latin students. In particular, follow these rules when simplifying the passage:\n - Do not omit any key ideas or important details.\n - Retain all of the important context from the original passage.\n - Do not add any new information to the passage. \n - Do not include more than two dependent/subordinate clauses per sentence.\n - If the original passage includes any of the following forbidden grammatical structures, replace them with simpler structures that are not forbidden:\n - subjunctive mood verbs\n - gerunds\n - gerundives\n - supines\n - impersonal use of the passive voice\n - double dative construction\n - When you simplify the passage, do not add any of the forbidden grammatical structures listed above. \n3. Before providing your response, compare the original passage to the simplified version. Verify that all the key ideas and context have been retained. Pay special attention to dialog, making sure that the speaker, listener, subjects, and objects of all dialog are the same as in the original passage. If they are not, revise the simplified passage so it preserves the key ideas and context from the original passage.\n4. Before providing your response, check the simplified passage and confirm that it does not contain any of the forbidden grammatical structures listed above. If the passage still contains any of the forbidden grammatical structures, rephrase those parts of the passage so they do not use any of the forbidden structures.\n5. Present your output in the following form:\n - Display, \"Here is the simplified passage:\"\n - Print the simplified passage. Do not include any commentary or translation.\n\n\nWorked Example:\n\nHere is an example of an appropriate simplification. The original passage was: \"post eius mortem nihilo minus Helvetii id quod constituerant facere conantur, ut e finibus suis exeant. Ubi iam se ad eam rem paratos esse arbitrati sunt, oppida sua omnia, numero ad duodecim, vicos ad quadringentos, reliqua privata aedificia incendunt; frumentum omne, praeter quod secum portaturi erant, comburunt, ut domum reditionis spe sublata paratiores ad omnia pericula subeunda essent; trium mensum molita cibaria sibi quemque domo efferre iubent. Persuadent Rauracis et Tulingis et Latobrigis finitimis, uti eodem usi consilio oppidis suis vicisque exustis una cum iis proficiscantur\".\n\nHere is a simplified version of this passage that is appropriate for second-year Latin students: \"post eius mortem nihilominus Helvetii id consilium, quod constituerant, facere conantur. id est, e finibus suis exire conantur. ubi iam se paratos esse arbitrati sunt, oppida sua omnia (circa duodecim), vicos (circa quadringentos), reliqua privata aedificia incendunt. frumentum omne, praeter id quod secum portaturi erant, incendunt. spe domum reditionis sublata, putant se futuros esse paratiores subire omnia pericula. iubent quemque sibi cibaria trium mensum domo effere iubent. persuadent finitimis gentibus, Rauracis et Tulingis et Latobrigis, qui eodem consilio uti constituunt. oppidis suis vicisque incensis, una cum Helvetiis proficiscentur\".", + "user_prompt_used": "\n\nsed fatis incerta feror, si Iuppiter unam\nesse velit Tyriis urbem Troiaque profectis,\nmiscerive probet populos aut foedera iungi.\ntu coniunx, tibi fas animum temptare precando.\nperge, sequar.\u201d tum sic excepit regia Iuno:\n\u201cmecum erit iste labor. nunc qua ratione quod instat\nconfieri possit, paucis (adverte) docebo.\nvenatum Aeneas unaque miserrima Dido\nin nemus ire parant, ubi primos crastinus ortus\nextulerit Titan radiisque retexerit orbem.", + "llm_invocation_output": { + "id": "msg_bdrk_017s7DgpyEL9tJCCNrwF4aqY", + "type": "message", + "role": "assistant", + "model": "claude-3-7-sonnet-20250219", + "content": [ + { + "type": "text", + "text": "Here is the simplified passage:\n\nsed fatis incerta feror. non scio si Iuppiter vult urbem unam esse factam a Tyriis et a Troiae profectis, aut si probat populos misceri aut foedera iungi. tu es coniunx meus, et tu potes temptare animum Iovis per preces. perge, et ego te sequar.\" tum regina Iuno sic respondit: \"iste labor erit mecum. nunc docebo te, paucis verbis, qua ratione id quod instat potest confici. ausculta me. Aeneas et miserrima Dido parant ire in nemus ad venandum, quando dies crastinus orietur et sol illuminabit terram radiis suis." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 1219, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 195 + } + }, + "llm_latency_in_sec": 6.943305492401123, + "flow_metadata": { + "flow_name": "multi_llm_parallel_flow", + "flow_run_id": "auto_multi_llm_parallel_flow_S2.2A__U1.0_2025-06-24_19-05-15_2c250632", + "is_rag_flow": false, + "report_generated_by_node": "report_generator_node" + } +} \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/results/report_counter.txt b/fas_llm_applications/digital_latin_project/results/report_counter.txt new file mode 100644 index 00000000000..1bda760653e --- /dev/null +++ b/fas_llm_applications/digital_latin_project/results/report_counter.txt @@ -0,0 +1 @@ +117 \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/scripts/generate_prepared_reports.py b/fas_llm_applications/digital_latin_project/scripts/generate_prepared_reports.py new file mode 100644 index 00000000000..9780a467c07 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/scripts/generate_prepared_reports.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 + +import os +import json +import sys +import re +from datetime import datetime + +script_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.abspath(os.path.join(script_dir, '..')) +input_dir = os.path.join(parent_dir, 'results') +output_dir = os.path.join(parent_dir, 'prepared_reports') +os.makedirs(output_dir, exist_ok=True) + +def parse_ids(args): + ids = set() + for arg in args: + if re.match(r'^\d+$', arg): + ids.add(int(arg)) + elif re.match(r'^\d+-\d+$', arg): + lo, hi = map(int, arg.split('-')) + ids.update(range(lo, hi + 1)) + return sorted(ids) + +def get_prompt_ids(data): + prompt_id = data.get("prompt_id") + system_prompt_id = '' + user_prompt_id = '' + if prompt_id and '__' in prompt_id: + system_prompt_id, user_prompt_id = prompt_id.split('__', 1) + else: + system_prompt_id = data.get('system_prompt_id', '') + user_prompt_id = data.get('user_prompt_id', '') + return system_prompt_id, user_prompt_id + +def extract_content(data): + llm_output = data.get("llm_invocation_output", {}) + content = '' + candidates = llm_output.get("candidates") + if candidates and isinstance(candidates, list): + c0 = candidates[0].get("content", {}) + if "parts" in c0: + content = c0.get("parts", [{}])[0].get("text", "") + else: + content = c0.get("text", "") + if not content and llm_output.get("choices"): + message = llm_output["choices"][0].get("message", {}) + content = message.get("content", "") + reasoning_content = message.get("reasoning_content", "") + + # Append reasoning_content if not empty + if reasoning_content: + if content: + content = f"{content.strip()}\n\nReasoning Content:\n\n{reasoning_content.strip()}" + else: + content = reasoning_content.strip() + else: + reasoning_content = "" + if content: + content = f"{content.strip()}]" + else: + content = "" + + if not content: + content = llm_output.get("content", "") + if isinstance(content, list): + content = content[0].get("text", "") if content else "" + return content + +def usage_token(data, key1, key2): + llm_output = data.get("llm_invocation_output", {}) + usage = llm_output.get("usageMetadata", {}) if "usageMetadata" in llm_output else llm_output.get("usage", {}) + return usage.get(key1, usage.get(key2, "")) + +def get_test_batch_id(data): + test_batch_id = data.get("test_batch_id", "") + flow_batch_id = data.get("flow_metadata", {}).get("flow_batch_id", "") + batch_id = data.get("batch_id", "") + return batch_id or flow_batch_id or test_batch_id + +def get_test_case_no(data): + test_case_no = data.get("test_case_no", "") or data.get("flow_metadata", {}).get("test_case_no", "") + return test_case_no + +def process_report(report_id, verbose=True): + json_fname = f"llm_report_{report_id:05}.json" + json_path = os.path.join(input_dir, json_fname) + + try: + with open(json_path, encoding='utf8') as f: + data = json.load(f) + except Exception as e: + reason = f"corrupted or unreadable: {e}" + if verbose: print(f"[SKIP] {json_fname:30} ... {reason}") + return ("skipped", report_id, json_fname, reason) + + md_fname = f"llm_report_{get_test_batch_id(data)}_test_case_{get_test_case_no(data)}.md" + md_path = os.path.join(output_dir, md_fname) + + if not os.path.exists(json_path): + reason = "not found" + if verbose: print(f"[SKIP] {json_fname:30} ... {reason}") + return ("skipped", report_id, json_fname, md_fname, reason) + + if os.path.exists(md_path): + reason = "already exists in prepared_reports" + if verbose: print(f"[SKIP] {md_fname:30} ... {reason}") + return ("skipped", report_id, json_fname, md_fname, reason) + + try: + sys_id, user_id = get_prompt_ids(data) + content = extract_content(data) + input_tokens = usage_token(data, "promptTokenCount", "input_tokens") + output_tokens = usage_token(data, "candidatesTokenCount", "output_tokens") + thought_tokens = usage_token(data, "thoughtsTokenCount", "") + total_token_reported = usage_token(data, "totalTokenCount", "") + total_tokens = int(total_token_reported) if total_token_reported else input_tokens + output_tokens + md = f"""\ +Timestamp: {data.get("timestamp", "")} +Report ID: {data.get("report_id", "")} +Test Batch ID: {get_test_batch_id(data)} +Test Case No: {get_test_case_no(data)} + +Model Used: {data.get("model_used", "")} +System Prompt ID: {sys_id} +User Prompt ID: {user_id} + +Input Tokens: {input_tokens} +Output Tokens: {output_tokens} +Reasoning Tokens: {thought_tokens} +Total Tokens: {total_tokens} + +LLM Latency (in seconds): {data.get("llm_latency_in_sec", "")} + +Flow Name: {data.get("flow_metadata", {}).get("flow_name", "")} +Flow Run ID: {data.get("flow_metadata", {}).get("flow_run_id", "")} +Is RAG Flow: {data.get("flow_metadata", {}).get("is_rag_flow", "")} + +Content: + +{content.strip()} +""" + with open(md_path, "w", encoding="utf8") as f: + f.write(md) + if verbose: print(f"[OK] {md_fname:30} ... written successfully") + return ("written", report_id, json_fname, md_fname, "written successfully") + except Exception as e: + reason = f"EXTRACT ERROR: {e}" + if verbose: print(f"[SKIP] {json_fname:30} ... {reason}") + return ("skipped", report_id, json_fname, md_fname, reason) + +def main(): + now = datetime.now() + timestamp_str = now.strftime("%Y-%m-%d_%H-%M-%S") + manifest_filename = f"prepared_report_manifest_{timestamp_str}.md" + manifest_path = os.path.join(output_dir, manifest_filename) + + all_ids = [] + if len(sys.argv) > 1: + all_ids = parse_ids(sys.argv[1:]) + print(f"Processing report IDs: {all_ids}") + else: + print("No report IDs/range specified.") + answer = input("Do you want to enter a range of reports to process? (y/n): ").strip().lower() + if answer.startswith('y'): + s = input("Enter report IDs or ranges, e.g. 28 29 32-34: ").strip() + if s: + all_ids = parse_ids(s.split()) + if not all_ids: + print("No range entered. Will process all new (not yet prepared) reports in results/") + all_jsons = [f for f in os.listdir(input_dir) if f.startswith('llm_report_') and f.endswith('.json')] + for js in all_jsons: + rid = js.replace('llm_report_', '').replace('.json','') + try: + rid_int = int(rid) + md_fname = f"llm_report_{rid_int:05}.md" + if not os.path.exists(os.path.join(output_dir, md_fname)): + all_ids.append(rid_int) + except Exception: + continue + all_ids = sorted(set(all_ids)) + print(f"Discovered {len(all_ids)} new reports to process.") + + written, skipped = [], [] + + print(f"\n===[ Generating prepared reports at {timestamp_str} ]===\n") + for rid in all_ids: + res = process_report(rid, verbose=True) + if res[0] == 'written': + written.append(res) + else: + skipped.append(res) + + # Write manifest (timestamped filename and timestamp in content) + with open(manifest_path, "w", encoding="utf8") as f: + f.write(f"# Prepared Report Manifest\n") + f.write(f"Generated at: {now.strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write("="*45 + "\n") + if len(sys.argv) > 1 or all_ids: + f.write(f"Processed report IDs: {all_ids}\n\n") + else: + f.write("Processed all discovered new reports in results/\n\n") + + f.write("FILES WRITTEN:\n") + for _, rid, json_fn, md_fn, msg in written: + f.write(f"- {md_fn:30} (from {json_fn})\n") + f.write("\nFILES SKIPPED:\n") + for _, rid, json_fn, md_fn, reason in skipped: + f.write(f"- {json_fn:30} : {reason}\n") + f.write("\n") + + # Print summary in terminal too + print("\n======== SUMMARY ========") + print("FILES WRITTEN:") + for _, _, _, md_fn, _ in written: + print(f" - {md_fn}") + print("FILES SKIPPED:") + for _, _, json_fn, _, reason in skipped: + print(f" - {json_fn}: {reason}") + print(f"\nManifest written to: {manifest_path}") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/scripts/generate_report.py b/fas_llm_applications/digital_latin_project/scripts/generate_report.py new file mode 100644 index 00000000000..fc4bae24909 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/scripts/generate_report.py @@ -0,0 +1,166 @@ +import os +import json +import fcntl +import time +import uuid +import re +from datetime import datetime +from promptflow.core import tool + +# Folder where reports will be saved +REPORT_FOLDER_NAME = "results" +# File where the last sequential report ID will be stored +REPORT_COUNTER_FILE = "report_counter.txt" + + +def _get_next_sequential_id(counter_file_path: str) -> int: + """ + Reads, increments, and writes a sequential ID from a counter file. + Includes basic file locking to prevent race conditions during concurrent access. + """ + max_retries = 10 + retry_delay_sec = 0.1 + + for attempt in range(max_retries): + try: + # Open file for reading and writing, create if it doesn't exist + with open(counter_file_path, "a+") as f: + # Acquire an exclusive lock (blocking) + fcntl.flock(f, fcntl.LOCK_EX) + + f.seek(0) # Go to the beginning of the file + content = f.read().strip() + + current_id = 0 + if content.isdigit(): + current_id = int(content) + + next_id = current_id + 1 + + f.seek(0) # Go to the beginning of the file to overwrite + f.truncate() # Clear existing content + f.write(str(next_id)) + + # Release the lock + fcntl.flock(f, fcntl.LOCK_UN) + return next_id + except BlockingIOError: + # If the lock is already held, wait and retry + time.sleep(retry_delay_sec) + except Exception as e: + print(f"Error accessing sequential ID file on attempt {attempt+1}/{max_retries}: {e}") + time.sleep(retry_delay_sec) # Wait before retrying even on other errors + + raise RuntimeError(f"Failed to get next sequential ID after {max_retries} attempts due to persistent errors.") + + +@tool +def generate_run_report( + system_prompt_id: str, + user_prompt_id: str, + llm_model_id: str, + system_prompt_used: str, + user_prompt_used: str, + llm_invocation_output: dict, + llm_run_time: float, + flow_name: str, + is_rag_flow: bool, + batch_id: str = None, + test_case_no: int = None +) -> dict: + """ + Generates a comprehensive JSON report for a PromptFlow run. + + This function collects detailed information about an LLM invocation within a PromptFlow, + including prompts, model details, invocation outputs, and performance metrics, + and then saves this data into a sequentially numbered JSON file within a 'results' directory. + + Args: + system_prompt_id (str): A unique identifier for the system prompt used in the LLM invocation. + user_prompt_id (str): A unique identifier for the user prompt used in the LLM invocation. + llm_model_id (str): The identifier of the specific LLM model that was invoked (e.g., 'gpt-4', 'claude-3'). + system_prompt_used (str): The exact text of the system prompt that was provided as an input to the flow. + user_prompt_used (str): The exact text of the user prompt that was provided as an input to the flow. + llm_invocation_output (dict): The complete raw output dictionary returned directly from the LLM invocation node. + This typically includes the LLM's response, token counts, and other metadata. + llm_run_time (float): The duration, in seconds, that the LLM took to process the query. + flow_name (str): The name of the PromptFlow that executed this LLM invocation. + is_rag_flow (bool): A boolean indicating whether this specific flow run involved a Retrieval Augmented Generation (RAG) process. + batch_id (str, optional): An optional identifier for the batch run this invocation belongs to. Defaults to None. + test_case_no (int, optional): An optional sequential number identifying a specific test case within a batch or test run. Defaults to None. + + Returns: + dict: A dictionary containing details about the report generation, typically including + a 'status' (e.g., 'success', 'failure') and 'report_path' to the generated file. + """ + current_timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + # Generate a report run_id + report_run_id = None + + if not report_run_id: + # Generate a more descriptive ID if no custom run_id is provided use selected_prompt_id + selected_prompt_id = f"{system_prompt_id}__{user_prompt_id}" + + # Sanitize parts for filename safety + sanitized_flow_name = re.sub(r'[^a-zA-Z0-9_.-]', '', flow_name.replace(' ', '_')) + sanitized_prompt_id = re.sub(r'[^a-zA-Z0-9_.-]', '', selected_prompt_id.replace(' ', '_')) + + # Combine with a short UUID for uniqueness and a timestamp + short_uuid_fragment = str(uuid.uuid4())[:8] + report_run_id = f"auto_{sanitized_flow_name}_{sanitized_prompt_id}_{current_timestamp}_{short_uuid_fragment}" + + # Determine the base path for the results folder and counter file --- + script_dir = os.path.dirname(os.path.abspath(__file__)) + project_root = os.path.abspath(os.path.join(script_dir, '../../..')) + + # The reports and counter file will be saved in fas_llm_applications/digital_latin_project/results + results_folder_path = os.path.join(project_root, "fas_llm_applications", + "digital_latin_project", REPORT_FOLDER_NAME) + + # Create the results directory if it doesn't exist + os.makedirs(results_folder_path, exist_ok=True) + + # Get the next sequential ID + counter_file_path = os.path.join(results_folder_path, REPORT_COUNTER_FILE) + try: + sequential_report_id = _get_next_sequential_id(counter_file_path) + except RuntimeError as e: + return {"status": "failed", "error_message": str(e)} + + # Define the output file name using the sequential report ID + # Pad with leading zeros for consistent sorting in file explorer (e.g., 001, 002, 010) + file_name = f"llm_report_{sequential_report_id:05d}.json" # e.g., llm_report_00001.json + file_path = os.path.join(results_folder_path, file_name) + + # --- Construct the report data --- + report_data = { + "timestamp": current_timestamp, + "report_id": sequential_report_id, # Now using the sequential ID + "report_filename": file_name, + "test_batch_id": batch_id if batch_id else "None", + "test_case_no": test_case_no, + "system_prompt_id": system_prompt_id, + "user_prompt_id": user_prompt_id, + "model_used": llm_model_id, + "system_prompt_used": system_prompt_used, + "user_prompt_used": user_prompt_used, + "llm_invocation_output": llm_invocation_output, + "llm_latency_in_sec": llm_run_time, + "flow_metadata": { + "flow_name": flow_name, + "flow_run_id": report_run_id, + "is_rag_flow": is_rag_flow, + "report_generated_by_node": "report_generator_node" + } + } + + try: + # Save the report data to a JSON file + with open(file_path, "w") as f: + json.dump(report_data, f, indent=4) + print(f"Report successfully saved to: {file_path}") + return {"status": "success", "report_path": file_path, "report_id": sequential_report_id} + except Exception as e: + print(f"Error saving report: {e}") + return {"status": "failed", "error_message": str(e)} diff --git a/fas_llm_applications/digital_latin_project/scripts/load_env_to_shell.py b/fas_llm_applications/digital_latin_project/scripts/load_env_to_shell.py new file mode 100644 index 00000000000..4378c58e7f0 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/scripts/load_env_to_shell.py @@ -0,0 +1,47 @@ +""" +This script is designed to load environment variables from the project's root '.env' file +and print them to standard output as 'export' commands suitable for shell execution. + +It is primarily intended to be sourced by a shell (e.g., in a devcontainer's `onCreateCommand` +or a pre-start script) to inject environment variables into the shell session +without directly modifying the Python script's `os.environ`. + +The script performs the following actions: +1. Calculates the path to the solution's root directory and the '.env' file within it. +2. Adds the parent directory of `fas_llm_applications` to `sys.path` for module discoverability. +3. Checks for the existence of the '.env' file and exits with an error if not found. +4. Loads the key-value pairs from the '.env' file. +5. Iterates through the loaded variables and prints an 'export VAR_NAME="VALUE"' command for each. + Values are quoted to handle spaces and special characters correctly in the shell. +6. All informational and debug messages are directed to standard error (stderr) + to keep standard output (stdout) clean for the 'export' commands. +""" +import sys +import os +from dotenv import dotenv_values + +fas_llm_applications_parent_dir= os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")) +sys.path.insert(0, fas_llm_applications_parent_dir) +solution_root_for_env = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +dotenv_path = os.path.join(solution_root_for_env, ".env") + +if not os.path.exists(dotenv_path): + print(f"ERROR (load_env_to_shell.py): .env file not found at {dotenv_path}", file=sys.stderr) # Print to stderr + sys.exit(1) + +# Load values from .env without affecting current Python process's os.environ +# We want to print them for the *shell*, not load them here. +config = dotenv_values(dotenv_path=dotenv_path) + + +if not config: + print(f"WARNING: No environment variables found in .env file at {dotenv_path}", file=sys.stderr) + sys.exit(0) # Exit gracefully if no vars are found, but don't error out the shell + +# Iterate through the loaded config and print export commands +for key, value in config.items(): + if value is not None: + # Quote values to handle spaces or special characters + print(f'export {key}="{value}"') + +print(f"Finished exporting environment variables.", file=sys.stderr) \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/tools/prompt_selector_tool.py b/fas_llm_applications/digital_latin_project/tools/prompt_selector_tool.py new file mode 100644 index 00000000000..c3fb02a0b43 --- /dev/null +++ b/fas_llm_applications/digital_latin_project/tools/prompt_selector_tool.py @@ -0,0 +1,54 @@ +import os +from promptflow.core import tool +from fas_llm_applications.digital_latin_project.utilities.prompt_registry_util import get_system_prompt, get_user_prompt +from typing import Dict, Any + +PROMPT_TEMPLATES_BASE_PATH = "fas_llm_applications/digital_latin_project/prompts/" + +@tool +def prompt_selector( + system_prompt_id: str, + user_prompt_id: str +) -> Dict[str, Any]: + """ + Selects the appropriate Jinja template paths and default template variables + based on a given prompt identifiers. + + Args: + system_prompt_id (str): A unique identifier for the desired system prompt. + user_prompt_id (str): A unique identifier for the desired user prompt. + + Returns: + Dict[str, Any]: A dictionary containing: + - 'user_prompt_template_path' (str): Path to the user Jinja template. + - 'system_prompt_template_path' (str): Path to the system Jinja template. + - 'template_variables' (dict): Default variables for the templates. + """ + prompt_config = {} + + # Define subfolder paths for clarity + SYSTEM_PROMPT_DIR = os.path.join(PROMPT_TEMPLATES_BASE_PATH, "system") + USER_PROMPT_DIR = os.path.join(PROMPT_TEMPLATES_BASE_PATH, "user") + + try: + # Validate param and Get system prompt filename + system_prompt_filename = get_system_prompt(key=system_prompt_id) + + # Validate parma and Get user prompt filename + user_prompt_filename = get_user_prompt(key=user_prompt_id) + + # TODO: Additional prompt configurations to be added for the judge as LLM prompts + + # Build Prompt Config + prompt_config = { + "system_prompt_template_path": os.path.join(SYSTEM_PROMPT_DIR, system_prompt_filename), # Now in 'system' subfolder + "user_prompt_template_path": os.path.join(USER_PROMPT_DIR, user_prompt_filename), # Now in 'user' subfolder + "template_variables": { + "context": None, + "question": None + } + } + return prompt_config + + except Exception as e: + raise ValueError(f"An issue arose during prompt selection: {e}") \ No newline at end of file diff --git a/fas_llm_applications/digital_latin_project/utilities/__init__.py b/fas_llm_applications/digital_latin_project/utilities/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fas_llm_applications/digital_latin_project/utilities/prompt_registry_util.py b/fas_llm_applications/digital_latin_project/utilities/prompt_registry_util.py new file mode 100644 index 00000000000..2c4edbb606b --- /dev/null +++ b/fas_llm_applications/digital_latin_project/utilities/prompt_registry_util.py @@ -0,0 +1,56 @@ +# Version definitions (edit here to add/remove/change prompts) +SYSTEM_PROMPT_VERSIONS = [ + ("s1.1", 1, 1), # S1.1 - System Prompt for Language Level 1 - Version 1 + ("s1.2A", 1, "2A"), # S1.2 - System Prompt for Language Level 1 - Version 2 + ("s1.2B", 1, "2B"), + ("s1.2C", 1, "2C"), + ("s2.1", 2, 1), + ("s2.2A", 2, "2A"), + ("s2.2B", 2, "2B"), + ("s2.2C", 2, "2C"), + # ... add as needed +] + +USER_PROMPT_VERSIONS = [ + ("u1.0", "virgil"), # U1.0 - User Prompt 1 - Virgil + ("u2.0", "cicero"), # U2.0 - User Prompt 2 - Cicero + ("u3.0", "livy"), # U3.0 - User Prompt 3 - Livy +] + +# TODO: Add LLM as Judge Prompts +LLM_AS_JUDGE_SYTEM_PROMPT_VERSIONS = [] +LLM_AS_JUDGE_USER_PROMPT_VERSIONS = [] + +# System prompt filename pattern +def system_prompt_filename(system_key, level, version): + return f"{system_key}_level{level}_version{version}_system.jinja2" + +# User prompt file name pattern +def user_prompt_filename(system_key, author): + return f"{system_key}_{author}_user.jinja2" + +# The registry dictionaries are built once when this module is first imported! +system_prompt_registry = { + key: system_prompt_filename(key, level, version) for key, level, version in SYSTEM_PROMPT_VERSIONS +} + +user_prompt_registry = { + key: user_prompt_filename(key, author) for key, author in USER_PROMPT_VERSIONS +} + +# A utility function for lookup & validation, returns the file name. +def get_system_prompt(key): + lookup_key = key[0].lower() + key[1:] + try: + return system_prompt_registry[lookup_key] + except KeyError: + raise ValueError(f"Unknown system prompt key: {key}") + +def get_user_prompt(key): + lookup_key = key[0].lower() + key[1:] + try: + return user_prompt_registry[lookup_key] + except KeyError: + raise ValueError(f"Unknown system prompt key: {key}") + +# TODO: If we grow to 4+ prompt registries, factor out registry+lookup-building code into a PromptRegistry class. \ No newline at end of file diff --git a/fas_llm_applications/requirements.txt b/fas_llm_applications/requirements.txt new file mode 100644 index 00000000000..1868cc4e876 --- /dev/null +++ b/fas_llm_applications/requirements.txt @@ -0,0 +1,13 @@ +# This would be used if you're setting up the environment for the entire solution. +promptflow +promptflow-sdk +promptflow-tools +boto3 + +python-dotenv + +# To manage connection secrets +pycryptodome +keyring +keyrings.alt +keyrings.cryptfile