Merge branch 'main' into fix-xlsb-cultural-category-commas

Author: hannahwestra25

.github/instructions/scorers.instructions.md

@@ -13,18 +13,16 @@ Scorers evaluate model responses against an objective and live under `pyrit/scor
 `Scorer` subclasses MUST use the keyword-only constructor shape:
 
 ```python
-class MyScorer(Scorer):
+class MyScorer(MessageTrueFalseScorer):
     def __init__(
         self,
         *,
-        chat_target: PromptTarget | None = None,
-        threshold: float = 0.5,
+        chat_target: PromptTarget,
         validator: ScorerPromptValidator | None = None,
     ) -> None:
-        super().__init__(
-            validator=validator or self._DEFAULT_VALIDATOR,
-            chat_target=chat_target,
-        )
+        super().__init__(validator=validator or self._DEFAULT_VALIDATOR)
+        self._prompt_target = chat_target
+        self._judge = TargetJudge(target=chat_target, requirements=self.TARGET_REQUIREMENTS)
 ```
 
 Requirements:
@@ -34,9 +32,20 @@ Requirements:
   `Scorer.__init_subclass__` calling `enforce_keyword_only_init`
   (see `pyrit/common/brick_contract.py`). Non-conforming subclasses
   raise `TypeError` at import time.
-- ``super().__init__(validator=..., chat_target=...)`` is required so the
-  base class wires the validator and validates ``TARGET_REQUIREMENTS``
-  against any provided ``chat_target``.
+- Message-family bases wire the validator. Their deprecated `chat_target` parameter and the
+  one on `Scorer` only validate `TARGET_REQUIREMENTS` until removal in 1.4.0; they do not store
+  a target or create a judge. New concrete target-backed scorers compose `TargetJudge`, which
+  validates the requirements. Specialized service scorers validate at their concrete owner.
+- Scorers render prompts, pass the effective expectation explicitly in `JudgmentRequest`, and
+  convert the returned judgment. The judge delegates transport and retries; the response handler
+  owns parsing. Raw `ObservationSource` implementations acquire evidence without criteria.
+- `JudgmentRequest` is data only. Message scorers call `_capture_judgment_evidence` before
+  sending it; other callers supply evidence references directly. The exchange consumes the request
+  without reading the active message or expectation context.
+- Preserve `get_chat_target()` for target discovery. Use `_score_piece_with_expectation_async`
+  for migrated judge consumers; do not replace it with an objective-only hook.
+- A legacy `_score_piece_async` override below a typed scorer raises `TypeError` at construction.
+  Keep this fail-fast check: implicit dispatch through both hooks can skip or repeat custom policy.
 
 ## Condition contract
 

.github/workflows/codeql.yml

@@ -0,0 +1,46 @@
+name: CodeQL
+
+on:
+  push:
+    branches:
+      - "main"
+      - "releases/v*"
+  pull_request:
+    branches:
+      - "main"
+      - "releases/**"
+  merge_group:
+  workflow_dispatch:
+  schedule:
+    - cron: "23 2 * * 1" # Mondays at 02:23 UTC
+
+permissions: {}
+
+jobs:
+  analyze:
+    name: CodeQL (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [actions, javascript-typescript, python]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: none
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/docker_build.yml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
     inputs:
       pypiVersion:
-        description: 'Exact stamped PyPI release (overrides PYRIT_PYPI_VERSION)'
+        description: 'Exact published PyPI version (defaults to the latest stable release)'
         required: false
         type: string
 
@@ -31,6 +31,45 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  validate-compose-bindings:
+    name: Validate Compose Bindings
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Require loopback-only published ports
+        working-directory: docker
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $env:PYRIT_SOURCE_COMMIT = git rev-parse --verify HEAD
+          if ($LASTEXITCODE -ne 0) {
+            throw "Cannot determine source commit."
+          }
+          $sourceStatus = git status --porcelain
+          if ($LASTEXITCODE -ne 0) {
+            throw "Cannot determine source dirty state."
+          }
+          $env:PYRIT_SOURCE_DIRTY = if ($sourceStatus) { 'true' } else { 'false' }
+          $json = docker compose --file docker-compose.yaml --profile gui --profile jupyter config --format json
+          if ($LASTEXITCODE -ne 0) {
+            throw "Docker Compose configuration resolution failed."
+          }
+          $services = ($json | ConvertFrom-Json).services
+          $expectedPorts = @{ 'pyrit-gui' = 8000; 'pyrit-jupyter' = 8888 }
+          foreach ($service in $expectedPorts.Keys) {
+            $port = $expectedPorts[$service]
+            $bindings = @($services.$service.ports)
+            if ($bindings.Count -ne 1 -or
+                $bindings[0].host_ip -ne '127.0.0.1' -or
+                $bindings[0].published -ne "$port" -or
+                $bindings[0].target -ne $port -or
+                $bindings[0].protocol -ne 'tcp') {
+              throw "Expected only 127.0.0.1:${port}:${port}/tcp for ${service}; got $($bindings | ConvertTo-Json -Compress)."
+            }
+            Write-Output "${service}: verified 127.0.0.1:${port}:${port}/tcp"
+          }
+
   build-and-test-local:
     name: Build and test (local)
     runs-on: ubuntu-latest
@@ -108,7 +147,8 @@ jobs:
   build-and-test-pypi:
     name: Build and test (PyPI)
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
+    # Restore main-push checks after validating 1.2.0: https://github.com/microsoft/PyRIT/issues/3007
+    if: github.event_name == 'workflow_dispatch'
     outputs:
       production: ${{ steps.production.outcome }}
       import: ${{ steps.import.outcome }}
@@ -117,20 +157,17 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - name: Select coordinated PyPI release
+      - name: Select PyPI release
         id: pypi-version
+        timeout-minutes: 5
         env:
-          PYRIT_PYPI_VERSION: ${{ inputs.pypiVersion || vars.PYRIT_PYPI_VERSION }}
+          PYRIT_PYPI_VERSION: ${{ inputs.pypiVersion }}
         shell: bash
         run: |
           set -euo pipefail
-          version_pattern='^[0-9]+\.[0-9]+\.[0-9]+((a|b|rc)[0-9]+)?(\.post[0-9]+)?(\.dev[0-9]+)?$'
-          if [[ ! "${PYRIT_PYPI_VERSION:-}" =~ $version_pattern ]]; then
-            echo "::error::Set pypiVersion or repository variable PYRIT_PYPI_VERSION to an exact coordinated PyPI release."
-            exit 1
-          fi
-          echo "version=$PYRIT_PYPI_VERSION" >> "$GITHUB_OUTPUT"
-          echo "Testing coordinated PyPI release: $PYRIT_PYPI_VERSION"
+          version=$(python3 build_scripts/select_pypi_version.py --version "${PYRIT_PYPI_VERSION:-}")
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+          echo "Testing published PyPI release: $version"
 
       - name: Set up cached devcontainer builder
         uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
@@ -219,7 +256,7 @@ jobs:
   pypi-production-check:
     name: Build Production (PyPI)
     needs: build-and-test-pypi
-    if: ${{ always() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+    if: ${{ always() && (github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-latest
     steps:
       - name: Check execution and stage results
@@ -237,7 +274,7 @@ jobs:
   pypi-import-check:
     name: Test Import (PyPI)
     needs: build-and-test-pypi
-    if: ${{ always() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+    if: ${{ always() && (github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-latest
     steps:
       - name: Check execution and stage results
@@ -255,7 +292,7 @@ jobs:
   pypi-gui-check:
     name: Test GUI (PyPI)
     needs: build-and-test-pypi
-    if: ${{ always() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+    if: ${{ always() && (github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-latest
     steps:
       - name: Check execution and stage results
@@ -273,7 +310,7 @@ jobs:
   pypi-jupyter-check:
     name: Test Jupyter (PyPI)
     needs: build-and-test-pypi
-    if: ${{ always() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+    if: ${{ always() && (github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-latest
     steps:
       - name: Check execution and stage results

.github/workflows/docs.yml

@@ -87,7 +87,7 @@ jobs:
           python-version: "3.13"
 
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: "0.9.17"
           enable-cache: false
@@ -230,7 +230,7 @@ jobs:
           python-version: "3.13"
 
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: "0.9.17"
           enable-cache: false

.pre-commit-config.yaml

@@ -67,7 +67,7 @@ repos:
       - id: detect-private-key
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.16.9
+    rev: v0.16.10
     hooks:
       - id: ruff-format
       - id: ruff-check

build_scripts/inject_version_picker.py

@@ -12,6 +12,7 @@
     * <script>...inlined picker.js...</script>  (CSS is bundled inside)
 
 Idempotent: files that already contain our marker are skipped.
+Embedded players opt out with the ``pyrit-no-version-picker`` HTML comment.
 
 Why inline the JS instead of <script src="..."> AND why no <link>?
 
@@ -40,6 +41,7 @@
 from pathlib import Path
 
 INJECT_MARKER = "<!-- pyrit-version-picker -->"
+SKIP_MARKER = "<!-- pyrit-no-version-picker -->"
 ASSETS_SOURCE_DIR = Path(__file__).resolve().parent / "version_picker_assets"
 
 # The same marker string is used in compose_docs_dist.py's 404 template. Both
@@ -54,7 +56,7 @@ def _head_block(base: str, picker_js: str) -> str:
 
 
 def _inject(html: str, base: str, picker_js: str) -> tuple[str, bool]:
-    if INJECT_MARKER in html:
+    if INJECT_MARKER in html or SKIP_MARKER in html:
         return html, False
     head_block = _head_block(base, picker_js)
     new = html.replace("</head>", f"{head_block}</head>", 1) if "</head>" in html else head_block + html

build_scripts/select_pypi_version.py

@@ -0,0 +1,96 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Select a published PyRIT release without installing project dependencies."""
+
+import argparse
+import json
+import re
+from urllib.request import urlopen
+
+
+def _validate_version(value: object) -> str:
+    if (
+        not isinstance(value, str)
+        or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:(?:a|b|rc)[0-9]+)?(?:\.post[0-9]+)?(?:\.dev[0-9]+)?", value) is None
+    ):
+        raise ValueError("PyRIT version must be an exact release, such as X.Y.Z or X.Y.Zrc1")
+    return value
+
+
+def _validate_distributions(*, files: object, version: str) -> None:
+    if not isinstance(files, list) or any(
+        not isinstance(file, dict) or not isinstance(file.get("yanked"), bool) for file in files
+    ):
+        raise ValueError(f"Invalid distribution metadata for PyPI release {version}")
+    if not any(
+        file["yanked"] is False
+        and isinstance(file.get("filename"), str)
+        and (
+            (
+                file.get("packagetype") == "bdist_wheel"
+                and file["filename"].startswith(f"pyrit-{version}-")
+                and file["filename"].endswith(".whl")
+            )
+            or (
+                file.get("packagetype") == "sdist"
+                and file["filename"] in {f"pyrit-{version}.tar.gz", f"pyrit-{version}.zip"}
+            )
+        )
+        and isinstance(file.get("url"), str)
+        and file["url"].startswith("https://files.pythonhosted.org/")
+        and file["url"].endswith(f"/{file['filename']}")
+        for file in files
+    ):
+        raise ValueError(f"PyPI release {version} has no non-yanked published wheel or sdist")
+
+
+def resolve_pypi_version(version: str = "") -> str:
+    """Resolve PyPI's latest stable release, or validate an explicit published version.
+
+    Args:
+        version: Exact manual override. An empty string selects the latest stable release.
+
+    Returns:
+        str: The exact published version to install and test.
+
+    Raises:
+        OSError: If the PyPI lookup fails.
+        ValueError: If the version, metadata, or published distributions are invalid.
+    """
+    if version:
+        _validate_version(version)
+    url = f"https://pypi.org/pypi/pyrit/{version}/json" if version else "https://pypi.org/pypi/pyrit/json"
+    with urlopen(url, timeout=30) as response:
+        metadata = json.load(response)
+    info = metadata.get("info") if isinstance(metadata, dict) else None
+    if not isinstance(info, dict) or not isinstance(info.get("name"), str) or info["name"].lower() != "pyrit":
+        raise ValueError("Invalid PyPI project metadata: expected PyRIT release information")
+    selected = _validate_version(info.get("version"))
+    if version and selected != version:
+        raise ValueError(f"Requested PyPI release {version}, but PyPI returned {selected}")
+    # The project endpoint orders non-yanked, stable releases using PyPI's version ordering.
+    if not version and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:\.post[0-9]+)?", selected) is None:
+        raise ValueError("PyPI did not return a stable, non-yanked PyRIT release")
+    if info.get("yanked") is not False:
+        raise ValueError(f"PyPI release {selected} is yanked or has invalid yank metadata")
+    _validate_distributions(files=metadata.get("urls"), version=selected)
+    return selected
+
+
+def main() -> None:
+    """Print only the validated version, or fail without supplying a fallback."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--version", default="", help="Exact published override; defaults to PyPI's latest stable release"
+    )
+    args = parser.parse_args()
+    try:
+        version = resolve_pypi_version(args.version)
+    except (OSError, ValueError) as exc:
+        parser.exit(1, f"::error::PyPI release selection failed: {exc}\n")
+    print(version)
+
+
+if __name__ == "__main__":
+    main()

doc/assets/videos/copyrit-player.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<!-- pyrit-no-version-picker -->
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>CoPyRIT GUI walkthrough</title>
+  <link rel="stylesheet" href="player.css">
+  <script src="player.js" defer></script>
+</head>
+<body>
+  <video controls playsinline preload="none" src="copyrit-walkthrough.mp4"
+         poster="../copyrit-demo.png" aria-label="CoPyRIT GUI walkthrough">
+    <a href="copyrit-walkthrough.mp4">Open the CoPyRIT GUI walkthrough directly</a>.
+  </video>
+</body>
+</html>