diff --git a/docs/integrations/capability-matrix.md b/docs/integrations/capability-matrix.md new file mode 100644 index 000000000..977064c1c --- /dev/null +++ b/docs/integrations/capability-matrix.md @@ -0,0 +1,720 @@ +--- +title: "Provider Capability Matrix - Instructor" +description: "Complete capability matrix showing which LLM providers support Streaming, Vision, Tool Calling, JSON Mode, Structured Outputs, Async, Images, Audio, and PDF across OpenAI, Anthropic, Gemini, Bedrock, Mistral, Groq and more." +--- + +# Provider Capability Matrix + +> **Discover incompatibilities before you hit them in production.** +> +> This page is the single reference for every supported provider's features. +> Use it to pick the right provider for your use-case or to understand why a +> specific combination might not work. + +## Quick Reference + +| Provider | Stream | Async | Tools | Parallel Tools | JSON Mode | JSON Schema | Vision | Audio | PDF | Responses API | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| **OpenAI** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| **Anthropic** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | +| **Google GenAI** | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | +| **Vertex AI** | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ⚠️ | ⚠️ | ❌ | +| **AWS Bedrock** | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ⚠️ | ❌ | ❌ | ❌ | +| **Azure OpenAI** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ⚠️ | ✅ | ❌ | +| **Mistral** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **Cohere** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| **Groq** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **Fireworks AI** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **Cerebras** | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| **Writer** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| **xAI (Grok)** | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **Perplexity** | ⚠️ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **OpenRouter** | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **DeepSeek** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **Together AI** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **Ollama (local)** | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **llama-cpp-python** | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ⚠️ | ❌ | ❌ | ❌ | +| **LiteLLM** | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ❌ | +| **Gemini (legacy)** | ⚠️ | ⚠️ | ✅ | ❌ | ❌ | ❌ | ✅ | ⚠️ | ⚠️ | ❌ | + +**Legend:** ✅ = Full support ⚠️ = Partial / model-dependent ❌ = Not supported + +--- + +## Mode Support by Provider + +Each provider supports a subset of Instructor output modes. Choosing the right +mode is critical — some modes silently fall back to others if not natively supported. + +| Provider | `TOOLS` | `PARALLEL_TOOLS` | `JSON` | `JSON_SCHEMA` | `MD_JSON` | `RESPONSES_TOOLS` | +|---|:---:|:---:|:---:|:---:|:---:|:---:| +| **OpenAI** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| **Anthropic** | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| **Google GenAI** | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | +| **Vertex AI** | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | +| **AWS Bedrock** | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | +| **Mistral** | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | +| **Cohere** | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | +| **Groq** | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | +| **Fireworks AI** | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | +| **Cerebras** | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | +| **Writer** | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | +| **xAI (Grok)** | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | +| **Perplexity** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | +| **OpenRouter** | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | +| **DeepSeek** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| **Together AI** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| **Ollama** | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | +| **llama-cpp-python** | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | +| **Gemini (legacy)** | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | + +!!! tip + When in doubt, use `Mode.TOOLS` — it is supported by every provider that has + tool/function-calling capability and produces the most reliable structured outputs. + +--- + +## Detailed Provider Profiles + +### OpenAI + +```python +client = instructor.from_provider("openai/gpt-4o-mini") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ✅ | `Mode.PARALLEL_TOOLS` | +| JSON Mode | ✅ | `Mode.JSON` | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision (Images) | ✅ | GPT-4o-class models; `Image.from_url()` | +| Audio | ✅ | GPT-4o-audio; `Audio.from_url()` | +| PDF | ✅ | GPT-4o-class models; `PDF.from_url()` | +| Responses API | ✅ | `Mode.RESPONSES_TOOLS` | + +**Install:** `pip install instructor` +**Docs:** [OpenAI Integration](./openai.md) + +--- + +### Anthropic + +```python +client = instructor.from_provider("anthropic/claude-3-5-haiku-latest") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ✅ | `Mode.PARALLEL_TOOLS` | +| JSON Mode | ✅ | `Mode.JSON` | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision (Images) | ✅ | All Claude 3+ models; `Image.from_url()` | +| Audio | ❌ | Not supported by Claude models | +| PDF | ✅ | Claude 3+ with `PDF.from_url()` | +| Responses API | ❌ | Not applicable | + +!!! note + `Mode.MD_JSON` is not supported on Anthropic. Use `Mode.TOOLS` or `Mode.JSON`. + Extended thinking is available via `Mode.TOOLS` with the `thinking={"type": "enabled"}` parameter. + +**Install:** `pip install "instructor[anthropic]"` +**Docs:** [Anthropic Integration](./anthropic.md) + +--- + +### Google GenAI (Recommended) + +```python +client = instructor.from_provider("google/gemini-2.5-flash") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ✅ | `Mode.JSON` | +| JSON Schema | ❌ | Not supported | +| Vision (Images) | ✅ | All Gemini 1.5+ models | +| Audio | ✅ | Gemini 1.5+ models | +| PDF | ✅ | Gemini 1.5+ models | +| Responses API | ❌ | Not applicable | + +!!! warning + Gemini does **not** support `Union` or `Optional` types in structured output schemas. + Instructor raises an error when these are detected. Use concrete types instead. + + Gemini 2.5 models emit internal reasoning "thought parts" — Instructor filters these automatically. + +**Install:** `pip install "instructor[google-genai]"` +**Docs:** [Google GenAI Integration](./genai.md) + +--- + +### Vertex AI + +```python +client = instructor.from_provider("vertexai/gemini-2.5-flash") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ✅ | `Mode.PARALLEL_TOOLS` | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ❌ | Not supported | +| Vision (Images) | ✅ | Gemini models on Vertex | +| Audio | ⚠️ | Model-dependent | +| PDF | ⚠️ | Model-dependent | +| Responses API | ❌ | Not applicable | + +!!! note + Requires GCP project credentials. Set `GOOGLE_APPLICATION_CREDENTIALS` or use ADC. + +**Install:** `pip install "instructor[vertexai]"` +**Docs:** [Vertex AI Integration](./vertex.md) + +--- + +### AWS Bedrock + +```python +client = instructor.from_provider("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ❌ | Not supported | +| Vision (Images) | ⚠️ | Only Claude models on Bedrock | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +!!! note + Requires AWS credentials with Bedrock access. Supports `Mode.TOOLS` and `Mode.MD_JSON` only. + +**Install:** `pip install "instructor[bedrock]"` +**Docs:** [AWS Bedrock Integration](./bedrock.md) + +--- + +### Azure OpenAI + +```python +from openai import AzureOpenAI +client = instructor.from_openai(AzureOpenAI(azure_endpoint=..., api_key=..., api_version=...)) +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | Mirrors OpenAI | +| Async | ✅ | Mirrors OpenAI | +| Tool Calling | ✅ | Mirrors OpenAI | +| Parallel Tools | ✅ | Mirrors OpenAI | +| JSON Mode | ✅ | Mirrors OpenAI | +| JSON Schema | ✅ | Mirrors OpenAI | +| Vision (Images) | ✅ | Requires GPT-4o deployment | +| Audio | ⚠️ | Depends on deployed model version | +| PDF | ✅ | Requires GPT-4o deployment | +| Responses API | ❌ | Not available on Azure | + +!!! note + Capabilities depend on the specific model deployed in your Azure resource. + Not all OpenAI model versions are immediately available on Azure. + +**Docs:** [Azure Integration](./azure.md) + +--- + +### Mistral + +```python +client = instructor.from_provider("mistral/mistral-large-latest") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision (Images) | ⚠️ | Pixtral models only | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +**Install:** `pip install "instructor[mistral]"` +**Docs:** [Mistral Integration](./mistral.md) + +--- + +### Cohere + +```python +client = instructor.from_provider("cohere/command-a-03-2025") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision | ❌ | Not supported | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +**Install:** `pip install "instructor[cohere]"` +**Docs:** [Cohere Integration](./cohere.md) + +--- + +### Groq + +```python +client = instructor.from_provider("groq/llama-3.3-70b-versatile") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision | ⚠️ | Vision-enabled models (e.g. llava-v1.5) | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +**Install:** `pip install "instructor[groq]"` +**Docs:** [Groq Integration](./groq.md) + +--- + +### Cerebras + +```python +client = instructor.from_provider("cerebras/gpt-oss-120b") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ✅ | `Mode.PARALLEL_TOOLS` | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision | ❌ | Not supported | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +**Install:** `pip install "instructor[cerebras]"` +**Docs:** [Cerebras Integration](./cerebras.md) + +--- + +### xAI (Grok) + +```python +client = instructor.from_provider("xai/grok-beta") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` (default) | +| Parallel Tools | ✅ | `Mode.PARALLEL_TOOLS` | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision | ⚠️ | Grok Vision models only | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +**Install:** `pip install instructor` +**Docs:** [xAI Integration](./xai.md) + +--- + +### Perplexity + +```python +client = instructor.from_provider("perplexity/sonar-small-online") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ⚠️ | Limited streaming support | +| Async | ✅ | `async_client=True` | +| Tool Calling | ❌ | Not supported | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ❌ | Not supported | +| Vision | ❌ | Not supported | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +!!! warning + Perplexity only supports `Mode.MD_JSON`. It is primarily designed for + search-augmented Q&A and has very limited structured output capabilities. + Consider using OpenAI or Anthropic for complex structured extraction tasks. + +**Docs:** [Perplexity Integration](./perplexity.md) + +--- + +### OpenRouter + +```python +client = instructor.from_provider("openrouter/anthropic/claude-3.5-haiku") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` | +| Parallel Tools | ✅ | `Mode.PARALLEL_TOOLS` | +| JSON Mode | ❌ | Not supported | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision | ⚠️ | Depends on the routed model | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +!!! note + All capabilities ultimately depend on the underlying model that OpenRouter routes to. + Check the specific model capabilities in the OpenRouter docs. + +**Docs:** [OpenRouter Integration](./openrouter.md) + +--- + +### Ollama (Local) + +```python +client = instructor.from_provider("ollama/llama3") +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ✅ | `create_partial`, `create_iterable` | +| Async | ✅ | `async_client=True` | +| Tool Calling | ✅ | `Mode.TOOLS` | +| Parallel Tools | ❌ | Not supported | +| JSON Mode | ✅ | `Mode.JSON` | +| JSON Schema | ✅ | `Mode.JSON_SCHEMA` | +| Vision | ⚠️ | Requires llava, bakllava, or other vision models | +| Audio | ❌ | Not supported | +| PDF | ❌ | Not supported | +| Responses API | ❌ | Not applicable | + +!!! note + Ollama uses the OpenAI-compatible interface. Run `ollama serve` before connecting. + Vision requires downloading a vision-capable model (e.g. `ollama pull llava`). + +**Docs:** [Ollama Integration](./ollama.md) + +--- + +### LiteLLM + +```python +import litellm +client = instructor.from_litellm(litellm.completion) +``` + +| Capability | Support | Notes | +|---|:---:|---| +| Streaming | ⚠️ | Depends on underlying provider | +| Async | ⚠️ | Depends on underlying provider | +| Tool Calling | ⚠️ | Depends on underlying provider | +| Parallel Tools | ⚠️ | Depends on underlying provider | +| JSON Mode | ⚠️ | Depends on underlying provider | +| JSON Schema | ⚠️ | Depends on underlying provider | +| Vision | ⚠️ | Depends on underlying provider | +| Audio | ⚠️ | Depends on underlying provider | +| PDF | ⚠️ | Depends on underlying provider | +| Responses API | ❌ | Not supported | + +!!! note + LiteLLM is a proxy/router. All capabilities depend on the underlying provider model. + Check LiteLLM model list for per-model capability information. + +**Docs:** [LiteLLM Integration](./litellm.md) + +--- + +## Capability Deep Dives + +### Streaming + +All major providers support streaming through Instructor's unified API: + +```python +# Partial streaming - stream a single object field-by-field +for partial_user in client.create_partial( + response_model=User, + messages=[{"role": "user", "content": "..."}], +): + print(partial_user) + +# Iterable streaming - stream a list of objects one-by-one +for user in client.create_iterable( + response_model=User, + messages=[{"role": "user", "content": "Extract multiple users: ..."}], +): + print(user) +``` + +- **Full streaming:** OpenAI, Anthropic, Google GenAI, Vertex AI, Bedrock, Mistral, Cohere, Groq, Cerebras, Writer, xAI, OpenRouter, DeepSeek, Together AI, Ollama +- **Partial only:** Perplexity (MD_JSON) +- **Model-dependent:** LiteLLM, Gemini (legacy) + +--- + +### Vision / Images + +Instructor provides a unified `Image` class that handles all provider-specific formatting: + +```python +from instructor.processing.multimodal import Image + +response = client.create( + response_model=ImageDescription, + messages=[{ + "role": "user", + "content": [ + "What is in this image?", + Image.from_url("https://example.com/image.jpg"), + # Image.from_path("local/image.png"), + # Image.from_base64("base64_data_here"), + ], + }], +) +``` + +| Provider | Vision Support | Notes | +|---|:---:|---| +| OpenAI | ✅ Full | GPT-4o, GPT-4-turbo models | +| Anthropic | ✅ Full | All Claude 3+ models | +| Google GenAI | ✅ Full | All Gemini 1.5+ models | +| Vertex AI | ✅ Full | Gemini models on Vertex | +| AWS Bedrock | ⚠️ Partial | Claude models on Bedrock only | +| Mistral | ⚠️ Partial | Pixtral models only | +| Groq | ⚠️ Partial | LLaVA and other vision models | +| xAI | ⚠️ Partial | Grok Vision models | +| Ollama | ⚠️ Partial | llava, bakllava, moondream models | +| DeepSeek | ⚠️ Partial | DeepSeek-VL2 models | +| Cohere | ❌ | Not supported | +| Perplexity | ❌ | Not supported | +| Cerebras | ❌ | Not supported | +| Writer | ❌ | Not supported | + +--- + +### Audio + +Instructor provides a unified `Audio` class for audio input: + +```python +from instructor.processing.multimodal import Audio + +response = client.create( + response_model=AudioTranscription, + messages=[{ + "role": "user", + "content": [ + "Transcribe this audio:", + Audio.from_url("https://example.com/audio.wav"), + # Audio.from_path("local/audio.mp3"), + ], + }], +) +``` + +| Provider | Audio Support | Notes | +|---|:---:|---| +| OpenAI | ✅ Full | GPT-4o-audio models | +| Google GenAI | ✅ Full | Gemini 1.5+ models | +| Anthropic | ❌ | Not supported | +| AWS Bedrock | ❌ | Not supported | +| Mistral | ❌ | Not supported | +| All Others | ❌ | Not supported | + +--- + +### PDF + +Instructor provides a unified `PDF` class for PDF document input: + +```python +from instructor.processing.multimodal import PDF + +response = client.create( + response_model=Invoice, + messages=[{ + "role": "user", + "content": [ + "Extract the invoice details:", + PDF.from_url("https://example.com/invoice.pdf"), + # PDF.from_path("local/document.pdf"), + ], + }], +) +``` + +| Provider | PDF Support | Notes | +|---|:---:|---| +| OpenAI | ✅ Full | GPT-4o models | +| Anthropic | ✅ Full | All Claude 3+ models | +| Google GenAI | ✅ Full | Gemini 1.5+ models | +| Azure OpenAI | ✅ Full | GPT-4o deployments | +| Vertex AI | ⚠️ Partial | Gemini models | +| Bedrock | ❌ | Not supported | +| All Others | ❌ | Not supported | + +--- + +### Async + +All providers that support async work with `async_client=True`: + +```python +import asyncio +import instructor + +async def main(): + client = instructor.from_provider("openai/gpt-4o-mini", async_client=True) + result = await client.create( + response_model=User, + messages=[{"role": "user", "content": "..."}], + ) + return result + +asyncio.run(main()) +``` + +**Not supported:** llama-cpp-python + +--- + +## Programmatic Access + +The capability matrix is also importable as a Python module for runtime queries: + +```python +from instructor.v2.core.capability_matrix import ( + CAPABILITY_MATRIX, + get_provider_capabilities, + providers_supporting, + print_capability_matrix, +) + +# Get capabilities for a specific provider +caps = get_provider_capabilities("anthropic") +print(caps.vision) # "yes" +print(caps.audio) # "no" +print(caps.notes) # ["Vision (images & PDFs) supported...", ...] + +# Find all providers that fully support a capability +vision_providers = providers_supporting("vision") +# -> ["openai", "anthropic", "google", "vertexai", ...] + +# Print the full matrix to stdout +print_capability_matrix() + +# Print with caveats / notes +print_capability_matrix(show_notes=True) + +# Filter to a subset of providers +print_capability_matrix(providers=["openai", "anthropic", "google"]) +``` + +--- + +## Choosing the Right Provider + +### For structured extraction from text + +Any provider with `Mode.TOOLS` or `Mode.JSON_SCHEMA`. +**Recommended:** OpenAI, Anthropic, or Google GenAI. + +### For document / image analysis + +You need **vision** support. +**Recommended:** OpenAI (GPT-4o), Anthropic (Claude 3+), Google GenAI (Gemini 1.5+). + +### For audio transcription + extraction + +You need **audio** support. +**Recommended:** OpenAI (GPT-4o-audio), Google GenAI (Gemini 1.5+). + +### For local / offline use + +Use **Ollama** with an appropriate model, or **llama-cpp-python**. No API key required. + +### For routing across multiple providers + +Use **LiteLLM** or **OpenRouter** to switch providers without code changes. + +### For lowest cost / highest speed + +**Groq** (fast inference), **Ollama** (free, local), or **Cerebras** (fast inference). + +### For search-augmented Q&A + +Use **Perplexity** but note its very limited structured output support (MD_JSON only). + +--- + +## Known Incompatibilities + +!!! warning + These combinations will raise errors or produce unexpected results: + +| Combination | Issue | +|---|---| +| `Anthropic` + `Mode.MD_JSON` | Not supported; use `TOOLS` or `JSON` | +| `Perplexity` + `Mode.TOOLS` | Tool calling not supported | +| `Google GenAI` + `Union[A, B]` schema | Gemini rejects Union types | +| `Google GenAI` + `Mode.JSON_SCHEMA` | Not supported; use `JSON` or `TOOLS` | +| `Bedrock` + `Mode.JSON_SCHEMA` | Not supported; use `TOOLS` or `MD_JSON` | +| `Bedrock` + `Mode.PARALLEL_TOOLS` | Not supported | +| `llama-cpp-python` + `async_client=True` | Async not natively supported | +| `Gemini (legacy)` + `Mode.JSON` | Not supported; use `MD_JSON` or `TOOLS` | +| Any provider + `Audio` (except OpenAI / Google GenAI) | Audio not supported | + +--- + +## Related Resources + +- [Modes Comparison Guide](../modes-comparison.md) — In-depth comparison of all `Mode.*` values +- [Multimodal Concepts](../concepts/multimodal.md) — Image, Audio, PDF usage details +- [from_provider Guide](../concepts/from_provider.md) — Client setup reference +- [Provider Examples](./index.md) — Quick-start code for every provider \ No newline at end of file diff --git a/docs/integrations/index.md b/docs/integrations/index.md index b22784b34..9cdc6c893 100644 --- a/docs/integrations/index.md +++ b/docs/integrations/index.md @@ -66,7 +66,11 @@ All integrations support these core features: | **Streaming** | Stream partial or iterative responses | [Partial](../concepts/partial.md), [Iterable](../concepts/iterable.md) | | **Hooks** | Add callbacks for monitoring and debugging | [Hooks](../concepts/hooks.md) | -However, each provider has different capabilities and limitations. Refer to the specific provider documentation for details. +However, each provider has different capabilities and limitations. Refer to the +[**Provider Capability Matrix**](./capability-matrix.md) for a complete, side-by-side +comparison of Streaming, Vision, Audio, Tool Calling, JSON Mode, and more across +all providers — including known incompatibilities. + ## Provider Modes diff --git a/instructor/v2/core/__init__.py b/instructor/v2/core/__init__.py index 59125e06d..5fb895d3b 100644 --- a/instructor/v2/core/__init__.py +++ b/instructor/v2/core/__init__.py @@ -15,6 +15,11 @@ "ReaskHandler", "ResponseParser", "normalize_mode", + "ProviderCapabilities", + "CAPABILITY_MATRIX", + "get_provider_capabilities", + "providers_supporting", + "print_capability_matrix", ] _LAZY_ATTRS: dict[str, tuple[str, str]] = { @@ -27,6 +32,11 @@ "ReaskHandler": ("instructor.v2.core.protocols", "ReaskHandler"), "ResponseParser": ("instructor.v2.core.protocols", "ResponseParser"), "normalize_mode": ("instructor.v2.core.registry", "normalize_mode"), + "ProviderCapabilities": ("instructor.v2.core.capability_matrix", "ProviderCapabilities"), + "CAPABILITY_MATRIX": ("instructor.v2.core.capability_matrix", "CAPABILITY_MATRIX"), + "get_provider_capabilities": ("instructor.v2.core.capability_matrix", "get_provider_capabilities"), + "providers_supporting": ("instructor.v2.core.capability_matrix", "providers_supporting"), + "print_capability_matrix": ("instructor.v2.core.capability_matrix", "print_capability_matrix"), } diff --git a/instructor/v2/core/capability_matrix.py b/instructor/v2/core/capability_matrix.py new file mode 100644 index 000000000..7f9106e7f --- /dev/null +++ b/instructor/v2/core/capability_matrix.py @@ -0,0 +1,615 @@ +"""Provider capability matrix for Instructor. + +Tracks what each supported LLM provider can do when used via Instructor. +'partial' means the feature works but only on specific models or with caveats. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +Support = Literal["yes", "no", "partial"] + + +@dataclass(frozen=True) +class ProviderCapabilities: + """Capability profile for a single provider.""" + + name: str + provider_key: str # matches the key used in from_provider() + + streaming: Support = "no" + async_support: Support = "no" + tool_calling: Support = "no" + parallel_tools: Support = "no" + json_mode: Support = "no" + json_schema_mode: Support = "no" + vision: Support = "no" + audio: Support = "no" + pdf: Support = "no" + responses_api: Support = "no" + + notes: tuple[str, ...] = () # known limitations or caveats + install_extra: str | None = None + example_model: str | None = None + docs_url: str | None = None + + +CAPABILITY_MATRIX: dict[str, ProviderCapabilities] = { + "openai": ProviderCapabilities( + name="OpenAI", + provider_key="openai", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="yes", + json_schema_mode="yes", + vision="yes", + audio="yes", + pdf="yes", + responses_api="yes", + notes=( + "Vision / audio / PDF require GPT-4o-class models.", + "Audio input available via Audio helper class.", + "Responses API available via Mode.RESPONSES_TOOLS.", + "Parallel tools via Mode.PARALLEL_TOOLS.", + ), + install_extra="instructor", + example_model="openai/gpt-4o-mini", + docs_url="integrations/openai.md", + ), + "anthropic": ProviderCapabilities( + name="Anthropic", + provider_key="anthropic", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="yes", + json_schema_mode="yes", + vision="yes", + audio="no", + pdf="yes", + responses_api="no", + notes=( + "Vision (images & PDFs) supported via Image / PDF helpers.", + "Audio input not supported by Claude models.", + "Extended thinking available via Mode.TOOLS with thinking param.", + "MD_JSON mode not supported; use TOOLS or JSON instead.", + ), + install_extra="instructor[anthropic]", + example_model="anthropic/claude-3-5-haiku-latest", + docs_url="integrations/anthropic.md", + ), + # Preferred Google integration — uses the google-genai SDK + "google": ProviderCapabilities( + name="Google GenAI (google-genai SDK)", + provider_key="google", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="yes", + json_schema_mode="no", + vision="yes", + audio="yes", + pdf="yes", + responses_api="no", + notes=( + "Uses Mode.TOOLS or Mode.JSON; JSON_SCHEMA and MD_JSON not supported.", + "No support for Union / Optional types in structured outputs.", + "Thought parts from Gemini 2.5 models are filtered automatically.", + "Audio and video inputs supported via the genai SDK.", + "Parallel tool calls not supported.", + ), + install_extra="instructor[google-genai]", + example_model="google/gemini-2.5-flash", + docs_url="integrations/genai.md", + ), + # Legacy google-generativeai SDK — prefer "google" for new projects + "gemini": ProviderCapabilities( + name="Google Gemini (google-generativeai SDK)", + provider_key="gemini", + streaming="partial", + async_support="partial", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="no", + vision="yes", + audio="partial", + pdf="partial", + responses_api="no", + notes=( + "Legacy SDK – prefer 'google' (google-genai) for new projects.", + "Supports Mode.TOOLS and Mode.MD_JSON only.", + "JSON and JSON_SCHEMA modes not supported.", + "Multimodal support depends on model version.", + ), + install_extra="instructor[google-generativeai]", + example_model=None, + docs_url="integrations/google.md", + ), + "vertexai": ProviderCapabilities( + name="Vertex AI", + provider_key="vertexai", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="no", + json_schema_mode="no", + vision="yes", + audio="partial", + pdf="partial", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.MD_JSON, Mode.PARALLEL_TOOLS.", + "JSON and JSON_SCHEMA modes not supported.", + "Multimodal inputs depend on underlying model (Gemini recommended).", + "Requires GCP project and credentials.", + ), + install_extra="instructor[vertexai]", + example_model="vertexai/gemini-2.5-flash", + docs_url="integrations/vertex.md", + ), + "bedrock": ProviderCapabilities( + name="AWS Bedrock", + provider_key="bedrock", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="no", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS and Mode.MD_JSON only.", + "JSON_SCHEMA and PARALLEL_TOOLS not supported.", + "Vision available on Claude models hosted via Bedrock.", + "Requires AWS credentials and Bedrock access.", + ), + install_extra="instructor[bedrock]", + example_model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + docs_url="integrations/bedrock.md", + ), + "mistral": ProviderCapabilities( + name="Mistral", + provider_key="mistral", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON.", + "Vision available on Pixtral models only.", + "No audio or PDF support.", + "PARALLEL_TOOLS not supported.", + ), + install_extra="instructor[mistral]", + example_model="mistral/ministral-8b-latest", + docs_url="integrations/mistral.md", + ), + "cohere": ProviderCapabilities( + name="Cohere", + provider_key="cohere", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="yes", + vision="no", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON.", + "No vision, audio, or PDF support.", + "PARALLEL_TOOLS not supported.", + ), + install_extra="instructor[cohere]", + example_model="cohere/command-a-03-2025", + docs_url="integrations/cohere.md", + ), + "groq": ProviderCapabilities( + name="Groq", + provider_key="groq", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON.", + "Vision available on select vision-enabled models (e.g. llava).", + "No audio or PDF support.", + "PARALLEL_TOOLS not supported.", + ), + install_extra="instructor[groq]", + example_model="groq/llama-3.3-70b-versatile", + docs_url="integrations/groq.md", + ), + "fireworks": ProviderCapabilities( + name="Fireworks AI", + provider_key="fireworks", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Uses OpenAI-compatible handler.", + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON.", + "Vision depends on the specific hosted model.", + "PARALLEL_TOOLS not supported.", + ), + install_extra="instructor[fireworks]", + example_model="fireworks/accounts/fireworks/models/kimi-k2p5", + docs_url="integrations/fireworks.md", + ), + "cerebras": ProviderCapabilities( + name="Cerebras", + provider_key="cerebras", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="no", + json_schema_mode="yes", + vision="no", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON, Mode.PARALLEL_TOOLS.", + "No vision, audio, or PDF support.", + "Requires 'cerebras' extra: pip install 'instructor[cerebras]'.", + ), + install_extra="instructor[cerebras]", + example_model="cerebras/gpt-oss-120b", + docs_url="integrations/cerebras.md", + ), + "writer": ProviderCapabilities( + name="Writer", + provider_key="writer", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="no", + json_schema_mode="yes", + vision="no", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON.", + "No vision, audio, PDF, or parallel tools.", + ), + install_extra="instructor[writer]", + example_model="writer/palmyra-x5", + docs_url="integrations/writer.md", + ), + "xai": ProviderCapabilities( + name="xAI (Grok)", + provider_key="xai", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="no", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON, Mode.PARALLEL_TOOLS.", + "Vision supported on Grok Vision models.", + "No audio or PDF support.", + ), + install_extra="instructor[xai]", + example_model="xai/grok-4.20-reasoning", + docs_url="integrations/xai.md", + ), + "perplexity": ProviderCapabilities( + name="Perplexity", + provider_key="perplexity", + streaming="partial", + async_support="yes", + tool_calling="no", + parallel_tools="no", + json_mode="no", + json_schema_mode="no", + vision="no", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Only Mode.MD_JSON supported.", + "No tool calling, JSON, or JSON_SCHEMA modes.", + "Primarily suited for search-augmented Q&A, not structured extraction.", + "No vision, audio, or PDF support.", + ), + install_extra="instructor[perplexity]", + example_model="perplexity/sonar-small-online", + docs_url="integrations/perplexity.md", + ), + "openrouter": ProviderCapabilities( + name="OpenRouter", + provider_key="openrouter", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="no", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Capabilities depend on the underlying model routed to.", + "Supports Mode.TOOLS, Mode.JSON_SCHEMA, Mode.MD_JSON, Mode.PARALLEL_TOOLS.", + "Vision availability varies by model.", + ), + install_extra="instructor", + example_model="openrouter/anthropic/claude-3.5-haiku", + docs_url="integrations/openrouter.md", + ), + # All capabilities are partial — LiteLLM proxies to other providers + "litellm": ProviderCapabilities( + name="LiteLLM", + provider_key="litellm", + streaming="partial", + async_support="partial", + tool_calling="partial", + parallel_tools="partial", + json_mode="partial", + json_schema_mode="partial", + vision="partial", + audio="partial", + pdf="partial", + responses_api="no", + notes=( + "All capabilities depend on the underlying provider model selected.", + "LiteLLM is a proxy/router; Instructor delegates to its OpenAI-compat interface.", + "Check per-model capability in the LiteLLM docs.", + ), + install_extra="instructor[litellm]", + example_model=None, + docs_url="integrations/litellm.md", + ), + "ollama": ProviderCapabilities( + name="Ollama (local)", + provider_key="ollama", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="no", + json_mode="yes", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Uses OpenAI-compatible interface (ollama serve).", + "Vision requires a vision-capable model (e.g. llava, bakllava).", + "No audio or PDF support.", + "PARALLEL_TOOLS not supported.", + "Run 'ollama serve' before connecting.", + ), + install_extra="instructor", + example_model="ollama/llama3", + docs_url="integrations/ollama.md", + ), + "azure_openai": ProviderCapabilities( + name="Azure OpenAI", + provider_key="azure_openai", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="yes", + json_schema_mode="yes", + vision="yes", + audio="partial", + pdf="yes", + responses_api="no", + notes=( + "Capabilities mirror OpenAI but depend on the deployed model version.", + "Requires Azure endpoint URL, API key, and deployment name.", + "Audio support depends on the deployed model.", + "Use instructor.from_openai(AzureOpenAI(...)) for setup.", + ), + install_extra="instructor", + example_model=None, + docs_url="integrations/azure.md", + ), + "deepseek": ProviderCapabilities( + name="DeepSeek", + provider_key="deepseek", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="yes", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Uses OpenAI-compatible interface.", + "Vision available on DeepSeek-VL2 models.", + "No audio or PDF support.", + ), + install_extra="instructor", + example_model="deepseek/deepseek-chat", + docs_url="integrations/deepseek.md", + ), + "together": ProviderCapabilities( + name="Together AI", + provider_key="together", + streaming="yes", + async_support="yes", + tool_calling="yes", + parallel_tools="yes", + json_mode="yes", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Uses OpenAI-compatible interface.", + "Vision depends on the specific hosted model.", + "No audio or PDF support.", + ), + install_extra="instructor", + example_model="together/meta-llama/Llama-3-70b-chat-hf", + docs_url="integrations/together.md", + ), + "llama_cpp": ProviderCapabilities( + name="llama-cpp-python (local)", + provider_key="llama_cpp", + streaming="yes", + async_support="no", + tool_calling="yes", + parallel_tools="no", + json_mode="yes", + json_schema_mode="yes", + vision="partial", + audio="no", + pdf="no", + responses_api="no", + notes=( + "Uses OpenAI-server interface (llama_cpp.server).", + "Vision requires LLaVA or similar multimodal GGUF models.", + "Async not natively supported via instructor wrapper.", + "No audio or PDF support.", + ), + install_extra="instructor", + example_model=None, + docs_url="integrations/llama-cpp-python.md", + ), +} + + +def get_provider_capabilities(provider_key: str) -> ProviderCapabilities | None: + """Return capabilities for the given provider key, or None if not found.""" + return CAPABILITY_MATRIX.get(provider_key) + + +def providers_supporting(capability: str) -> list[str]: + """Return provider keys where the given capability is fully supported ('yes'). + + Raises ValueError for unrecognised capability names. + """ + valid = { + "streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", + "responses_api", + } + if capability not in valid: + raise ValueError( + f"Unknown capability {capability!r}. Valid options: {sorted(valid)}" + ) + return [ + key + for key, caps in CAPABILITY_MATRIX.items() + if getattr(caps, capability) == "yes" + ] + + +def print_capability_matrix( + *, + providers: list[str] | None = None, + show_notes: bool = False, +) -> None: + """Print the capability matrix to stdout. + + Args: + providers: Subset of provider keys to display. Defaults to all. + show_notes: Also print per-provider caveats beneath each row. + """ + _COLS = [ + ("streaming", "Stream"), + ("async_support", "Async"), + ("tool_calling", "Tools"), + ("parallel_tools", "Parallel"), + ("json_mode", "JSON"), + ("json_schema_mode", "JSONSchema"), + ("vision", "Vision"), + ("audio", "Audio"), + ("pdf", "PDF"), + ("responses_api", "Responses API"), + ] + _ICON_UNICODE = {"yes": "✅", "no": "❌", "partial": "⚠️ "} + _ICON_ASCII = {"yes": " Y ", "no": " N ", "partial": " ~ "} + + # Fall back to ASCII symbols when the terminal encoding can't render emoji + _stdout_enc = getattr(__import__("sys").stdout, "encoding", "utf-8") or "utf-8" + try: + "✅❌⚠️".encode(_stdout_enc) + _ICON = _ICON_UNICODE + except (UnicodeEncodeError, LookupError): + _ICON = _ICON_ASCII + + keys = providers or list(CAPABILITY_MATRIX.keys()) + caps_list = [(k, CAPABILITY_MATRIX[k]) for k in keys if k in CAPABILITY_MATRIX] + + name_w = max(len(c.name) for _, c in caps_list) + col_ws = [max(len(h), 4) for _, h in _COLS] + + header = f"{'Provider':<{name_w}} " + " ".join( + f"{h:^{w}}" for (_, h), w in zip(_COLS, col_ws) + ) + sep = "-" * len(header) + print(sep) + print(header) + print(sep) + for key, caps in caps_list: + row = f"{caps.name:<{name_w}} " + " ".join( + f"{_ICON[getattr(caps, attr)]:^{w}}" for (attr, _), w in zip(_COLS, col_ws) + ) + try: + print(row) + except UnicodeEncodeError: + print(row.encode("ascii", errors="replace").decode("ascii")) + if show_notes and caps.notes: + for note in caps.notes: + print(f" {'':>{name_w}} * {note}") + print(sep) + if _ICON is _ICON_UNICODE: + print("Legend: \u2705 = Yes \u274c = No \u26a0\ufe0f = Partial / model-dependent") + else: + print("Legend: Y = Yes N = No ~ = Partial / model-dependent") + print(sep) + + +if __name__ == "__main__": + print_capability_matrix(show_notes=True) diff --git a/mkdocs.yml b/mkdocs.yml index 8487fdac3..c7e686dfe 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -164,6 +164,7 @@ nav: - Field-level Validation: 'learning/validation/field_level_validation.md' - Integrations: - Overview: 'integrations/index.md' + - Capability Matrix: 'integrations/capability-matrix.md' # Major cloud providers - OpenAI: 'integrations/openai.md' - OpenAI Responses: 'integrations/openai-responses.md' diff --git a/tests/v2/run_capability_matrix_checks.py b/tests/v2/run_capability_matrix_checks.py new file mode 100644 index 000000000..0dab23241 --- /dev/null +++ b/tests/v2/run_capability_matrix_checks.py @@ -0,0 +1,499 @@ +"""Standalone verification script for capability_matrix.py. No pytest required.""" + +from __future__ import annotations +import io, sys, pathlib, re +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +PASS = 0 +FAIL = 0 +RESULTS: list[tuple[str, str, str]] = [] +_current_section = "UNKNOWN" + + +def section(name: str) -> None: + global _current_section + _current_section = name + print(f"\n{'='*72}") + print(f" {name}") + print(f"{'='*72}") + + +def check(label: str, condition: bool, detail: str = "") -> None: + global PASS, FAIL + if condition: + print(f" [PASS] {label}") + PASS += 1 + RESULTS.append(("PASS", _current_section, label)) + else: + msg = f" [FAIL] {label}" + if detail: + msg += f" [{detail}]" + print(msg) + FAIL += 1 + RESULTS.append(("FAIL", _current_section, label + (f" [{detail}]" if detail else ""))) + + +from instructor.v2.core.capability_matrix import ( + CAPABILITY_MATRIX, ProviderCapabilities, Support, + get_provider_capabilities, providers_supporting, print_capability_matrix, +) +from instructor.v2.core.provider_specs import PROVIDER_SPECS, ProviderSpec +from instructor.v2.core.mode import Mode +from instructor.v2.core.providers import Provider +from dataclasses import fields as dc_fields +from typing import get_args + +CAP_FIELDS = [ + "streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", "responses_api", +] + +EXPECTED_PROVIDERS = { + "openai", "anthropic", "google", "gemini", "vertexai", "bedrock", + "mistral", "cohere", "groq", "fireworks", "cerebras", "writer", + "xai", "perplexity", "openrouter", "litellm", "ollama", "azure_openai", + "deepseek", "together", "llama_cpp", +} + +# SECTION 1 – MODULE STRUCTURE +section("1. MODULE STRUCTURE") +import instructor.v2.core.capability_matrix as _m +for sym in ("CAPABILITY_MATRIX", "ProviderCapabilities", "Support", + "get_provider_capabilities", "providers_supporting", + "print_capability_matrix"): + check(f"Symbol '{sym}' exported", hasattr(_m, sym)) +check("Support literal = {yes,no,partial}", set(get_args(Support)) == {"yes", "no", "partial"}) + +# SECTION 2 – DATACLASS INTEGRITY +section("2. DATACLASS INTEGRITY") +field_names = {f.name for f in dc_fields(ProviderCapabilities)} +for cap in CAP_FIELDS: + check(f"Field '{cap}' exists", cap in field_names) +for req in ("name", "provider_key", "notes", "install_extra", "example_model", "docs_url"): + check(f"Field '{req}' exists", req in field_names) + +pc = ProviderCapabilities(name="Test", provider_key="test") +try: + # frozen dataclasses raise FrozenInstanceError on assignment + pc.streaming = "yes" # type: ignore + check("Dataclass is frozen", False, "assignment succeeded") +except Exception: + check("Dataclass is frozen", True) + + +check("Default caps are all 'no'", all(getattr(pc, c) == "no" for c in CAP_FIELDS)) +check("Default notes = ()", pc.notes == ()) +# hashable? (only works if notes is tuple, not list) +try: + hash(pc) + check("Dataclass is hashable", True) +except TypeError as e: + check("Dataclass is hashable", False, str(e)) +# equality +pc2 = ProviderCapabilities(name="Test", provider_key="test") +pc3 = ProviderCapabilities(name="Other", provider_key="other") +check("Equal instances compare equal", pc == pc2) +check("Unequal instances compare unequal", pc != pc3) + +# SECTION 3 – CAPABILITY MATRIX DATA +section("3. CAPABILITY MATRIX DATA") +check(f"21 providers in matrix", len(CAPABILITY_MATRIX) == 21, f"got {len(CAPABILITY_MATRIX)}") +check("Exactly expected providers", + EXPECTED_PROVIDERS == set(CAPABILITY_MATRIX.keys()), + str(EXPECTED_PROVIDERS.symmetric_difference(set(CAPABILITY_MATRIX.keys())))) + +valid_vals = {"yes", "no", "partial"} +errors: list[str] = [] +for key, caps in CAPABILITY_MATRIX.items(): + for cap in CAP_FIELDS: + val = getattr(caps, cap) + if val not in valid_vals: + errors.append(f"{key}.{cap}={val!r}") + if caps.provider_key != key: + errors.append(f"{key}.provider_key={caps.provider_key!r}") + if not (caps.name and caps.name.strip()): + errors.append(f"{key}.name empty") + if not isinstance(caps.notes, tuple): + errors.append(f"{key}.notes not tuple (type={type(caps.notes).__name__})") + elif len(caps.notes) == 0: + errors.append(f"{key} has no notes") + for note in caps.notes: + if not isinstance(note, str): + errors.append(f"{key}: non-str note: {note!r}") + if caps.install_extra and not caps.install_extra.startswith("instructor"): + errors.append(f"{key}.install_extra={caps.install_extra!r}") + if caps.example_model and "/" not in caps.example_model: + errors.append(f"{key}.example_model missing slash: {caps.example_model!r}") + if caps.docs_url and not caps.docs_url.endswith(".md"): + errors.append(f"{key}.docs_url no .md: {caps.docs_url!r}") + +check("All capability values valid & fields well-formed", len(errors) == 0, "; ".join(errors[:5])) + +# Spot-checks +c = CAPABILITY_MATRIX +check("OpenAI: all 10 caps = yes", all(getattr(c["openai"], cap) == "yes" for cap in CAP_FIELDS)) +check("Anthropic: audio=no", c["anthropic"].audio == "no") +check("Anthropic: vision=yes, pdf=yes", c["anthropic"].vision == "yes" and c["anthropic"].pdf == "yes") +check("Anthropic: responses_api=no", c["anthropic"].responses_api == "no") +check("Google: parallel_tools=no", c["google"].parallel_tools == "no") +check("Google: json_schema_mode=no", c["google"].json_schema_mode == "no") +check("Google: audio=yes", c["google"].audio == "yes") +check("Bedrock: json_mode=no, json_schema_mode=no", c["bedrock"].json_mode == "no" and c["bedrock"].json_schema_mode == "no") +check("Bedrock: parallel_tools=no", c["bedrock"].parallel_tools == "no") +check("Bedrock: vision=partial", c["bedrock"].vision == "partial") +check("Perplexity: tool_calling=no, parallel_tools=no", c["perplexity"].tool_calling == "no" and c["perplexity"].parallel_tools == "no") +check("Perplexity: no multimodal at all", all(getattr(c["perplexity"], cap) == "no" for cap in ["vision","audio","pdf"])) +check("Cerebras: parallel_tools=yes", c["cerebras"].parallel_tools == "yes") +check("Cerebras: vision=no", c["cerebras"].vision == "no") +check("LiteLLM: all caps except responses_api = partial", + all(getattr(c["litellm"], cap) == "partial" for cap in CAP_FIELDS[:-1])) +check("LiteLLM: responses_api=no", c["litellm"].responses_api == "no") +check("Gemini legacy: streaming=partial, async=partial", + c["gemini"].streaming == "partial" and c["gemini"].async_support == "partial") +check("llama_cpp: async_support=no", c["llama_cpp"].async_support == "no") +check("Mistral: json_mode=no, json_schema_mode=yes", + c["mistral"].json_mode == "no" and c["mistral"].json_schema_mode == "yes") +check("Cohere: vision=no", c["cohere"].vision == "no") +check("xAI: parallel_tools=yes", c["xai"].parallel_tools == "yes") +check("Only openai+google have audio='yes'", + {k for k,v in c.items() if v.audio == "yes"} == {"openai","google"}) +check("Only openai has responses_api='yes'", + {k for k,v in c.items() if v.responses_api == "yes"} == {"openai"}) + +# SECTION 4 – CROSS-REFERENCE WITH provider_specs.py +section("4. CROSS-REFERENCE WITH provider_specs.py") +cross_checks = [ + (Provider.OPENAI, "openai", Mode.TOOLS, "tool_calling", "yes"), + (Provider.OPENAI, "openai", Mode.PARALLEL_TOOLS, "parallel_tools", "yes"), + (Provider.OPENAI, "openai", Mode.JSON_SCHEMA, "json_schema_mode", "yes"), + (Provider.OPENAI, "openai", Mode.JSON, "json_mode", "yes"), + (Provider.OPENAI, "openai", Mode.RESPONSES_TOOLS, "responses_api", "yes"), + (Provider.ANTHROPIC, "anthropic", Mode.TOOLS, "tool_calling", "yes"), + (Provider.ANTHROPIC, "anthropic", Mode.PARALLEL_TOOLS, "parallel_tools", "yes"), + (Provider.ANTHROPIC, "anthropic", Mode.JSON_SCHEMA, "json_schema_mode", "yes"), + (Provider.GROQ, "groq", Mode.PARALLEL_TOOLS, "parallel_tools", "no"), + (Provider.GROQ, "groq", Mode.JSON_SCHEMA, "json_schema_mode", "yes"), + (Provider.BEDROCK, "bedrock", Mode.PARALLEL_TOOLS, "parallel_tools", "no"), + (Provider.BEDROCK, "bedrock", Mode.JSON_SCHEMA, "json_schema_mode", "no"), + (Provider.GENAI, "google", Mode.JSON, "json_mode", "yes"), + (Provider.GENAI, "google", Mode.JSON_SCHEMA, "json_schema_mode", "no"), + (Provider.GENAI, "google", Mode.PARALLEL_TOOLS, "parallel_tools", "no"), + (Provider.VERTEXAI, "vertexai", Mode.JSON, "json_mode", "no"), + (Provider.VERTEXAI, "vertexai", Mode.PARALLEL_TOOLS, "parallel_tools", "yes"), + (Provider.CEREBRAS, "cerebras", Mode.PARALLEL_TOOLS, "parallel_tools", "yes"), + (Provider.CEREBRAS, "cerebras", Mode.JSON_SCHEMA, "json_schema_mode", "yes"), + (Provider.PERPLEXITY, "perplexity", Mode.TOOLS, "tool_calling", "no"), + (Provider.XAI, "xai", Mode.PARALLEL_TOOLS, "parallel_tools", "yes"), + (Provider.MISTRAL, "mistral", Mode.JSON_SCHEMA, "json_schema_mode", "yes"), + (Provider.COHERE, "cohere", Mode.JSON_SCHEMA, "json_schema_mode", "yes"), + (Provider.GEMINI, "gemini", Mode.JSON, "json_mode", "no"), +] +for prov_enum, cap_key, mode, cap_field, expected in cross_checks: + spec = PROVIDER_SPECS[prov_enum] + actual = getattr(CAPABILITY_MATRIX[cap_key], cap_field) + # verify the spec side + if expected == "yes": + spec_ok = mode in spec.supported_modes + else: + spec_ok = mode not in spec.supported_modes or mode in spec.unsupported_modes + check( + f"Cross-ref {cap_key}.{cap_field}={expected!r} (spec consistent)", + actual == expected and spec_ok, + f"matrix={actual!r}, spec_ok={spec_ok}", + ) + +# Verify async_modes -> async_support consistency for providers we track +async_map = { + Provider.OPENAI: "openai", + Provider.ANTHROPIC: "anthropic", + Provider.GENAI: "google", + Provider.COHERE: "cohere", + Provider.GROQ: "groq", + Provider.MISTRAL: "mistral", + Provider.CEREBRAS: "cerebras", + Provider.BEDROCK: "bedrock", + Provider.VERTEXAI: "vertexai", + Provider.XAI: "xai", + Provider.PERPLEXITY:"perplexity", +} +for prov_enum, cap_key in async_map.items(): + spec = PROVIDER_SPECS[prov_enum] + caps = CAPABILITY_MATRIX[cap_key] + if len(spec.async_modes) > 0: + check(f"async_modes->async_support: {cap_key}", + caps.async_support in ("yes", "partial"), + f"got {caps.async_support!r}") + +# SECTION 5 – QUERY HELPERS +section("5. QUERY HELPERS — get_provider_capabilities") +check("openai -> not None", get_provider_capabilities("openai") is not None) +check("anthropic -> not None", get_provider_capabilities("anthropic") is not None) +check("unknown_xyz -> None", get_provider_capabilities("unknown_xyz") is None) +check("empty string -> None", get_provider_capabilities("") is None) +check("OpenAI (capitalized) -> None", get_provider_capabilities("OpenAI") is None) +for pk in EXPECTED_PROVIDERS: + check(f"get_provider_capabilities({pk!r})", get_provider_capabilities(pk) is not None) + +section("5. QUERY HELPERS — providers_supporting") +for cap in CAP_FIELDS: + res = providers_supporting(cap) + check(f"providers_supporting({cap!r}) returns list", isinstance(res, list)) + check(f"providers_supporting({cap!r}) subset of matrix", + set(res).issubset(set(CAPABILITY_MATRIX.keys())), + str(set(res) - set(CAPABILITY_MATRIX.keys()))) + for k in res: + check(f" {cap!r}: {k} has 'yes' (not partial)", + getattr(CAPABILITY_MATRIX[k], cap) == "yes", + f"got {getattr(CAPABILITY_MATRIX[k], cap)!r}") + +# specific expected results +check("audio providers = {openai,google}", set(providers_supporting("audio")) == {"openai","google"}) +check("responses_api providers = [openai]", providers_supporting("responses_api") == ["openai"]) +check("tool_calling excludes perplexity", "perplexity" not in providers_supporting("tool_calling")) +check("parallel_tools includes openai,anthropic,cerebras,xai", + {"openai","anthropic","cerebras","xai"}.issubset(set(providers_supporting("parallel_tools")))) +check("parallel_tools excludes groq,mistral,bedrock", + not {"groq","mistral","bedrock"}.intersection(set(providers_supporting("parallel_tools")))) +check("vision includes openai,anthropic,google,vertexai", + {"openai","anthropic","google","vertexai"}.issubset(set(providers_supporting("vision")))) +check("vision excludes cohere,perplexity,cerebras,writer", + not {"cohere","perplexity","cerebras","writer"}.intersection(set(providers_supporting("vision")))) +check("pdf includes openai,anthropic,google", + {"openai","anthropic","google"}.issubset(set(providers_supporting("pdf")))) +check("pdf excludes bedrock,mistral,cohere", + not {"bedrock","mistral","cohere"}.intersection(set(providers_supporting("pdf")))) + +try: + providers_supporting("turbo_mode_xyz") + check("ValueError on invalid capability", False) +except ValueError as e: + check("ValueError on invalid capability", "Unknown capability" in str(e)) + +try: + providers_supporting("") + check("ValueError on empty capability", False) +except ValueError: + check("ValueError on empty capability", True) + +# SECTION 6 – PRINT HELPER +section("6. PRINT HELPER") + +def capture(**kwargs) -> str: + buf = io.StringIO() + old = sys.stdout + sys.stdout = buf + try: + print_capability_matrix(**kwargs) + finally: + sys.stdout = old + return buf.getvalue() + +out_all = capture() +check("Full output non-empty", len(out_all) > 200) +check("Header has 'Provider'", "Provider" in out_all) +check("Header has 'Stream'", "Stream" in out_all) +check("Header has 'Vision'", "Vision" in out_all) +check("All provider names in output", all(caps.name in out_all for caps in CAPABILITY_MATRIX.values())) +check("Legend line present", "Legend" in out_all) +check("Separator lines present", "---" in out_all) + +out_filtered = capture(providers=["openai","anthropic"]) +check("Filtered: OpenAI present", "OpenAI" in out_filtered) +check("Filtered: Anthropic present", "Anthropic" in out_filtered) +check("Filtered: Groq absent", "Groq" not in out_filtered) +check("Filtered: Bedrock absent", "AWS Bedrock" not in out_filtered) + +out_notes = capture(providers=["openai"], show_notes=True) +check("show_notes=True: note text appears", + any(kw in out_notes for kw in ["GPT-4o", "Vision", "Audio", "Responses"])) + +out_no_notes = capture(providers=["openai"], show_notes=False) +check("show_notes=False: specific note text absent", "GPT-4o-class" not in out_no_notes) + +# Unknown provider in filter -> silently skipped +out_unk = capture(providers=["openai","nonexistent_xyz"]) +check("Unknown key in filter: OpenAI still shown", "OpenAI" in out_unk) +check("Unknown key in filter: unknown key not shown", "nonexistent_xyz" not in out_unk) + +# SECTION 7 – EDGE CASES +section("7. EDGE CASES") +check("Matrix is not empty", len(CAPABILITY_MATRIX) > 0) +check("All dict keys are lowercase", all(k == k.lower() for k in CAPABILITY_MATRIX)) +check("All dict keys are strings", all(isinstance(k, str) for k in CAPABILITY_MATRIX)) +check("Import is idempotent (same object)", + CAPABILITY_MATRIX is __import__("instructor.v2.core.capability_matrix", + fromlist=["CAPABILITY_MATRIX"]).CAPABILITY_MATRIX) +# Total data points +total_dp = len(CAPABILITY_MATRIX) * len(CAP_FIELDS) +check(f"Total data points = 21×10 = 210", total_dp == 210, f"got {total_dp}") + +# SECTION 8 – v2.core __init__ EXPORTS +section("8. v2.core __init__ EXPORTS") +from instructor.v2.core import ( + ProviderCapabilities as PC2, + CAPABILITY_MATRIX as CM2, + get_provider_capabilities as gpc2, + providers_supporting as ps2, + print_capability_matrix as pmat2, +) +check("ProviderCapabilities identity", PC2 is ProviderCapabilities) +check("CAPABILITY_MATRIX identity", CM2 is CAPABILITY_MATRIX) +check("get_provider_capabilities identity", gpc2 is get_provider_capabilities) +check("providers_supporting identity", ps2 is providers_supporting) +check("print_capability_matrix identity", pmat2 is print_capability_matrix) +check("Exported gpc2 works: openai streaming=yes", gpc2("openai").streaming == "yes") +check("Exported ps2 works: audio={openai,google}", set(ps2("audio")) == {"openai","google"}) + +# SECTION 9 – DOCS FILE INTEGRITY +section("9. DOCS FILE INTEGRITY") +docs = pathlib.Path(r"D:\VN-Aspect\coding\Open Sources Conti\instructor\docs\integrations\capability-matrix.md") +check("capability-matrix.md exists", docs.exists()) +content = docs.read_text(encoding="utf-8") +check("File > 5000 chars", len(content) > 5000, f"got {len(content)}") +check("Has YAML frontmatter title:", "title:" in content) +check("Quick Reference section", "Quick Reference" in content) +check("Mode Support section", "Mode Support" in content) +check("Detailed Provider Profiles section", "Provider Profiles" in content) +check("Known Incompatibilities section", "Incompatib" in content) +check("Audio section", "### Audio" in content or "## Audio" in content or "Audio" in content) +check("PDF section", "PDF" in content) +check("Programmatic Access section", "Programmatic" in content) +check("get_provider_capabilities in doc", "get_provider_capabilities" in content) +check("providers_supporting in doc", "providers_supporting" in content) +check("Choosing the Right Provider", "Choosing the Right Provider" in content or "right provider" in content.lower()) +check("Emoji check present", "\u2705" in content) +check("Emoji cross present", "\u274c" in content) +check("Warning emoji present", "\u26a0\ufe0f" in content) +check("Anthropic link present", "./anthropic.md" in content) +check("OpenAI link present", "./openai.md" in content) +check("MD_JSON incompatibility mentioned", "MD_JSON" in content) +check("Union type warning mentioned", "Union" in content) +check("Perplexity warning mentioned", "Perplexity" in content) +for pname in ["OpenAI","Anthropic","Google GenAI","Vertex AI","AWS Bedrock", + "Mistral","Cohere","Groq","Cerebras","Perplexity","Ollama","LiteLLM"]: + check(f"Doc mentions '{pname}'", pname in content) + +# SECTION 10 – mkdocs.yml NAV +section("10. mkdocs.yml NAV") +mkdocs_path = pathlib.Path(r"D:\VN-Aspect\coding\Open Sources Conti\instructor\mkdocs.yml") +check("mkdocs.yml exists", mkdocs_path.exists()) +mkdocs = mkdocs_path.read_text(encoding="utf-8") +check("capability-matrix.md in nav", "capability-matrix.md" in mkdocs) +check("Capability Matrix label in nav", + bool(re.search(r"Capability Matrix.*capability-matrix\.md", mkdocs))) +check("Integrations section present", "Integrations:" in mkdocs) +# Must appear BEFORE other integration pages (it's the second entry after Overview) +idx_overview = mkdocs.find("integrations/index.md") +idx_cap = mkdocs.find("capability-matrix.md") +idx_openai = mkdocs.find("integrations/openai.md") +check("Capability Matrix appears before OpenAI entry", 0 < idx_cap < idx_openai, + f"cap_idx={idx_cap}, openai_idx={idx_openai}") + +# SECTION 11 – integrations/index.md LINKS +section("11. integrations/index.md LINKS") +idx_path = pathlib.Path(r"D:\VN-Aspect\coding\Open Sources Conti\instructor\docs\integrations\index.md") +check("integrations/index.md exists", idx_path.exists()) +idx_content = idx_path.read_text(encoding="utf-8") +check("index.md links to capability-matrix", "capability-matrix" in idx_content) +check("Meaningful link text in index", "Capability Matrix" in idx_content) +check("Link is formatted as markdown link", + bool(re.search(r"\[.*Capability Matrix.*\]\(.*capability-matrix.*\)", idx_content, re.IGNORECASE))) + +# SECTION 12 – BEFORE / AFTER COMPARISON +section("12. BEFORE / AFTER REGRESSION COMPARISON") + +spec_a = PROVIDER_SPECS[Provider.ANTHROPIC] +check("BEFORE: Anthropic MD_JSON in unsupported_modes (spec truth)", + Mode.MD_JSON in spec_a.unsupported_modes) + +spec_p = PROVIDER_SPECS[Provider.PERPLEXITY] +check("BEFORE: Perplexity has NO TOOLS in supported_modes (spec truth)", + Mode.TOOLS not in spec_p.supported_modes) + +spec_b = PROVIDER_SPECS[Provider.BEDROCK] +check("BEFORE: Bedrock has NO PARALLEL_TOOLS (spec truth)", + Mode.PARALLEL_TOOLS not in spec_b.supported_modes) + +spec_gem = PROVIDER_SPECS[Provider.GEMINI] +check("BEFORE: Gemini JSON mode in unsupported_modes (spec truth)", + Mode.JSON in spec_gem.unsupported_modes) + +# ProviderSpec has no audio/vision fields (that's what the matrix ADDS) +spec_field_names = {f.name for f in dc_fields(ProviderSpec)} +check("BEFORE: ProviderSpec has no 'audio' field (matrix adds this)", + "audio" not in spec_field_names) +check("BEFORE: ProviderSpec has no 'vision' field (matrix adds this)", + "vision" not in spec_field_names) +check("BEFORE: ProviderSpec has no 'pdf' field (matrix adds this)", + "pdf" not in spec_field_names) + +cm_a = CAPABILITY_MATRIX["anthropic"] +check("AFTER: Matrix notes document Anthropic MD_JSON incompatibility", + any("MD_JSON" in n for n in cm_a.notes)) +check("AFTER: Matrix marks perplexity.tool_calling=no", + CAPABILITY_MATRIX["perplexity"].tool_calling == "no") +check("AFTER: Matrix marks bedrock.parallel_tools=no", + CAPABILITY_MATRIX["bedrock"].parallel_tools == "no") +check("AFTER: Matrix marks gemini.json_mode=no", + CAPABILITY_MATRIX["gemini"].json_mode == "no") +check("AFTER: Matrix marks llama_cpp.async_support=no", + CAPABILITY_MATRIX["llama_cpp"].async_support == "no") + +google_notes = CAPABILITY_MATRIX["google"].notes +check("AFTER: Google notes document Union/Optional restriction", + any("Union" in n or "Optional" in n for n in google_notes)) + +audio_yes = set(providers_supporting("audio")) +check("AFTER: Audio providers exactly {openai, google}", audio_yes == {"openai","google"}) + +pdf_yes = set(providers_supporting("pdf")) +check("AFTER: PDF providers include openai,anthropic,google", + {"openai","anthropic","google"}.issubset(pdf_yes)) +check("AFTER: PDF providers exclude bedrock,mistral", + not {"bedrock","mistral"}.intersection(pdf_yes)) + +vision_yes = set(providers_supporting("vision")) +check("AFTER: Vision full-support: openai,anthropic,google,vertexai", + {"openai","anthropic","google","vertexai"}.issubset(vision_yes)) +check("AFTER: Vision excludes cohere,perplexity,cerebras,writer", + not {"cohere","perplexity","cerebras","writer"}.intersection(vision_yes)) + +check("AFTER: 210 total capability data points (21 providers × 10 caps)", + len(CAPABILITY_MATRIX) * len(CAP_FIELDS) == 210) + +check("AFTER: Programmatic query API works (no such API existed before)", + callable(providers_supporting) and callable(get_provider_capabilities)) + +# FINAL SUMMARY +print() +print("=" * 72) +print(" FINAL TEST SUMMARY") +print("=" * 72) + +# Group by section +from collections import defaultdict +by_section: dict[str, list[tuple[str,str]]] = defaultdict(list) +for status, sec, label in RESULTS: + by_section[sec].append((status, label)) + +for sec, items in by_section.items(): + sec_pass = sum(1 for s,_ in items if s == "PASS") + sec_fail = sum(1 for s,_ in items if s == "FAIL") + bar = "PASS" if sec_fail == 0 else "FAIL" + print(f" [{bar}] {sec}: {sec_pass} passed, {sec_fail} failed") + +print() +print(f" TOTAL: {PASS} PASSED / {FAIL} FAILED / {PASS+FAIL} TOTAL") +print("=" * 72) + +if FAIL > 0: + print() + print(" FAILING TESTS:") + for status, sec, label in RESULTS: + if status == "FAIL": + print(f" [{sec}] {label}") + sys.exit(1) +else: + print() + print(" ALL TESTS PASSED!") + sys.exit(0) diff --git a/tests/v2/test_capability_matrix.py b/tests/v2/test_capability_matrix.py new file mode 100644 index 000000000..ee82b3b64 --- /dev/null +++ b/tests/v2/test_capability_matrix.py @@ -0,0 +1,823 @@ +"""Tests for instructor.v2.core.capability_matrix.""" + +from __future__ import annotations + +import io +import sys +import pathlib +import re +from dataclasses import fields +from typing import get_args + +import pytest + +REPO_ROOT = pathlib.Path(__file__).parents[3] +CAPABILITY_MATRIX_PY = REPO_ROOT / "instructor" / "v2" / "core" / "capability_matrix.py" +DOCS_MATRIX_MD = REPO_ROOT / "docs" / "integrations" / "capability-matrix.md" +MKDOCS_YML = REPO_ROOT / "mkdocs.yml" +INTEGRATIONS_INDEX_MD = REPO_ROOT / "docs" / "integrations" / "index.md" + + +def _import_matrix(): + from instructor.v2.core.capability_matrix import ( + CAPABILITY_MATRIX, + ProviderCapabilities, + Support, + get_provider_capabilities, + providers_supporting, + print_capability_matrix, + ) + return ( + CAPABILITY_MATRIX, + ProviderCapabilities, + Support, + get_provider_capabilities, + providers_supporting, + print_capability_matrix, + ) + + +EXPECTED_PROVIDERS = { + "openai", "anthropic", "google", "gemini", "vertexai", "bedrock", + "mistral", "cohere", "groq", "fireworks", "cerebras", "writer", + "xai", "perplexity", "openrouter", "litellm", "ollama", "azure_openai", + "deepseek", "together", "llama_cpp", +} + + +class TestModuleStructure: + def test_module_file_exists(self): + assert CAPABILITY_MATRIX_PY.exists() + + def test_module_imports_without_error(self): + CM, PC, Sup, get_caps, prov_sup, print_mat = _import_matrix() + assert CM is not None + assert PC is not None + + def test_public_api_symbols_present(self): + import instructor.v2.core.capability_matrix as m + for sym in ("CAPABILITY_MATRIX", "ProviderCapabilities", "Support", + "get_provider_capabilities", "providers_supporting", + "print_capability_matrix"): + assert hasattr(m, sym), f"Missing public symbol: {sym}" + + def test_support_literal_values(self): + from instructor.v2.core.capability_matrix import Support + assert set(get_args(Support)) == {"yes", "no", "partial"} + + +class TestProviderCapabilitiesDataclass: + CAPABILITY_FIELDS = [ + "streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", + "responses_api", + ] + REQUIRED_FIELDS = ["name", "provider_key"] + OPTIONAL_FIELDS = ["notes", "install_extra", "example_model", "docs_url"] + + def test_all_fields_exist(self): + from instructor.v2.core.capability_matrix import ProviderCapabilities + field_names = {f.name for f in fields(ProviderCapabilities)} + for cap in self.CAPABILITY_FIELDS + self.REQUIRED_FIELDS + self.OPTIONAL_FIELDS: + assert cap in field_names, f"Missing field: {cap}" + + def test_frozen(self): + from instructor.v2.core.capability_matrix import ProviderCapabilities + pc = ProviderCapabilities(name="Test", provider_key="test") + with pytest.raises((AttributeError, TypeError)): + pc.streaming = "yes" # type: ignore + + def test_default_capabilities_are_no(self): + from instructor.v2.core.capability_matrix import ProviderCapabilities + pc = ProviderCapabilities(name="Test", provider_key="test") + for cap in self.CAPABILITY_FIELDS: + assert getattr(pc, cap) == "no", f"{cap} default should be 'no'" + + def test_default_notes_is_empty_tuple(self): + from instructor.v2.core.capability_matrix import ProviderCapabilities + pc = ProviderCapabilities(name="Test", provider_key="test") + assert pc.notes == () + + def test_capability_values_constrained(self): + CM, PC, Sup, *_ = _import_matrix() + valid = {"yes", "no", "partial"} + for key, caps in CM.items(): + for cap in self.CAPABILITY_FIELDS: + val = getattr(caps, cap) + assert val in valid, f"Provider '{key}' field '{cap}' has invalid value '{val}'" + + def test_notes_is_tuple_of_strings(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + assert isinstance(caps.notes, tuple), f"Provider '{key}' notes must be tuple" + for note in caps.notes: + assert isinstance(note, str), f"Provider '{key}' note is not a string: {note!r}" + + def test_provider_key_matches_dict_key(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + assert caps.provider_key == key + + def test_name_is_non_empty_string(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + assert isinstance(caps.name, str) and caps.name.strip() + + +class TestCapabilityMatrixData: + + def test_expected_providers_present(self): + CM, *_ = _import_matrix() + for p in EXPECTED_PROVIDERS: + assert p in CM, f"Expected provider '{p}' missing from CAPABILITY_MATRIX" + + def test_no_unexpected_providers(self): + CM, *_ = _import_matrix() + extra = set(CM.keys()) - EXPECTED_PROVIDERS + assert not extra, f"Unexpected providers in matrix: {extra}" + + def test_provider_count(self): + CM, *_ = _import_matrix() + assert len(CM) == len(EXPECTED_PROVIDERS) + + def test_openai_is_fully_capable(self): + CM, *_ = _import_matrix() + c = CM["openai"] + for cap in ["streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", "responses_api"]: + assert getattr(c, cap) == "yes", f"OpenAI should fully support '{cap}'" + + def test_anthropic_no_audio(self): + CM, *_ = _import_matrix() + assert CM["anthropic"].audio == "no" + + def test_anthropic_has_vision_and_pdf(self): + CM, *_ = _import_matrix() + assert CM["anthropic"].vision == "yes" + assert CM["anthropic"].pdf == "yes" + + def test_anthropic_no_responses_api(self): + CM, *_ = _import_matrix() + assert CM["anthropic"].responses_api == "no" + + def test_google_genai_no_parallel_tools(self): + CM, *_ = _import_matrix() + assert CM["google"].parallel_tools == "no" + + def test_google_genai_no_json_schema(self): + CM, *_ = _import_matrix() + assert CM["google"].json_schema_mode == "no" + + def test_google_genai_has_audio(self): + CM, *_ = _import_matrix() + assert CM["google"].audio == "yes" + + def test_bedrock_no_json_modes(self): + CM, *_ = _import_matrix() + assert CM["bedrock"].json_mode == "no" + assert CM["bedrock"].json_schema_mode == "no" + + def test_bedrock_no_parallel_tools(self): + CM, *_ = _import_matrix() + assert CM["bedrock"].parallel_tools == "no" + + def test_bedrock_vision_is_partial(self): + CM, *_ = _import_matrix() + assert CM["bedrock"].vision == "partial" + + def test_perplexity_no_tools(self): + CM, *_ = _import_matrix() + assert CM["perplexity"].tool_calling == "no" + assert CM["perplexity"].parallel_tools == "no" + + def test_perplexity_no_multimodal(self): + CM, *_ = _import_matrix() + assert CM["perplexity"].vision == "no" + assert CM["perplexity"].audio == "no" + assert CM["perplexity"].pdf == "no" + + def test_cerebras_has_parallel_tools(self): + CM, *_ = _import_matrix() + assert CM["cerebras"].parallel_tools == "yes" + + def test_cerebras_no_vision(self): + CM, *_ = _import_matrix() + assert CM["cerebras"].vision == "no" + + def test_xai_has_parallel_tools(self): + CM, *_ = _import_matrix() + assert CM["xai"].parallel_tools == "yes" + + def test_gemini_legacy_is_partial(self): + # Legacy SDK — streaming and async are degraded + CM, *_ = _import_matrix() + assert CM["gemini"].streaming == "partial" + assert CM["gemini"].async_support == "partial" + + def test_litellm_all_partial_except_responses(self): + CM, *_ = _import_matrix() + c = CM["litellm"] + for cap in ["streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf"]: + assert getattr(c, cap) == "partial", f"LiteLLM '{cap}' should be partial" + assert c.responses_api == "no" + + def test_llama_cpp_no_async(self): + CM, *_ = _import_matrix() + assert CM["llama_cpp"].async_support == "no" + + def test_mistral_no_json_mode_but_has_json_schema(self): + # Mistral supports Mode.JSON_SCHEMA but not Mode.JSON + CM, *_ = _import_matrix() + assert CM["mistral"].json_mode == "no" + assert CM["mistral"].json_schema_mode == "yes" + + def test_cohere_no_vision(self): + CM, *_ = _import_matrix() + assert CM["cohere"].vision == "no" + + def test_only_openai_and_google_have_audio(self): + CM, *_ = _import_matrix() + audio_yes = {k for k, c in CM.items() if c.audio == "yes"} + assert audio_yes == {"openai", "google"} + + def test_only_openai_has_responses_api(self): + CM, *_ = _import_matrix() + assert {k for k, c in CM.items() if c.responses_api == "yes"} == {"openai"} + + def test_install_extra_format(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + if caps.install_extra is not None: + assert caps.install_extra.startswith("instructor") + + def test_example_model_format(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + if caps.example_model is not None: + assert "/" in caps.example_model + + def test_docs_url_format(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + if caps.docs_url is not None: + assert caps.docs_url.endswith(".md") + + def test_at_least_one_note_per_provider(self): + CM, *_ = _import_matrix() + for key, caps in CM.items(): + assert len(caps.notes) > 0, f"Provider '{key}' has no notes" + + +class TestCrossReferenceWithProviderSpecs: + + def _get_specs(self): + from instructor.v2.core.provider_specs import PROVIDER_SPECS + from instructor.v2.core.mode import Mode + return PROVIDER_SPECS, Mode + + def _get_matrix(self): + from instructor.v2.core.capability_matrix import CAPABILITY_MATRIX + return CAPABILITY_MATRIX + + def test_openai_tool_calling_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.OPENAI] + CM = self._get_matrix() + assert CM["openai"].tool_calling == ("yes" if Mode.TOOLS in spec.supported_modes else "no") + + def test_anthropic_tool_calling_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.ANTHROPIC] + CM = self._get_matrix() + assert CM["anthropic"].tool_calling == ("yes" if Mode.TOOLS in spec.supported_modes else "no") + + def test_perplexity_no_tool_calling_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.PERPLEXITY] + CM = self._get_matrix() + assert Mode.TOOLS not in spec.supported_modes + assert CM["perplexity"].tool_calling == "no" + + def test_openai_parallel_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.OPENAI] + CM = self._get_matrix() + assert Mode.PARALLEL_TOOLS in spec.supported_modes + assert CM["openai"].parallel_tools == "yes" + + def test_groq_no_parallel_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.GROQ] + CM = self._get_matrix() + assert Mode.PARALLEL_TOOLS not in spec.supported_modes + assert CM["groq"].parallel_tools == "no" + + def test_cerebras_parallel_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.CEREBRAS] + CM = self._get_matrix() + assert Mode.PARALLEL_TOOLS in spec.supported_modes + assert CM["cerebras"].parallel_tools == "yes" + + def test_bedrock_no_parallel_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.BEDROCK] + CM = self._get_matrix() + assert Mode.PARALLEL_TOOLS not in spec.supported_modes + assert CM["bedrock"].parallel_tools == "no" + + def test_openai_json_schema_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.OPENAI] + CM = self._get_matrix() + assert Mode.JSON_SCHEMA in spec.supported_modes + assert CM["openai"].json_schema_mode == "yes" + + def test_anthropic_json_schema_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.ANTHROPIC] + CM = self._get_matrix() + assert Mode.JSON_SCHEMA in spec.supported_modes + assert CM["anthropic"].json_schema_mode == "yes" + + def test_bedrock_no_json_schema_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.BEDROCK] + CM = self._get_matrix() + assert Mode.JSON_SCHEMA in spec.unsupported_modes + assert CM["bedrock"].json_schema_mode == "no" + + def test_google_genai_no_json_schema_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.GENAI] + CM = self._get_matrix() + assert Mode.JSON_SCHEMA in spec.unsupported_modes + assert CM["google"].json_schema_mode == "no" + + def test_openai_json_mode_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.OPENAI] + CM = self._get_matrix() + assert Mode.JSON in spec.supported_modes + assert CM["openai"].json_mode == "yes" + + def test_google_json_mode_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.GENAI] + CM = self._get_matrix() + assert Mode.JSON in spec.supported_modes + assert CM["google"].json_mode == "yes" + + def test_vertexai_no_json_mode_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.VERTEXAI] + CM = self._get_matrix() + assert Mode.JSON in spec.unsupported_modes + assert CM["vertexai"].json_mode == "no" + + def test_openai_responses_api_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.OPENAI] + CM = self._get_matrix() + assert Mode.RESPONSES_TOOLS in spec.supported_modes + assert CM["openai"].responses_api == "yes" + + def test_anthropic_no_responses_api_matches_spec(self): + PROVIDER_SPECS, Mode = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.ANTHROPIC] + CM = self._get_matrix() + assert Mode.RESPONSES_TOOLS not in spec.supported_modes + assert CM["anthropic"].responses_api == "no" + + def test_openai_has_async_modes(self): + PROVIDER_SPECS, _ = self._get_specs() + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.OPENAI] + CM = self._get_matrix() + assert len(spec.async_modes) > 0 + assert CM["openai"].async_support == "yes" + + def test_all_providers_with_async_modes_marked_as_yes(self): + PROVIDER_SPECS, _ = self._get_specs() + CM = self._get_matrix() + # Map provider_specs enum values to capability_matrix keys where they differ + spec_to_cap_key = { + "openai": "openai", "anthropic": "anthropic", "genai": "google", + "cohere": "cohere", "groq": "groq", "mistral": "mistral", + "fireworks": "fireworks", "cerebras": "cerebras", "writer": "writer", + "bedrock": "bedrock", "vertexai": "vertexai", "xai": "xai", + "perplexity": "perplexity", + } + for provider_enum, spec in PROVIDER_SPECS.items(): + cap_key = spec_to_cap_key.get(provider_enum.value) + if cap_key is None or cap_key not in CM: + continue + if len(spec.async_modes) > 0: + caps = CM[cap_key] + assert caps.async_support in ("yes", "partial"), ( + f"'{cap_key}' has async_modes in spec but async_support='{caps.async_support}'" + ) + + +class TestGetProviderCapabilities: + + def test_returns_correct_object_for_known_provider(self): + CM, PC, _, get_caps, *_ = _import_matrix() + assert get_caps("openai") is CM["openai"] + assert get_caps("anthropic") is CM["anthropic"] + + def test_returns_none_for_unknown_provider(self): + CM, PC, _, get_caps, *_ = _import_matrix() + assert get_caps("nonexistent_provider_xyz") is None + + def test_returns_none_for_empty_string(self): + _, _, _, get_caps, *_ = _import_matrix() + assert get_caps("") is None + + def test_case_sensitive(self): + _, _, _, get_caps, *_ = _import_matrix() + assert get_caps("OpenAI") is None # keys are lowercase + assert get_caps("openai") is not None + + @pytest.mark.parametrize("provider_key", list(EXPECTED_PROVIDERS)) + def test_all_expected_providers_return_non_none(self, provider_key): + _, _, _, get_caps, *_ = _import_matrix() + assert get_caps(provider_key) is not None + + +class TestProvidersSupporting: + + VALID_CAPS = [ + "streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", + "responses_api", + ] + + @pytest.mark.parametrize("cap", VALID_CAPS) + def test_returns_list_for_valid_capability(self, cap): + _, _, _, _, prov_sup, _ = _import_matrix() + assert isinstance(prov_sup(cap), list) + + def test_raises_value_error_for_invalid_capability(self): + _, _, _, _, prov_sup, _ = _import_matrix() + with pytest.raises(ValueError, match="Unknown capability"): + prov_sup("turbo_mode") + + def test_raises_value_error_for_empty_string(self): + _, _, _, _, prov_sup, _ = _import_matrix() + with pytest.raises(ValueError): + prov_sup("") + + def test_audio_returns_only_openai_and_google(self): + _, _, _, _, prov_sup, _ = _import_matrix() + assert set(prov_sup("audio")) == {"openai", "google"} + + def test_responses_api_returns_only_openai(self): + _, _, _, _, prov_sup, _ = _import_matrix() + assert prov_sup("responses_api") == ["openai"] + + def test_streaming_includes_major_providers(self): + _, _, _, _, prov_sup, _ = _import_matrix() + result = set(prov_sup("streaming")) + for p in ["openai", "anthropic", "google", "groq", "mistral", "bedrock"]: + assert p in result + + def test_tool_calling_excludes_perplexity(self): + _, _, _, _, prov_sup, _ = _import_matrix() + assert "perplexity" not in prov_sup("tool_calling") + + def test_parallel_tools_includes_correct_providers(self): + _, _, _, _, prov_sup, _ = _import_matrix() + result = set(prov_sup("parallel_tools")) + for p in {"openai", "anthropic", "vertexai", "cerebras", "xai", + "openrouter", "deepseek", "together", "azure_openai"}: + assert p in result + + def test_parallel_tools_excludes_groq_and_mistral(self): + _, _, _, _, prov_sup, _ = _import_matrix() + result = prov_sup("parallel_tools") + assert "groq" not in result + assert "mistral" not in result + + def test_vision_includes_full_support_providers(self): + _, _, _, _, prov_sup, _ = _import_matrix() + result = set(prov_sup("vision")) + for p in ["openai", "anthropic", "google", "vertexai"]: + assert p in result + + def test_vision_excludes_cohere_and_perplexity(self): + _, _, _, _, prov_sup, _ = _import_matrix() + result = prov_sup("vision") + assert "cohere" not in result + assert "perplexity" not in result + + def test_result_is_subset_of_all_providers(self): + _, _, _, _, prov_sup, _ = _import_matrix() + CM, *_ = _import_matrix() + for cap in self.VALID_CAPS: + assert set(prov_sup(cap)).issubset(set(CM.keys())) + + def test_no_partial_in_result(self): + """providers_supporting should only return 'yes' entries, not 'partial'.""" + _, _, _, _, prov_sup, _ = _import_matrix() + CM, *_ = _import_matrix() + for cap in self.VALID_CAPS: + for key in prov_sup(cap): + assert getattr(CM[key], cap) == "yes" + + +class TestPrintCapabilityMatrix: + + def _capture_print(self, **kwargs) -> str: + CM, PC, Sup, get_caps, prov_sup, print_mat = _import_matrix() + buf = io.StringIO() + orig_stdout = sys.stdout + sys.stdout = buf + try: + print_mat(**kwargs) + finally: + sys.stdout = orig_stdout + return buf.getvalue() + + def test_produces_output(self): + assert len(self._capture_print()) > 100 + + def test_header_line_present(self): + out = self._capture_print() + assert "Provider" in out + assert "Stream" in out + assert "Tools" in out + + def test_all_providers_appear_in_output(self): + CM, *_ = _import_matrix() + out = self._capture_print() + for key, caps in CM.items(): + assert caps.name in out + + def test_filter_by_providers(self): + out = self._capture_print(providers=["openai", "anthropic"]) + assert "OpenAI" in out + assert "Anthropic" in out + assert "Groq" not in out + + def test_show_notes_adds_note_text(self): + out = self._capture_print(providers=["openai"], show_notes=True) + assert "GPT-4o" in out or "Vision" in out or "Responses" in out + + def test_show_notes_false_omits_note_text(self): + out = self._capture_print(providers=["openai"], show_notes=False) + assert "GPT-4o-class" not in out + + def test_legend_line_present(self): + assert "Legend" in self._capture_print() + + def test_unknown_provider_key_in_filter_is_ignored(self): + out = self._capture_print(providers=["openai", "nonexistent_xyz"]) + assert "OpenAI" in out + assert "nonexistent_xyz" not in out + + +class TestEdgeCases: + + def test_capability_matrix_is_not_empty(self): + CM, *_ = _import_matrix() + assert len(CM) > 0 + + def test_all_keys_are_lowercase_strings(self): + CM, *_ = _import_matrix() + for k in CM.keys(): + assert isinstance(k, str) + assert k == k.lower() + + def test_no_none_names(self): + CM, *_ = _import_matrix() + for k, v in CM.items(): + assert v.name is not None and v.name != "" + + def test_module_import_is_idempotent(self): + from instructor.v2.core.capability_matrix import CAPABILITY_MATRIX as CM1 + from instructor.v2.core.capability_matrix import CAPABILITY_MATRIX as CM2 + assert CM1 is CM2 + + def test_ProviderCapabilities_is_hashable(self): + from instructor.v2.core.capability_matrix import ProviderCapabilities + pc = ProviderCapabilities(name="Test", provider_key="test") + hash(pc) + + def test_ProviderCapabilities_equality(self): + from instructor.v2.core.capability_matrix import ProviderCapabilities + pc1 = ProviderCapabilities(name="A", provider_key="a", streaming="yes") + pc2 = ProviderCapabilities(name="A", provider_key="a", streaming="yes") + pc3 = ProviderCapabilities(name="B", provider_key="b") + assert pc1 == pc2 + assert pc1 != pc3 + + +class TestV2CoreExports: + + def test_exports_are_accessible(self): + from instructor.v2.core import ( + ProviderCapabilities, + CAPABILITY_MATRIX, + get_provider_capabilities, + providers_supporting, + print_capability_matrix, + ) + assert isinstance(CAPABILITY_MATRIX, dict) + assert callable(get_provider_capabilities) + assert callable(providers_supporting) + assert callable(print_capability_matrix) + + def test_exported_functions_work(self): + from instructor.v2.core import get_provider_capabilities, providers_supporting + caps = get_provider_capabilities("openai") + assert caps is not None + assert caps.streaming == "yes" + assert "openai" in providers_supporting("audio") + + +class TestDocsFile: + + def test_file_exists(self): + assert DOCS_MATRIX_MD.exists() + + def test_file_is_non_empty(self): + assert len(DOCS_MATRIX_MD.read_text(encoding="utf-8")) > 1000 + + def test_required_sections_present(self): + content = DOCS_MATRIX_MD.read_text(encoding="utf-8") + for section in ["title:", "Quick Reference", "Mode Support", "Provider Profiles", + "Known Incompatib", "Programmatic", "get_provider_capabilities", + "providers_supporting"]: + assert section in content, f"Missing section: {section!r}" + + def test_all_major_providers_mentioned(self): + content = DOCS_MATRIX_MD.read_text(encoding="utf-8") + for name in ["OpenAI", "Anthropic", "Google GenAI", "Vertex AI", "AWS Bedrock", + "Mistral", "Cohere", "Groq", "Cerebras", "Perplexity", "Ollama", "LiteLLM"]: + assert name in content, f"Provider '{name}' not mentioned in capability-matrix.md" + + def test_symbols_and_links(self): + content = DOCS_MATRIX_MD.read_text(encoding="utf-8") + assert "✅" in content + assert "❌" in content + assert "./openai.md" in content + assert "./anthropic.md" in content + + def test_known_incompatibilities_documented(self): + content = DOCS_MATRIX_MD.read_text(encoding="utf-8") + # Anthropic + MD_JSON and Google Union restriction are the main ones + assert "MD_JSON" in content + assert "Union" in content + + +class TestMkdocsNav: + + def test_capability_matrix_in_nav(self): + content = MKDOCS_YML.read_text(encoding="utf-8") + assert "capability-matrix.md" in content + assert re.search(r"Capability Matrix.*capability-matrix\.md", content) + + +class TestIntegrationsIndexLinks: + + def test_index_links_to_capability_matrix(self): + content = INTEGRATIONS_INDEX_MD.read_text(encoding="utf-8") + assert "capability-matrix" in content + assert "Capability Matrix" in content + + +class TestBeforeAfterComparison: + """Regression checks: things that would have been silent runtime failures before this PR.""" + + def test_anthropic_md_json_in_unsupported_modes(self): + from instructor.v2.core.provider_specs import PROVIDER_SPECS + from instructor.v2.core.mode import Mode + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.ANTHROPIC] + assert Mode.MD_JSON in spec.unsupported_modes + + def test_matrix_documents_anthropic_md_json_incompatibility(self): + from instructor.v2.core.capability_matrix import get_provider_capabilities + caps = get_provider_capabilities("anthropic") + assert any("MD_JSON" in note for note in caps.notes) + + def test_perplexity_no_tools_in_spec(self): + from instructor.v2.core.provider_specs import PROVIDER_SPECS + from instructor.v2.core.mode import Mode + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.PERPLEXITY] + assert Mode.TOOLS in spec.unsupported_modes + + def test_matrix_marks_perplexity_tool_calling_no(self): + from instructor.v2.core.capability_matrix import get_provider_capabilities + assert get_provider_capabilities("perplexity").tool_calling == "no" + + def test_provider_spec_has_no_audio_or_vision_field(self): + # These fields didn't exist before — the matrix adds them + from instructor.v2.core.provider_specs import ProviderSpec + spec_fields = {f.name for f in fields(ProviderSpec)} + assert "audio" not in spec_fields + assert "vision" not in spec_fields + + def test_matrix_exposes_audio_support(self): + from instructor.v2.core.capability_matrix import providers_supporting + audio = providers_supporting("audio") + assert "openai" in audio + assert "google" in audio + assert "anthropic" not in audio + + def test_bedrock_no_parallel_tools_in_spec(self): + from instructor.v2.core.provider_specs import PROVIDER_SPECS + from instructor.v2.core.mode import Mode + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.BEDROCK] + assert Mode.PARALLEL_TOOLS not in spec.supported_modes + + def test_matrix_marks_bedrock_parallel_no(self): + from instructor.v2.core.capability_matrix import get_provider_capabilities + assert get_provider_capabilities("bedrock").parallel_tools == "no" + + def test_matrix_documents_google_union_restriction(self): + from instructor.v2.core.capability_matrix import get_provider_capabilities + caps = get_provider_capabilities("google") + assert any("Union" in note or "Optional" in note for note in caps.notes) + + def test_gemini_json_mode_in_unsupported_modes(self): + from instructor.v2.core.provider_specs import PROVIDER_SPECS + from instructor.v2.core.mode import Mode + from instructor.v2.core.providers import Provider + spec = PROVIDER_SPECS[Provider.GEMINI] + assert Mode.JSON in spec.unsupported_modes + + def test_matrix_marks_gemini_json_mode_no(self): + from instructor.v2.core.capability_matrix import get_provider_capabilities + assert get_provider_capabilities("gemini").json_mode == "no" + + def test_matrix_marks_llama_cpp_async_no(self): + from instructor.v2.core.capability_matrix import get_provider_capabilities + assert get_provider_capabilities("llama_cpp").async_support == "no" + + def test_programmatic_vision_query(self): + from instructor.v2.core.capability_matrix import providers_supporting + vision = set(providers_supporting("vision")) + assert {"openai", "anthropic", "google", "vertexai"}.issubset(vision) + for no_vision in ["cohere", "perplexity", "writer", "cerebras"]: + assert no_vision not in vision + + def test_programmatic_pdf_query(self): + from instructor.v2.core.capability_matrix import providers_supporting + pdf = set(providers_supporting("pdf")) + assert {"openai", "anthropic", "google"}.issubset(pdf) + assert "bedrock" not in pdf + assert "mistral" not in pdf + + def test_total_capability_data_points(self): + from instructor.v2.core.capability_matrix import CAPABILITY_MATRIX + cap_fields = [ + "streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", "responses_api", + ] + # 21 providers × 10 capabilities = 210 data points + assert len(CAPABILITY_MATRIX) * len(cap_fields) == len(EXPECTED_PROVIDERS) * 10 + + def test_before_after_summary(self, capsys): + from instructor.v2.core.capability_matrix import CAPABILITY_MATRIX, providers_supporting + cap_fields = [ + "streaming", "async_support", "tool_calling", "parallel_tools", + "json_mode", "json_schema_mode", "vision", "audio", "pdf", "responses_api", + ] + print("\n" + "=" * 70) + print("BEFORE / AFTER SUMMARY") + print("=" * 70) + print("BEFORE: No programmatic capability check existed.") + print(" Developers discovered incompatibilities at runtime.") + print() + print(f"AFTER: CAPABILITY_MATRIX covers {len(CAPABILITY_MATRIX)} providers, " + f"{len(cap_fields)} capabilities = {len(CAPABILITY_MATRIX) * len(cap_fields)} data points") + print(f" Audio providers: {sorted(providers_supporting('audio'))}") + print(f" Responses API: {providers_supporting('responses_api')}") + no_tools = sorted(set(CAPABILITY_MATRIX) - set(providers_supporting("tool_calling"))) + print(f" No tool calling (MD_JSON only): {no_tools}") + print("=" * 70) + + out = capsys.readouterr().out + assert "BEFORE" in out + assert "AFTER" in out