-
Notifications
You must be signed in to change notification settings - Fork 2
Feat/extend llm apis #7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
3f8fafc
WIP: extend authentication to cover multiple LLM providers
will-fawcett 4f747bb
Fix import of new llm init function. Cleanup code smells
will-fawcett b916172
Extract the cashing as a separate class
will-fawcett 9940138
Use decorator for LLMProvider registry
will-fawcett 1a490b9
Split genai module into llm package
will-fawcett ff753e4
Add llm package tests; document LLM provider config
will-fawcett a82c17e
Bump version to 2.0.0
will-fawcett File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| # Import provider modules for their import-time side effect: each one's | ||
| # @register_llm decorator populates base._LLM_REGISTRY. Without these imports | ||
| # the registry is empty and init_llm_client() can't resolve a provider. | ||
| from .anthropic_provider import AnthropicLLM | ||
| from .base import BaseLLM, LLMProvider, register_llm | ||
| from .cache import JSONFileCache, NullCache, ResponseCache | ||
| from .factory import init_llm_client | ||
| from .google_provider import GoogleLLM | ||
| from .openai_provider import OpenAILLM | ||
| from .types import LLMCallReturn | ||
|
|
||
| __all__ = [ | ||
| "BaseLLM", | ||
| "LLMProvider", | ||
| "LLMCallReturn", | ||
| "ResponseCache", | ||
| "NullCache", | ||
| "JSONFileCache", | ||
| "register_llm", | ||
| "init_llm_client", | ||
| "AnthropicLLM", | ||
| "GoogleLLM", | ||
| "OpenAILLM", | ||
| ] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| import os | ||
|
|
||
| import anthropic | ||
|
|
||
| from .base import BaseLLM, LLMProvider, register_llm | ||
| from .cache import ResponseCache | ||
| from .types import LLMCallReturn | ||
|
|
||
|
|
||
| @register_llm(LLMProvider.ANTHROPIC) | ||
| class AnthropicLLM(BaseLLM): | ||
| # Anthropic requires an explicit max output token count; adaptive thinking | ||
| # plus tool-free prose answers fit comfortably under this. | ||
| MAX_TOKENS = 16000 | ||
|
|
||
| def __init__( | ||
| self, | ||
| api_key, | ||
| model, | ||
| cache: ResponseCache | None = None, | ||
| ): | ||
| super().__init__(api_key=api_key, model=model, cache=cache) | ||
| self.client = anthropic.Anthropic(api_key=api_key) | ||
|
|
||
| @classmethod | ||
| def from_env( | ||
| cls, model: str, cache: ResponseCache | None = None | ||
| ) -> "AnthropicLLM": | ||
| api_key = os.getenv("ANTHROPIC_API_KEY") | ||
| if api_key is None: | ||
| raise ValueError("API key for Anthropic not found.") | ||
| return cls(api_key=api_key, model=model, cache=cache) | ||
|
|
||
| def _call(self, text: str, include_thinking: bool = True) -> LLMCallReturn: | ||
| thinking_conf: anthropic.types.ThinkingConfigParam | ||
| if include_thinking: | ||
| # Adaptive thinking is the only supported "on" mode on Opus 4.7; | ||
| # "summarized" surfaces the reasoning text instead of omitting it. | ||
| thinking_conf = {"type": "adaptive", "display": "summarized"} | ||
| else: | ||
| thinking_conf = {"type": "disabled"} | ||
|
|
||
| messages: list[anthropic.types.MessageParam] = [ | ||
| {"role": "user", "content": text} | ||
| ] | ||
| response = self.client.messages.create( | ||
| model=self.model, | ||
| max_tokens=self.MAX_TOKENS, | ||
| thinking=thinking_conf, | ||
| messages=messages, | ||
| ) | ||
|
|
||
| thought = None | ||
| answer = None | ||
|
|
||
| for block in response.content: | ||
| if block.type == "thinking": | ||
| thought = block.thinking | ||
| elif block.type == "text": | ||
| answer = block.text | ||
|
|
||
| return {"answer": answer, "thought": thought, "response": response} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| from abc import ABC, abstractmethod | ||
| from enum import Enum | ||
|
|
||
| from .cache import NullCache, ResponseCache | ||
| from .types import LLMCallReturn | ||
|
|
||
|
|
||
| class LLMProvider(Enum): | ||
| GOOGLE = "google" | ||
| ANTHROPIC = "anthropic" | ||
| OPENAI = "openai" | ||
|
|
||
|
|
||
| # provider -> implementing class, populated by the @register_llm decorator | ||
| _LLM_REGISTRY: dict[LLMProvider, type["BaseLLM"]] = {} | ||
|
|
||
|
|
||
| def register_llm(provider: LLMProvider): | ||
| """Class decorator: tag a `BaseLLM` subclass with its provider and register it.""" | ||
|
|
||
| def deco(cls: type["BaseLLM"]) -> type["BaseLLM"]: | ||
| cls._provider = provider | ||
| _LLM_REGISTRY[provider] = cls | ||
| return cls | ||
|
|
||
| return deco | ||
|
|
||
|
|
||
| class BaseLLM(ABC): | ||
| _provider: LLMProvider | None = None | ||
|
|
||
| def __init__( | ||
| self, | ||
| api_key: str | None, | ||
| model: str, | ||
| cache: ResponseCache | None = None, | ||
| ): | ||
| if self._provider is None: | ||
| raise RuntimeError( | ||
| f"{type(self).__name__} must set a class-level `_provider`" | ||
| ) | ||
|
|
||
| self.api_key = api_key | ||
| self.model = model | ||
| self.cache: ResponseCache = cache if cache is not None else NullCache() | ||
|
|
||
| @classmethod | ||
| @abstractmethod | ||
| def from_env( | ||
| cls, model: str, cache: ResponseCache | None = None | ||
| ) -> "BaseLLM": | ||
| """Build an instance using credentials/config from environment variables.""" | ||
|
|
||
| def _cache_key(self, text: str, include_thinking: bool) -> str: | ||
| return f"{self.model}::{include_thinking}::{text}" | ||
|
|
||
| def call(self, text: str, include_thinking: bool = True) -> LLMCallReturn: | ||
| key = self._cache_key(text, include_thinking) | ||
| cached = self.cache.get(key) | ||
| if cached is not None: | ||
| return cached | ||
| call_return = self._call(text=text, include_thinking=include_thinking) | ||
| self.cache.put(key, call_return) | ||
| return call_return | ||
|
|
||
| @abstractmethod | ||
| def _call(self, text: str, include_thinking: bool = True) -> LLMCallReturn: | ||
| pass |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.