diff --git a/lib/gepa/adapters/generic_rag/vector_stores/qdrant.ex b/lib/gepa/adapters/generic_rag/vector_stores/qdrant.ex index ec50715..9fbfe0a 100644 --- a/lib/gepa/adapters/generic_rag/vector_stores/qdrant.ex +++ b/lib/gepa/adapters/generic_rag/vector_stores/qdrant.ex @@ -384,8 +384,10 @@ defmodule GEPA.Adapters.GenericRAG.VectorStores.Qdrant do defp headers(%__MODULE__{api_key: nil}), do: [] defp headers(%__MODULE__{api_key: api_key}), do: [{"api-key", api_key}] - defp result_embedding_dimension(%{"config" => %{"params" => %{"vectors" => %{"size" => size}}}}), - do: size + defp result_embedding_dimension(%{ + "config" => %{"params" => %{"vectors" => %{"size" => size}}} + }), + do: size defp result_embedding_dimension(_result), do: nil diff --git a/lib/gepa/adapters/mcp.ex b/lib/gepa/adapters/mcp.ex index d36dd6f..a00d339 100644 --- a/lib/gepa/adapters/mcp.ex +++ b/lib/gepa/adapters/mcp.ex @@ -399,11 +399,12 @@ defmodule GEPA.Adapters.MCP do defp normalize_tool_selection({:ok, tool, args}), do: {:ok, to_string(tool), stringify_map(args || %{})} + defp normalize_tool_selection({:error, reason}), do: {:error, reason} + defp normalize_tool_selection({tool, args}), do: {:ok, to_string(tool), stringify_map(args || %{})} defp normalize_tool_selection(tool) when is_binary(tool), do: {:ok, tool, %{}} - defp normalize_tool_selection({:error, reason}), do: {:error, reason} defp normalize_example(%DataInst{} = example), do: Map.from_struct(example) diff --git a/lib/gepa/evaluation_batch.ex b/lib/gepa/evaluation_batch.ex index 045c173..724f1d9 100644 --- a/lib/gepa/evaluation_batch.ex +++ b/lib/gepa/evaluation_batch.ex @@ -132,7 +132,6 @@ defmodule GEPA.EvaluationBatch do defp optional_list_length_matches?(_values, _expected), do: false - defp optional_length(nil), do: nil defp optional_length(values) when is_list(values), do: length(values) defp optional_length(_), do: :not_a_list @@ -153,6 +152,4 @@ defmodule GEPA.EvaluationBatch do true end) end - - defp has_invalid_objective_scores?(_objective_scores), do: true end diff --git a/lib/gepa/llm.ex b/lib/gepa/llm.ex index bec26f4..8e11caf 100644 --- a/lib/gepa/llm.ex +++ b/lib/gepa/llm.ex @@ -8,7 +8,7 @@ defmodule GEPA.LLM do of that surface they support. """ - alias GEPA.LLM.{Adapters, Client, Mock, ReqLLM, Request, Response, Tracking} + alias GEPA.LLM.{Client, Mock, Request, Response, Tracking} @type prompt :: String.t() | [map()] @type t :: module() | map() | Client.t() | function() @@ -25,33 +25,33 @@ defmodule GEPA.LLM do @optional_callbacks complete_structured: 3 - @doc "Builds a normalized GEPA LLM client." - @spec new(:req_llm | :agent_session_manager | :asm, keyword()) :: Client.t() - def new(:req_llm, opts) do - provider = Keyword.fetch!(opts, :provider) - req_llm(provider, Keyword.delete(opts, :provider)) - end + @instruction_schema [ + instruction: [type: :string, required: true, doc: "The improved instruction text."] + ] - def new(adapter, opts) when adapter in [:agent_session_manager, :asm] do - provider = Keyword.fetch!(opts, :provider) - agent(provider, Keyword.delete(opts, :provider)) - end + @removed_provider_msg """ + Built-in GEPA.LLM providers (ReqLLM and Agent Session Manager) were removed because \ + they depended on the unpublished `:inference` package. Supply your own LLM instead: a \ + `fn prompt -> {:ok, text} end` callable, a `%GEPA.LLM.Client{}`, or `GEPA.LLM.Mock`.\ + """ - @doc "Builds a hosted-provider client backed by ReqLLM." - @spec req_llm(atom(), keyword()) :: Client.t() - def req_llm(provider, opts \\ []) when is_atom(provider) do - opts - |> Keyword.put(:provider, provider) - |> Adapters.ReqLLM.new!() - end + @doc """ + Removed. The built-in ReqLLM and Agent-Session-Manager providers depended on the + unpublished `:inference` package. Inject your own LLM (a callable, a + `%GEPA.LLM.Client{}`, or `GEPA.LLM.Mock`) wherever an `t:t/0` is expected. + """ + @spec new(atom(), keyword()) :: no_return() + def new(_adapter, _opts), do: raise(ArgumentError, @removed_provider_msg) - @doc "Builds a local CLI/agent client backed by Agent Session Manager." - @spec agent(atom(), keyword()) :: Client.t() - def agent(provider, opts \\ []) when is_atom(provider) do - opts - |> Keyword.put(:provider, provider) - |> Adapters.AgentSessionManager.new!() - end + @doc "Removed. Provide your own LLM callable. See `new/2`." + @spec req_llm(atom()) :: no_return() + @spec req_llm(atom(), keyword()) :: no_return() + def req_llm(_provider, _opts \\ []), do: raise(ArgumentError, @removed_provider_msg) + + @doc "Removed. Provide your own LLM callable. See `new/2`." + @spec agent(atom()) :: no_return() + @spec agent(atom(), keyword()) :: no_return() + def agent(_provider, _opts \\ []), do: raise(ArgumentError, @removed_provider_msg) @doc "Wrap a one- or two-arity callable in a cost/token tracking LLM." @spec track(function() | t()) :: Tracking.t() | t() @@ -114,7 +114,7 @@ defmodule GEPA.LLM do request = Request.structured( prompt, - Keyword.put_new(opts, :schema, Adapters.ReqLLM.instruction_schema()) + Keyword.put_new(opts, :schema, @instruction_schema) ) with {:ok, %Response{object: object} = response} <- client.adapter.complete(client, request) do @@ -192,17 +192,21 @@ defmodule GEPA.LLM do defp normalize_callable_response(%{text: text}) when is_binary(text), do: {:ok, text} defp normalize_callable_response(other), do: {:ok, to_string(other)} - @doc "Returns the default LLM provider based on application configuration." + @doc """ + Returns the default LLM. With no configured provider this is `GEPA.LLM.Mock`; + there are no built-in network providers, so inject your own callable for those. + """ @spec default() :: t() def default do config = Application.get_env(:gepa_ex, :llm, []) - provider = Keyword.get(config, :provider, :openai) - build_default_llm(provider, config) - end - defp build_default_llm(:mock, _config), do: Mock.new() + case Keyword.get(config, :provider, :mock) do + :mock -> + Mock.new() - defp build_default_llm(provider, config) do - ReqLLM.new(Keyword.put(config, :provider, provider)) + other -> + raise ArgumentError, + "no built-in LLM provider #{inspect(other)}. " <> @removed_provider_msg + end end end diff --git a/lib/gepa/llm/adapters/agent_session_manager.ex b/lib/gepa/llm/adapters/agent_session_manager.ex deleted file mode 100644 index 5655b2e..0000000 --- a/lib/gepa/llm/adapters/agent_session_manager.ex +++ /dev/null @@ -1,275 +0,0 @@ -defmodule GEPA.LLM.Adapters.AgentSessionManager do - @moduledoc """ - GEPA compatibility adapter backed by the shared `:inference` ASM adapter. - - This module keeps the `GEPA.LLM.agent/2` API stable while delegating - Agent Session Manager query and stream behavior to `Inference.Adapters.ASM`. - """ - - alias GEPA.LLM.{Client, Request, Response} - - defstruct [ - :provider, - :lane, - :session, - :asm_module, - :inference_client, - session_opts: [], - query_opts: [], - stream_opts: [], - provider_opts: [] - ] - - @type provider :: :claude | :codex | :codex_exec | :gemini | :amp - @type lane :: :auto | :core | :sdk - - @type t :: %__MODULE__{ - provider: provider(), - lane: lane(), - session: term(), - session_opts: keyword(), - query_opts: keyword(), - stream_opts: keyword(), - provider_opts: keyword(), - asm_module: module(), - inference_client: Inference.Client.t() - } - - @providers [:claude, :codex, :codex_exec, :gemini, :amp] - @lanes [:auto, :core, :sdk] - @default_models %{codex: "gpt-5.4-mini", gemini: "gemini-3.1-flash-lite-preview"} - - @spec new(keyword()) :: {:ok, Client.t()} | {:error, term()} - def new(opts \\ []) do - with {:ok, state} <- build_state(opts) do - {:ok, - %Client{ - adapter: __MODULE__, - adapter_state: state, - provider: state.provider, - model: Keyword.get(state.provider_opts, :model), - defaults: [lane: state.lane], - capabilities: MapSet.new([:text, :stream, :session, :session_resume, :cost]), - metadata: %{inference_adapter: Inference.Adapters.ASM} - }} - end - end - - @spec new!(keyword()) :: Client.t() - def new!(opts \\ []) do - case new(opts) do - {:ok, client} -> client - {:error, reason} -> raise ArgumentError, "invalid ASM adapter options: #{inspect(reason)}" - end - end - - @spec build_state(keyword()) :: {:ok, t()} | {:error, term()} - def build_state(opts) do - with {:ok, provider} <- fetch_provider(opts), - {:ok, lane} <- fetch_lane(opts) do - state = %__MODULE__{ - provider: provider, - lane: lane, - session: Keyword.get(opts, :session), - session_opts: Keyword.get(opts, :session_opts, []), - query_opts: Keyword.get(opts, :query_opts, []), - stream_opts: Keyword.get(opts, :stream_opts, []), - provider_opts: - opts - |> Keyword.get(:provider_opts, []) - |> normalize_provider_opts() - |> put_default_model(provider) - |> maybe_put_external_model_payload(provider), - asm_module: Keyword.get(opts, :asm_module, ASM) - } - - {:ok, %{state | inference_client: inference_client(state)}} - end - end - - @spec complete(Client.t(), Request.t()) :: {:ok, Response.t()} | {:error, term()} - def complete(%Client{} = client, %Request{schema: schema}) when not is_nil(schema) do - {:error, - {:unsupported_capability, :structured_output, - %{adapter: __MODULE__, provider: client.provider, schema: schema}}} - end - - def complete(%Client{adapter_state: %__MODULE__{} = state}, %Request{} = request) do - state - |> run_complete(request) - |> map_result(__MODULE__, state, request) - end - - @spec stream(Client.t(), Request.t()) :: {:ok, Enumerable.t()} | {:error, term()} - def stream(%Client{adapter_state: %__MODULE__{} = state}, %Request{} = request) do - opts = inference_request_opts(state, request) - - case Inference.stream(state.inference_client, request_text(request), opts) do - {:ok, stream} -> - {:ok, Stream.flat_map(stream, &stream_text_chunks/1)} - - {:error, error} -> - {:error, error} - end - rescue - error -> {:error, Exception.message(error)} - end - - @spec capabilities(Client.t()) :: MapSet.t(atom()) - def capabilities(%Client{capabilities: capabilities}), do: capabilities - - @spec close(Client.t()) :: :ok | {:error, term()} - def close(%Client{adapter_state: %__MODULE__{session: nil}}), do: :ok - - def close(%Client{adapter_state: %__MODULE__{session: session, asm_module: asm_module}}) do - asm_module.stop_session(session) - end - - defp run_complete(%__MODULE__{} = state, %Request{} = request) do - Inference.complete( - state.inference_client, - request_text(request), - inference_request_opts(state, request) - ) - rescue - error -> {:error, Exception.message(error)} - end - - defp map_result( - {:ok, %Inference.Response{} = response}, - adapter, - %__MODULE__{} = state, - %Request{} = request - ) do - {:ok, - %Response{ - text: Inference.Response.text(response), - messages: response.raw |> value(:messages), - cost: response.metadata[:cost] || value(response.raw, :cost), - stop_reason: response.finish_reason, - adapter: adapter, - provider: state.provider, - model: request.model || Keyword.get(state.provider_opts, :model), - session_ref: request.session || state.session, - raw: response.raw, - metadata: response.metadata - }} - end - - defp map_result({:error, error}, _adapter, _state, _request), do: {:error, error} - - defp inference_client(%__MODULE__{} = state) do - Inference.Client.new!( - adapter: Inference.Adapters.ASM, - provider: state.provider, - model: Keyword.get(state.provider_opts, :model), - defaults: [lane: state.lane] ++ state.provider_opts, - adapter_opts: [ - asm_module: state.asm_module, - session: state.session, - session_opts: state.session_opts, - query_opts: state.query_opts, - stream_opts: state.stream_opts - ] - ) - end - - defp inference_request_opts(%__MODULE__{} = state, %Request{} = request) do - provider_opts = - state.provider_opts - |> Keyword.merge(request.provider_opts) - |> normalize_provider_opts() - |> maybe_put(:model, request.model) - |> rename_timeout(request.timeout) - |> maybe_put(:prompt, request_text(request)) - - [ - model: request.model, - session: request.session, - options: provider_opts - ] - |> Enum.reject(fn {_key, value} -> is_nil(value) or value == [] end) - end - - defp request_text(%Request{} = request), do: Request.to_text(request) - - defp normalize_provider_opts(provider_opts) do - provider_opts - |> Keyword.delete(:temperature) - |> Keyword.delete(:max_tokens) - |> Keyword.delete(:top_p) - |> rename_timeout() - end - - defp rename_timeout(provider_opts) do - case Keyword.pop(provider_opts, :timeout) do - {nil, opts} -> opts - {timeout, opts} -> Keyword.put_new(opts, :transport_timeout_ms, timeout) - end - end - - defp rename_timeout(provider_opts, nil), do: provider_opts - - defp rename_timeout(provider_opts, timeout), - do: Keyword.put(provider_opts, :transport_timeout_ms, timeout) - - defp put_default_model(provider_opts, provider) do - case {Keyword.get(provider_opts, :model), Map.get(@default_models, provider)} do - {model, _default} when is_binary(model) and model != "" -> provider_opts - {_model, nil} -> provider_opts - {_model, default} -> Keyword.put(provider_opts, :model, default) - end - end - - defp maybe_put_external_model_payload(provider_opts, :gemini) do - case {Keyword.get(provider_opts, :model_payload), Keyword.get(provider_opts, :model)} do - {payload, _model} when is_map(payload) -> - provider_opts - - {_payload, model} when is_binary(model) and model != "" -> - Keyword.put(provider_opts, :model_payload, %{ - provider: :gemini, - requested_model: model, - resolved_model: model, - resolution_source: :explicit, - model_source: :external - }) - - _other -> - provider_opts - end - end - - defp maybe_put_external_model_payload(provider_opts, _provider), do: provider_opts - - defp fetch_provider(opts) do - case Keyword.fetch(opts, :provider) do - {:ok, provider} when provider in @providers -> {:ok, provider} - {:ok, provider} -> {:error, {:invalid_provider, provider, @providers}} - :error -> {:error, :missing_provider} - end - end - - defp fetch_lane(opts) do - case Keyword.get(opts, :lane, :auto) do - lane when lane in @lanes -> {:ok, lane} - lane -> {:error, {:invalid_lane, lane, @lanes}} - end - end - - defp stream_text_chunks(%Inference.StreamEvent{data: text}) when is_binary(text) and text != "", - do: [text] - - defp stream_text_chunks(text) when is_binary(text) and text != "", do: [text] - defp stream_text_chunks(_chunk), do: [] - - defp value(%{__struct__: _} = map, key), do: Map.get(map, key) - - defp value(map, key) when is_map(map), - do: Map.get(map, key) || Map.get(map, Atom.to_string(key)) - - defp value(_other, _key), do: nil - - defp maybe_put(opts, _key, nil), do: opts - defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value) -end diff --git a/lib/gepa/llm/adapters/req_llm.ex b/lib/gepa/llm/adapters/req_llm.ex deleted file mode 100644 index 5da295c..0000000 --- a/lib/gepa/llm/adapters/req_llm.ex +++ /dev/null @@ -1,297 +0,0 @@ -defmodule GEPA.LLM.Adapters.ReqLLM do - @moduledoc """ - GEPA compatibility adapter backed by the shared `:inference` ReqLLM adapter. - - This module preserves the public `GEPA.LLM.Client` surface while moving - provider-specific ReqLLM behavior into `Inference.Adapters.ReqLLM`. - """ - - alias GEPA.LLM.{Client, Request, Response} - - defstruct [ - :provider, - :model, - :api_key, - :temperature, - :max_tokens, - :top_p, - :timeout, - :req_llm_module, - :response_module, - :env, - :inference_client, - req_options: [], - provider_opts: [] - ] - - @type provider :: :openai | :gemini | :anthropic - @type t :: %__MODULE__{ - provider: provider(), - model: String.t(), - api_key: String.t() | nil, - temperature: float() | nil, - max_tokens: pos_integer() | nil, - top_p: float() | nil, - timeout: pos_integer() | nil, - req_options: keyword(), - provider_opts: keyword(), - req_llm_module: module(), - response_module: module(), - env: (String.t() -> String.t() | nil), - inference_client: Inference.Client.t() | nil - } - - @providers [:openai, :gemini, :anthropic] - - @default_models %{ - openai: "gpt-5.4-mini", - gemini: "gemini-3.1-flash-lite-preview", - anthropic: "claude-haiku-4-5" - } - - @default_temperature 0.7 - @default_max_tokens 2000 - @default_timeout 60_000 - - @instruction_schema [ - instruction: [type: :string, required: true, doc: "The improved instruction text."] - ] - - @spec instruction_schema() :: keyword() - def instruction_schema, do: @instruction_schema - - @spec new(keyword()) :: {:ok, Client.t()} | {:error, term()} - def new(opts \\ []) do - with {:ok, state} <- build_state(opts) do - {:ok, - %Client{ - adapter: __MODULE__, - adapter_state: state, - provider: state.provider, - model: state.model, - defaults: state_defaults(state), - capabilities: MapSet.new([:text, :messages, :structured_output, :tools, :cost]), - metadata: %{inference_adapter: Inference.Adapters.ReqLLM} - }} - end - end - - @spec new!(keyword()) :: Client.t() - def new!(opts \\ []) do - case new(opts) do - {:ok, client} -> - client - - {:error, reason} -> - raise ArgumentError, "invalid ReqLLM adapter options: #{inspect(reason)}" - end - end - - @spec build_state(keyword()) :: {:ok, t()} | {:error, term()} - def build_state(opts) do - with {:ok, provider} <- fetch_provider(opts) do - state = %__MODULE__{ - provider: provider, - model: Keyword.get(opts, :model, @default_models[provider]), - api_key: Keyword.get(opts, :api_key), - temperature: Keyword.get(opts, :temperature, @default_temperature), - max_tokens: Keyword.get(opts, :max_tokens, @default_max_tokens), - top_p: Keyword.get(opts, :top_p), - timeout: Keyword.get(opts, :timeout, @default_timeout), - req_options: Keyword.get(opts, :req_options, []), - provider_opts: Keyword.get(opts, :provider_opts, []), - req_llm_module: Keyword.get(opts, :req_llm_module, ReqLLM), - response_module: Keyword.get(opts, :response_module, ReqLLM.Response), - env: Keyword.get(opts, :env, &System.get_env/1) - } - - {:ok, %{state | inference_client: inference_client(state)}} - end - end - - @spec complete(Client.t(), Request.t()) :: {:ok, Response.t()} | {:error, term()} - def complete(%Client{adapter_state: %__MODULE__{} = state}, %Request{} = request) do - state - |> run_inference(request, nil) - |> map_result(__MODULE__) - end - - @spec stream(Client.t(), Request.t()) :: {:error, term()} - def stream(%Client{} = client, %Request{} = _request) do - {:error, - {:unsupported_capability, :stream, %{adapter: client.adapter, provider: client.provider}}} - end - - @spec capabilities(Client.t()) :: MapSet.t(atom()) - def capabilities(%Client{capabilities: capabilities}), do: capabilities - - @spec close(Client.t()) :: :ok - def close(%Client{}), do: :ok - - @spec complete_legacy(t(), String.t(), keyword()) :: {:ok, String.t()} | {:error, term()} - def complete_legacy(%__MODULE__{} = state, prompt, opts \\ []) when is_binary(prompt) do - request = Request.from_prompt(prompt, opts) - - with {:ok, %Response{} = response} <- - state - |> run_inference(request, nil) - |> map_result(__MODULE__) do - {:ok, Response.text(response)} - end - end - - @spec complete_structured_legacy(t(), String.t(), keyword()) :: {:ok, map()} | {:error, term()} - def complete_structured_legacy(%__MODULE__{} = state, prompt, opts \\ []) - when is_binary(prompt) do - request = Request.structured(prompt, Keyword.put_new(opts, :schema, @instruction_schema)) - - case state - |> run_inference(request, request.schema || @instruction_schema) - |> map_result(__MODULE__) do - {:ok, %Response{object: object}} when is_map(object) -> {:ok, object} - {:ok, %Response{} = response} -> {:ok, %{"instruction" => Response.text(response)}} - {:error, _} = error -> error - end - end - - defp run_inference(%__MODULE__{} = state, %Request{} = request, response_format) do - opts = inference_request_opts(state, request, response_format) - Inference.complete(inference_client(state, request), request_input(request), opts) - rescue - error -> {:error, Exception.message(error)} - end - - defp map_result({:ok, %Inference.Response{} = response}, adapter) do - {:ok, - %Response{ - text: Inference.Response.text(response), - object: response.object, - tool_calls: response.tool_calls, - tool_results: value(response.raw, :tool_results) || [], - usage: response.usage, - cost: response.cost || response.metadata[:cost] || value(response.raw, :cost), - stop_reason: response.finish_reason, - adapter: adapter, - provider: response.provider, - model: response.model, - raw: response.raw, - metadata: response.metadata - }} - end - - defp map_result({:error, error}, _adapter), do: {:error, error} - - defp inference_client(%__MODULE__{} = state) do - Inference.Client.new!( - adapter: Inference.Adapters.ReqLLM, - provider: state.provider, - model: state.model, - defaults: inference_defaults(state), - adapter_opts: [ - req_llm_module: state.req_llm_module, - response_module: state.response_module, - api_key: state.api_key, - env: state.env, - model_spec: model_spec(state.provider, state.model) - ] - ) - end - - defp inference_client(%__MODULE__{} = state, %Request{} = request) do - client = state.inference_client || inference_client(state) - model = request.model || state.model - - %{ - client - | model: model, - adapter_opts: - Keyword.put(client.adapter_opts, :model_spec, model_spec(state.provider, model)) - } - end - - defp inference_defaults(%__MODULE__{} = state) do - [ - temperature: state.temperature, - max_tokens: state.max_tokens, - top_p: state.top_p, - receive_timeout: state.timeout - ] - |> Keyword.merge(state.req_options) - |> Keyword.merge(state.provider_opts) - |> Enum.reject(fn {_key, value} -> is_nil(value) end) - end - - defp inference_request_opts(%__MODULE__{} = state, %Request{} = request, response_format) do - provider_opts = - state.provider_opts - |> Keyword.merge(request.provider_opts) - |> maybe_put(:api_key, request_api_key(state, request)) - |> maybe_put(:tools, request.tools) - |> maybe_put(:tool_choice, request.tool_choice) - |> maybe_put(:prompt, request_input(request)) - - [ - temperature: request.temperature, - model: request.model, - max_tokens: request.max_tokens, - top_p: request.top_p, - response_format: response_format || request.schema, - options: provider_opts - ] - |> Enum.reject(fn {_key, value} -> is_nil(value) or value == [] end) - end - - defp request_api_key(%__MODULE__{} = state, %Request{} = request) do - Keyword.get(request.provider_opts, :api_key) || state.api_key - end - - defp request_input(%Request{system: system} = request) - when is_binary(system) and system != "" do - case Request.prompt(request) do - messages when is_list(messages) -> - [%{role: :system, content: system} | messages] - - prompt when is_binary(prompt) -> - [%{role: :system, content: system}, %{role: :user, content: prompt}] - - nil -> - [%{role: :system, content: system}] - end - end - - defp request_input(%Request{} = request), do: Request.prompt(request) - - defp state_defaults(%__MODULE__{} = state) do - [ - model: state.model, - temperature: state.temperature, - max_tokens: state.max_tokens, - top_p: state.top_p, - timeout: state.timeout - ] - |> Enum.reject(fn {_key, value} -> is_nil(value) end) - end - - defp fetch_provider(opts) do - case Keyword.fetch(opts, :provider) do - {:ok, provider} when provider in @providers -> {:ok, provider} - {:ok, provider} -> {:error, {:invalid_provider, provider, @providers}} - :error -> {:error, :missing_provider} - end - end - - defp model_spec(:openai, model), do: "openai:#{model}" - defp model_spec(:gemini, model), do: "google:#{model}" - defp model_spec(:anthropic, model), do: "anthropic:#{model}" - - defp maybe_put(opts, _key, nil), do: opts - defp maybe_put(opts, _key, []), do: opts - defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value) - - defp value(%{__struct__: _} = map, key), do: Map.get(map, key) - - defp value(map, key) when is_map(map), - do: Map.get(map, key) || Map.get(map, Atom.to_string(key)) - - defp value(_other, _key), do: nil -end diff --git a/lib/gepa/llm/req_llm.ex b/lib/gepa/llm/req_llm.ex deleted file mode 100644 index 68bde98..0000000 --- a/lib/gepa/llm/req_llm.ex +++ /dev/null @@ -1,115 +0,0 @@ -defmodule GEPA.LLM.ReqLLM do - @moduledoc """ - Backward-compatible ReqLLM implementation. - - New code should prefer `GEPA.LLM.req_llm/2`, which returns a normalized - `GEPA.LLM.Client`. This module preserves the original struct-returning API - used by existing examples and tests while delegating all provider behavior to - `GEPA.LLM.Adapters.ReqLLM`. - """ - - @behaviour GEPA.LLM - - alias GEPA.LLM.Adapters.ReqLLM, as: Adapter - - defstruct [ - :provider, - :model, - :api_key, - :temperature, - :max_tokens, - :top_p, - :timeout, - :req_llm_module, - :response_module, - :env, - req_options: [], - provider_opts: [] - ] - - @type provider :: :openai | :gemini | :anthropic - @type t :: %__MODULE__{ - provider: provider(), - model: String.t(), - api_key: String.t() | nil, - temperature: float() | nil, - max_tokens: pos_integer() | nil, - top_p: float() | nil, - timeout: pos_integer() | nil, - req_options: keyword(), - provider_opts: keyword(), - req_llm_module: module(), - response_module: module(), - env: (String.t() -> String.t() | nil) | nil - } - - @doc """ - Creates a backward-compatible ReqLLM provider struct. - - Supported providers are `:openai`, `:gemini`, and `:anthropic`. - """ - @spec new(keyword()) :: t() - def new(opts \\ []) do - case Adapter.build_state(opts) do - {:ok, %Adapter{} = state} -> - from_adapter_state(state) - - {:error, {:invalid_provider, provider, providers}} -> - raise ArgumentError, - "provider must be one of #{inspect(providers)}, got: #{inspect(provider)}" - - {:error, :missing_provider} -> - raise KeyError, key: :provider, term: opts - end - end - - @impl GEPA.LLM - @spec complete(t(), String.t(), keyword()) :: {:ok, String.t()} | {:error, term()} - def complete(%__MODULE__{} = llm, prompt, opts \\ []) when is_binary(prompt) do - llm - |> to_adapter_state() - |> Adapter.complete_legacy(prompt, opts) - end - - @impl GEPA.LLM - @spec complete_structured(t(), String.t(), keyword()) :: {:ok, map()} | {:error, term()} - def complete_structured(%__MODULE__{} = llm, prompt, opts \\ []) when is_binary(prompt) do - llm - |> to_adapter_state() - |> Adapter.complete_structured_legacy(prompt, opts) - end - - defp from_adapter_state(%Adapter{} = state) do - %__MODULE__{ - provider: state.provider, - model: state.model, - api_key: state.api_key, - temperature: state.temperature, - max_tokens: state.max_tokens, - top_p: state.top_p, - timeout: state.timeout, - req_options: state.req_options, - provider_opts: state.provider_opts, - req_llm_module: state.req_llm_module, - response_module: state.response_module, - env: state.env - } - end - - defp to_adapter_state(%__MODULE__{} = llm) do - %Adapter{ - provider: llm.provider, - model: llm.model, - api_key: llm.api_key, - temperature: llm.temperature, - max_tokens: llm.max_tokens, - top_p: llm.top_p, - timeout: llm.timeout, - req_options: llm.req_options, - provider_opts: llm.provider_opts, - req_llm_module: llm.req_llm_module || ReqLLM, - response_module: llm.response_module || ReqLLM.Response, - env: llm.env || (&System.get_env/1) - } - end -end diff --git a/lib/gepa/optimize_anything.ex b/lib/gepa/optimize_anything.ex index 6d3af99..dbaa2ff 100644 --- a/lib/gepa/optimize_anything.ex +++ b/lib/gepa/optimize_anything.ex @@ -1109,18 +1109,19 @@ defmodule GEPA.OptimizeAnything do Seed.generate(lm, opts) end - @doc "Build a normalized hosted-provider LM client for optimize-anything helpers." - @spec make_litellm_lm(String.t(), keyword()) :: GEPA.LLM.Client.t() - def make_litellm_lm(model_name, opts \\ []) when is_binary(model_name) do - case String.split(model_name, "/", parts: 2) do - [provider, model] -> - provider - |> provider_atom() - |> GEPA.LLM.req_llm([{:model, model} | opts]) - - [model] -> - GEPA.LLM.req_llm(:openai, [{:model, model} | opts]) - end + @doc """ + Removed. This built a hosted-provider client through the ReqLLM provider, which + depended on the unpublished `:inference` package. Inject your own LLM instead: a + `fn prompt -> {:ok, text} end` callable, a `%GEPA.LLM.Client{}`, or `GEPA.LLM.Mock`. + """ + @spec make_litellm_lm(String.t()) :: no_return() + @spec make_litellm_lm(String.t(), keyword()) :: no_return() + def make_litellm_lm(model_name, _opts \\ []) when is_binary(model_name) do + raise ArgumentError, """ + GEPA.OptimizeAnything.make_litellm_lm/2 was removed: it built a ReqLLM provider \ + that depended on the unpublished `:inference` package. Inject your own LLM (a \ + `fn prompt -> {:ok, text} end` callable, a `%GEPA.LLM.Client{}`, or `GEPA.LLM.Mock`).\ + """ end @doc "Append a diagnostic message to the process-local optimize-anything log." @@ -1203,7 +1204,7 @@ defmodule GEPA.OptimizeAnything do resolve_num_parallel_proposals( config.engine.num_parallel_proposals, workers, - reflection_minibatch_size || 1 + reflection_minibatch_size ), reflection_minibatch_size: reflection_minibatch_size, run_dir: config.engine.run_dir, diff --git a/mix.exs b/mix.exs index 6308996..75f6d45 100644 --- a/mix.exs +++ b/mix.exs @@ -8,7 +8,7 @@ defmodule GepaEx.MixProject do [ app: :gepa_ex, version: @version, - elixir: "~> 1.15", + elixir: "~> 1.20", start_permanent: Mix.env() == :prod, elixirc_paths: elixirc_paths(Mix.env()), deps: deps(), @@ -19,15 +19,20 @@ defmodule GepaEx.MixProject do package: package(), docs: docs(), test_coverage: [tool: ExCoveralls], - preferred_cli_env: [ + dialyzer: [ + plt_file: {:no_warn, "priv/plts/dialyzer.plt"}, + plt_add_apps: [:mix] + ] + ] + end + + def cli do + [ + preferred_envs: [ coveralls: :test, "coveralls.detail": :test, "coveralls.post": :test, "coveralls.html": :test - ], - dialyzer: [ - plt_file: {:no_warn, "priv/plts/dialyzer.plt"}, - plt_add_apps: [:mix] ] ] end @@ -48,11 +53,13 @@ defmodule GepaEx.MixProject do {:jason, "~> 1.4"}, {:telemetry, "~> 1.4"}, - # LLM integration - {:inference, path: "../inference/apps/inference"}, - {:req_llm, "~> 1.10"}, - {:agent_session_manager, "~> 0.9.2"}, - {:req, "~> 0.5.17"}, + # LLM integration (optional). The optimizer core is provider-agnostic and + # talks to any model through the GEPA.LLM facade: a plain function, a + # GEPA.LLM.Client, or GEPA.LLM.Mock. req_llm/req are only pulled in for the + # optional embeddings + RAG/Qdrant adapters, so consumers that inject their + # own LLM callable take on neither dependency. + {:req_llm, "~> 1.10", optional: true}, + {:req, "~> 0.5.17", optional: true}, # Development and testing {:mox, "~> 1.2", only: :test}, @@ -217,9 +224,6 @@ defmodule GepaEx.MixProject do GEPA.LLM.Response, GEPA.LLM.Capabilities, GEPA.LLM.Tool, - GEPA.LLM.Adapters.ReqLLM, - GEPA.LLM.Adapters.AgentSessionManager, - GEPA.LLM.ReqLLM, GEPA.LLM.Mock, GEPA.Embeddings, GEPA.Embeddings.ReqLLM, @@ -304,15 +308,14 @@ defmodule GepaEx.MixProject do [ name: "gepa_ex", description: description(), - files: - ~w(lib mix.exs README.md LICENSE guides examples livebooks gepa/LICENSE gepa/README.md assets), + files: ~w(lib mix.exs README.md LICENSE guides examples livebooks assets), licenses: ["MIT"], links: %{ "GitHub" => @source_url, "Online documentation" => "https://hexdocs.pm/gepa_ex", "Python reference implementation" => "https://github.com/gepa-ai/gepa" }, - maintainers: ["Lakshya A Agrawal"], + maintainers: ["nshkrdotcom"], exclude_patterns: [ "priv/plts", ".DS_Store" diff --git a/test/gepa/adapters/basic_test.exs b/test/gepa/adapters/basic_test.exs index 665bf47..ba0ae8f 100644 --- a/test/gepa/adapters/basic_test.exs +++ b/test/gepa/adapters/basic_test.exs @@ -4,15 +4,6 @@ defmodule GEPA.Adapters.BasicTest do alias GEPA.Adapters.Basic alias GEPA.LLM.Mock - defmodule BasicFakeReqLLM do - def put_key(_key, _value), do: :ok - def generate_text(_model_spec, _prompt, _opts), do: {:ok, %{text: "The answer is A1."}} - end - - defmodule BasicFakeReqLLMResponse do - def text(%{text: text}), do: text - end - describe "evaluate/3" do test "scores correctly when answer found in response" do adapter = Basic.new() @@ -62,13 +53,8 @@ defmodule GEPA.Adapters.BasicTest do assert hd(result.trajectories).response == "The answer is A1." end - test "uses a normalized GEPA LLM client, not only test mock structs" do - llm = - GEPA.LLM.req_llm(:openai, - api_key: "explicit-key", - req_llm_module: BasicFakeReqLLM, - response_module: BasicFakeReqLLMResponse - ) + test "uses a normalized GEPA LLM client (a plain function), not only test mock structs" do + llm = fn _prompt -> {:ok, "The answer is A1."} end adapter = Basic.new(llm: llm) diff --git a/test/gepa/code_execution_test.exs b/test/gepa/code_execution_test.exs index 2812f3f..f0f8ea4 100644 --- a/test/gepa/code_execution_test.exs +++ b/test/gepa/code_execution_test.exs @@ -17,7 +17,9 @@ defmodule GEPA.CodeExecutionTest do end test "executes code in subprocess" do - result = GEPA.CodeExecution.execute("IO.puts(\"sub\")", mode: :subprocess) + # Subprocess mode boots a fresh BEAM; the default 5s timeout can race cold-boot + # on loaded machines. Give it headroom — this asserts behavior, not speed. + result = GEPA.CodeExecution.execute("IO.puts(\"sub\")", mode: :subprocess, timeout: 30_000) assert result.ok assert result.stdout =~ "sub" diff --git a/test/gepa/examples/live_cli_test.exs b/test/gepa/examples/live_cli_test.exs deleted file mode 100644 index 7f2b8bb..0000000 --- a/test/gepa/examples/live_cli_test.exs +++ /dev/null @@ -1,288 +0,0 @@ -defmodule GEPA.Examples.LiveCLITest do - use GEPA.SupertesterCase, isolation: :full_isolation - - alias GEPA.LLM.Adapters - - Code.require_file("../../../examples/support/live_cli.exs", __DIR__) - - @example [ - name: "Example Test", - script: "examples/test.exs", - summary: "Test help output.", - required: [:train_jsonl, :val_jsonl] - ] - - @prompt_example [ - name: "Prompt Example Test", - script: "examples/prompt_test.exs", - summary: "Test prompt help output.", - required: [:input] - ] - - describe "parse/2 help and validation" do - test "prints help for --help without building a client" do - assert {:help, help} = LiveCLI.parse(["--help"], no_env(@example)) - assert help =~ "Usage:" - assert help =~ "Sensible Defaults" - assert help =~ "--simple" - assert help =~ "Default Models:" - assert help =~ "gpt-5.4-mini" - assert help =~ "gemini-3.1-flash-lite-preview" - assert help =~ "--adapter asm --provider codex -> gpt-5.4-mini" - assert help =~ "GEMINI_API_KEY" - assert help =~ "--adapter asm --provider gemini" - end - - test "accepts the Mix argv separator before --help" do - assert {:help, help} = LiveCLI.parse(["--", "--help"], no_env(@example)) - assert help =~ "Usage:" - end - - test "rejects missing defaults with useful help" do - assert {:error, message} = LiveCLI.parse([], no_env(@example)) - assert message =~ "could not infer a default provider" - assert message =~ "Usage:" - assert message =~ "Sensible Defaults" - end - - test "rejects missing ReqLLM key when no explicit key or default env key is available" do - assert {:error, message} = - LiveCLI.parse( - ["--adapter", "req_llm", "--provider", "gemini", "--input", "hello"], - no_env(@prompt_example) - ) - - assert message =~ "missing API key for req_llm/gemini" - assert message =~ "GEMINI_API_KEY or GOOGLE_API_KEY" - end - - test "rejects missing provider even when req_llm adapter is explicit and no key-backed default exists" do - assert {:error, message} = - LiveCLI.parse(["--adapter", "req_llm"], no_env(@prompt_example)) - - assert message =~ "could not infer a default provider" - assert message =~ "Usage:" - end - end - - describe "parse/2 client construction" do - test "builds a ReqLLM client from explicit args" do - train = jsonl_fixture([%{input: "real question", answer: "real answer"}]) - val = jsonl_fixture([%{input: "real validation", answer: "real answer"}]) - - assert {:ok, config} = - LiveCLI.parse( - [ - "--adapter", - "req_llm", - "--provider", - "openai", - "--api-key", - "explicit-key", - "--train-jsonl", - train, - "--val-jsonl", - val - ], - no_env(@example) - ) - - assert config.adapter == :req_llm - assert config.provider == :openai - assert config.client.adapter == Adapters.ReqLLM - assert config.client.model == "gpt-5.4-mini" - assert config.trainset == [%{input: "real question", answer: "real answer"}] - assert config.valset == [%{input: "real validation", answer: "real answer"}] - end - - test "builds an ASM client from explicit args" do - train = jsonl_fixture([%{input: "real question", answer: "real answer"}]) - val = jsonl_fixture([%{input: "real validation", answer: "real answer"}]) - - assert {:ok, config} = - LiveCLI.parse( - [ - "--adapter", - "asm", - "--provider", - "codex", - "--lane", - "core", - "--session", - "gepa-test", - "--model", - "codex-model", - "--train-jsonl", - train, - "--val-jsonl", - val - ], - no_env(@example) - ) - - assert config.adapter == :asm - assert config.provider == :codex - assert config.client.adapter == Adapters.AgentSessionManager - assert config.client.model == "codex-model" - assert config.lane == :core - assert config.session == "gepa-test" - end - - test "supports inline prompt and expected output as user-provided data" do - assert {:ok, config} = - LiveCLI.parse( - [ - "--adapter", - "asm", - "--provider", - "codex", - "--input", - "Summarize this real repo", - "--expected", - "summary" - ], - no_env(Keyword.put(@example, :required, [:input, :expected])) - ) - - assert config.input == "Summarize this real repo" - assert config.expected == "summary" - end - - test "simple mode defaults to ReqLLM Gemini from GEMINI_API_KEY" do - assert {:ok, config} = - LiveCLI.parse(["--simple"], with_env(@prompt_example, %{"GEMINI_API_KEY" => "g"})) - - assert config.simple? - assert config.adapter == :req_llm - assert config.provider == :gemini - assert config.input =~ "Reply with exactly" - assert config.client.adapter == Adapters.ReqLLM - assert config.client.provider == :gemini - assert config.client.adapter_state.api_key == "g" - end - - test "simple mode defaults to ReqLLM Gemini from GOOGLE_API_KEY when GEMINI_API_KEY is absent" do - assert {:ok, config} = - LiveCLI.parse(["--simple"], with_env(@prompt_example, %{"GOOGLE_API_KEY" => "g"})) - - assert config.adapter == :req_llm - assert config.provider == :gemini - assert config.client.adapter_state.api_key == "g" - end - - test "simple mode falls back to OpenAI when Gemini keys are absent" do - assert {:ok, config} = - LiveCLI.parse(["--simple"], with_env(@prompt_example, %{"OPENAI_API_KEY" => "o"})) - - assert config.adapter == :req_llm - assert config.provider == :openai - assert config.client.adapter_state.api_key == "o" - end - - test "explicit CLI API key overrides default env key" do - assert {:ok, config} = - LiveCLI.parse( - [ - "--simple", - "--provider", - "gemini", - "--api-key", - "explicit" - ], - with_env(@prompt_example, %{"GEMINI_API_KEY" => "env"}) - ) - - assert config.provider == :gemini - assert config.client.adapter_state.api_key == "explicit" - end - - test "explicit ASM provider infers ASM adapter" do - assert {:ok, config} = - LiveCLI.parse( - ["--provider", "codex", "--input", "Summarize this real repo"], - no_env(@prompt_example) - ) - - assert config.adapter == :asm - assert config.provider == :codex - assert config.model == "gpt-5.4-mini" - assert config.client.model == "gpt-5.4-mini" - end - - test "simple mode supplies built-in live demo data for optimization examples" do - assert {:ok, config} = - LiveCLI.parse(["--simple"], with_env(@example, %{"GEMINI_API_KEY" => "g"})) - - assert config.simple? - assert [%{input: _, answer: _} | _] = config.trainset - assert [%{input: _, answer: _} | _] = config.valset - assert config.max_metric_calls == 2 - assert config.minibatch_size == 1 - end - - test "simple mode supplies ARC grid fixtures for the ARC example" do - example = Keyword.put(@example, :script, "examples/14_arc_grid.exs") - - assert {:ok, config} = - LiveCLI.parse(["--simple"], with_env(example, %{"GEMINI_API_KEY" => "g"})) - - assert [%{input: input, answer: answer} | _] = config.trainset - assert input =~ "ARC" - assert input =~ "[[" - assert answer =~ "[[" - end - - test "simple ASM Codex uses gpt-5.4-mini without --model" do - assert {:ok, config} = - LiveCLI.parse( - ["--simple", "--adapter", "asm", "--provider", "codex"], - no_env(@prompt_example) - ) - - assert config.adapter == :asm - assert config.provider == :codex - assert config.model == "gpt-5.4-mini" - assert config.client.model == "gpt-5.4-mini" - end - - test "simple ASM defaults to Gemini model without --provider" do - assert {:ok, config} = - LiveCLI.parse(["--simple", "--adapter", "asm"], no_env(@prompt_example)) - - assert config.adapter == :asm - assert config.provider == :gemini - assert config.model == "gemini-3.1-flash-lite-preview" - assert config.client.model == "gemini-3.1-flash-lite-preview" - end - end - - describe "messages" do - test "cost warning names the example and adapter" do - warning = LiveCLI.cost_warning("Example Test", :asm, :codex, 7) - assert warning =~ "LIVE LLM CALL WARNING" - assert warning =~ "Example Test" - assert warning =~ "asm/codex" - assert warning =~ "up to 7" - end - end - - defp jsonl_fixture(rows) do - path = - Path.join(System.tmp_dir!(), "gepa-live-cli-#{System.unique_integer([:positive])}.jsonl") - - content = - rows - |> Enum.map_join("\n", &Jason.encode!/1) - - File.write!(path, content <> "\n") - path - end - - defp no_env(example), do: with_env(example, %{}) - - defp with_env(example, env) do - example - |> Keyword.put(:env, fn key -> Map.get(env, key) end) - |> Keyword.put(:app_config, fn _key -> nil end) - end -end diff --git a/test/gepa/llm/adapter_facade_test.exs b/test/gepa/llm/adapter_facade_test.exs deleted file mode 100644 index 3835c7a..0000000 --- a/test/gepa/llm/adapter_facade_test.exs +++ /dev/null @@ -1,332 +0,0 @@ -defmodule GEPA.LLM.AdapterFacadeTest do - use GEPA.SupertesterCase, isolation: :full_isolation - - alias GEPA.LLM.{Client, Request, Tool} - - defmodule FakeReqLLM do - def put_key(key, value) do - send(self(), {:put_key, key, value}) - :ok - end - - def generate_text(model_spec, prompt, opts) do - {:ok, - %{ - text: "text:#{model_spec}:#{prompt_label(prompt)}", - usage: %{tokens: 7}, - cost: %{usd: 0.001}, - tool_calls: [%{id: "call-1", name: "lookup", arguments: %{"query" => "hello"}}], - opts: opts - }} - end - - def generate_object(model_spec, prompt, schema, opts) do - {:ok, - %{ - object: %{"instruction" => "structured:#{model_spec}:#{prompt}"}, - schema: schema, - opts: opts - }} - end - - defp prompt_label(prompt) when is_binary(prompt), do: prompt - defp prompt_label(prompt), do: inspect(prompt) - end - - defmodule FakeReqLLMResponse do - def text(%{text: text}), do: text - def unwrap_object(%{object: object}), do: {:ok, object} - end - - defmodule FakeASM do - def query(provider, prompt, opts) when is_atom(provider) and is_binary(prompt) do - {:ok, - %{ - run_id: "run-1", - session_id: "session-1", - text: "asm:#{inspect(provider)}:#{prompt}", - cost: %{usd: 0.0}, - stop_reason: :stop, - metadata: %{target: provider, opts: opts} - }} - end - - def query(session, prompt, opts) when is_pid(session) and is_binary(prompt) do - {:ok, - %{ - run_id: "run-1", - session_id: "session-1", - text: "asm:pid:#{prompt}", - cost: %{usd: 0.0}, - stop_reason: :stop, - metadata: %{target: session, opts: opts} - }} - end - - def start_session(opts) do - send(self(), {:start_session, opts}) - {:ok, self()} - end - - def stream(session, _prompt, _opts) when is_pid(session) do - [ - %ASM.Event{ - id: "event-1", - run_id: "run-1", - session_id: "session-1", - provider: :codex, - kind: :assistant_delta, - payload: %CliSubprocessCore.Payload.AssistantDelta{content: "a", metadata: %{}}, - timestamp: DateTime.utc_now() - }, - %ASM.Event{ - id: "event-2", - run_id: "run-1", - session_id: "session-1", - provider: :codex, - kind: :run_started, - payload: nil, - timestamp: DateTime.utc_now() - }, - "b" - ] - end - - def stop_session(session) do - send(self(), {:stop_session, session}) - :ok - end - end - - describe "ReqLLM client facade" do - test "builds a normalized client for Anthropic through ReqLLM" do - client = - GEPA.LLM.req_llm(:anthropic, - api_key: "sk-test", - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse - ) - - assert %Client{} = client - assert client.provider == :anthropic - assert client.model == "claude-haiku-4-5" - assert MapSet.member?(client.capabilities, :structured_output) - end - - test "complete/3 dispatches a normalized client through the adapter" do - client = - GEPA.LLM.req_llm(:anthropic, - api_key: "sk-test", - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse - ) - - assert {:ok, "text:anthropic:claude-haiku-4-5:hello"} = - GEPA.LLM.complete(client, "hello") - end - - test "complete_structured/3 uses ReqLLM object generation" do - client = - GEPA.LLM.req_llm(:openai, - api_key: "sk-test", - model: "gpt-test", - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse - ) - - assert {:ok, %{"instruction" => "structured:openai:gpt-test:prompt"}} = - GEPA.LLM.complete_structured(client, "prompt") - end - - test "Gemini resolves GEMINI_API_KEY alias and writes ReqLLM Google key config" do - client = - GEPA.LLM.req_llm(:gemini, - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse, - env: fn - "GEMINI_API_KEY" -> "gemini-env-key" - _key -> nil - end - ) - - request = Request.from_prompt("hello") - - assert {:ok, response} = client.adapter.complete(client, request) - assert response.raw.opts[:api_key] == "gemini-env-key" - assert_received {:put_key, :google_api_key, "gemini-env-key"} - end - - test "explicit API key overrides ReqLLM provider key aliases" do - client = - GEPA.LLM.req_llm(:gemini, - api_key: "explicit-key", - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse, - env: fn - "GEMINI_API_KEY" -> "gemini-env-key" - _key -> nil - end - ) - - request = Request.from_prompt("hello") - - assert {:ok, response} = client.adapter.complete(client, request) - assert response.raw.opts[:api_key] == "explicit-key" - assert_received {:put_key, :google_api_key, "explicit-key"} - end - - test "converts portable tools to ReqLLM tools" do - client = - GEPA.LLM.req_llm(:openai, - api_key: "sk-test", - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse - ) - - tool = - Tool.new( - name: "lookup", - description: "Look up a value", - input_schema: [query: [type: :string, required: true]], - run: fn args, _context -> {:ok, args} end - ) - - request = Request.from_prompt("hello", tools: [tool]) - - assert {:ok, response} = client.adapter.complete(client, request) - assert [%ReqLLM.Tool{name: "lookup"}] = response.raw.opts[:tools] - end - - test "propagates system prompts, tool choice, cost, and tool calls through inference" do - client = - GEPA.LLM.req_llm(:openai, - api_key: "sk-test", - req_llm_module: FakeReqLLM, - response_module: FakeReqLLMResponse - ) - - request = - Request.from_prompt("hello", - system: "be exact", - tool_choice: :required, - provider_opts: [api_key: "request-key"] - ) - - assert {:ok, response} = client.adapter.complete(client, request) - assert response.text =~ "text:openai:gpt-5.4-mini:" - assert response.text =~ "%{role: :system, content: \"be exact\"}" - assert response.text =~ "%{role: :user, content: \"hello\"}" - assert response.raw.opts[:api_key] == "request-key" - assert response.raw.opts[:tool_choice] == :required - assert response.cost == %{usd: 0.001} - - assert response.tool_calls == [ - %{id: "call-1", name: "lookup", arguments: %{"query" => "hello"}} - ] - end - end - - describe "ASM client facade" do - test "builds a normalized agent client" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core) - - assert %Client{} = client - assert client.provider == :codex - assert client.model == "gpt-5.4-mini" - assert MapSet.member?(client.capabilities, :stream) - refute MapSet.member?(client.capabilities, :structured_output) - end - - test "Gemini ASM client defaults to the integration-foundation model" do - client = GEPA.LLM.agent(:gemini, asm_module: FakeASM, lane: :core) - - assert client.model == "gemini-3.1-flash-lite-preview" - - request = Request.from_prompt("hello") - assert {:ok, response} = client.adapter.complete(client, request) - assert response.metadata.opts[:model] == "gemini-3.1-flash-lite-preview" - - assert response.metadata.opts[:model_payload].resolved_model == - "gemini-3.1-flash-lite-preview" - end - - test "complete/3 dispatches to ASM query and normalizes metadata" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core) - - assert {:ok, "asm::codex:hello"} = GEPA.LLM.complete(client, "hello") - - request = Request.from_prompt("hello") - assert {:ok, response} = client.adapter.complete(client, request) - assert response.metadata.run_id == "run-1" - assert response.metadata.session_id == "session-1" - assert response.metadata.lane == :core - assert response.metadata.opts[:model] == "gpt-5.4-mini" - end - - test "per-call ASM model overrides the Codex default model" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core) - request = Request.from_prompt("hello", model: "gpt-explicit") - - assert {:ok, response} = client.adapter.complete(client, request) - assert response.metadata.opts[:model] == "gpt-explicit" - end - - test "ASM adapter filters unsupported portable generation options" do - client = - GEPA.LLM.agent(:gemini, - asm_module: FakeASM, - lane: :core, - provider_opts: [model: "gemini-3.1-flash-lite-preview", max_tokens: 20, timeout: 90_000] - ) - - request = Request.from_prompt("hello", max_tokens: 10, temperature: 0.1, timeout: 120_000) - - assert {:ok, response} = client.adapter.complete(client, request) - refute Keyword.has_key?(response.metadata.opts, :max_tokens) - refute Keyword.has_key?(response.metadata.opts, :temperature) - refute Keyword.has_key?(response.metadata.opts, :top_p) - assert response.metadata.opts[:transport_timeout_ms] == 120_000 - end - - test "complete/3 passes named ASM sessions as session_id query options" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core, session: "gepa-test") - request = Request.from_prompt("hello") - - assert {:ok, response} = client.adapter.complete(client, request) - assert response.text == "asm::codex:hello" - assert response.session_ref == "gepa-test" - assert response.metadata.opts[:session_id] == "gepa-test" - assert response.metadata.opts[:model] == "gpt-5.4-mini" - assert response.metadata.target == :codex - end - - test "stream/3 starts and closes a managed ASM session for named sessions" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core, session: "gepa-stream") - - assert {:ok, stream} = GEPA.LLM.stream(client, "hello") - assert Enum.to_list(stream) == ["a", "b"] - assert_received {:start_session, opts} - assert opts[:provider] == :codex - assert opts[:session_id] == "gepa-stream" - assert opts[:lane] == :core - assert_received {:stop_session, pid} when pid == self() - end - - test "stream/3 closes managed ASM sessions when consumers halt early" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core, session: "gepa-stream") - - assert {:ok, stream} = GEPA.LLM.stream(client, "hello") - assert Enum.take(stream, 1) == ["a"] - assert_received {:start_session, opts} - assert opts[:session_id] == "gepa-stream" - assert_received {:stop_session, pid} when pid == self() - end - - test "structured output fails closed for ASM" do - client = GEPA.LLM.agent(:codex, asm_module: FakeASM, lane: :core) - - assert {:error, {:unsupported_capability, :structured_output, _}} = - GEPA.LLM.complete_structured(client, "prompt") - end - end -end diff --git a/test/gepa/llm/req_llm_error_test.exs b/test/gepa/llm/req_llm_error_test.exs deleted file mode 100644 index cc42513..0000000 --- a/test/gepa/llm/req_llm_error_test.exs +++ /dev/null @@ -1,190 +0,0 @@ -defmodule GEPA.LLM.ReqLLMErrorTest do - use GEPA.SupertesterCase, isolation: :full_isolation - - alias GEPA.LLM.ReqLLM - - describe "error handling" do - test "returns error or success depending on ReqLLM availability" do - llm = ReqLLM.new(provider: :openai, api_key: "test-key") - - # This may succeed if ReqLLM modules are available, or error if not - # We're testing that it doesn't crash - result = ReqLLM.complete(llm, "test prompt") - - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - - test "handles missing API key gracefully" do - llm = ReqLLM.new(provider: :openai, api_key: nil) - - result = ReqLLM.complete(llm, "test prompt") - - # Should error due to missing API key, or if modules available might succeed - # Key point: doesn't crash - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - - test "handles invalid prompt type gracefully" do - llm = ReqLLM.new(provider: :openai, api_key: "test") - - # Passing non-string should raise FunctionClauseError - assert_raise FunctionClauseError, fn -> - ReqLLM.complete(llm, 123) - end - - assert_raise FunctionClauseError, fn -> - ReqLLM.complete(llm, nil) - end - end - - test "empty string prompt is valid" do - llm = ReqLLM.new(provider: :openai, api_key: "test") - - # Should not raise for empty string (though API might reject it) - result = ReqLLM.complete(llm, "") - # May succeed if ReqLLM available, or error if not - key is no crash - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - - test "handles Gemini provider gracefully" do - llm = ReqLLM.new(provider: :gemini, api_key: "test-key") - - result = ReqLLM.complete(llm, "test prompt") - - # May succeed if ReqLLM is available, or error if not - # Key point: doesn't crash - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - - test "handles Anthropic provider gracefully through ReqLLM" do - llm = ReqLLM.new(provider: :anthropic, api_key: "test-key") - - result = ReqLLM.complete(llm, "test prompt") - - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - end - - describe "timeout handling" do - test "timeout option is properly set" do - llm = ReqLLM.new(provider: :openai, api_key: "test", timeout: 5000) - - assert llm.timeout == 5000 - - # Actual timeout handling happens in HTTP layer (ReqLLM) - # We verify the configuration is passed through - end - - test "default timeout is reasonable" do - llm = ReqLLM.new(provider: :openai) - - # Default should be 60 seconds - assert llm.timeout == 60_000 - end - end - - describe "parameter validation" do - test "accepts valid temperature range" do - llm = ReqLLM.new(provider: :openai, temperature: 0.0) - assert llm.temperature == 0.0 - - llm = ReqLLM.new(provider: :openai, temperature: 1.0) - assert llm.temperature == 1.0 - - llm = ReqLLM.new(provider: :openai, temperature: 0.5) - assert llm.temperature == 0.5 - end - - test "accepts valid max_tokens" do - llm = ReqLLM.new(provider: :openai, max_tokens: 1) - assert llm.max_tokens == 1 - - llm = ReqLLM.new(provider: :openai, max_tokens: 10_000) - assert llm.max_tokens == 10_000 - end - - test "accepts optional top_p parameter" do - llm = ReqLLM.new(provider: :openai, top_p: 0.9) - assert llm.top_p == 0.9 - - llm = ReqLLM.new(provider: :openai) - assert llm.top_p == nil - end - end - - describe "req_options passthrough" do - test "custom req_options are stored" do - custom_opts = [ - connect_timeout: 5000, - receive_timeout: 30_000, - retry: false - ] - - llm = ReqLLM.new(provider: :openai, req_options: custom_opts) - - assert llm.req_options == custom_opts - end - - test "req_options default to empty list" do - llm = ReqLLM.new(provider: :openai) - assert llm.req_options == [] - end - end - - describe "API key retrieval" do - test "no API key results in nil" do - llm = ReqLLM.new(provider: :openai) - assert llm.api_key == nil - end - - test "explicit nil API key is preserved" do - llm = ReqLLM.new(provider: :openai, api_key: nil) - assert llm.api_key == nil - end - - test "empty string API key is preserved" do - llm = ReqLLM.new(provider: :openai, api_key: "") - assert llm.api_key == "" - end - end - - describe "model specification" do - test "accepts any string as model name" do - # OpenAI models - llm = ReqLLM.new(provider: :openai, model: "gpt-4") - assert llm.model == "gpt-4" - - llm = ReqLLM.new(provider: :openai, model: "gpt-4-turbo") - assert llm.model == "gpt-4-turbo" - - # Gemini models - llm = ReqLLM.new(provider: :gemini, model: "gemini-pro") - assert llm.model == "gemini-pro" - - # Anthropic models - llm = ReqLLM.new(provider: :anthropic, model: "claude-sonnet-4-5") - assert llm.model == "claude-sonnet-4-5" - - # Future/unknown models should work - llm = ReqLLM.new(provider: :openai, model: "gpt-5-ultra") - assert llm.model == "gpt-5-ultra" - end - end - - describe "complete/3 opts parameter" do - test "accepts opts parameter" do - llm = ReqLLM.new(provider: :openai, api_key: "test") - - # Should not raise - may succeed or error - result = ReqLLM.complete(llm, "prompt", temperature: 0.9, max_tokens: 100) - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - - test "empty opts list works" do - llm = ReqLLM.new(provider: :openai, api_key: "test") - - result = ReqLLM.complete(llm, "prompt", []) - assert match?({:ok, _}, result) or match?({:error, _}, result) - end - end -end diff --git a/test/gepa/llm/req_llm_integration_test.exs b/test/gepa/llm/req_llm_integration_test.exs deleted file mode 100644 index 28e20b1..0000000 --- a/test/gepa/llm/req_llm_integration_test.exs +++ /dev/null @@ -1,86 +0,0 @@ -defmodule GEPA.LLM.ReqLLMIntegrationTest do - use GEPA.SupertesterCase, isolation: :full_isolation, async: false - - # These tests demonstrate the complete/3 function behavior - # They don't make real HTTP calls, but test the request building logic - - alias GEPA.LLM.ReqLLM - - @moduletag :integration - - describe "complete/3 request building (without HTTP)" do - test "builds correct request structure for OpenAI" do - llm = - ReqLLM.new( - provider: :openai, - model: "gpt-5.4-mini", - api_key: "test-key", - temperature: 0.8, - max_tokens: 500 - ) - - # We can verify the struct is properly configured - assert llm.provider == :openai - assert llm.model == "gpt-5.4-mini" - assert llm.api_key == "test-key" - assert llm.temperature == 0.8 - assert llm.max_tokens == 500 - - # The actual complete call would fail without mocking ReqLLM.OpenAI - # but we've verified the configuration is correct - end - - test "builds correct request structure for Gemini" do - llm = - ReqLLM.new( - provider: :gemini, - model: "gemini-3.1-flash-lite-preview", - api_key: "test-key", - temperature: 0.8, - max_tokens: 500 - ) - - assert llm.provider == :gemini - assert llm.model == "gemini-3.1-flash-lite-preview" - assert llm.api_key == "test-key" - assert llm.temperature == 0.8 - assert llm.max_tokens == 500 - end - - test "per-call options would override instance options" do - llm = - ReqLLM.new( - provider: :openai, - model: "gpt-5.4-mini", - temperature: 0.7 - ) - - # Verify instance defaults - assert llm.temperature == 0.7 - - # In actual call, opts would override: - # complete(llm, "prompt", temperature: 0.9) - # This would use temperature: 0.9 for that specific call - end - end - - describe "error handling structure" do - test "handles missing API key case" do - llm = ReqLLM.new(provider: :openai, api_key: nil) - - # Without API key, the actual call would fail - # Testing that the struct allows nil api_key (fail at call time, not creation) - assert llm.api_key == nil - end - - test "validates provider at creation time, not call time" do - # This should fail immediately - assert_raise ArgumentError, fn -> - ReqLLM.new(provider: :invalid_provider) - end - end - end - - # Real hosted-provider smoke checks live in examples and take explicit CLI - # credentials. The unit/integration suite remains deterministic. -end diff --git a/test/gepa/llm/req_llm_test.exs b/test/gepa/llm/req_llm_test.exs deleted file mode 100644 index f2f84be..0000000 --- a/test/gepa/llm/req_llm_test.exs +++ /dev/null @@ -1,233 +0,0 @@ -defmodule GEPA.LLM.ReqLLMTest do - use GEPA.SupertesterCase, isolation: :full_isolation - - import ExUnit.CaptureIO - - alias GEPA.LLM.ReqLLM - - describe "new/1" do - test "creates OpenAI instance with defaults" do - llm = ReqLLM.new(provider: :openai) - - assert llm.provider == :openai - assert llm.model == "gpt-5.4-mini" - assert llm.temperature == 0.7 - assert llm.max_tokens == 2000 - assert llm.timeout == 60_000 - end - - test "creates Gemini instance with defaults" do - llm = ReqLLM.new(provider: :gemini) - - assert llm.provider == :gemini - assert llm.model == "gemini-3.1-flash-lite-preview" - assert llm.temperature == 0.7 - assert llm.max_tokens == 2000 - end - - test "creates Anthropic instance with defaults" do - llm = ReqLLM.new(provider: :anthropic) - - assert llm.provider == :anthropic - assert llm.model == "claude-haiku-4-5" - assert llm.temperature == 0.7 - assert llm.max_tokens == 2000 - end - - test "creates instance with custom options" do - llm = - ReqLLM.new( - provider: :openai, - model: "gpt-4o", - temperature: 0.9, - max_tokens: 1000, - top_p: 0.95, - timeout: 30_000 - ) - - assert llm.model == "gpt-4o" - assert llm.temperature == 0.9 - assert llm.max_tokens == 1000 - assert llm.top_p == 0.95 - assert llm.timeout == 30_000 - end - - test "does not infer API key for OpenAI" do - llm = ReqLLM.new(provider: :openai) - assert llm.api_key == nil - end - - test "does not infer API key for Gemini" do - llm = ReqLLM.new(provider: :gemini) - assert llm.api_key == nil - end - - test "stores explicit API key" do - llm = ReqLLM.new(provider: :openai, api_key: "explicit-key") - assert llm.api_key == "explicit-key" - end - - test "raises on invalid provider" do - assert_raise ArgumentError, ~r/provider must be/, fn -> - ReqLLM.new(provider: :invalid) - end - end - - test "requires provider option" do - assert_raise KeyError, fn -> - ReqLLM.new([]) - end - end - end - - describe "complete/3 option merging" do - test "per-call options override instance options" do - llm = - ReqLLM.new( - provider: :openai, - model: "gpt-5.4-mini", - temperature: 0.7, - api_key: "default-key" - ) - - # Test that complete would merge options correctly - # We can't actually call complete without mocking HTTP, but we can test the struct - assert llm.model == "gpt-5.4-mini" - assert llm.temperature == 0.7 - end - end - - describe "model defaults" do - test "OpenAI default model is gpt-5.4-mini" do - llm = ReqLLM.new(provider: :openai) - assert llm.model == "gpt-5.4-mini" - end - - test "Gemini default model is gemini-3.1-flash-lite-preview" do - llm = ReqLLM.new(provider: :gemini) - assert llm.model == "gemini-3.1-flash-lite-preview" - end - - test "Gemini default model resolves through ReqLLM catalog without fallback warning" do - llm = ReqLLM.new(provider: :gemini) - - warning = - capture_io(:stderr, fn -> - assert {:ok, model} = Elixir.ReqLLM.model("google:" <> llm.model) - assert model.id == "gemini-3.1-flash-lite-preview" - end) - - assert warning == "" - end - - test "Anthropic default model is claude-haiku-4-5" do - llm = ReqLLM.new(provider: :anthropic) - assert llm.model == "claude-haiku-4-5" - end - - test "can override default models" do - llm1 = ReqLLM.new(provider: :openai, model: "gpt-4o") - assert llm1.model == "gpt-4o" - - llm2 = ReqLLM.new(provider: :gemini, model: "gemini-1.5-pro") - assert llm2.model == "gemini-1.5-pro" - - llm3 = ReqLLM.new(provider: :anthropic, model: "claude-sonnet-4-5") - assert llm3.model == "claude-sonnet-4-5" - end - end - - describe "configuration" do - test "stores all configuration options" do - llm = - ReqLLM.new( - provider: :openai, - model: "gpt-4o", - api_key: "sk-test", - temperature: 0.8, - max_tokens: 500, - top_p: 0.9, - timeout: 45_000, - req_options: [connect_timeout: 10_000] - ) - - assert llm.provider == :openai - assert llm.model == "gpt-4o" - assert llm.api_key == "sk-test" - assert llm.temperature == 0.8 - assert llm.max_tokens == 500 - assert llm.top_p == 0.9 - assert llm.timeout == 45_000 - assert llm.req_options == [connect_timeout: 10_000] - end - - test "nil values are allowed for optional parameters" do - llm = ReqLLM.new(provider: :openai, top_p: nil) - assert llm.top_p == nil - end - end - - # Note: Integration tests with actual HTTP calls would go in a separate - # integration test file and would be optional (require API keys). - # These tests cover the structure and configuration of ReqLLM without - # requiring network calls or complex mocking of external libraries. - - describe "documentation and examples" do - test "example from module docs works" do - # OpenAI example - llm = ReqLLM.new(provider: :openai) - assert %ReqLLM{provider: :openai} = llm - - # Gemini example - llm = ReqLLM.new(provider: :gemini) - assert %ReqLLM{provider: :gemini} = llm - - # Custom options example - llm = - ReqLLM.new( - provider: :openai, - model: "gpt-4o", - temperature: 0.9, - max_tokens: 1000 - ) - - assert llm.model == "gpt-4o" - assert llm.temperature == 0.9 - end - end - - describe "struct creation and validation" do - test "creates valid struct with all fields" do - llm = ReqLLM.new(provider: :openai, api_key: "test") - - assert is_struct(llm, ReqLLM) - assert is_atom(llm.provider) - assert is_binary(llm.model) - assert is_float(llm.temperature) - assert is_integer(llm.max_tokens) - assert is_integer(llm.timeout) - end - - test "handles missing API key gracefully (returns nil)" do - llm = ReqLLM.new(provider: :openai) - assert llm.api_key == nil - - llm = ReqLLM.new(provider: :gemini) - assert llm.api_key == nil - end - end - - describe "type specifications" do - test "provider type is enforced" do - # Valid providers - assert %ReqLLM{provider: :openai} = ReqLLM.new(provider: :openai) - assert %ReqLLM{provider: :gemini} = ReqLLM.new(provider: :gemini) - assert %ReqLLM{provider: :anthropic} = ReqLLM.new(provider: :anthropic) - - # Invalid providers raise - assert_raise ArgumentError, fn -> - ReqLLM.new(provider: :invalid) - end - end - end -end diff --git a/test/gepa/llm_test.exs b/test/gepa/llm_test.exs index e1314d4..f9b6beb 100644 --- a/test/gepa/llm_test.exs +++ b/test/gepa/llm_test.exs @@ -3,7 +3,6 @@ defmodule GEPA.LLMTest do doctest GEPA.LLM alias GEPA.LLM.Mock - alias GEPA.LLM.ReqLLM describe "GEPA.LLM behavior" do test "complete/3 delegates to module's implementation" do @@ -12,30 +11,34 @@ defmodule GEPA.LLMTest do assert response == "Test response" end - test "default/0 returns ReqLLM with OpenAI by default" do - llm = GEPA.LLM.default() - assert %ReqLLM{provider: :openai} = llm + test "default/0 returns a Mock when no provider is configured" do + original = Application.get_env(:gepa_ex, :llm, []) + + try do + Application.delete_env(:gepa_ex, :llm) + assert %Mock{} = GEPA.LLM.default() + after + Application.put_env(:gepa_ex, :llm, original) + end end - test "default/0 respects application config" do + test "default/0 returns a Mock when the provider is :mock" do original = Application.get_env(:gepa_ex, :llm, []) try do - Application.put_env(:gepa_ex, :llm, provider: :gemini) - llm = GEPA.LLM.default() - assert %ReqLLM{provider: :gemini} = llm + Application.put_env(:gepa_ex, :llm, provider: :mock) + assert %Mock{} = GEPA.LLM.default() after Application.put_env(:gepa_ex, :llm, original) end end - test "default/0 routes Anthropic through ReqLLM compatibility wrapper" do + test "default/0 raises for a non-mock provider (built-in network providers were removed)" do original = Application.get_env(:gepa_ex, :llm, []) try do - Application.put_env(:gepa_ex, :llm, provider: :anthropic) - llm = GEPA.LLM.default() - assert %ReqLLM{provider: :anthropic} = llm + Application.put_env(:gepa_ex, :llm, provider: :openai) + assert_raise ArgumentError, fn -> GEPA.LLM.default() end after Application.put_env(:gepa_ex, :llm, original) end @@ -85,16 +88,5 @@ defmodule GEPA.LLMTest do assert {:ok, %{"instruction" => "fallback response"}} = GEPA.LLM.complete_structured(llm, "prompt") end - - test "dispatches to ReqLLM Anthropic wrapper when configured" do - llm = ReqLLM.new(provider: :anthropic, api_key: nil) - - assert {:error, reason} = GEPA.LLM.complete_structured(llm, "hello") - reason_text = reason |> inspect() |> String.downcase() - assert String.contains?(reason_text, "api") - - assert String.contains?(reason_text, "key") or - String.contains?(reason_text, "authentication") - end end end diff --git a/test/gepa/optimize_anything_evaluator_wrapper_parity_test.exs b/test/gepa/optimize_anything_evaluator_wrapper_parity_test.exs index 657969e..6098389 100644 --- a/test/gepa/optimize_anything_evaluator_wrapper_parity_test.exs +++ b/test/gepa/optimize_anything_evaluator_wrapper_parity_test.exs @@ -4,23 +4,9 @@ defmodule GEPA.OptimizeAnythingEvaluatorWrapperParityTest do import ExUnit.CaptureIO import ExUnit.CaptureLog - alias GEPA.LLM.Client alias GEPA.OptimizeAnything alias GEPA.OptimizeAnything.{Adapter, EvaluatorWrapper, OptimizationState} - defmodule FakeReqLLM do - def put_key(_provider, _api_key), do: :ok - - def generate_text(model_spec, prompt, opts) do - send(Keyword.fetch!(opts, :test_pid), {:llm_request, model_spec, prompt, opts}) - {:ok, %{text: "response text", finish_reason: "stop"}} - end - end - - defmodule FakeResponse do - def text(%{text: text}), do: text - end - describe "oa.log parity" do test "log basic capture" do evaluator = fn _candidate -> @@ -467,32 +453,6 @@ defmodule GEPA.OptimizeAnythingEvaluatorWrapperParityTest do end end - describe "make_litellm_lm parity" do - test "returns LM client instance" do - lm = OptimizeAnything.make_litellm_lm("test-model") - assert %Client{provider: :openai, model: "test-model"} = lm - end - - test "string prompt" do - lm = fake_lm() - - assert {:ok, "response text"} = GEPA.LLM.complete(lm, "hello") - - assert_received {:llm_request, "openai:test-model", "hello", opts} - assert Keyword.fetch!(opts, :test_pid) == self() - end - - test "messages prompt" do - lm = fake_lm() - messages = [%{role: "system", content: "sys"}, %{role: "user", content: "hi"}] - - assert {:ok, "response text"} = GEPA.LLM.complete(lm, messages) - - assert_received {:llm_request, "openai:test-model", ^messages, opts} - assert Keyword.fetch!(opts, :test_pid) == self() - end - end - describe "BEAM stdio capture utility parity" do test "passthrough when not capturing" do assert capture_io(fn -> IO.write("hello") end) == "hello" @@ -564,16 +524,6 @@ defmodule GEPA.OptimizeAnythingEvaluatorWrapperParityTest do end end - defp fake_lm do - OptimizeAnything.make_litellm_lm( - "test-model", - req_llm_module: FakeReqLLM, - response_module: FakeResponse, - provider_opts: [test_pid: self()], - env: fn _name -> nil end - ) - end - defp capture_stdout(text) do EvaluatorWrapper.evaluate( fn _candidate -> diff --git a/test/gepa/proposer/instruction_proposal_test.exs b/test/gepa/proposer/instruction_proposal_test.exs index 7f07c2d..5f7947f 100644 --- a/test/gepa/proposer/instruction_proposal_test.exs +++ b/test/gepa/proposer/instruction_proposal_test.exs @@ -10,7 +10,7 @@ defmodule GEPA.Proposer.InstructionProposalTest do proposal = InstructionProposal.new(llm: llm) assert proposal.llm == llm - assert proposal.template != nil + assert is_binary(proposal.template) assert String.contains?(proposal.template, "") assert String.contains?(proposal.template, "") end diff --git a/test/gepa/strategies/batch_sampler_test.exs b/test/gepa/strategies/batch_sampler_test.exs index a077e2f..ec06d44 100644 --- a/test/gepa/strategies/batch_sampler_test.exs +++ b/test/gepa/strategies/batch_sampler_test.exs @@ -65,7 +65,7 @@ defmodule GEPA.Strategies.BatchSamplerTest do {batch1, sampler} = EpochShuffled.next_batch(sampler, loader, state) assert length(batch1) == 3 - assert sampler.shuffled_ids != nil + assert is_list(sampler.shuffled_ids) assert sampler.epoch == 0 # All elements should be from the original list (0-indexed) assert Enum.all?(batch1, &(&1 in 0..5))