diff --git a/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt new file mode 100644 index 000000000..a44460f5b --- /dev/null +++ b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.anthropic + +import com.embabel.agent.api.models.AnthropicModels +import com.embabel.agent.spi.support.springai.SpringAiLlmService +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.ai.anthropic.AnthropicChatOptions +import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage +import org.springframework.ai.chat.metadata.ChatResponseMetadata +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.Prompt + +/** + * End-to-end regression for the Anthropic half of the Spring AI 2.0 model-binding bug + * (see #1815 for the OpenAI/Gemini equivalent). + * + * [AnthropicOptionsConverter] never sets a model, so its [AnthropicChatOptions] carry the + * SDK's baked-in `DEFAULT_MODEL` (`claude-haiku-4-5`). Because Embabel always passes these + * per-request options and Spring AI 2.0 no longer merges the model bean's configured model, + * every call would silently target Haiku. [SpringAiLlmService] now binds the configured + * model generically; this test proves the selected model reaches the wire while the concrete + * [AnthropicChatOptions] type and other fields are preserved. + */ +class AnthropicModelBindingTest { + + @Test + fun `sends the selected Anthropic model rather than the converter default`() { + // Prepare: bean configured with Sonnet; converter (unchanged) bakes in Haiku default. + val configuredOptions = AnthropicChatOptions.builder().model("claude-sonnet-4-5").build() + val capturedPrompt = slot() + val generation = Generation(SpringAiAssistantMessage("done")) + val chatResponse = mockk { + every { result } returns generation + every { results } returns listOf(generation) + every { metadata } returns mockk(relaxed = true) { every { usage } returns null } + } + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "claude-sonnet-4-5", + provider = AnthropicModels.PROVIDER, + chatModel = chatModel, + optionsConverter = AnthropicOptionsConverter, + thinkingSupported = true, + ) + + // Execute + service.createMessageSender(LlmOptions().withMaxTokens(500)) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: concrete Anthropic options carry the selected model, maxTokens preserved. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions).isInstanceOf(AnthropicChatOptions::class.java) + assertThat(sentOptions.model).isEqualTo("claude-sonnet-4-5") + assertThat(sentOptions.maxTokens).isEqualTo(500) + } +} diff --git a/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt new file mode 100644 index 000000000..8df9d172c --- /dev/null +++ b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.anthropic + +import com.embabel.agent.api.models.AnthropicModels +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.opentest4j.TestAbortedException + +/** + * Live regression for the Anthropic same-provider silent variant of #1815: select Sonnet and + * assert Anthropic reports a sonnet model back. If binding regressed, the request would run on + * the AnthropicChatOptions default ("claude-haiku-4-5") and the served model would be a haiku. + * + * Requires ANTHROPIC_API_KEY. Skipped (aborted) if the key lacks access to the chosen model. + */ +class AnthropicServedModelBindingIT { + + @Test + @EnabledIfEnvironmentVariable(named = "ANTHROPIC_API_KEY", matches = ".+") + fun `real Anthropic serves the selected model, not the default`() { + // Prepare + val model = AnthropicModels.CLAUDE_SONNET_4_5 + val llm = AnthropicModelFactory(apiKey = System.getenv("ANTHROPIC_API_KEY")).build(model) + + // Execute + val response = try { + llm.createMessageSender(LlmOptions()) + .call(listOf(UserMessage("Reply with exactly the word READY.")), emptyList()) + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("ANTHROPIC_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: Anthropic served the selected Sonnet, not the claude-haiku-4-5 default. + assertThat(response.textContent).isNotBlank() + assertThat(response.model) + .withFailMessage("Served model was '%s', expected a sonnet model", response.model) + .isNotNull() + .contains("sonnet") + assertThat(response.model).doesNotContain("haiku") + } + + /** Broad check for a provider "no access to this model" error, so the IT aborts rather than fails. */ + private fun isModelAccessError(ex: Throwable): Boolean { + val message = generateSequence(ex) { it.cause } + .mapNotNull { it.message } + .joinToString(" | ") + .lowercase() + return listOf( + "not_found", + "does not exist", + "not found", + "permission", + "invalid model", + "unsupported model", + "does not have access", + ).any { message.contains(it) } + } +} diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt index 7f0a51181..c98d626a5 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt @@ -68,11 +68,14 @@ data class LlmMessageRequest @JvmOverloads constructor( * @param message The full message object from the LLM * @param textContent The text content of the message * @param usage Optional usage information (tokens, etc.) + * @param model The model that actually served the request, as reported by the provider + * (from the response metadata). Null/blank if the provider does not report it. */ data class LlmMessageResponse( val message: Message, val textContent: String, val usage: Usage? = null, + val model: String? = null, ) /** diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt index c4bb7769d..f77b155bb 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt @@ -139,4 +139,3 @@ class UnfoldingToolInjectionStrategy : ToolInjectionStrategy { val INSTANCE = UnfoldingToolInjectionStrategy() } } - diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt index a3e0f89dc..904713906 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt @@ -111,6 +111,7 @@ internal class SpringAiLlmMessageSender( message = embabelMessage, textContent = assistantMessage.text ?: "", usage = usage, + model = response.metadata?.model, ) } diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt index 8f5fe2e41..f4a24bf99 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt @@ -28,6 +28,7 @@ import org.springframework.ai.chat.client.ChatClient import org.springframework.ai.chat.messages.UserMessage import org.springframework.ai.chat.model.ChatModel import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.prompt.ChatOptions import org.springframework.ai.chat.prompt.Prompt import reactor.core.publisher.Flux import java.time.Duration @@ -123,8 +124,22 @@ data class SpringAiLlmService @JvmOverloads constructor( */ override val model: ChatModel get() = chatModel + /** + * Binds the model configured on the underlying [ChatModel] onto per-request options. + * + * Spring AI 2.0 no longer merges a model's configured options into a prompt that already + * carries options (OpenAiChatModel/AnthropicChatModel `buildRequestPrompt` returns the + * prompt unchanged when `getOptions() != null`). Because Embabel always supplies + * per-request options — and provider option types coerce a null model to a hard-coded + * default in their constructor (e.g. `gpt-5-mini`, `claude-haiku-4-5`) — the selected + * model would otherwise be ignored on every call. Binding the [ChatModel]'s own + * configured model restores the pre-2.0 merge semantics for every provider at once. + */ + private fun bindConfiguredModel(chatOptions: ChatOptions): ChatOptions = + bindModel(chatOptions, chatModel.options?.model) + override fun createMessageSender(options: LlmOptions): LlmMessageSender { - val chatOptions = optionsConverter.convertOptions(options) + val chatOptions = bindConfiguredModel(optionsConverter.convertOptions(options)) return SpringAiLlmMessageSender( chatModel = chatModel, chatOptions = chatOptions, @@ -136,7 +151,7 @@ data class SpringAiLlmService @JvmOverloads constructor( } override fun createMessageStreamer(options: LlmOptions): LlmMessageStreamer { - val chatOptions = optionsConverter.convertOptions(options) + val chatOptions = bindConfiguredModel(optionsConverter.convertOptions(options)) val chatClient = ChatClient.create(chatModel) return SpringAiLlmMessageStreamer(chatClient, chatOptions) } @@ -160,3 +175,13 @@ data class SpringAiLlmService @JvmOverloads constructor( fun withOptionsConverter(converter: OptionsConverter<*>): SpringAiLlmService = copy(optionsConverter = converter) } + +/** + * Binds [model] onto [options], preserving the concrete provider option type and all other + * fields via each provider's overridden `mutate()` (dynamic dispatch keeps e.g. + * `OpenAiChatOptions`/`AnthropicChatOptions` intact). Returns [options] unchanged when + * [model] is null or blank, so models that expose no configured default (bare test doubles, + * exotic providers) keep the converter's output. + */ +internal fun bindModel(options: ChatOptions, model: String?): ChatOptions = + if (model.isNullOrBlank()) options else options.mutate().model(model).build() diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt index 8c0a005cc..facecaf3c 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt @@ -51,7 +51,7 @@ class SpringAiLlmMessageSenderTest { val configuredOptions = testChatOptions() val capturedPrompt = slot() val generation = Generation(SpringAiAssistantMessage("done")) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { @@ -106,7 +106,7 @@ class SpringAiLlmMessageSenderTest { val configuredOptions = testChatOptions() val capturedPrompt = slot() val generation = Generation(SpringAiAssistantMessage("done")) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { @@ -157,7 +157,7 @@ class SpringAiLlmMessageSenderTest { val originalOptions = testChatOptions() val capturedPrompt = slot() val generation = Generation(SpringAiAssistantMessage("done")) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { @@ -240,7 +240,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -313,7 +313,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -374,7 +374,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -439,7 +439,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -503,7 +503,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -581,7 +581,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt new file mode 100644 index 000000000..e77871102 --- /dev/null +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.spi.support.springai + +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.OptionsConverter +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage +import org.springframework.ai.chat.metadata.ChatResponseMetadata +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.ChatOptions +import org.springframework.ai.chat.prompt.Prompt +import org.springframework.ai.model.tool.ToolCallingChatOptions +import reactor.core.publisher.Flux + +/** + * Regression tests for the generic model-binding performed by [SpringAiLlmService]. + * + * Spring AI 2.0 stopped merging a [ChatModel]'s configured options into a prompt that + * already carries per-request options (see OpenAiChatModel/AnthropicChatModel + * `buildRequestPrompt`). Embabel always supplies per-request options via its + * [OptionsConverter], and those options carry the provider's baked-in default model + * (e.g. `gpt-5-mini`, `claude-haiku-4-5`). Without re-binding, every call would ignore + * the selected model. [SpringAiLlmService] closes this generically by binding the model + * configured on the underlying [ChatModel] onto the converted request options. + */ +class SpringAiLlmServiceModelBindingTest { + + private fun chatResponseStub(): ChatResponse { + val generation = Generation(SpringAiAssistantMessage("done")) + return mockk { + every { result } returns generation + every { results } returns listOf(generation) + every { metadata } returns mockk(relaxed = true) { every { usage } returns null } + } + } + + @Test + fun `createMessageSender binds the chat model's configured model onto the request`() { + // Prepare: the bean is configured with the selected model, but the converter + // returns options carrying a different (provider-default) model. + val configuredOptions = ChatOptions.builder().model("gpt-4.1").build() + val converterOutput = ChatOptions.builder() + .model("gpt-5-mini") + .temperature(0.5) + .build() + val capturedPrompt = slot() + val chatResponse = chatResponseStub() + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "gpt-4.1", + provider = "Test", + chatModel = chatModel, + optionsConverter = OptionsConverter { converterOutput }, + ) + + // Execute + service.createMessageSender(LlmOptions()) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: the request carries the configured model, other options preserved. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions.model).isEqualTo("gpt-4.1") + assertThat(sentOptions.temperature).isEqualTo(0.5) + } + + @Test + fun `binds model onto a generic ToolCallingChatOptions converter (Bedrock-shape)`() { + // Bedrock's converter returns a generic ToolCallingChatOptions carrying no model; the + // model lives only on the ChatModel. Verify binding sets it while preserving the type. + val configuredOptions = ToolCallingChatOptions.builder().model("anthropic.claude-sonnet").build() + val converterOutput = ToolCallingChatOptions.builder().build() + val capturedPrompt = slot() + val chatResponse = chatResponseStub() + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "anthropic.claude-sonnet", + provider = "Bedrock", + chatModel = chatModel, + optionsConverter = OptionsConverter { converterOutput }, + ) + + // Execute + service.createMessageSender(LlmOptions()) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: concrete generic type preserved and the configured model bound. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions).isInstanceOf(ToolCallingChatOptions::class.java) + assertThat(sentOptions.model).isEqualTo("anthropic.claude-sonnet") + } + + @Test + fun `createMessageStreamer binds the configured model onto the streamed request`() { + // Prepare + val configuredOptions = ChatOptions.builder().model("gpt-4.1").build() + val converterOutput = ChatOptions.builder().model("gpt-5-mini").build() + val capturedPrompt = slot() + // Use a real ChatResponse: the ChatClient streaming aggregator reads metadata the mock + // stub does not provide. + val chatModel = mockk { + every { options } returns configuredOptions + every { stream(capture(capturedPrompt)) } returns + Flux.just(ChatResponse(listOf(Generation(SpringAiAssistantMessage("chunk"))))) + } + val service = SpringAiLlmService( + name = "gpt-4.1", + provider = "Test", + chatModel = chatModel, + optionsConverter = OptionsConverter { converterOutput }, + ) + + // Execute + service.createMessageStreamer(LlmOptions()) + .stream(messages = listOf(UserMessage("Hi")), tools = emptyList(), toolCallInspectors = emptyList()) + .blockLast() + + // Verify + assertThat(capturedPrompt.captured.options.model).isEqualTo("gpt-4.1") + } + + @Test + fun `bindModel overrides the converted model and preserves other fields`() { + // Prepare + val converted = ChatOptions.builder() + .model("gpt-5-mini") + .temperature(0.5) + .maxTokens(1000) + .topP(0.9) + .build() + + // Execute + val bound = bindModel(converted, "gpt-4.1") + + // Verify + assertThat(bound.model).isEqualTo("gpt-4.1") + assertThat(bound.temperature).isEqualTo(0.5) + assertThat(bound.maxTokens).isEqualTo(1000) + assertThat(bound.topP).isEqualTo(0.9) + } + + @Test + fun `bindModel passes options through unchanged when configured model is null`() { + // Prepare + val converted = ChatOptions.builder().model("converter-model").build() + + // Execute + val bound = bindModel(converted, null) + + // Verify + assertThat(bound).isSameAs(converted) + assertThat(bound.model).isEqualTo("converter-model") + } + + @Test + fun `bindModel passes options through unchanged when configured model is blank`() { + // Prepare + val converted = ChatOptions.builder().model("converter-model").build() + + // Execute + val bound = bindModel(converted, " ") + + // Verify + assertThat(bound).isSameAs(converted) + assertThat(bound.model).isEqualTo("converter-model") + } +} diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt index 5eccb4de9..6e01f838a 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt @@ -24,6 +24,7 @@ import com.embabel.common.ai.model.OptionsConverter import com.embabel.common.ai.model.PricingModel import com.embabel.common.ai.prompt.KnowledgeCutoffDate import com.embabel.common.ai.prompt.PromptContributor +import io.mockk.every import io.mockk.mockk import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested @@ -34,7 +35,11 @@ import java.time.LocalDate class SpringAiLlmServiceTest { - private val mockChatModel: ChatModel = mockk() + private val mockChatModel: ChatModel = mockk { + // SpringAiLlmService reads the model configured on the ChatModel to bind it onto + // request options; empty options (null model) make binding a no-op for these tests. + every { options } returns ChatOptions.builder().build() + } @Nested inner class ConstructorTests { diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt index a2787f667..4aee72e13 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt @@ -25,7 +25,9 @@ import com.embabel.common.ai.autoconfig.RegisteredModel import com.embabel.common.ai.model.* import com.embabel.common.util.ExcludeFromJacocoGeneratedReport import com.openai.client.OpenAIClient +import com.openai.client.OpenAIClientAsync import com.openai.client.okhttp.OpenAIOkHttpClient +import com.openai.client.okhttp.OpenAIOkHttpClientAsync import io.micrometer.observation.ObservationRegistry import org.slf4j.LoggerFactory import org.springframework.ai.document.MetadataMode @@ -140,6 +142,20 @@ class DockerLocalModelsConfig( .build() } + /** + * Async counterpart to [openAiClient]. Spring AI 2.0's `OpenAiChatModel.Builder.build()` + * builds an async client via `OpenAiSetup.setupAsyncClient(...)` unless one is supplied, + * and that fallback requires a credential from the environment (`OPENAI_API_KEY`). Docker + * local endpoints have no key, so we build the async client explicitly with the same + * placeholder and wire it into the chat model to stay credential-independent. + */ + private val openAiClientAsync: OpenAIClientAsync by lazy { + OpenAIOkHttpClientAsync.builder() + .baseUrl(dockerConnectionProperties.baseUrl) + .apiKey("no-auth") + .build() + } + private fun loadModels(): List = try { val restClient = RestClient.create() @@ -230,6 +246,7 @@ class DockerLocalModelsConfig( private fun dockerLlmOf(model: Model): SpringAiLlmService { val chatModel = OpenAiChatModel.builder() .openAiClient(openAiClient) + .openAiClientAsync(openAiClientAsync) .observationRegistry(observationRegistry.getIfUnique { ObservationRegistry.NOOP }) .toolCallingManager( ToolCallingManager.builder() diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java index f3fd0b531..bdce9ca47 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java @@ -18,11 +18,17 @@ import com.embabel.agent.config.models.docker.DockerConnectionProperties; import com.embabel.agent.config.models.docker.DockerLocalModelsConfig; import com.embabel.agent.config.models.docker.DockerRetryProperties; +import com.embabel.agent.spi.support.springai.SpringAiLlmService; import com.embabel.common.ai.autoconfig.ProviderInitialization; +import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -157,6 +163,40 @@ void providerInitializationHasEmptyModelListWhenDockerUnavailable() { }); } + /** + * Verifies that a model discovered from a Docker OpenAI-compatible endpoint configures its discovered id on the chat model, which + * {@link SpringAiLlmService} then binds onto request-level options at call time. + */ + @Test + void discoveredDockerModelServiceBindsConfiguredModelOnRequestOptions() throws IOException { + // Prepare + final HttpServer server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + server.createContext("/engines/v1/models", exchange -> { + final byte[] body = """ + {"object":"list","data":[{"id":"docker-test-model"}]} + """.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + try { + // Execute + final String baseUrl = "http://localhost:" + server.getAddress().getPort() + "/engines"; + contextRunner.withPropertyValues("embabel.agent.models.docker.base-url=" + baseUrl).run(context -> { + final SpringAiLlmService service = context.getBean("dockerModel-docker-test-model", SpringAiLlmService.class); + + // Verify: the discovered id is configured on the chat model, which SpringAiLlmService + // binds onto request options at call time. + assertThat(service.getChatModel().getOptions().getModel()).isEqualTo("docker-test-model"); + }); + } + finally { + server.stop(0); + } + } + @Override public String toString() { diff --git a/embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt b/embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt new file mode 100644 index 000000000..25431a46d --- /dev/null +++ b/embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.config.models.ollama + +import com.embabel.agent.spi.support.springai.SpringAiLlmService +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage +import org.springframework.ai.chat.metadata.ChatResponseMetadata +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.Prompt +import org.springframework.ai.ollama.api.OllamaChatOptions + +/** + * End-to-end regression that a native (non-OpenAI-compatible) provider reaches the wire with + * the selected model. [OllamaOptionsConverter] never sets a model, so before generic binding + * in [SpringAiLlmService] its [OllamaChatOptions] carried no model. This proves the model + * configured on the [ChatModel] is bound onto request options while the concrete + * [OllamaChatOptions] type and other fields survive. + */ +class OllamaModelBindingTest { + + @Test + fun `sends the selected Ollama model rather than the converter default`() { + // Prepare: bean configured with a specific Ollama model; converter sets no model. + val configuredOptions = OllamaChatOptions.builder().model("llama3.2").build() + val capturedPrompt = slot() + val generation = Generation(SpringAiAssistantMessage("done")) + val chatResponse = mockk { + every { result } returns generation + every { results } returns listOf(generation) + every { metadata } returns mockk(relaxed = true) { every { usage } returns null } + } + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "llama3.2", + provider = "Ollama", + chatModel = chatModel, + optionsConverter = OllamaOptionsConverter, + ) + + // Execute + service.createMessageSender(LlmOptions().withTopP(0.8)) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: concrete Ollama options carry the selected model, topP preserved. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions).isInstanceOf(OllamaChatOptions::class.java) + assertThat(sentOptions.model).isEqualTo("llama3.2") + assertThat(sentOptions.topP).isEqualTo(0.8) + } +} diff --git a/embabel-agent-autoconfigure/models/embabel-agent-openai-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/AgentOpenAiAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-openai-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/AgentOpenAiAutoConfigurationTest.java new file mode 100644 index 000000000..12d2b530e --- /dev/null +++ b/embabel-agent-autoconfigure/models/embabel-agent-openai-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/AgentOpenAiAutoConfigurationTest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.autoconfigure.models.openai; + +import com.embabel.agent.spi.support.springai.SpringAiLlmService; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies OpenAI model auto-configuration wiring that is specific to the Spring AI OpenAI provider. + */ +class AgentOpenAiAutoConfigurationTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(AgentOpenAiAutoConfiguration.class)) + .withPropertyValues("embabel.agent.platform.models.openai.api-key=test-key"); + + /** + * Verifies that the standard OpenAI auto-configuration configures the catalog model id on the chat model, which + * {@link SpringAiLlmService} binds onto request-level options at call time. + */ + @Test + void openAiModelServiceBindsConfiguredModelOnRequestOptions() { + // Prepare + final String beanName = "gpt41"; + final String expectedModel = "gpt-4.1"; + + // Execute + contextRunner.run(context -> { + final SpringAiLlmService service = context.getBean(beanName, SpringAiLlmService.class); + + // Verify + assertThat(service.getChatModel().getOptions().getModel()).isEqualTo(expectedModel); + }); + } +} diff --git a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java index 95be75946..71fc8b509 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java +++ b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java @@ -16,6 +16,7 @@ package com.embabel.agent.autoconfigure.models.openai.custom; import com.embabel.common.ai.autoconfig.ProviderInitialization; +import com.embabel.agent.spi.support.springai.SpringAiLlmService; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -42,14 +43,35 @@ class AgentOpenAiCustomAutoConfigurationTest { */ @Test void registersCustomModelAndInitialization() { - // Act + // Prepare + final String modelBeanName = "test-model"; + + // Execute contextRunner.run(context -> { - // Assert + // Verify assertThat(context).hasSingleBean(ProviderInitialization.class); - assertThat(context).hasBean("test-model"); + assertThat(context).hasBean(modelBeanName); assertThat(context.getBean(ProviderInitialization.class).getRegisteredLlms()) .extracting(registeredModel -> registeredModel.getBeanName()) - .contains("test-model"); + .contains(modelBeanName); + }); + } + + /** + * Verifies that custom OpenAI-compatible models configure their declared model id on the chat model, which + * {@link SpringAiLlmService} binds onto request-level options at call time. + */ + @Test + void customModelServiceBindsConfiguredModelOnRequestOptions() { + // Prepare + final String modelBeanName = "test-model"; + + // Execute + contextRunner.run(context -> { + final SpringAiLlmService service = context.getBean(modelBeanName, SpringAiLlmService.class); + + // Verify + assertThat(service.getChatModel().getOptions().getModel()).isEqualTo(modelBeanName); }); } } diff --git a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt index 0c0c32fb3..9c99dbaf7 100644 --- a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt +++ b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt @@ -377,7 +377,6 @@ object OpenAiChatOptionsConverter : OptionsConverter { .maxTokens(options.maxTokens) .presencePenalty(options.presencePenalty) .frequencyPenalty(options.frequencyPenalty) - .topP(options.topP) //.streamUsage(true) additional feature note .build() } diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt new file mode 100644 index 000000000..aa3cc94ac --- /dev/null +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt @@ -0,0 +1,93 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.openai + +import com.embabel.agent.api.models.GoogleGenAiModels +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.PricingModel +import com.embabel.agent.spi.LlmService +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.opentest4j.TestAbortedException + +/** + * Live regression for the reported #1815 scenario: Gemini via the OpenAI-compatible path. This + * is the cross-endpoint variant — the Gemini endpoint *rejects* an OpenAI model id, so if model + * binding regressed (sending "gpt-5-mini"), the call would error. A successful call that reports + * a gemini model back is therefore the regression guard, on both the blocking and streaming paths. + * + * Requires GEMINI_API_KEY. Skipped (aborted) if the key lacks access to the chosen model. + */ +class GeminiServedModelBindingIT { + + private val geminiBaseUrl = "https://generativelanguage.googleapis.com/v1beta/openai" + private val model = GoogleGenAiModels.GEMINI_2_5_FLASH + + private fun geminiLlm(): LlmService<*> = + OpenAiCompatibleModelFactory(apiKey = System.getenv("GEMINI_API_KEY"), baseUrl = geminiBaseUrl) + .openAiCompatibleLlm( + model = model, + pricingModel = PricingModel.ALL_YOU_CAN_EAT, + provider = GoogleGenAiModels.PROVIDER, + knowledgeCutoffDate = null, + ) + + @Test + @EnabledIfEnvironmentVariable(named = "GEMINI_API_KEY", matches = ".+") + fun `real Gemini serves the selected model over the OpenAI-compatible endpoint`() { + // Execute + val response = try { + geminiLlm().createMessageSender(LlmOptions()) + .call(listOf(UserMessage("Reply with exactly the word READY.")), emptyList()) + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("GEMINI_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: a successful call proves the gemini model (not gpt-5-mini) reached the endpoint. + assertThat(response.textContent).isNotBlank() + assertThat(response.model) + .withFailMessage("Served model was '%s', expected a gemini model", response.model) + .isNotNull() + .contains("gemini") + } + + @Test + @EnabledIfEnvironmentVariable(named = "GEMINI_API_KEY", matches = ".+") + fun `real Gemini streaming reaches the endpoint with the selected model`() { + // Execute + val chunks = try { + geminiLlm().createMessageStreamer(LlmOptions()) + .stream(listOf(UserMessage("Reply with exactly the word READY.")), emptyList(), emptyList()) + .collectList() + .block() + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("GEMINI_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: streamed content proves the streaming path bound the gemini model (a wrong model + // id would be rejected by the Gemini endpoint before any content streamed). + assertThat(chunks).isNotNull() + assertThat(chunks!!.joinToString("")).isNotBlank() + } +} diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt index 45a882340..41bc77fac 100644 --- a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt @@ -16,15 +16,25 @@ package com.embabel.agent.openai import com.embabel.agent.spi.support.springai.SpringAiLlmService +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.OptionsConverter import com.embabel.common.ai.model.PricingModel import io.mockk.Runs import io.mockk.every import io.mockk.just import io.mockk.mockk +import io.mockk.slot import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.springframework.ai.chat.messages.AssistantMessage +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.Prompt import org.springframework.ai.openai.OpenAiChatModel +import org.springframework.ai.openai.OpenAiChatOptions import org.springframework.beans.factory.ObjectProvider import org.springframework.web.client.RestClient import java.util.function.Supplier @@ -80,4 +90,66 @@ class OpenAiCompatibleModelFactoryTest { assertTrue(llm.model is OpenAiChatModel) } + @Test + fun `factory bakes the service model into the chat model options`() { + // SpringAiLlmService binds the model configured on the ChatModel onto request options, + // so the factory must set the selected model there for it to reach the wire. + val llm = openAiCompatibleLlm( + model = "gemini-2.5-flash", + optionsConverter = OpenAiChatOptionsConverter, + ) + + assertEquals("gemini-2.5-flash", llm.model.options.model) + } + + @Test + fun `message sender prompt includes the service model`() { + // Prepare + val promptSlot = slot() + val chatModel = mockk { + // Mirror the production chat model, which carries the selected model in its options. + every { options } returns OpenAiChatOptions.builder().model("gemini-2.5-flash").build() + every { call(capture(promptSlot)) } returns ChatResponse( + listOf(Generation(AssistantMessage("done"))) + ) + } + val llm = SpringAiLlmService( + name = "gemini-2.5-flash", + provider = "Test", + chatModel = chatModel, + optionsConverter = OpenAiChatOptionsConverter, + ) + + // Execute + llm.createMessageSender(LlmOptions()).call(listOf(UserMessage("Hi")), emptyList()) + + // Verify + assertEquals("gemini-2.5-flash", promptSlot.captured.options.model) + } + + /** + * Creates an OpenAI-compatible [SpringAiLlmService] through the production factory path + * while allowing tests to supply the delegate options converter under test. + */ + private fun openAiCompatibleLlm( + model: String, + optionsConverter: OptionsConverter<*>, + ): SpringAiLlmService { + val mf = OpenAiCompatibleModelFactory( + baseUrl = "http://foobar.example", + apiKey = null, + completionsPath = null, + embeddingsPath = null, + observationRegistry = mockk(), + restClientBuilder = restClientBuilder, + ) + return mf.openAiCompatibleLlm( + model = model, + pricingModel = PricingModel.ALL_YOU_CAN_EAT, + provider = "Test", + knowledgeCutoffDate = null, + optionsConverter = optionsConverter, + ) as SpringAiLlmService + } + } diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt new file mode 100644 index 000000000..68fe12bc9 --- /dev/null +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.openai + +import com.embabel.agent.api.models.OpenAiModels +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.PricingModel +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.opentest4j.TestAbortedException + +/** + * Live regression for the same-provider silent variant of #1815: with real OpenAI, select a + * non-default model and assert the provider reports that model back (via response metadata). + * If model binding regressed, the request would silently run on the OpenAiChatOptions default + * ("gpt-5-mini") and the served model would not match the selection. + * + * Requires OPENAI_API_KEY. Skipped (aborted) if the key lacks access to the chosen model. + */ +class OpenAiServedModelBindingIT { + + @Test + @EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+") + fun `real OpenAI serves the selected model, not the default`() { + // Prepare + val model = OpenAiModels.GPT_54 + val llm = OpenAiCompatibleModelFactory(apiKey = System.getenv("OPENAI_API_KEY"), baseUrl = null) + .openAiCompatibleLlm( + model = model, + pricingModel = PricingModel.ALL_YOU_CAN_EAT, + provider = OpenAiModels.PROVIDER, + knowledgeCutoffDate = null, + ) + + // Execute + val response = try { + llm.createMessageSender(LlmOptions()) + .call(listOf(UserMessage("Reply with exactly the word READY.")), emptyList()) + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("OPENAI_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: the provider served the selected model family, not the gpt-5-mini default. + assertThat(response.textContent).isNotBlank() + assertThat(response.model) + .withFailMessage("Served model was '%s', expected to contain '%s'", response.model, model) + .isNotNull() + .contains(model) + assertThat(response.model).doesNotContain("mini") + } +} + +/** Broad check for a provider "you don't have access to this model" error, so ITs abort rather than fail. */ +internal fun isModelAccessError(ex: Throwable): Boolean { + val message = generateSequence(ex) { it.cause } + .mapNotNull { it.message } + .joinToString(" | ") + .lowercase() + return listOf( + "does not have access to model", + "model_not_found", + "does not exist", + "not found", + "permission", + "invalid model", + "unsupported model", + ).any { message.contains(it) } +}