From 2fd425111bbf0428b5eec1c2a14cfa40ca7f3a9a Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 12:17:28 -0700 Subject: [PATCH 1/7] #1815 - Fix Gemini OpenAI-compatible guardrail IT sends gpt-5-mini instead of selected Gemini model Bind the resolved service model onto request-level OpenAI chat options so Spring AI 2.0 does not fall back to `gpt-5-mini` for Gemini, MiniMax, LM Studio, Docker, custom OpenAI-compatible, or OpenAI services. Add regression coverage for converter wrapping, factory-created services, and direct OpenAI-compatible autoconfiguration paths. Signed-off-by: Slava Imeshev --- .../models/docker/DockerLocalModelsConfig.kt | 3 +- ...gentDockerModelsAutoConfigurationTest.java | 41 ++++++ .../minimax/MiniMaxOptionsConverterTest.kt | 15 ++ .../models/openai/OpenAiModelsConfig.kt | 3 +- .../AgentOpenAiAutoConfigurationTest.java | 54 +++++++ .../openai/custom/OpenAiCustomModelsConfig.kt | 3 +- ...gentOpenAiCustomAutoConfigurationTest.java | 32 ++++- .../openai/OpenAiCompatibleModelFactory.kt | 3 +- .../com/embabel/agent/openai/converters.kt | 30 ++++ .../OpenAiCompatibleModelFactoryTest.kt | 99 +++++++++++++ .../OpenAiModelBindingOptionsConverterTest.kt | 133 ++++++++++++++++++ 11 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 embabel-agent-autoconfigure/models/embabel-agent-openai-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/AgentOpenAiAutoConfigurationTest.java create mode 100644 embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt index a2787f667..b23f357d4 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt @@ -18,6 +18,7 @@ package com.embabel.agent.config.models.docker import com.embabel.agent.api.models.DockerLocalModels.Companion.PROVIDER import com.embabel.agent.config.models.docker.DockerRetryProperties.Companion.PREFIX import com.embabel.agent.openai.OpenAiChatOptionsConverter +import com.embabel.agent.openai.withOpenAiModel import com.embabel.agent.spi.common.RetryProperties import com.embabel.agent.spi.support.springai.SpringAiLlmService import com.embabel.common.ai.autoconfig.ProviderInitialization @@ -246,7 +247,7 @@ class DockerLocalModelsConfig( name = model.id, chatModel = chatModel, provider = PROVIDER, - optionsConverter = OpenAiChatOptionsConverter, + optionsConverter = OpenAiChatOptionsConverter.withOpenAiModel(model.id), knowledgeCutoffDate = null, pricingModel = PricingModel.ALL_YOU_CAN_EAT, ) diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java index f3fd0b531..1755b59c1 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java @@ -18,11 +18,19 @@ import com.embabel.agent.config.models.docker.DockerConnectionProperties; import com.embabel.agent.config.models.docker.DockerLocalModelsConfig; import com.embabel.agent.config.models.docker.DockerRetryProperties; +import com.embabel.agent.spi.support.springai.SpringAiLlmService; import com.embabel.common.ai.autoconfig.ProviderInitialization; +import com.embabel.common.ai.model.LlmOptions; +import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; +import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -157,6 +165,39 @@ void providerInitializationHasEmptyModelListWhenDockerUnavailable() { }); } + /** + * Verifies that a model discovered from a Docker OpenAI-compatible endpoint binds its discovered id onto request-level options. + */ + @Test + void discoveredDockerModelServiceBindsConfiguredModelOnRequestOptions() throws IOException { + // Prepare + final HttpServer server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + server.createContext("/engines/v1/models", exchange -> { + final byte[] body = """ + {"object":"list","data":[{"id":"docker-test-model"}]} + """.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + try { + // Execute + final String baseUrl = "http://localhost:" + server.getAddress().getPort() + "/engines"; + contextRunner.withPropertyValues("embabel.agent.models.docker.base-url=" + baseUrl).run(context -> { + final SpringAiLlmService service = context.getBean("dockerModel-docker-test-model", SpringAiLlmService.class); + final OpenAiChatOptions options = (OpenAiChatOptions) service.getOptionsConverter().convertOptions(new LlmOptions()); + + // Verify + assertThat(options.getModel()).isEqualTo("docker-test-model"); + }); + } + finally { + server.stop(0); + } + } + @Override public String toString() { diff --git a/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt b/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt index 955f08e91..2662eccae 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt @@ -15,6 +15,7 @@ */ package com.embabel.agent.config.models.minimax +import com.embabel.agent.openai.withOpenAiModel import com.embabel.agent.test.models.OptionsConverterTestSupport import com.embabel.common.ai.model.LlmOptions import org.junit.jupiter.api.Assertions.assertEquals @@ -77,4 +78,18 @@ class MiniMaxOptionsConverterTest : OptionsConverterTestSupport { + final SpringAiLlmService service = context.getBean(beanName, SpringAiLlmService.class); + final OpenAiChatOptions options = (OpenAiChatOptions) service.getOptionsConverter().convertOptions(new LlmOptions()); + + // Verify + assertThat(options.getModel()).isEqualTo(expectedModel); + }); + } +} diff --git a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt index 56bbd900f..4059bc904 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt @@ -18,6 +18,7 @@ package com.embabel.agent.config.models.openai.custom import com.embabel.agent.config.models.openai.custom.OpenAiCustomProperties.Companion.PREFIX import com.embabel.agent.openai.OpenAiCompatibleModelFactory import com.embabel.agent.openai.StandardOpenAiOptionsConverter +import com.embabel.agent.openai.withOpenAiModel import com.embabel.agent.spi.LlmService import com.embabel.agent.spi.common.RetryProperties import com.embabel.agent.spi.support.springai.SpringAiLlmService @@ -209,7 +210,7 @@ class OpenAiCustomModelsConfig( name = modelId, chatModel = chatModel, provider = CUSTOM_PROVIDER, - optionsConverter = StandardOpenAiOptionsConverter, + optionsConverter = StandardOpenAiOptionsConverter.withOpenAiModel(modelId), ) } diff --git a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java index 95be75946..7c3ea898a 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java +++ b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java @@ -16,7 +16,10 @@ package com.embabel.agent.autoconfigure.models.openai.custom; import com.embabel.common.ai.autoconfig.ProviderInitialization; +import com.embabel.agent.spi.support.springai.SpringAiLlmService; +import com.embabel.common.ai.model.LlmOptions; import org.junit.jupiter.api.Test; +import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -42,14 +45,35 @@ class AgentOpenAiCustomAutoConfigurationTest { */ @Test void registersCustomModelAndInitialization() { - // Act + // Prepare + final String modelBeanName = "test-model"; + + // Execute contextRunner.run(context -> { - // Assert + // Verify assertThat(context).hasSingleBean(ProviderInitialization.class); - assertThat(context).hasBean("test-model"); + assertThat(context).hasBean(modelBeanName); assertThat(context.getBean(ProviderInitialization.class).getRegisteredLlms()) .extracting(registeredModel -> registeredModel.getBeanName()) - .contains("test-model"); + .contains(modelBeanName); + }); + } + + /** + * Verifies that custom OpenAI-compatible models bind their declared model id onto request-level options. + */ + @Test + void customModelServiceBindsConfiguredModelOnRequestOptions() { + // Prepare + final String modelBeanName = "test-model"; + + // Execute + contextRunner.run(context -> { + final SpringAiLlmService service = context.getBean(modelBeanName, SpringAiLlmService.class); + final OpenAiChatOptions options = (OpenAiChatOptions) service.getOptionsConverter().convertOptions(new LlmOptions()); + + // Verify + assertThat(options.getModel()).isEqualTo(modelBeanName); }); } } diff --git a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt index 0c0c32fb3..e3ba57364 100644 --- a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt +++ b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt @@ -266,7 +266,7 @@ open class OpenAiCompatibleModelFactory( name = model, chatModel = chatModelOf(model), provider = provider, - optionsConverter = optionsConverter, + optionsConverter = optionsConverter.withOpenAiModel(model), pricingModel = pricingModel, knowledgeCutoffDate = knowledgeCutoffDate, ) @@ -377,7 +377,6 @@ object OpenAiChatOptionsConverter : OptionsConverter { .maxTokens(options.maxTokens) .presencePenalty(options.presencePenalty) .frequencyPenalty(options.frequencyPenalty) - .topP(options.topP) //.streamUsage(true) additional feature note .build() } diff --git a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt index bc68e1d6f..3f56af20b 100644 --- a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt +++ b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt @@ -15,11 +15,41 @@ */ package com.embabel.agent.openai +import com.embabel.agent.spi.support.springai.SpringAiLlmService import com.embabel.common.ai.model.LlmOptions import com.embabel.common.ai.model.OptionsConverter import com.embabel.common.util.loggerFor +import org.springframework.ai.openai.OpenAiChatModel import org.springframework.ai.openai.OpenAiChatOptions +/** + * Wrap this converter so every request-level [OpenAiChatOptions] carries the resolved + * OpenAI-compatible wire model id for its [SpringAiLlmService]. + * + * Spring AI 2.0 does not merge an [OpenAiChatModel]'s default model when runtime + * [OpenAiChatOptions] are present, so OpenAI-compatible providers need the model bound + * directly onto the request options produced by Embabel converters. + * + * @param model provider wire model id to force onto every converted request. + * @throws IllegalArgumentException if [model] is blank or the delegate does not return + * [OpenAiChatOptions]. + */ +fun OptionsConverter<*>.withOpenAiModel(model: String): OptionsConverter { + require(model.isNotBlank()) { + "OpenAI-compatible model must not be blank" + } + return OptionsConverter { options -> + val converted: Any = convertOptions(options) + require(converted is OpenAiChatOptions) { + val type = converted.let { it::class.qualifiedName } ?: "null" + "OpenAI-compatible options converter must return OpenAiChatOptions for model '$model', but returned $type" + } + converted.mutate() + .model(model) + .build() + } +} + /** * Options converter for GPT-5 models that don't support temperature adjustment. */ diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt index 45a882340..40684ccc0 100644 --- a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt @@ -16,15 +16,25 @@ package com.embabel.agent.openai import com.embabel.agent.spi.support.springai.SpringAiLlmService +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.OptionsConverter import com.embabel.common.ai.model.PricingModel import io.mockk.Runs import io.mockk.every import io.mockk.just import io.mockk.mockk +import io.mockk.slot import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.springframework.ai.chat.messages.AssistantMessage +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.Prompt import org.springframework.ai.openai.OpenAiChatModel +import org.springframework.ai.openai.OpenAiChatOptions import org.springframework.beans.factory.ObjectProvider import org.springframework.web.client.RestClient import java.util.function.Supplier @@ -80,4 +90,93 @@ class OpenAiCompatibleModelFactoryTest { assertTrue(llm.model is OpenAiChatModel) } + @Test + fun `request options include the service model`() { + // Prepare + val llm = openAiCompatibleLlm( + model = "gemini-2.5-flash", + optionsConverter = OpenAiChatOptionsConverter, + ) + + // Execute + val chatOptions = llm.optionsConverter.convertOptions(LlmOptions()) as OpenAiChatOptions + + // Verify + assertEquals("gemini-2.5-flash", chatOptions.model) + } + + @Test + fun `request options preserve delegate values when binding service model`() { + // Prepare + val llm = openAiCompatibleLlm( + model = "gemini-2.5-flash", + optionsConverter = OpenAiChatOptionsConverter, + ) + val llmOptions = LlmOptions() + .withTemperature(0.7) + .withTopP(0.9) + .withMaxTokens(1000) + .withPresencePenalty(0.5) + .withFrequencyPenalty(0.3) + + // Execute + val chatOptions = llm.optionsConverter.convertOptions(llmOptions) as OpenAiChatOptions + + // Verify + assertEquals("gemini-2.5-flash", chatOptions.model) + assertEquals(0.7, chatOptions.temperature) + assertEquals(0.9, chatOptions.topP) + assertEquals(1000, chatOptions.maxTokens) + assertEquals(0.5, chatOptions.presencePenalty) + assertEquals(0.3, chatOptions.frequencyPenalty) + } + + @Test + fun `message sender prompt includes the service model`() { + // Prepare + val promptSlot = slot() + val chatModel = mockk { + every { call(capture(promptSlot)) } returns ChatResponse( + listOf(Generation(AssistantMessage("done"))) + ) + } + val llm = SpringAiLlmService( + name = "gemini-2.5-flash", + provider = "Test", + chatModel = chatModel, + optionsConverter = OpenAiChatOptionsConverter.withOpenAiModel("gemini-2.5-flash"), + ) + + // Execute + llm.createMessageSender(LlmOptions()).call(listOf(UserMessage("Hi")), emptyList()) + + // Verify + assertEquals("gemini-2.5-flash", promptSlot.captured.options.model) + } + + /** + * Creates an OpenAI-compatible [SpringAiLlmService] through the production factory path + * while allowing tests to supply the delegate options converter under test. + */ + private fun openAiCompatibleLlm( + model: String, + optionsConverter: OptionsConverter<*>, + ): SpringAiLlmService { + val mf = OpenAiCompatibleModelFactory( + baseUrl = "http://foobar.example", + apiKey = null, + completionsPath = null, + embeddingsPath = null, + observationRegistry = mockk(), + restClientBuilder = restClientBuilder, + ) + return mf.openAiCompatibleLlm( + model = model, + pricingModel = PricingModel.ALL_YOU_CAN_EAT, + provider = "Test", + knowledgeCutoffDate = null, + optionsConverter = optionsConverter, + ) as SpringAiLlmService + } + } diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt new file mode 100644 index 000000000..5054d3e6d --- /dev/null +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt @@ -0,0 +1,133 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.openai + +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.OptionsConverter +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.ai.chat.prompt.ChatOptions +import org.springframework.ai.model.tool.ToolCallingChatOptions +import java.lang.reflect.Proxy + +/** + * Regression tests for [withOpenAiModel], which pins the resolved service model onto + * request-level OpenAI chat options while preserving delegate converter output. + */ +class OpenAiModelBindingOptionsConverterTest { + + @Test + fun `binds model for OpenAI compatible converter and preserves options`() { + // Prepare + val converter = OpenAiChatOptionsConverter.withOpenAiModel("gemini-2.5-flash") + val llmOptions = LlmOptions() + .withTemperature(0.7) + .withTopP(0.9) + .withMaxTokens(1000) + .withPresencePenalty(0.5) + .withFrequencyPenalty(0.3) + + // Execute + val chatOptions = converter.convertOptions(llmOptions) + + // Verify + assertEquals("gemini-2.5-flash", chatOptions.model) + assertEquals(0.7, chatOptions.temperature) + assertEquals(0.9, chatOptions.topP) + assertEquals(1000, chatOptions.maxTokens) + assertEquals(0.5, chatOptions.presencePenalty) + assertEquals(0.3, chatOptions.frequencyPenalty) + } + + @Test + fun `binds model for standard OpenAI converter`() { + // Prepare + val converter = StandardOpenAiOptionsConverter.withOpenAiModel("gpt-4.1") + + // Execute + val chatOptions = converter.convertOptions(LlmOptions().withTemperature(0.4)) + + // Verify + assertEquals("gpt-4.1", chatOptions.model) + assertEquals(0.4, chatOptions.temperature) + } + + @Test + fun `binds model for GPT-5 converter while preserving temperature omission`() { + // Prepare + val converter = Gpt5ChatOptionsConverter.withOpenAiModel("gpt-5") + + // Execute + val chatOptions = converter.convertOptions(LlmOptions().withTemperature(0.4)) + + // Verify + assertEquals("gpt-5", chatOptions.model) + val temperature: Double? = chatOptions.temperature + assertNull(temperature) + } + + @Test + fun `rejects blank model`() { + // Prepare + val blankModel = " " + + // Execute + val thrown = assertThrows(IllegalArgumentException::class.java) { + OpenAiChatOptionsConverter.withOpenAiModel(blankModel) + } + + // Verify + assertEquals("OpenAI-compatible model must not be blank", thrown.message) + } + + @Test + fun `rejects non OpenAI chat options`() { + // Prepare + val converter = OptionsConverter { + ToolCallingChatOptions.builder().build() + } + + // Execute + val thrown = assertThrows(IllegalArgumentException::class.java) { + converter.withOpenAiModel("gemini-2.5-flash").convertOptions(LlmOptions()) + } + + // Verify + assertTrue(thrown.message!!.contains("gemini-2.5-flash")) + assertTrue(thrown.message!!.contains("ToolCallingChatOptions")) + } + + @Test + fun `rejects null converted options without masking the error`() { + // Prepare + val converter = Proxy.newProxyInstance( + OptionsConverter::class.java.classLoader, + arrayOf(OptionsConverter::class.java), + ) { _, _, _ -> null } as OptionsConverter + + // Execute + val thrown = assertThrows(IllegalArgumentException::class.java) { + converter.withOpenAiModel("gemini-2.5-flash").convertOptions(LlmOptions()) + } + + // Verify + assertTrue(thrown.message!!.contains("gemini-2.5-flash")) + assertTrue(thrown.message!!.contains("null")) + } +} From 9162b3991ace5ac4b3935e22ebf4cd8da1dae837 Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 12:59:00 -0700 Subject: [PATCH 2/7] #1815 - Fix nullability possibility reported by Github feedback. Signed-off-by: Slava Imeshev --- .../src/main/kotlin/com/embabel/agent/openai/converters.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt index 3f56af20b..2a64d0e58 100644 --- a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt +++ b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt @@ -39,9 +39,9 @@ fun OptionsConverter<*>.withOpenAiModel(model: String): OptionsConverter - val converted: Any = convertOptions(options) + val converted: Any? = convertOptions(options) require(converted is OpenAiChatOptions) { - val type = converted.let { it::class.qualifiedName } ?: "null" + val type = converted?.let { it::class.qualifiedName } ?: "null" "OpenAI-compatible options converter must return OpenAiChatOptions for model '$model', but returned $type" } converted.mutate() From 276baf32311759c34000232114abea08d6ec56d5 Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 14:26:31 -0700 Subject: [PATCH 3/7] #1815 - Test fix: Docker local models failing to register without OPENAI_API_KEY DockerLocalModelsConfig.dockerLlmOf built its OpenAiChatModel with only the sync client: OpenAiChatModel.builder() .openAiClient(openAiClient) // sync client only ... .build() In Spring AI 2.0, OpenAiChatModel.Builder.build() also requires an async client. When .openAiClientAsync(...) is not supplied, the builder falls back to OpenAiSetup.setupAsyncClient(...), which constructs a fresh client and demands a credential from the environment (OPENAI_API_KEY). Docker local endpoints have no API key, so on any environment where OPENAI_API_KEY is unset (including CI) that fallback threw: IllegalStateException: At least one credential source must be specified: credential (apiKey), workloadIdentity, or adminApiKey The per-model catch in dockerLocalModelsInitializer swallowed the exception ("Failed to register Docker model ..."), so the model bean was never registered. This was a latent production bug: any real Docker model would silently fail to register whenever OPENAI_API_KEY was absent. The new AgentDockerModelsAutoConfigurationTest regression test made the failure visible, failing at context.getBean("dockerModel-docker-test-model") with NoSuchBeanDefinitionException. Root cause is a Spring AI 2.0 migration gap. The old OpenAiApi was replaced by the openai-java SDK, which has separate sync/async clients. OpenAiCompatibleModelFactory already handles this correctly by building both clients from resolved credentials and wiring both into the chat model; the Docker config was only updated to build the sync client and so inherited the async-fallback trap. Fix mirrors the factory: build an openAiClientAsync with the same "no-auth" placeholder key and wire .openAiClientAsync(openAiClientAsync) into the OpenAiChatModel builder, keeping model construction independent of ambient environment credentials. Signed-off-by: Slava Imeshev --- .../models/docker/DockerLocalModelsConfig.kt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt index b23f357d4..98cdbd7a0 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt @@ -26,7 +26,9 @@ import com.embabel.common.ai.autoconfig.RegisteredModel import com.embabel.common.ai.model.* import com.embabel.common.util.ExcludeFromJacocoGeneratedReport import com.openai.client.OpenAIClient +import com.openai.client.OpenAIClientAsync import com.openai.client.okhttp.OpenAIOkHttpClient +import com.openai.client.okhttp.OpenAIOkHttpClientAsync import io.micrometer.observation.ObservationRegistry import org.slf4j.LoggerFactory import org.springframework.ai.document.MetadataMode @@ -141,6 +143,20 @@ class DockerLocalModelsConfig( .build() } + /** + * Async counterpart to [openAiClient]. Spring AI 2.0's `OpenAiChatModel.Builder.build()` + * builds an async client via `OpenAiSetup.setupAsyncClient(...)` unless one is supplied, + * and that fallback requires a credential from the environment (`OPENAI_API_KEY`). Docker + * local endpoints have no key, so we build the async client explicitly with the same + * placeholder and wire it into the chat model to stay credential-independent. + */ + private val openAiClientAsync: OpenAIClientAsync by lazy { + OpenAIOkHttpClientAsync.builder() + .baseUrl(dockerConnectionProperties.baseUrl) + .apiKey("no-auth") + .build() + } + private fun loadModels(): List = try { val restClient = RestClient.create() @@ -231,6 +247,7 @@ class DockerLocalModelsConfig( private fun dockerLlmOf(model: Model): SpringAiLlmService { val chatModel = OpenAiChatModel.builder() .openAiClient(openAiClient) + .openAiClientAsync(openAiClientAsync) .observationRegistry(observationRegistry.getIfUnique { ObservationRegistry.NOOP }) .toolCallingManager( ToolCallingManager.builder() From 71340b8c5d6fb3282628a285c5874da2900bf6cc Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 21:30:10 -0700 Subject: [PATCH 4/7] #1815 - Bind selected model generically in SpringAiLlmService Spring AI 2.0 stopped merging a ChatModel's configured options into a prompt that already carries per-request options: OpenAiChatModel and AnthropicChatModel buildRequestPrompt return the prompt unchanged when getOptions() != null. Embabel always supplies per-request options via its OptionsConverter, and provider option types coerce a null model to a baked-in default in their constructor (OpenAiChatOptions -> gpt-5-mini, AnthropicChatOptions -> claude-haiku-4-5). Net effect: every call ignored the selected model. This is the same root cause as #1815 (OpenAI/Gemini) and it also affects Anthropic and any OpenAI-compatible provider. Fix it once at the single chokepoint. SpringAiLlmService now binds the model configured on the underlying ChatModel (chatModel.getOptions().getModel()) onto the converted request options in both createMessageSender and createMessageStreamer: ``` internal fun bindModel(options: ChatOptions, model: String?): ChatOptions = if (model.isNullOrBlank()) options else options.mutate().model(model).build() ``` - Source is the bean's configured model, not the service name, so there is no name == wire-id assumption; it restores the pre-2.0 merge semantics. - mutate() dispatches to each provider's overridden builder, preserving the concrete option type (OpenAiChatOptions/AnthropicChatOptions) and all fields. - Null/blank guard leaves converter output untouched for models that expose no configured default (bare doubles, exotic providers). The per-site withOpenAiModel() wrappers (the #1815 fix) are now redundant but harmless: they bind model = modelId, then this layer re-binds the identical id read from the same bean, so the operation is idempotent. They can be retired in a follow-up; left in place here to keep the change focused. Tests (test-first): SpringAiLlmServiceModelBindingTest reproduces the bug (red: expected gpt-4.1 but was gpt-5-mini) then passes, plus direct bindModel unit tests; AnthropicModelBindingTest proves end-to-end that the selected Sonnet model reaches the wire via the unchanged AnthropicOptionsConverter. Updated three tests whose strict ChatModel mocks now need a getOptions() stub. Signed-off-by: Slava Imeshev --- .../anthropic/AnthropicModelBindingTest.kt | 81 +++++++++++ .../loop/UnfoldingToolInjectionStrategy.kt | 1 - .../support/springai/SpringAiLlmService.kt | 29 +++- .../SpringAiLlmServiceModelBindingTest.kt | 133 ++++++++++++++++++ .../springai/SpringAiLlmServiceTest.kt | 7 +- .../OpenAiCompatibleModelFactoryTest.kt | 2 + 6 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt create mode 100644 embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt diff --git a/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt new file mode 100644 index 000000000..6dadd0bc4 --- /dev/null +++ b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.anthropic + +import com.embabel.agent.api.models.AnthropicModels +import com.embabel.agent.spi.support.springai.SpringAiLlmService +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.ai.anthropic.AnthropicChatOptions +import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage +import org.springframework.ai.chat.metadata.ChatResponseMetadata +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.Prompt + +/** + * End-to-end regression for the Anthropic half of the Spring AI 2.0 model-binding bug + * (see #1815 for the OpenAI/Gemini equivalent). + * + * [AnthropicOptionsConverter] never sets a model, so its [AnthropicChatOptions] carry the + * SDK's baked-in `DEFAULT_MODEL` (`claude-haiku-4-5`). Because Embabel always passes these + * per-request options and Spring AI 2.0 no longer merges the model bean's configured model, + * every call would silently target Haiku. [SpringAiLlmService] now binds the configured + * model generically; this test proves the selected model reaches the wire while the concrete + * [AnthropicChatOptions] type and other fields are preserved. + */ +class AnthropicModelBindingTest { + + @Test + fun `sends the selected Anthropic model rather than the converter default`() { + // Prepare: bean configured with Sonnet; converter (unchanged) bakes in Haiku default. + val configuredOptions = AnthropicChatOptions.builder().model("claude-sonnet-4-5").build() + val capturedPrompt = slot() + val generation = Generation(SpringAiAssistantMessage("done")) + val chatResponse = mockk { + every { result } returns generation + every { results } returns listOf(generation) + every { metadata } returns mockk { every { usage } returns null } + } + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "claude-sonnet-4-5", + provider = AnthropicModels.PROVIDER, + chatModel = chatModel, + optionsConverter = AnthropicOptionsConverter, + thinkingSupported = true, + ) + + // Execute + service.createMessageSender(LlmOptions().withMaxTokens(500)) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: concrete Anthropic options carry the selected model, maxTokens preserved. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions).isInstanceOf(AnthropicChatOptions::class.java) + assertThat(sentOptions.model).isEqualTo("claude-sonnet-4-5") + assertThat(sentOptions.maxTokens).isEqualTo(500) + } +} diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt index c4bb7769d..f77b155bb 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/UnfoldingToolInjectionStrategy.kt @@ -139,4 +139,3 @@ class UnfoldingToolInjectionStrategy : ToolInjectionStrategy { val INSTANCE = UnfoldingToolInjectionStrategy() } } - diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt index 8f5fe2e41..f4a24bf99 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmService.kt @@ -28,6 +28,7 @@ import org.springframework.ai.chat.client.ChatClient import org.springframework.ai.chat.messages.UserMessage import org.springframework.ai.chat.model.ChatModel import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.prompt.ChatOptions import org.springframework.ai.chat.prompt.Prompt import reactor.core.publisher.Flux import java.time.Duration @@ -123,8 +124,22 @@ data class SpringAiLlmService @JvmOverloads constructor( */ override val model: ChatModel get() = chatModel + /** + * Binds the model configured on the underlying [ChatModel] onto per-request options. + * + * Spring AI 2.0 no longer merges a model's configured options into a prompt that already + * carries options (OpenAiChatModel/AnthropicChatModel `buildRequestPrompt` returns the + * prompt unchanged when `getOptions() != null`). Because Embabel always supplies + * per-request options — and provider option types coerce a null model to a hard-coded + * default in their constructor (e.g. `gpt-5-mini`, `claude-haiku-4-5`) — the selected + * model would otherwise be ignored on every call. Binding the [ChatModel]'s own + * configured model restores the pre-2.0 merge semantics for every provider at once. + */ + private fun bindConfiguredModel(chatOptions: ChatOptions): ChatOptions = + bindModel(chatOptions, chatModel.options?.model) + override fun createMessageSender(options: LlmOptions): LlmMessageSender { - val chatOptions = optionsConverter.convertOptions(options) + val chatOptions = bindConfiguredModel(optionsConverter.convertOptions(options)) return SpringAiLlmMessageSender( chatModel = chatModel, chatOptions = chatOptions, @@ -136,7 +151,7 @@ data class SpringAiLlmService @JvmOverloads constructor( } override fun createMessageStreamer(options: LlmOptions): LlmMessageStreamer { - val chatOptions = optionsConverter.convertOptions(options) + val chatOptions = bindConfiguredModel(optionsConverter.convertOptions(options)) val chatClient = ChatClient.create(chatModel) return SpringAiLlmMessageStreamer(chatClient, chatOptions) } @@ -160,3 +175,13 @@ data class SpringAiLlmService @JvmOverloads constructor( fun withOptionsConverter(converter: OptionsConverter<*>): SpringAiLlmService = copy(optionsConverter = converter) } + +/** + * Binds [model] onto [options], preserving the concrete provider option type and all other + * fields via each provider's overridden `mutate()` (dynamic dispatch keeps e.g. + * `OpenAiChatOptions`/`AnthropicChatOptions` intact). Returns [options] unchanged when + * [model] is null or blank, so models that expose no configured default (bare test doubles, + * exotic providers) keep the converter's output. + */ +internal fun bindModel(options: ChatOptions, model: String?): ChatOptions = + if (model.isNullOrBlank()) options else options.mutate().model(model).build() diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt new file mode 100644 index 000000000..7246d2a55 --- /dev/null +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt @@ -0,0 +1,133 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.spi.support.springai + +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.OptionsConverter +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage +import org.springframework.ai.chat.metadata.ChatResponseMetadata +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.ChatOptions +import org.springframework.ai.chat.prompt.Prompt + +/** + * Regression tests for the generic model-binding performed by [SpringAiLlmService]. + * + * Spring AI 2.0 stopped merging a [ChatModel]'s configured options into a prompt that + * already carries per-request options (see OpenAiChatModel/AnthropicChatModel + * `buildRequestPrompt`). Embabel always supplies per-request options via its + * [OptionsConverter], and those options carry the provider's baked-in default model + * (e.g. `gpt-5-mini`, `claude-haiku-4-5`). Without re-binding, every call would ignore + * the selected model. [SpringAiLlmService] closes this generically by binding the model + * configured on the underlying [ChatModel] onto the converted request options. + */ +class SpringAiLlmServiceModelBindingTest { + + private fun chatResponseStub(): ChatResponse { + val generation = Generation(SpringAiAssistantMessage("done")) + return mockk { + every { result } returns generation + every { results } returns listOf(generation) + every { metadata } returns mockk { every { usage } returns null } + } + } + + @Test + fun `createMessageSender binds the chat model's configured model onto the request`() { + // Prepare: the bean is configured with the selected model, but the converter + // returns options carrying a different (provider-default) model. + val configuredOptions = ChatOptions.builder().model("gpt-4.1").build() + val converterOutput = ChatOptions.builder() + .model("gpt-5-mini") + .temperature(0.5) + .build() + val capturedPrompt = slot() + val chatResponse = chatResponseStub() + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "gpt-4.1", + provider = "Test", + chatModel = chatModel, + optionsConverter = OptionsConverter { converterOutput }, + ) + + // Execute + service.createMessageSender(LlmOptions()) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: the request carries the configured model, other options preserved. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions.model).isEqualTo("gpt-4.1") + assertThat(sentOptions.temperature).isEqualTo(0.5) + } + + @Test + fun `bindModel overrides the converted model and preserves other fields`() { + // Prepare + val converted = ChatOptions.builder() + .model("gpt-5-mini") + .temperature(0.5) + .maxTokens(1000) + .topP(0.9) + .build() + + // Execute + val bound = bindModel(converted, "gpt-4.1") + + // Verify + assertThat(bound.model).isEqualTo("gpt-4.1") + assertThat(bound.temperature).isEqualTo(0.5) + assertThat(bound.maxTokens).isEqualTo(1000) + assertThat(bound.topP).isEqualTo(0.9) + } + + @Test + fun `bindModel passes options through unchanged when configured model is null`() { + // Prepare + val converted = ChatOptions.builder().model("converter-model").build() + + // Execute + val bound = bindModel(converted, null) + + // Verify + assertThat(bound).isSameAs(converted) + assertThat(bound.model).isEqualTo("converter-model") + } + + @Test + fun `bindModel passes options through unchanged when configured model is blank`() { + // Prepare + val converted = ChatOptions.builder().model("converter-model").build() + + // Execute + val bound = bindModel(converted, " ") + + // Verify + assertThat(bound).isSameAs(converted) + assertThat(bound.model).isEqualTo("converter-model") + } +} diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt index 5eccb4de9..6e01f838a 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceTest.kt @@ -24,6 +24,7 @@ import com.embabel.common.ai.model.OptionsConverter import com.embabel.common.ai.model.PricingModel import com.embabel.common.ai.prompt.KnowledgeCutoffDate import com.embabel.common.ai.prompt.PromptContributor +import io.mockk.every import io.mockk.mockk import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Nested @@ -34,7 +35,11 @@ import java.time.LocalDate class SpringAiLlmServiceTest { - private val mockChatModel: ChatModel = mockk() + private val mockChatModel: ChatModel = mockk { + // SpringAiLlmService reads the model configured on the ChatModel to bind it onto + // request options; empty options (null model) make binding a no-op for these tests. + every { options } returns ChatOptions.builder().build() + } @Nested inner class ConstructorTests { diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt index 40684ccc0..dcfb5537d 100644 --- a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt @@ -136,6 +136,8 @@ class OpenAiCompatibleModelFactoryTest { // Prepare val promptSlot = slot() val chatModel = mockk { + // Mirror the production chat model, which carries the selected model in its options. + every { options } returns OpenAiChatOptions.builder().model("gemini-2.5-flash").build() every { call(capture(promptSlot)) } returns ChatResponse( listOf(Generation(AssistantMessage("done"))) ) From d8742825f68559d8059784944645e399003f1d0c Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 21:45:05 -0700 Subject: [PATCH 5/7] #1815 - Remove per-site withOpenAiModel wrapper; rely on generic model binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SpringAiLlmService now binds the selected model onto request options for every provider (reading it from the ChatModel's configured options), so the OpenAI-compatible withOpenAiModel() wrapper is redundant: each call site was just re-binding, idempotently, the same model id the generic layer already reads from the same bean. Two mechanisms doing one job is worse than one. Production: - Delete the withOpenAiModel extension and pass the raw converter at all four call sites: OpenAiCompatibleModelFactory (Gemini/MiniMax/LM Studio/BYOK), OpenAiModelsConfig, OpenAiCustomModelsConfig, DockerLocalModelsConfig. Tests: - Delete OpenAiModelBindingOptionsConverterTest (covered the wrapper; its override/field-preservation coverage now lives in SpringAiLlmServiceModelBindingTest). - OpenAiCompatibleModelFactoryTest: replace the two converter-level binding tests with one asserting the load-bearing invariant — the factory bakes the model into ChatModel.getOptions(), which the generic binder reads. - Retarget the Docker/OpenAI/OpenAI-custom autoconfig regression tests to assert service.getChatModel().getOptions().getModel() instead of converter output, keeping config-site coverage at the correct layer. - Drop the wrapper test from MiniMaxOptionsConverterTest. No behavior change: binding moves from the converter to the shared call-time path. Model construction still bakes the selected model into every ChatModel bean, so the wire model is unchanged. Signed-off-by: Slava Imeshev --- .../models/docker/DockerLocalModelsConfig.kt | 3 +- ...gentDockerModelsAutoConfigurationTest.java | 11 +- .../minimax/MiniMaxOptionsConverterTest.kt | 15 -- .../models/openai/OpenAiModelsConfig.kt | 3 +- .../AgentOpenAiAutoConfigurationTest.java | 8 +- .../openai/custom/OpenAiCustomModelsConfig.kt | 3 +- ...gentOpenAiCustomAutoConfigurationTest.java | 8 +- .../openai/OpenAiCompatibleModelFactory.kt | 2 +- .../com/embabel/agent/openai/converters.kt | 30 ---- .../OpenAiCompatibleModelFactoryTest.kt | 39 +---- .../OpenAiModelBindingOptionsConverterTest.kt | 133 ------------------ 11 files changed, 20 insertions(+), 235 deletions(-) delete mode 100644 embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt index 98cdbd7a0..4aee72e13 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/docker/DockerLocalModelsConfig.kt @@ -18,7 +18,6 @@ package com.embabel.agent.config.models.docker import com.embabel.agent.api.models.DockerLocalModels.Companion.PROVIDER import com.embabel.agent.config.models.docker.DockerRetryProperties.Companion.PREFIX import com.embabel.agent.openai.OpenAiChatOptionsConverter -import com.embabel.agent.openai.withOpenAiModel import com.embabel.agent.spi.common.RetryProperties import com.embabel.agent.spi.support.springai.SpringAiLlmService import com.embabel.common.ai.autoconfig.ProviderInitialization @@ -264,7 +263,7 @@ class DockerLocalModelsConfig( name = model.id, chatModel = chatModel, provider = PROVIDER, - optionsConverter = OpenAiChatOptionsConverter.withOpenAiModel(model.id), + optionsConverter = OpenAiChatOptionsConverter, knowledgeCutoffDate = null, pricingModel = PricingModel.ALL_YOU_CAN_EAT, ) diff --git a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java index 1755b59c1..bdce9ca47 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java +++ b/embabel-agent-autoconfigure/models/embabel-agent-dockermodels-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/docker/AgentDockerModelsAutoConfigurationTest.java @@ -20,10 +20,8 @@ import com.embabel.agent.config.models.docker.DockerRetryProperties; import com.embabel.agent.spi.support.springai.SpringAiLlmService; import com.embabel.common.ai.autoconfig.ProviderInitialization; -import com.embabel.common.ai.model.LlmOptions; import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; -import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -166,7 +164,8 @@ void providerInitializationHasEmptyModelListWhenDockerUnavailable() { } /** - * Verifies that a model discovered from a Docker OpenAI-compatible endpoint binds its discovered id onto request-level options. + * Verifies that a model discovered from a Docker OpenAI-compatible endpoint configures its discovered id on the chat model, which + * {@link SpringAiLlmService} then binds onto request-level options at call time. */ @Test void discoveredDockerModelServiceBindsConfiguredModelOnRequestOptions() throws IOException { @@ -187,10 +186,10 @@ void discoveredDockerModelServiceBindsConfiguredModelOnRequestOptions() throws I final String baseUrl = "http://localhost:" + server.getAddress().getPort() + "/engines"; contextRunner.withPropertyValues("embabel.agent.models.docker.base-url=" + baseUrl).run(context -> { final SpringAiLlmService service = context.getBean("dockerModel-docker-test-model", SpringAiLlmService.class); - final OpenAiChatOptions options = (OpenAiChatOptions) service.getOptionsConverter().convertOptions(new LlmOptions()); - // Verify - assertThat(options.getModel()).isEqualTo("docker-test-model"); + // Verify: the discovered id is configured on the chat model, which SpringAiLlmService + // binds onto request options at call time. + assertThat(service.getChatModel().getOptions().getModel()).isEqualTo("docker-test-model"); }); } finally { diff --git a/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt b/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt index 2662eccae..955f08e91 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-minimax-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/minimax/MiniMaxOptionsConverterTest.kt @@ -15,7 +15,6 @@ */ package com.embabel.agent.config.models.minimax -import com.embabel.agent.openai.withOpenAiModel import com.embabel.agent.test.models.OptionsConverterTestSupport import com.embabel.common.ai.model.LlmOptions import org.junit.jupiter.api.Assertions.assertEquals @@ -78,18 +77,4 @@ class MiniMaxOptionsConverterTest : OptionsConverterTestSupport { final SpringAiLlmService service = context.getBean(beanName, SpringAiLlmService.class); - final OpenAiChatOptions options = (OpenAiChatOptions) service.getOptionsConverter().convertOptions(new LlmOptions()); // Verify - assertThat(options.getModel()).isEqualTo(expectedModel); + assertThat(service.getChatModel().getOptions().getModel()).isEqualTo(expectedModel); }); } } diff --git a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt index 4059bc904..56bbd900f 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/main/kotlin/com/embabel/agent/config/models/openai/custom/OpenAiCustomModelsConfig.kt @@ -18,7 +18,6 @@ package com.embabel.agent.config.models.openai.custom import com.embabel.agent.config.models.openai.custom.OpenAiCustomProperties.Companion.PREFIX import com.embabel.agent.openai.OpenAiCompatibleModelFactory import com.embabel.agent.openai.StandardOpenAiOptionsConverter -import com.embabel.agent.openai.withOpenAiModel import com.embabel.agent.spi.LlmService import com.embabel.agent.spi.common.RetryProperties import com.embabel.agent.spi.support.springai.SpringAiLlmService @@ -210,7 +209,7 @@ class OpenAiCustomModelsConfig( name = modelId, chatModel = chatModel, provider = CUSTOM_PROVIDER, - optionsConverter = StandardOpenAiOptionsConverter.withOpenAiModel(modelId), + optionsConverter = StandardOpenAiOptionsConverter, ) } diff --git a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java index 7c3ea898a..71fc8b509 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java +++ b/embabel-agent-autoconfigure/models/embabel-agent-openai-custom-autoconfigure/src/test/java/com/embabel/agent/autoconfigure/models/openai/custom/AgentOpenAiCustomAutoConfigurationTest.java @@ -17,9 +17,7 @@ import com.embabel.common.ai.autoconfig.ProviderInitialization; import com.embabel.agent.spi.support.springai.SpringAiLlmService; -import com.embabel.common.ai.model.LlmOptions; import org.junit.jupiter.api.Test; -import org.springframework.ai.openai.OpenAiChatOptions; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; @@ -60,7 +58,8 @@ void registersCustomModelAndInitialization() { } /** - * Verifies that custom OpenAI-compatible models bind their declared model id onto request-level options. + * Verifies that custom OpenAI-compatible models configure their declared model id on the chat model, which + * {@link SpringAiLlmService} binds onto request-level options at call time. */ @Test void customModelServiceBindsConfiguredModelOnRequestOptions() { @@ -70,10 +69,9 @@ void customModelServiceBindsConfiguredModelOnRequestOptions() { // Execute contextRunner.run(context -> { final SpringAiLlmService service = context.getBean(modelBeanName, SpringAiLlmService.class); - final OpenAiChatOptions options = (OpenAiChatOptions) service.getOptionsConverter().convertOptions(new LlmOptions()); // Verify - assertThat(options.getModel()).isEqualTo(modelBeanName); + assertThat(service.getChatModel().getOptions().getModel()).isEqualTo(modelBeanName); }); } } diff --git a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt index e3ba57364..9c99dbaf7 100644 --- a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt +++ b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactory.kt @@ -266,7 +266,7 @@ open class OpenAiCompatibleModelFactory( name = model, chatModel = chatModelOf(model), provider = provider, - optionsConverter = optionsConverter.withOpenAiModel(model), + optionsConverter = optionsConverter, pricingModel = pricingModel, knowledgeCutoffDate = knowledgeCutoffDate, ) diff --git a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt index 2a64d0e58..bc68e1d6f 100644 --- a/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt +++ b/embabel-agent-openai/src/main/kotlin/com/embabel/agent/openai/converters.kt @@ -15,41 +15,11 @@ */ package com.embabel.agent.openai -import com.embabel.agent.spi.support.springai.SpringAiLlmService import com.embabel.common.ai.model.LlmOptions import com.embabel.common.ai.model.OptionsConverter import com.embabel.common.util.loggerFor -import org.springframework.ai.openai.OpenAiChatModel import org.springframework.ai.openai.OpenAiChatOptions -/** - * Wrap this converter so every request-level [OpenAiChatOptions] carries the resolved - * OpenAI-compatible wire model id for its [SpringAiLlmService]. - * - * Spring AI 2.0 does not merge an [OpenAiChatModel]'s default model when runtime - * [OpenAiChatOptions] are present, so OpenAI-compatible providers need the model bound - * directly onto the request options produced by Embabel converters. - * - * @param model provider wire model id to force onto every converted request. - * @throws IllegalArgumentException if [model] is blank or the delegate does not return - * [OpenAiChatOptions]. - */ -fun OptionsConverter<*>.withOpenAiModel(model: String): OptionsConverter { - require(model.isNotBlank()) { - "OpenAI-compatible model must not be blank" - } - return OptionsConverter { options -> - val converted: Any? = convertOptions(options) - require(converted is OpenAiChatOptions) { - val type = converted?.let { it::class.qualifiedName } ?: "null" - "OpenAI-compatible options converter must return OpenAiChatOptions for model '$model', but returned $type" - } - converted.mutate() - .model(model) - .build() - } -} - /** * Options converter for GPT-5 models that don't support temperature adjustment. */ diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt index dcfb5537d..41bc77fac 100644 --- a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiCompatibleModelFactoryTest.kt @@ -91,44 +91,15 @@ class OpenAiCompatibleModelFactoryTest { } @Test - fun `request options include the service model`() { - // Prepare + fun `factory bakes the service model into the chat model options`() { + // SpringAiLlmService binds the model configured on the ChatModel onto request options, + // so the factory must set the selected model there for it to reach the wire. val llm = openAiCompatibleLlm( model = "gemini-2.5-flash", optionsConverter = OpenAiChatOptionsConverter, ) - // Execute - val chatOptions = llm.optionsConverter.convertOptions(LlmOptions()) as OpenAiChatOptions - - // Verify - assertEquals("gemini-2.5-flash", chatOptions.model) - } - - @Test - fun `request options preserve delegate values when binding service model`() { - // Prepare - val llm = openAiCompatibleLlm( - model = "gemini-2.5-flash", - optionsConverter = OpenAiChatOptionsConverter, - ) - val llmOptions = LlmOptions() - .withTemperature(0.7) - .withTopP(0.9) - .withMaxTokens(1000) - .withPresencePenalty(0.5) - .withFrequencyPenalty(0.3) - - // Execute - val chatOptions = llm.optionsConverter.convertOptions(llmOptions) as OpenAiChatOptions - - // Verify - assertEquals("gemini-2.5-flash", chatOptions.model) - assertEquals(0.7, chatOptions.temperature) - assertEquals(0.9, chatOptions.topP) - assertEquals(1000, chatOptions.maxTokens) - assertEquals(0.5, chatOptions.presencePenalty) - assertEquals(0.3, chatOptions.frequencyPenalty) + assertEquals("gemini-2.5-flash", llm.model.options.model) } @Test @@ -146,7 +117,7 @@ class OpenAiCompatibleModelFactoryTest { name = "gemini-2.5-flash", provider = "Test", chatModel = chatModel, - optionsConverter = OpenAiChatOptionsConverter.withOpenAiModel("gemini-2.5-flash"), + optionsConverter = OpenAiChatOptionsConverter, ) // Execute diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt deleted file mode 100644 index 5054d3e6d..000000000 --- a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiModelBindingOptionsConverterTest.kt +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2024-2026 Embabel Pty Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.embabel.agent.openai - -import com.embabel.common.ai.model.LlmOptions -import com.embabel.common.ai.model.OptionsConverter -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Assertions.assertThrows -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.springframework.ai.chat.prompt.ChatOptions -import org.springframework.ai.model.tool.ToolCallingChatOptions -import java.lang.reflect.Proxy - -/** - * Regression tests for [withOpenAiModel], which pins the resolved service model onto - * request-level OpenAI chat options while preserving delegate converter output. - */ -class OpenAiModelBindingOptionsConverterTest { - - @Test - fun `binds model for OpenAI compatible converter and preserves options`() { - // Prepare - val converter = OpenAiChatOptionsConverter.withOpenAiModel("gemini-2.5-flash") - val llmOptions = LlmOptions() - .withTemperature(0.7) - .withTopP(0.9) - .withMaxTokens(1000) - .withPresencePenalty(0.5) - .withFrequencyPenalty(0.3) - - // Execute - val chatOptions = converter.convertOptions(llmOptions) - - // Verify - assertEquals("gemini-2.5-flash", chatOptions.model) - assertEquals(0.7, chatOptions.temperature) - assertEquals(0.9, chatOptions.topP) - assertEquals(1000, chatOptions.maxTokens) - assertEquals(0.5, chatOptions.presencePenalty) - assertEquals(0.3, chatOptions.frequencyPenalty) - } - - @Test - fun `binds model for standard OpenAI converter`() { - // Prepare - val converter = StandardOpenAiOptionsConverter.withOpenAiModel("gpt-4.1") - - // Execute - val chatOptions = converter.convertOptions(LlmOptions().withTemperature(0.4)) - - // Verify - assertEquals("gpt-4.1", chatOptions.model) - assertEquals(0.4, chatOptions.temperature) - } - - @Test - fun `binds model for GPT-5 converter while preserving temperature omission`() { - // Prepare - val converter = Gpt5ChatOptionsConverter.withOpenAiModel("gpt-5") - - // Execute - val chatOptions = converter.convertOptions(LlmOptions().withTemperature(0.4)) - - // Verify - assertEquals("gpt-5", chatOptions.model) - val temperature: Double? = chatOptions.temperature - assertNull(temperature) - } - - @Test - fun `rejects blank model`() { - // Prepare - val blankModel = " " - - // Execute - val thrown = assertThrows(IllegalArgumentException::class.java) { - OpenAiChatOptionsConverter.withOpenAiModel(blankModel) - } - - // Verify - assertEquals("OpenAI-compatible model must not be blank", thrown.message) - } - - @Test - fun `rejects non OpenAI chat options`() { - // Prepare - val converter = OptionsConverter { - ToolCallingChatOptions.builder().build() - } - - // Execute - val thrown = assertThrows(IllegalArgumentException::class.java) { - converter.withOpenAiModel("gemini-2.5-flash").convertOptions(LlmOptions()) - } - - // Verify - assertTrue(thrown.message!!.contains("gemini-2.5-flash")) - assertTrue(thrown.message!!.contains("ToolCallingChatOptions")) - } - - @Test - fun `rejects null converted options without masking the error`() { - // Prepare - val converter = Proxy.newProxyInstance( - OptionsConverter::class.java.classLoader, - arrayOf(OptionsConverter::class.java), - ) { _, _, _ -> null } as OptionsConverter - - // Execute - val thrown = assertThrows(IllegalArgumentException::class.java) { - converter.withOpenAiModel("gemini-2.5-flash").convertOptions(LlmOptions()) - } - - // Verify - assertTrue(thrown.message!!.contains("gemini-2.5-flash")) - assertTrue(thrown.message!!.contains("null")) - } -} From 97cd9ccb15337a0990ec01dea03b7028ab67b147 Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 22:51:34 -0700 Subject: [PATCH 6/7] #1815 - Add real-API integration tests proving the selected model reaches the wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests with mocks can prove request options carry the right model, but not that the provider actually served it. These live ITs close that gap and guard both variants of #1815: - cross-endpoint (Gemini via OpenAI-compat): a wrong model id is rejected by the endpoint, so a successful call is itself the proof; - same-provider silent (OpenAI, Anthropic): both the selected and the default model are valid, so the only tell is the model the provider reports back. To assert the served model, surface it: LlmMessageResponse gains a `model` field, populated in SpringAiLlmMessageSender from the response metadata (also useful observability — which model actually answered). ITs (gated by *_API_KEY, aborted rather than failed when the key lacks model access, so CI-safe): - OpenAiServedModelBindingIT: served model is the selected gpt-5.4, not the gpt-5-mini OpenAiChatOptions default. - GeminiServedModelBindingIT: blocking + streaming calls reach the Gemini OpenAI-compatible endpoint with a gemini model (the reported #1815 case). - AnthropicServedModelBindingIT: served a sonnet, not the claude-haiku-4-5 default. All four pass against live OpenAI/Gemini/Anthropic APIs. Reading response metadata.model tripped strict ChatResponseMetadata mocks that only stubbed usage; made those mocks relaxed in the four affected test files so getModel() returns "" instead of throwing. Signed-off-by: Slava Imeshev --- .../anthropic/AnthropicModelBindingTest.kt | 2 +- .../agent/spi/loop/LlmMessageSender.kt | 3 + .../springai/SpringAiLlmMessageSender.kt | 1 + .../springai/SpringAiLlmMessageSenderTest.kt | 18 +++--- .../SpringAiLlmServiceModelBindingTest.kt | 62 ++++++++++++++++++- 5 files changed, 75 insertions(+), 11 deletions(-) diff --git a/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt index 6dadd0bc4..a44460f5b 100644 --- a/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt +++ b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicModelBindingTest.kt @@ -54,7 +54,7 @@ class AnthropicModelBindingTest { val chatResponse = mockk { every { result } returns generation every { results } returns listOf(generation) - every { metadata } returns mockk { every { usage } returns null } + every { metadata } returns mockk(relaxed = true) { every { usage } returns null } } val chatModel = mockk { every { options } returns configuredOptions diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt index 7f0a51181..c98d626a5 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/loop/LlmMessageSender.kt @@ -68,11 +68,14 @@ data class LlmMessageRequest @JvmOverloads constructor( * @param message The full message object from the LLM * @param textContent The text content of the message * @param usage Optional usage information (tokens, etc.) + * @param model The model that actually served the request, as reported by the provider + * (from the response metadata). Null/blank if the provider does not report it. */ data class LlmMessageResponse( val message: Message, val textContent: String, val usage: Usage? = null, + val model: String? = null, ) /** diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt index a3e0f89dc..904713906 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt @@ -111,6 +111,7 @@ internal class SpringAiLlmMessageSender( message = embabelMessage, textContent = assistantMessage.text ?: "", usage = usage, + model = response.metadata?.model, ) } diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt index 8c0a005cc..facecaf3c 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt @@ -51,7 +51,7 @@ class SpringAiLlmMessageSenderTest { val configuredOptions = testChatOptions() val capturedPrompt = slot() val generation = Generation(SpringAiAssistantMessage("done")) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { @@ -106,7 +106,7 @@ class SpringAiLlmMessageSenderTest { val configuredOptions = testChatOptions() val capturedPrompt = slot() val generation = Generation(SpringAiAssistantMessage("done")) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { @@ -157,7 +157,7 @@ class SpringAiLlmMessageSenderTest { val originalOptions = testChatOptions() val capturedPrompt = slot() val generation = Generation(SpringAiAssistantMessage("done")) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { @@ -240,7 +240,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -313,7 +313,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -374,7 +374,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -439,7 +439,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -503,7 +503,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns mockk(relaxed = true) } val chatResponse = mockk { @@ -581,7 +581,7 @@ class SpringAiLlmMessageSenderTest { .build() ) - val mockMetadata = mockk { + val mockMetadata = mockk(relaxed = true) { every { usage } returns null } val chatResponse = mockk { diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt index 7246d2a55..e77871102 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmServiceModelBindingTest.kt @@ -30,6 +30,8 @@ import org.springframework.ai.chat.model.ChatResponse import org.springframework.ai.chat.model.Generation import org.springframework.ai.chat.prompt.ChatOptions import org.springframework.ai.chat.prompt.Prompt +import org.springframework.ai.model.tool.ToolCallingChatOptions +import reactor.core.publisher.Flux /** * Regression tests for the generic model-binding performed by [SpringAiLlmService]. @@ -49,7 +51,7 @@ class SpringAiLlmServiceModelBindingTest { return mockk { every { result } returns generation every { results } returns listOf(generation) - every { metadata } returns mockk { every { usage } returns null } + every { metadata } returns mockk(relaxed = true) { every { usage } returns null } } } @@ -85,6 +87,64 @@ class SpringAiLlmServiceModelBindingTest { assertThat(sentOptions.temperature).isEqualTo(0.5) } + @Test + fun `binds model onto a generic ToolCallingChatOptions converter (Bedrock-shape)`() { + // Bedrock's converter returns a generic ToolCallingChatOptions carrying no model; the + // model lives only on the ChatModel. Verify binding sets it while preserving the type. + val configuredOptions = ToolCallingChatOptions.builder().model("anthropic.claude-sonnet").build() + val converterOutput = ToolCallingChatOptions.builder().build() + val capturedPrompt = slot() + val chatResponse = chatResponseStub() + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "anthropic.claude-sonnet", + provider = "Bedrock", + chatModel = chatModel, + optionsConverter = OptionsConverter { converterOutput }, + ) + + // Execute + service.createMessageSender(LlmOptions()) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: concrete generic type preserved and the configured model bound. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions).isInstanceOf(ToolCallingChatOptions::class.java) + assertThat(sentOptions.model).isEqualTo("anthropic.claude-sonnet") + } + + @Test + fun `createMessageStreamer binds the configured model onto the streamed request`() { + // Prepare + val configuredOptions = ChatOptions.builder().model("gpt-4.1").build() + val converterOutput = ChatOptions.builder().model("gpt-5-mini").build() + val capturedPrompt = slot() + // Use a real ChatResponse: the ChatClient streaming aggregator reads metadata the mock + // stub does not provide. + val chatModel = mockk { + every { options } returns configuredOptions + every { stream(capture(capturedPrompt)) } returns + Flux.just(ChatResponse(listOf(Generation(SpringAiAssistantMessage("chunk"))))) + } + val service = SpringAiLlmService( + name = "gpt-4.1", + provider = "Test", + chatModel = chatModel, + optionsConverter = OptionsConverter { converterOutput }, + ) + + // Execute + service.createMessageStreamer(LlmOptions()) + .stream(messages = listOf(UserMessage("Hi")), tools = emptyList(), toolCallInspectors = emptyList()) + .blockLast() + + // Verify + assertThat(capturedPrompt.captured.options.model).isEqualTo("gpt-4.1") + } + @Test fun `bindModel overrides the converted model and preserves other fields`() { // Prepare From 7164aa21667c7a589303913b5a1b62323d83eaa5 Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 22:52:13 -0700 Subject: [PATCH 7/7] #1815 - Add real-API integration tests proving the selected model reaches the wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests with mocks can prove request options carry the right model, but not that the provider actually served it. These live ITs close that gap and guard both variants of #1815: - cross-endpoint (Gemini via OpenAI-compat): a wrong model id is rejected by the endpoint, so a successful call is itself the proof; - same-provider silent (OpenAI, Anthropic): both the selected and the default model are valid, so the only tell is the model the provider reports back. To assert the served model, surface it: LlmMessageResponse gains a `model` field, populated in SpringAiLlmMessageSender from the response metadata (also useful observability — which model actually answered). ITs (gated by *_API_KEY, aborted rather than failed when the key lacks model access, so CI-safe): - OpenAiServedModelBindingIT: served model is the selected gpt-5.4, not the gpt-5-mini OpenAiChatOptions default. - GeminiServedModelBindingIT: blocking + streaming calls reach the Gemini OpenAI-compatible endpoint with a gemini model (the reported #1815 case). - AnthropicServedModelBindingIT: served a sonnet, not the claude-haiku-4-5 default. All four pass against live OpenAI/Gemini/Anthropic APIs. Reading response metadata.model tripped strict ChatResponseMetadata mocks that only stubbed usage; made those mocks relaxed in the four affected test files so getModel() returns "" instead of throwing. Signed-off-by: Slava Imeshev --- .../AnthropicServedModelBindingIT.kt | 78 ++++++++++++++++ .../models/ollama/OllamaModelBindingTest.kt | 75 +++++++++++++++ .../openai/GeminiServedModelBindingIT.kt | 93 +++++++++++++++++++ .../openai/OpenAiServedModelBindingIT.kt | 86 +++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt create mode 100644 embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt create mode 100644 embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt create mode 100644 embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt diff --git a/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt new file mode 100644 index 000000000..8df9d172c --- /dev/null +++ b/embabel-agent-anthropic/src/test/kotlin/com/embabel/agent/anthropic/AnthropicServedModelBindingIT.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.anthropic + +import com.embabel.agent.api.models.AnthropicModels +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.opentest4j.TestAbortedException + +/** + * Live regression for the Anthropic same-provider silent variant of #1815: select Sonnet and + * assert Anthropic reports a sonnet model back. If binding regressed, the request would run on + * the AnthropicChatOptions default ("claude-haiku-4-5") and the served model would be a haiku. + * + * Requires ANTHROPIC_API_KEY. Skipped (aborted) if the key lacks access to the chosen model. + */ +class AnthropicServedModelBindingIT { + + @Test + @EnabledIfEnvironmentVariable(named = "ANTHROPIC_API_KEY", matches = ".+") + fun `real Anthropic serves the selected model, not the default`() { + // Prepare + val model = AnthropicModels.CLAUDE_SONNET_4_5 + val llm = AnthropicModelFactory(apiKey = System.getenv("ANTHROPIC_API_KEY")).build(model) + + // Execute + val response = try { + llm.createMessageSender(LlmOptions()) + .call(listOf(UserMessage("Reply with exactly the word READY.")), emptyList()) + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("ANTHROPIC_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: Anthropic served the selected Sonnet, not the claude-haiku-4-5 default. + assertThat(response.textContent).isNotBlank() + assertThat(response.model) + .withFailMessage("Served model was '%s', expected a sonnet model", response.model) + .isNotNull() + .contains("sonnet") + assertThat(response.model).doesNotContain("haiku") + } + + /** Broad check for a provider "no access to this model" error, so the IT aborts rather than fails. */ + private fun isModelAccessError(ex: Throwable): Boolean { + val message = generateSequence(ex) { it.cause } + .mapNotNull { it.message } + .joinToString(" | ") + .lowercase() + return listOf( + "not_found", + "does not exist", + "not found", + "permission", + "invalid model", + "unsupported model", + "does not have access", + ).any { message.contains(it) } + } +} diff --git a/embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt b/embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt new file mode 100644 index 000000000..25431a46d --- /dev/null +++ b/embabel-agent-autoconfigure/models/embabel-agent-ollama-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/ollama/OllamaModelBindingTest.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.config.models.ollama + +import com.embabel.agent.spi.support.springai.SpringAiLlmService +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage +import org.springframework.ai.chat.metadata.ChatResponseMetadata +import org.springframework.ai.chat.model.ChatModel +import org.springframework.ai.chat.model.ChatResponse +import org.springframework.ai.chat.model.Generation +import org.springframework.ai.chat.prompt.Prompt +import org.springframework.ai.ollama.api.OllamaChatOptions + +/** + * End-to-end regression that a native (non-OpenAI-compatible) provider reaches the wire with + * the selected model. [OllamaOptionsConverter] never sets a model, so before generic binding + * in [SpringAiLlmService] its [OllamaChatOptions] carried no model. This proves the model + * configured on the [ChatModel] is bound onto request options while the concrete + * [OllamaChatOptions] type and other fields survive. + */ +class OllamaModelBindingTest { + + @Test + fun `sends the selected Ollama model rather than the converter default`() { + // Prepare: bean configured with a specific Ollama model; converter sets no model. + val configuredOptions = OllamaChatOptions.builder().model("llama3.2").build() + val capturedPrompt = slot() + val generation = Generation(SpringAiAssistantMessage("done")) + val chatResponse = mockk { + every { result } returns generation + every { results } returns listOf(generation) + every { metadata } returns mockk(relaxed = true) { every { usage } returns null } + } + val chatModel = mockk { + every { options } returns configuredOptions + every { call(capture(capturedPrompt)) } returns chatResponse + } + val service = SpringAiLlmService( + name = "llama3.2", + provider = "Ollama", + chatModel = chatModel, + optionsConverter = OllamaOptionsConverter, + ) + + // Execute + service.createMessageSender(LlmOptions().withTopP(0.8)) + .call(messages = listOf(UserMessage("Hi")), tools = emptyList()) + + // Verify: concrete Ollama options carry the selected model, topP preserved. + val sentOptions = capturedPrompt.captured.options + assertThat(sentOptions).isInstanceOf(OllamaChatOptions::class.java) + assertThat(sentOptions.model).isEqualTo("llama3.2") + assertThat(sentOptions.topP).isEqualTo(0.8) + } +} diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt new file mode 100644 index 000000000..aa3cc94ac --- /dev/null +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/GeminiServedModelBindingIT.kt @@ -0,0 +1,93 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.openai + +import com.embabel.agent.api.models.GoogleGenAiModels +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.PricingModel +import com.embabel.agent.spi.LlmService +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.opentest4j.TestAbortedException + +/** + * Live regression for the reported #1815 scenario: Gemini via the OpenAI-compatible path. This + * is the cross-endpoint variant — the Gemini endpoint *rejects* an OpenAI model id, so if model + * binding regressed (sending "gpt-5-mini"), the call would error. A successful call that reports + * a gemini model back is therefore the regression guard, on both the blocking and streaming paths. + * + * Requires GEMINI_API_KEY. Skipped (aborted) if the key lacks access to the chosen model. + */ +class GeminiServedModelBindingIT { + + private val geminiBaseUrl = "https://generativelanguage.googleapis.com/v1beta/openai" + private val model = GoogleGenAiModels.GEMINI_2_5_FLASH + + private fun geminiLlm(): LlmService<*> = + OpenAiCompatibleModelFactory(apiKey = System.getenv("GEMINI_API_KEY"), baseUrl = geminiBaseUrl) + .openAiCompatibleLlm( + model = model, + pricingModel = PricingModel.ALL_YOU_CAN_EAT, + provider = GoogleGenAiModels.PROVIDER, + knowledgeCutoffDate = null, + ) + + @Test + @EnabledIfEnvironmentVariable(named = "GEMINI_API_KEY", matches = ".+") + fun `real Gemini serves the selected model over the OpenAI-compatible endpoint`() { + // Execute + val response = try { + geminiLlm().createMessageSender(LlmOptions()) + .call(listOf(UserMessage("Reply with exactly the word READY.")), emptyList()) + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("GEMINI_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: a successful call proves the gemini model (not gpt-5-mini) reached the endpoint. + assertThat(response.textContent).isNotBlank() + assertThat(response.model) + .withFailMessage("Served model was '%s', expected a gemini model", response.model) + .isNotNull() + .contains("gemini") + } + + @Test + @EnabledIfEnvironmentVariable(named = "GEMINI_API_KEY", matches = ".+") + fun `real Gemini streaming reaches the endpoint with the selected model`() { + // Execute + val chunks = try { + geminiLlm().createMessageStreamer(LlmOptions()) + .stream(listOf(UserMessage("Reply with exactly the word READY.")), emptyList(), emptyList()) + .collectList() + .block() + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("GEMINI_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: streamed content proves the streaming path bound the gemini model (a wrong model + // id would be rejected by the Gemini endpoint before any content streamed). + assertThat(chunks).isNotNull() + assertThat(chunks!!.joinToString("")).isNotBlank() + } +} diff --git a/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt new file mode 100644 index 000000000..68fe12bc9 --- /dev/null +++ b/embabel-agent-openai/src/test/kotlin/com/embabel/agent/openai/OpenAiServedModelBindingIT.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.openai + +import com.embabel.agent.api.models.OpenAiModels +import com.embabel.chat.UserMessage +import com.embabel.common.ai.model.LlmOptions +import com.embabel.common.ai.model.PricingModel +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.opentest4j.TestAbortedException + +/** + * Live regression for the same-provider silent variant of #1815: with real OpenAI, select a + * non-default model and assert the provider reports that model back (via response metadata). + * If model binding regressed, the request would silently run on the OpenAiChatOptions default + * ("gpt-5-mini") and the served model would not match the selection. + * + * Requires OPENAI_API_KEY. Skipped (aborted) if the key lacks access to the chosen model. + */ +class OpenAiServedModelBindingIT { + + @Test + @EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+") + fun `real OpenAI serves the selected model, not the default`() { + // Prepare + val model = OpenAiModels.GPT_54 + val llm = OpenAiCompatibleModelFactory(apiKey = System.getenv("OPENAI_API_KEY"), baseUrl = null) + .openAiCompatibleLlm( + model = model, + pricingModel = PricingModel.ALL_YOU_CAN_EAT, + provider = OpenAiModels.PROVIDER, + knowledgeCutoffDate = null, + ) + + // Execute + val response = try { + llm.createMessageSender(LlmOptions()) + .call(listOf(UserMessage("Reply with exactly the word READY.")), emptyList()) + } catch (ex: Exception) { + if (isModelAccessError(ex)) { + throw TestAbortedException("OPENAI_API_KEY is set but lacks access to $model", ex) + } + throw ex + } + + // Verify: the provider served the selected model family, not the gpt-5-mini default. + assertThat(response.textContent).isNotBlank() + assertThat(response.model) + .withFailMessage("Served model was '%s', expected to contain '%s'", response.model, model) + .isNotNull() + .contains(model) + assertThat(response.model).doesNotContain("mini") + } +} + +/** Broad check for a provider "you don't have access to this model" error, so ITs abort rather than fail. */ +internal fun isModelAccessError(ex: Throwable): Boolean { + val message = generateSequence(ex) { it.cause } + .mapNotNull { it.message } + .joinToString(" | ") + .lowercase() + return listOf( + "does not have access to model", + "model_not_found", + "does not exist", + "not found", + "permission", + "invalid model", + "unsupported model", + ).any { message.contains(it) } +}