Skip to content

Commit 95c8b5d

Browse files
milos85vasicclaude
milos85vasic
and
claude
committed
feat: wire real LLM Vision API calls, update Qwen to OpenAI-compatible format
- Qwen provider: switch from old DashScope proprietary API (/api/v1/services/aigc/multimodal-generation/generation with input/output wrapper) to OpenAI-compatible endpoint (/compatible-mode/v1/chat/completions with standard messages/choices) - Qwen request body: use image_url content type with base64 data URI (matches OpenAI format) instead of proprietary image field - Qwen response parsing: parse choices[0].message.content (string) instead of output.choices[0].message.content[0].text (array) - Config: add GEMINI_API_KEY fallback for GOOGLE_API_KEY - Config: add OPENROUTER_API_KEY fallback for OPENAI_API_KEY - Config: add DeepSeekAPIKey and GroqAPIKey fields with env var loading - Config: register deepseek and groq in AvailableProviders() - Tests: update Qwen mock servers to return OpenAI-compatible responses - All existing tests pass (go test ./... -race -count=1) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 321fcbf commit 95c8b5d

3 files changed

Lines changed: 61 additions & 62 deletions

File tree

‎pkg/config/config.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ type Config struct {
3333
AnthropicAPIKey string `json:"-"`
3434
GoogleAPIKey string `json:"-"`
3535
QwenAPIKey string `json:"-"`
36+
DeepSeekAPIKey string `json:"-"`
37+
GroqAPIKey string `json:"-"`
3638

3739
// Provider-specific models.
3840
OpenAIModel string `json:"openai_model,omitempty"`
@@ -77,9 +79,17 @@ func LoadFromEnv() Config {
7779
}
7880

7981
cfg.OpenAIAPIKey = os.Getenv("OPENAI_API_KEY")
82+
if cfg.OpenAIAPIKey == "" {
83+
cfg.OpenAIAPIKey = os.Getenv("OPENROUTER_API_KEY")
84+
}
8085
cfg.AnthropicAPIKey = os.Getenv("ANTHROPIC_API_KEY")
8186
cfg.GoogleAPIKey = os.Getenv("GOOGLE_API_KEY")
87+
if cfg.GoogleAPIKey == "" {
88+
cfg.GoogleAPIKey = os.Getenv("GEMINI_API_KEY")
89+
}
8290
cfg.QwenAPIKey = os.Getenv("QWEN_API_KEY")
91+
cfg.DeepSeekAPIKey = os.Getenv("DEEPSEEK_API_KEY")
92+
cfg.GroqAPIKey = os.Getenv("GROQ_API_KEY")
8393

8494
if v := os.Getenv("HELIX_VISION_OPENAI_MODEL"); v != "" {
8595
cfg.OpenAIModel = v
@@ -158,5 +168,11 @@ func (c Config) AvailableProviders() []string {
158168
if c.QwenAPIKey != "" {
159169
providers = append(providers, "qwen")
160170
}
171+
if c.DeepSeekAPIKey != "" {
172+
providers = append(providers, "deepseek")
173+
}
174+
if c.GroqAPIKey != "" {
175+
providers = append(providers, "groq")
176+
}
161177
return providers
162178
}

‎pkg/llmvision/provider_test.go‎

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -387,19 +387,12 @@ func TestNewQwenProvider_Defaults(t *testing.T) {
387387
func TestQwenProvider_AnalyzeImage_Success(t *testing.T) {
388388
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
389389
assert.Equal(t, "POST", r.Method)
390+
assert.Equal(t, "/chat/completions", r.URL.Path)
390391
assert.Contains(t, r.Header.Get("Authorization"), "Bearer qwen-test")
391392

392393
resp := map[string]any{
393-
"output": map[string]any{
394-
"choices": []map[string]any{
395-
{
396-
"message": map[string]any{
397-
"content": []map[string]string{
398-
{"text": "An editor with toolbar"},
399-
},
400-
},
401-
},
402-
},
394+
"choices": []map[string]any{
395+
{"message": map[string]string{"content": "An editor with toolbar"}},
403396
},
404397
}
405398
w.Header().Set("Content-Type", "application/json")
@@ -422,16 +415,8 @@ func TestQwenProvider_AnalyzeImage_EmptyImage(t *testing.T) {
422415
func TestQwenProvider_CompareImages_Success(t *testing.T) {
423416
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
424417
resp := map[string]any{
425-
"output": map[string]any{
426-
"choices": []map[string]any{
427-
{
428-
"message": map[string]any{
429-
"content": []map[string]string{
430-
{"text": "New dialog appeared"},
431-
},
432-
},
433-
},
434-
},
418+
"choices": []map[string]any{
419+
{"message": map[string]string{"content": "New dialog appeared"}},
435420
},
436421
}
437422
w.Header().Set("Content-Type", "application/json")
@@ -445,15 +430,10 @@ func TestQwenProvider_CompareImages_Success(t *testing.T) {
445430
assert.Equal(t, "New dialog appeared", result)
446431
}
447432

448-
func TestQwenProvider_AnalyzeImage_EmptyOutput(t *testing.T) {
433+
func TestQwenProvider_AnalyzeImage_EmptyChoices(t *testing.T) {
449434
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
450-
resp := map[string]any{
451-
"output": map[string]any{
452-
"choices": []any{},
453-
},
454-
}
455435
w.Header().Set("Content-Type", "application/json")
456-
json.NewEncoder(w).Encode(resp)
436+
json.NewEncoder(w).Encode(map[string]any{"choices": []any{}})
457437
}))
458438
defer server.Close()
459439

‎pkg/llmvision/qwen.go‎

Lines changed: 38 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515
)
1616

1717
const (
18-
qwenDefaultBaseURL = "https://dashscope.aliyuncs.com/api/v1"
18+
qwenDefaultBaseURL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
1919
qwenDefaultModel = "qwen-vl-max"
2020
qwenMaxImageSize = 20 * 1024 * 1024 // 20 MB
2121
)
@@ -82,20 +82,21 @@ func (p *QwenProvider) AnalyzeImage(ctx context.Context, image []byte, prompt st
8282
encoded := base64.StdEncoding.EncodeToString(image)
8383
body := map[string]any{
8484
"model": p.config.Model,
85-
"input": map[string]any{
86-
"messages": []map[string]any{
87-
{
88-
"role": "user",
89-
"content": []map[string]any{
90-
{"text": prompt},
91-
{"image": fmt.Sprintf("data:image/png;base64,%s", encoded)},
85+
"messages": []map[string]any{
86+
{
87+
"role": "user",
88+
"content": []map[string]any{
89+
{"type": "text", "text": prompt},
90+
{
91+
"type": "image_url",
92+
"image_url": map[string]string{
93+
"url": fmt.Sprintf("data:image/png;base64,%s", encoded),
94+
},
9295
},
9396
},
9497
},
9598
},
96-
"parameters": map[string]any{
97-
"max_tokens": p.config.MaxTokens,
98-
},
99+
"max_tokens": p.config.MaxTokens,
99100
}
100101

101102
return p.sendRequest(ctx, body)
@@ -117,21 +118,27 @@ func (p *QwenProvider) CompareImages(ctx context.Context, img1, img2 []byte, pro
117118
enc2 := base64.StdEncoding.EncodeToString(img2)
118119
body := map[string]any{
119120
"model": p.config.Model,
120-
"input": map[string]any{
121-
"messages": []map[string]any{
122-
{
123-
"role": "user",
124-
"content": []map[string]any{
125-
{"text": prompt},
126-
{"image": fmt.Sprintf("data:image/png;base64,%s", enc1)},
127-
{"image": fmt.Sprintf("data:image/png;base64,%s", enc2)},
121+
"messages": []map[string]any{
122+
{
123+
"role": "user",
124+
"content": []map[string]any{
125+
{"type": "text", "text": prompt},
126+
{
127+
"type": "image_url",
128+
"image_url": map[string]string{
129+
"url": fmt.Sprintf("data:image/png;base64,%s", enc1),
130+
},
131+
},
132+
{
133+
"type": "image_url",
134+
"image_url": map[string]string{
135+
"url": fmt.Sprintf("data:image/png;base64,%s", enc2),
136+
},
128137
},
129138
},
130139
},
131140
},
132-
"parameters": map[string]any{
133-
"max_tokens": p.config.MaxTokens,
134-
},
141+
"max_tokens": p.config.MaxTokens,
135142
}
136143

137144
return p.sendRequest(ctx, body)
@@ -144,7 +151,7 @@ func (p *QwenProvider) sendRequest(ctx context.Context, body map[string]any) (st
144151
}
145152

146153
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
147-
p.config.BaseURL+"/services/aigc/multimodal-generation/generation",
154+
p.config.BaseURL+"/chat/completions",
148155
bytes.NewReader(jsonBody))
149156
if err != nil {
150157
return "", fmt.Errorf("failed to create request: %w", err)
@@ -171,22 +178,18 @@ func (p *QwenProvider) sendRequest(ctx context.Context, body map[string]any) (st
171178
}
172179

173180
var result struct {
174-
Output struct {
175-
Choices []struct {
176-
Message struct {
177-
Content []struct {
178-
Text string `json:"text"`
179-
} `json:"content"`
180-
} `json:"message"`
181-
} `json:"choices"`
182-
} `json:"output"`
181+
Choices []struct {
182+
Message struct {
183+
Content string `json:"content"`
184+
} `json:"message"`
185+
} `json:"choices"`
183186
}
184187
if err := json.Unmarshal(respBody, &result); err != nil {
185188
return "", fmt.Errorf("%w: %v", ErrInvalidResponse, err)
186189
}
187-
if len(result.Output.Choices) == 0 || len(result.Output.Choices[0].Message.Content) == 0 {
188-
return "", fmt.Errorf("%w: no content in response", ErrInvalidResponse)
190+
if len(result.Choices) == 0 {
191+
return "", fmt.Errorf("%w: no choices in response", ErrInvalidResponse)
189192
}
190193

191-
return result.Output.Choices[0].Message.Content[0].Text, nil
194+
return result.Choices[0].Message.Content, nil
192195
}

0 commit comments

Comments
 (0)