feat(ai-core): real vision image analysis

- llm: Adapter.Vision sends multimodal chat requests (text + image_url,
  data URLs supported) and logs the call as purpose 'vision'
- gateway /images/analyze now accepts image_url or image_base64 and
  returns real analysis instead of a placeholder message
- main wires the default LLM adapter as the vision analyzer
- httptest coverage for the vision request/response shape
This commit is contained in:
Aska committed 2026-08-21 18:33:44 +08:00
1 parent 2104fc0c6c
commit ac0c2b2c73
5 files changed
+117 -7

No files matched your search

+36
View File
@@ -76,6 +76,42 @@ type CallRecord struct {
Error string
}
// Vision sends a prompt plus an image (URL or data URL) to a multimodal model
// and returns the analysis text.
func (a *Adapter) Vision(ctx context.Context, prompt, imageRef string) (*model.LLMResponse, error) {
start := time.Now()
body := map[string]any{
"model": a.cfg.Model,
"messages": []map[string]any{
{
"role": "user",
"content": []map[string]any{
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": map[string]string{"url": imageRef}},
},
},
},
}
resp := &model.LLMResponse{Model: a.cfg.Model}
err := a.doRequest(ctx, "POST", "/chat/completions", body, func(data map[string]any) error {
if choices, ok := data["choices"].([]any); ok && len(choices) > 0 {
choice := choices[0].(map[string]any)
if msg, ok := choice["message"].(map[string]any); ok {
if content, ok := msg["content"].(string); ok { resp.Content = content }
}
if reason, ok := choice["finish_reason"].(string); ok { resp.FinishReason = reason }
}
if usage, ok := data["usage"].(map[string]any); ok {
if total, ok := usage["total_tokens"].(float64); ok { resp.TokensUsed = int(total) }
}
return nil
})
a.recordCall(start, "vision", resp.TokensUsed, 0, err)
return resp, err
}
// ModelName returns the configured model name.
func (a *Adapter) ModelName() string { return a.cfg.Model }
+37
View File
@@ -0,0 +1,37 @@
package llm
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestVision(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
s := string(body)
if !strings.Contains(s, `"type":"image_url"`) || !strings.Contains(s, "data:image/jpeg;base64,AAAA") {
t.Errorf("unexpected vision request body: %s", s)
}
w.Write([]byte(`{"choices":[{"message":{"content":"图片里是一只猫"},"finish_reason":"stop"}],"usage":{"total_tokens":42}}`))
}))
defer srv.Close()
cfg := DefaultConfig()
cfg.BaseURL = srv.URL
a := NewAdapter(cfg)
resp, err := a.Vision(context.Background(), "这是什么?", "data:image/jpeg;base64,AAAA")
if err != nil {
t.Fatalf("Vision: %v", err)
}
if resp.Content != "图片里是一只猫" {
t.Errorf("content = %q", resp.Content)
}
if resp.TokensUsed != 42 {
t.Errorf("tokens = %d", resp.TokensUsed)
}
}