feat(ai-core): real vision image analysis

- llm: Adapter.Vision sends multimodal chat requests (text + image_url,
  data URLs supported) and logs the call as purpose 'vision'
- gateway /images/analyze now accepts image_url or image_base64 and
  returns real analysis instead of a placeholder message
- main wires the default LLM adapter as the vision analyzer
- httptest coverage for the vision request/response shape
This commit is contained in:
Aska committed 2026-08-21 18:33:44 +08:00
1 parent 2104fc0c6c
commit ac0c2b2c73
5 files changed
+117 -7

No files matched your search

+12
View File
@@ -315,6 +315,7 @@ func main() {
gwDeps.ChatStore = chatStore
}
gwDeps.VoiceSvc = voiceSvc
gwDeps.Vision = visionAdapter{llmSelector.DefaultAdapter()}
wsHub := ws.NewHub(orch)
gwDeps.WSHub = wsHub
@@ -462,6 +463,17 @@ func (a *perfAdapter) History() []map[string]any {
return result
}
// visionAdapter adapts the LLM adapter to api.VisionAnalyzer.
type visionAdapter struct{ llm *llm.Adapter }
func (v visionAdapter) AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error) {
resp, err := v.llm.Vision(ctx, prompt, imageRef)
if err != nil {
return "", err
}
return resp.Content, nil
}
// automationSourceAdapter adapts api.AutomationStore to engine.RuleSource.
type automationSourceAdapter struct{ s *api.AutomationStore }
+36
View File
@@ -76,6 +76,42 @@ type CallRecord struct {
Error string
}
// Vision sends a prompt plus an image (URL or data URL) to a multimodal model
// and returns the analysis text.
func (a *Adapter) Vision(ctx context.Context, prompt, imageRef string) (*model.LLMResponse, error) {
start := time.Now()
body := map[string]any{
"model": a.cfg.Model,
"messages": []map[string]any{
{
"role": "user",
"content": []map[string]any{
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": map[string]string{"url": imageRef}},
},
},
},
}
resp := &model.LLMResponse{Model: a.cfg.Model}
err := a.doRequest(ctx, "POST", "/chat/completions", body, func(data map[string]any) error {
if choices, ok := data["choices"].([]any); ok && len(choices) > 0 {
choice := choices[0].(map[string]any)
if msg, ok := choice["message"].(map[string]any); ok {
if content, ok := msg["content"].(string); ok { resp.Content = content }
}
if reason, ok := choice["finish_reason"].(string); ok { resp.FinishReason = reason }
}
if usage, ok := data["usage"].(map[string]any); ok {
if total, ok := usage["total_tokens"].(float64); ok { resp.TokensUsed = int(total) }
}
return nil
})
a.recordCall(start, "vision", resp.TokensUsed, 0, err)
return resp, err
}
// ModelName returns the configured model name.
func (a *Adapter) ModelName() string { return a.cfg.Model }
+37
View File
@@ -0,0 +1,37 @@
package llm
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestVision(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
s := string(body)
if !strings.Contains(s, `"type":"image_url"`) || !strings.Contains(s, "data:image/jpeg;base64,AAAA") {
t.Errorf("unexpected vision request body: %s", s)
}
w.Write([]byte(`{"choices":[{"message":{"content":"图片里是一只猫"},"finish_reason":"stop"}],"usage":{"total_tokens":42}}`))
}))
defer srv.Close()
cfg := DefaultConfig()
cfg.BaseURL = srv.URL
a := NewAdapter(cfg)
resp, err := a.Vision(context.Background(), "这是什么?", "data:image/jpeg;base64,AAAA")
if err != nil {
t.Fatalf("Vision: %v", err)
}
if resp.Content != "图片里是一只猫" {
t.Errorf("content = %q", resp.Content)
}
if resp.TokensUsed != 42 {
t.Errorf("tokens = %d", resp.TokensUsed)
}
}
+26 -7
View File
@@ -260,16 +260,35 @@ func (r *Router) sessionExportHandler(w http.ResponseWriter, req *http.Request)
func (r *Router) imageAnalyzeHandler(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" { writeJSON(w, 405, M{"error": "POST required"}); return }
if r.deps.Vision == nil { writeJSON(w, 503, M{"error": "vision not available"}); return }
var body struct {
ImageURL string `json:"image_url"`
Prompt string `json:"prompt"`
ImageURL string `json:"image_url"`
ImageBase64 string `json:"image_base64"` // raw base64 or data URL
Prompt string `json:"prompt"`
}
json.NewDecoder(req.Body).Decode(&body)
// Stub: requires multimodal LLM (GPT-4o / Qwen-VL)
writeJSON(w, 200, M{
"analysis": "图片分析需要多模态大模型。当前为预留接口。请配置支持 vision 的 LLM provider。",
"image_url": body.ImageURL, "prompt": body.Prompt,
})
imageRef := body.ImageURL
if imageRef == "" && body.ImageBase64 != "" {
imageRef = body.ImageBase64
if !strings.HasPrefix(imageRef, "data:") {
imageRef = "data:image/jpeg;base64," + imageRef
}
}
if imageRef == "" {
writeJSON(w, 400, M{"error": "missing image_url or image_base64"})
return
}
if body.Prompt == "" {
body.Prompt = "描述这张图片的内容。"
}
analysis, err := r.deps.Vision.AnalyzeImage(req.Context(), body.Prompt, imageRef)
if err != nil {
writeJSON(w, 500, M{"error": err.Error()})
return
}
writeJSON(w, 200, M{"analysis": analysis, "prompt": body.Prompt})
}
func (r *Router) hostExecHandler(w http.ResponseWriter, req *http.Request) {
+6
View File
@@ -91,6 +91,11 @@ type AutomationEngine interface {
Trigger(ctx context.Context, id string) error
}
// VisionAnalyzer analyzes images with a multimodal model.
type VisionAnalyzer interface {
AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error)
}
// ChatStorer persists chat sessions and messages.
type ChatStorer interface {
CreateSession(ctx context.Context, id, userID, title string) (*store.Session, error)
@@ -117,6 +122,7 @@ type Dependencies struct {
Automation *AutomationStore
AutomationEng AutomationEngine
Reminders *ReminderStore
Vision VisionAnalyzer
PluginList []map[string]string
}