From ac0c2b2c730ece46daf38cd65d5e4a9cd17b682c Mon Sep 17 00:00:00 2001 From: Aska Date: Fri, 21 Aug 2026 18:33:44 +0800 Subject: [PATCH] feat(ai-core): real vision image analysis - llm: Adapter.Vision sends multimodal chat requests (text + image_url, data URLs supported) and logs the call as purpose 'vision' - gateway /images/analyze now accepts image_url or image_base64 and returns real analysis instead of a placeholder message - main wires the default LLM adapter as the vision analyzer - httptest coverage for the vision request/response shape --- backend/cmd/cyrene/main.go | 12 ++++++ backend/services/ai-core/llm/llm.go | 36 ++++++++++++++++++ backend/services/ai-core/llm/llm_test.go | 37 +++++++++++++++++++ backend/services/gateway/api/handlers_more.go | 33 +++++++++++++---- backend/services/gateway/api/router.go | 6 +++ 5 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 backend/services/ai-core/llm/llm_test.go diff --git a/backend/cmd/cyrene/main.go b/backend/cmd/cyrene/main.go index ae1316f..6004645 100644 --- a/backend/cmd/cyrene/main.go +++ b/backend/cmd/cyrene/main.go @@ -315,6 +315,7 @@ func main() { gwDeps.ChatStore = chatStore } gwDeps.VoiceSvc = voiceSvc + gwDeps.Vision = visionAdapter{llmSelector.DefaultAdapter()} wsHub := ws.NewHub(orch) gwDeps.WSHub = wsHub @@ -462,6 +463,17 @@ func (a *perfAdapter) History() []map[string]any { return result } +// visionAdapter adapts the LLM adapter to api.VisionAnalyzer. +type visionAdapter struct{ llm *llm.Adapter } + +func (v visionAdapter) AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error) { + resp, err := v.llm.Vision(ctx, prompt, imageRef) + if err != nil { + return "", err + } + return resp.Content, nil +} + // automationSourceAdapter adapts api.AutomationStore to engine.RuleSource. type automationSourceAdapter struct{ s *api.AutomationStore } diff --git a/backend/services/ai-core/llm/llm.go b/backend/services/ai-core/llm/llm.go index c0ea633..f1b7b6b 100644 --- a/backend/services/ai-core/llm/llm.go +++ b/backend/services/ai-core/llm/llm.go @@ -76,6 +76,42 @@ type CallRecord struct { Error string } +// Vision sends a prompt plus an image (URL or data URL) to a multimodal model +// and returns the analysis text. +func (a *Adapter) Vision(ctx context.Context, prompt, imageRef string) (*model.LLMResponse, error) { + start := time.Now() + body := map[string]any{ + "model": a.cfg.Model, + "messages": []map[string]any{ + { + "role": "user", + "content": []map[string]any{ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": map[string]string{"url": imageRef}}, + }, + }, + }, + } + + resp := &model.LLMResponse{Model: a.cfg.Model} + err := a.doRequest(ctx, "POST", "/chat/completions", body, func(data map[string]any) error { + if choices, ok := data["choices"].([]any); ok && len(choices) > 0 { + choice := choices[0].(map[string]any) + if msg, ok := choice["message"].(map[string]any); ok { + if content, ok := msg["content"].(string); ok { resp.Content = content } + } + if reason, ok := choice["finish_reason"].(string); ok { resp.FinishReason = reason } + } + if usage, ok := data["usage"].(map[string]any); ok { + if total, ok := usage["total_tokens"].(float64); ok { resp.TokensUsed = int(total) } + } + return nil + }) + + a.recordCall(start, "vision", resp.TokensUsed, 0, err) + return resp, err +} + // ModelName returns the configured model name. func (a *Adapter) ModelName() string { return a.cfg.Model } diff --git a/backend/services/ai-core/llm/llm_test.go b/backend/services/ai-core/llm/llm_test.go new file mode 100644 index 0000000..3ab10c1 --- /dev/null +++ b/backend/services/ai-core/llm/llm_test.go @@ -0,0 +1,37 @@ +package llm + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestVision(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + s := string(body) + if !strings.Contains(s, `"type":"image_url"`) || !strings.Contains(s, "data:image/jpeg;base64,AAAA") { + t.Errorf("unexpected vision request body: %s", s) + } + w.Write([]byte(`{"choices":[{"message":{"content":"图片里是一只猫"},"finish_reason":"stop"}],"usage":{"total_tokens":42}}`)) + })) + defer srv.Close() + + cfg := DefaultConfig() + cfg.BaseURL = srv.URL + a := NewAdapter(cfg) + + resp, err := a.Vision(context.Background(), "这是什么？", "data:image/jpeg;base64,AAAA") + if err != nil { + t.Fatalf("Vision: %v", err) + } + if resp.Content != "图片里是一只猫" { + t.Errorf("content = %q", resp.Content) + } + if resp.TokensUsed != 42 { + t.Errorf("tokens = %d", resp.TokensUsed) + } +} diff --git a/backend/services/gateway/api/handlers_more.go b/backend/services/gateway/api/handlers_more.go index 478da26..0ae4950 100644 --- a/backend/services/gateway/api/handlers_more.go +++ b/backend/services/gateway/api/handlers_more.go @@ -260,16 +260,35 @@ func (r *Router) sessionExportHandler(w http.ResponseWriter, req *http.Request) func (r *Router) imageAnalyzeHandler(w http.ResponseWriter, req *http.Request) { if req.Method != "POST" { writeJSON(w, 405, M{"error": "POST required"}); return } + if r.deps.Vision == nil { writeJSON(w, 503, M{"error": "vision not available"}); return } var body struct { - ImageURL string `json:"image_url"` - Prompt string `json:"prompt"` + ImageURL string `json:"image_url"` + ImageBase64 string `json:"image_base64"` // raw base64 or data URL + Prompt string `json:"prompt"` } json.NewDecoder(req.Body).Decode(&body) - // Stub: requires multimodal LLM (GPT-4o / Qwen-VL) - writeJSON(w, 200, M{ - "analysis": "图片分析需要多模态大模型。当前为预留接口。请配置支持 vision 的 LLM provider。", - "image_url": body.ImageURL, "prompt": body.Prompt, - }) + + imageRef := body.ImageURL + if imageRef == "" && body.ImageBase64 != "" { + imageRef = body.ImageBase64 + if !strings.HasPrefix(imageRef, "data:") { + imageRef = "data:image/jpeg;base64," + imageRef + } + } + if imageRef == "" { + writeJSON(w, 400, M{"error": "missing image_url or image_base64"}) + return + } + if body.Prompt == "" { + body.Prompt = "描述这张图片的内容。" + } + + analysis, err := r.deps.Vision.AnalyzeImage(req.Context(), body.Prompt, imageRef) + if err != nil { + writeJSON(w, 500, M{"error": err.Error()}) + return + } + writeJSON(w, 200, M{"analysis": analysis, "prompt": body.Prompt}) } func (r *Router) hostExecHandler(w http.ResponseWriter, req *http.Request) { diff --git a/backend/services/gateway/api/router.go b/backend/services/gateway/api/router.go index a4b3ac5..609a29e 100644 --- a/backend/services/gateway/api/router.go +++ b/backend/services/gateway/api/router.go @@ -91,6 +91,11 @@ type AutomationEngine interface { Trigger(ctx context.Context, id string) error } +// VisionAnalyzer analyzes images with a multimodal model. +type VisionAnalyzer interface { + AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error) +} + // ChatStorer persists chat sessions and messages. type ChatStorer interface { CreateSession(ctx context.Context, id, userID, title string) (*store.Session, error) @@ -117,6 +122,7 @@ type Dependencies struct { Automation *AutomationStore AutomationEng AutomationEngine Reminders *ReminderStore + Vision VisionAnalyzer PluginList []map[string]string }