feat(ai-core): real vision image analysis
- llm: Adapter.Vision sends multimodal chat requests (text + image_url, data URLs supported) and logs the call as purpose 'vision' - gateway /images/analyze now accepts image_url or image_base64 and returns real analysis instead of a placeholder message - main wires the default LLM adapter as the vision analyzer - httptest coverage for the vision request/response shape
This commit is contained in:
1 parent
2104fc0c6c
commit
ac0c2b2c73
5 files changed
+117
-7
No files matched your search
@@ -315,6 +315,7 @@ func main() {
|
||||
gwDeps.ChatStore = chatStore
|
||||
}
|
||||
gwDeps.VoiceSvc = voiceSvc
|
||||
gwDeps.Vision = visionAdapter{llmSelector.DefaultAdapter()}
|
||||
|
||||
wsHub := ws.NewHub(orch)
|
||||
gwDeps.WSHub = wsHub
|
||||
@@ -462,6 +463,17 @@ func (a *perfAdapter) History() []map[string]any {
|
||||
return result
|
||||
}
|
||||
|
||||
// visionAdapter adapts the LLM adapter to api.VisionAnalyzer.
|
||||
type visionAdapter struct{ llm *llm.Adapter }
|
||||
|
||||
func (v visionAdapter) AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error) {
|
||||
resp, err := v.llm.Vision(ctx, prompt, imageRef)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Content, nil
|
||||
}
|
||||
|
||||
// automationSourceAdapter adapts api.AutomationStore to engine.RuleSource.
|
||||
type automationSourceAdapter struct{ s *api.AutomationStore }
|
||||
|
||||
|
||||
@@ -76,6 +76,42 @@ type CallRecord struct {
|
||||
Error string
|
||||
}
|
||||
|
||||
// Vision sends a prompt plus an image (URL or data URL) to a multimodal model
|
||||
// and returns the analysis text.
|
||||
func (a *Adapter) Vision(ctx context.Context, prompt, imageRef string) (*model.LLMResponse, error) {
|
||||
start := time.Now()
|
||||
body := map[string]any{
|
||||
"model": a.cfg.Model,
|
||||
"messages": []map[string]any{
|
||||
{
|
||||
"role": "user",
|
||||
"content": []map[string]any{
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": map[string]string{"url": imageRef}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resp := &model.LLMResponse{Model: a.cfg.Model}
|
||||
err := a.doRequest(ctx, "POST", "/chat/completions", body, func(data map[string]any) error {
|
||||
if choices, ok := data["choices"].([]any); ok && len(choices) > 0 {
|
||||
choice := choices[0].(map[string]any)
|
||||
if msg, ok := choice["message"].(map[string]any); ok {
|
||||
if content, ok := msg["content"].(string); ok { resp.Content = content }
|
||||
}
|
||||
if reason, ok := choice["finish_reason"].(string); ok { resp.FinishReason = reason }
|
||||
}
|
||||
if usage, ok := data["usage"].(map[string]any); ok {
|
||||
if total, ok := usage["total_tokens"].(float64); ok { resp.TokensUsed = int(total) }
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
a.recordCall(start, "vision", resp.TokensUsed, 0, err)
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// ModelName returns the configured model name.
|
||||
func (a *Adapter) ModelName() string { return a.cfg.Model }
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVision(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
s := string(body)
|
||||
if !strings.Contains(s, `"type":"image_url"`) || !strings.Contains(s, "data:image/jpeg;base64,AAAA") {
|
||||
t.Errorf("unexpected vision request body: %s", s)
|
||||
}
|
||||
w.Write([]byte(`{"choices":[{"message":{"content":"图片里是一只猫"},"finish_reason":"stop"}],"usage":{"total_tokens":42}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.BaseURL = srv.URL
|
||||
a := NewAdapter(cfg)
|
||||
|
||||
resp, err := a.Vision(context.Background(), "这是什么?", "data:image/jpeg;base64,AAAA")
|
||||
if err != nil {
|
||||
t.Fatalf("Vision: %v", err)
|
||||
}
|
||||
if resp.Content != "图片里是一只猫" {
|
||||
t.Errorf("content = %q", resp.Content)
|
||||
}
|
||||
if resp.TokensUsed != 42 {
|
||||
t.Errorf("tokens = %d", resp.TokensUsed)
|
||||
}
|
||||
}
|
||||
@@ -260,16 +260,35 @@ func (r *Router) sessionExportHandler(w http.ResponseWriter, req *http.Request)
|
||||
|
||||
func (r *Router) imageAnalyzeHandler(w http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != "POST" { writeJSON(w, 405, M{"error": "POST required"}); return }
|
||||
if r.deps.Vision == nil { writeJSON(w, 503, M{"error": "vision not available"}); return }
|
||||
var body struct {
|
||||
ImageURL string `json:"image_url"`
|
||||
Prompt string `json:"prompt"`
|
||||
ImageURL string `json:"image_url"`
|
||||
ImageBase64 string `json:"image_base64"` // raw base64 or data URL
|
||||
Prompt string `json:"prompt"`
|
||||
}
|
||||
json.NewDecoder(req.Body).Decode(&body)
|
||||
// Stub: requires multimodal LLM (GPT-4o / Qwen-VL)
|
||||
writeJSON(w, 200, M{
|
||||
"analysis": "图片分析需要多模态大模型。当前为预留接口。请配置支持 vision 的 LLM provider。",
|
||||
"image_url": body.ImageURL, "prompt": body.Prompt,
|
||||
})
|
||||
|
||||
imageRef := body.ImageURL
|
||||
if imageRef == "" && body.ImageBase64 != "" {
|
||||
imageRef = body.ImageBase64
|
||||
if !strings.HasPrefix(imageRef, "data:") {
|
||||
imageRef = "data:image/jpeg;base64," + imageRef
|
||||
}
|
||||
}
|
||||
if imageRef == "" {
|
||||
writeJSON(w, 400, M{"error": "missing image_url or image_base64"})
|
||||
return
|
||||
}
|
||||
if body.Prompt == "" {
|
||||
body.Prompt = "描述这张图片的内容。"
|
||||
}
|
||||
|
||||
analysis, err := r.deps.Vision.AnalyzeImage(req.Context(), body.Prompt, imageRef)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, M{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, M{"analysis": analysis, "prompt": body.Prompt})
|
||||
}
|
||||
|
||||
func (r *Router) hostExecHandler(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
@@ -91,6 +91,11 @@ type AutomationEngine interface {
|
||||
Trigger(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
// VisionAnalyzer analyzes images with a multimodal model.
|
||||
type VisionAnalyzer interface {
|
||||
AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error)
|
||||
}
|
||||
|
||||
// ChatStorer persists chat sessions and messages.
|
||||
type ChatStorer interface {
|
||||
CreateSession(ctx context.Context, id, userID, title string) (*store.Session, error)
|
||||
@@ -117,6 +122,7 @@ type Dependencies struct {
|
||||
Automation *AutomationStore
|
||||
AutomationEng AutomationEngine
|
||||
Reminders *ReminderStore
|
||||
Vision VisionAnalyzer
|
||||
PluginList []map[string]string
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user