feat(ai-core): real vision image analysis
- llm: Adapter.Vision sends multimodal chat requests (text + image_url, data URLs supported) and logs the call as purpose 'vision' - gateway /images/analyze now accepts image_url or image_base64 and returns real analysis instead of a placeholder message - main wires the default LLM adapter as the vision analyzer - httptest coverage for the vision request/response shape
This commit is contained in:
1 parent
2104fc0c6c
commit
ac0c2b2c73
5 files changed
+117
-7
No files matched your search
@@ -315,6 +315,7 @@ func main() {
|
|||||||
gwDeps.ChatStore = chatStore
|
gwDeps.ChatStore = chatStore
|
||||||
}
|
}
|
||||||
gwDeps.VoiceSvc = voiceSvc
|
gwDeps.VoiceSvc = voiceSvc
|
||||||
|
gwDeps.Vision = visionAdapter{llmSelector.DefaultAdapter()}
|
||||||
|
|
||||||
wsHub := ws.NewHub(orch)
|
wsHub := ws.NewHub(orch)
|
||||||
gwDeps.WSHub = wsHub
|
gwDeps.WSHub = wsHub
|
||||||
@@ -462,6 +463,17 @@ func (a *perfAdapter) History() []map[string]any {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// visionAdapter adapts the LLM adapter to api.VisionAnalyzer.
|
||||||
|
type visionAdapter struct{ llm *llm.Adapter }
|
||||||
|
|
||||||
|
func (v visionAdapter) AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error) {
|
||||||
|
resp, err := v.llm.Vision(ctx, prompt, imageRef)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return resp.Content, nil
|
||||||
|
}
|
||||||
|
|
||||||
// automationSourceAdapter adapts api.AutomationStore to engine.RuleSource.
|
// automationSourceAdapter adapts api.AutomationStore to engine.RuleSource.
|
||||||
type automationSourceAdapter struct{ s *api.AutomationStore }
|
type automationSourceAdapter struct{ s *api.AutomationStore }
|
||||||
|
|
||||||
|
|||||||
@@ -76,6 +76,42 @@ type CallRecord struct {
|
|||||||
Error string
|
Error string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Vision sends a prompt plus an image (URL or data URL) to a multimodal model
|
||||||
|
// and returns the analysis text.
|
||||||
|
func (a *Adapter) Vision(ctx context.Context, prompt, imageRef string) (*model.LLMResponse, error) {
|
||||||
|
start := time.Now()
|
||||||
|
body := map[string]any{
|
||||||
|
"model": a.cfg.Model,
|
||||||
|
"messages": []map[string]any{
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": []map[string]any{
|
||||||
|
{"type": "text", "text": prompt},
|
||||||
|
{"type": "image_url", "image_url": map[string]string{"url": imageRef}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
resp := &model.LLMResponse{Model: a.cfg.Model}
|
||||||
|
err := a.doRequest(ctx, "POST", "/chat/completions", body, func(data map[string]any) error {
|
||||||
|
if choices, ok := data["choices"].([]any); ok && len(choices) > 0 {
|
||||||
|
choice := choices[0].(map[string]any)
|
||||||
|
if msg, ok := choice["message"].(map[string]any); ok {
|
||||||
|
if content, ok := msg["content"].(string); ok { resp.Content = content }
|
||||||
|
}
|
||||||
|
if reason, ok := choice["finish_reason"].(string); ok { resp.FinishReason = reason }
|
||||||
|
}
|
||||||
|
if usage, ok := data["usage"].(map[string]any); ok {
|
||||||
|
if total, ok := usage["total_tokens"].(float64); ok { resp.TokensUsed = int(total) }
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
a.recordCall(start, "vision", resp.TokensUsed, 0, err)
|
||||||
|
return resp, err
|
||||||
|
}
|
||||||
|
|
||||||
// ModelName returns the configured model name.
|
// ModelName returns the configured model name.
|
||||||
func (a *Adapter) ModelName() string { return a.cfg.Model }
|
func (a *Adapter) ModelName() string { return a.cfg.Model }
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestVision(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
s := string(body)
|
||||||
|
if !strings.Contains(s, `"type":"image_url"`) || !strings.Contains(s, "data:image/jpeg;base64,AAAA") {
|
||||||
|
t.Errorf("unexpected vision request body: %s", s)
|
||||||
|
}
|
||||||
|
w.Write([]byte(`{"choices":[{"message":{"content":"图片里是一只猫"},"finish_reason":"stop"}],"usage":{"total_tokens":42}}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.BaseURL = srv.URL
|
||||||
|
a := NewAdapter(cfg)
|
||||||
|
|
||||||
|
resp, err := a.Vision(context.Background(), "这是什么?", "data:image/jpeg;base64,AAAA")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Vision: %v", err)
|
||||||
|
}
|
||||||
|
if resp.Content != "图片里是一只猫" {
|
||||||
|
t.Errorf("content = %q", resp.Content)
|
||||||
|
}
|
||||||
|
if resp.TokensUsed != 42 {
|
||||||
|
t.Errorf("tokens = %d", resp.TokensUsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -260,16 +260,35 @@ func (r *Router) sessionExportHandler(w http.ResponseWriter, req *http.Request)
|
|||||||
|
|
||||||
func (r *Router) imageAnalyzeHandler(w http.ResponseWriter, req *http.Request) {
|
func (r *Router) imageAnalyzeHandler(w http.ResponseWriter, req *http.Request) {
|
||||||
if req.Method != "POST" { writeJSON(w, 405, M{"error": "POST required"}); return }
|
if req.Method != "POST" { writeJSON(w, 405, M{"error": "POST required"}); return }
|
||||||
|
if r.deps.Vision == nil { writeJSON(w, 503, M{"error": "vision not available"}); return }
|
||||||
var body struct {
|
var body struct {
|
||||||
ImageURL string `json:"image_url"`
|
ImageURL string `json:"image_url"`
|
||||||
Prompt string `json:"prompt"`
|
ImageBase64 string `json:"image_base64"` // raw base64 or data URL
|
||||||
|
Prompt string `json:"prompt"`
|
||||||
}
|
}
|
||||||
json.NewDecoder(req.Body).Decode(&body)
|
json.NewDecoder(req.Body).Decode(&body)
|
||||||
// Stub: requires multimodal LLM (GPT-4o / Qwen-VL)
|
|
||||||
writeJSON(w, 200, M{
|
imageRef := body.ImageURL
|
||||||
"analysis": "图片分析需要多模态大模型。当前为预留接口。请配置支持 vision 的 LLM provider。",
|
if imageRef == "" && body.ImageBase64 != "" {
|
||||||
"image_url": body.ImageURL, "prompt": body.Prompt,
|
imageRef = body.ImageBase64
|
||||||
})
|
if !strings.HasPrefix(imageRef, "data:") {
|
||||||
|
imageRef = "data:image/jpeg;base64," + imageRef
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if imageRef == "" {
|
||||||
|
writeJSON(w, 400, M{"error": "missing image_url or image_base64"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if body.Prompt == "" {
|
||||||
|
body.Prompt = "描述这张图片的内容。"
|
||||||
|
}
|
||||||
|
|
||||||
|
analysis, err := r.deps.Vision.AnalyzeImage(req.Context(), body.Prompt, imageRef)
|
||||||
|
if err != nil {
|
||||||
|
writeJSON(w, 500, M{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 200, M{"analysis": analysis, "prompt": body.Prompt})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Router) hostExecHandler(w http.ResponseWriter, req *http.Request) {
|
func (r *Router) hostExecHandler(w http.ResponseWriter, req *http.Request) {
|
||||||
|
|||||||
@@ -91,6 +91,11 @@ type AutomationEngine interface {
|
|||||||
Trigger(ctx context.Context, id string) error
|
Trigger(ctx context.Context, id string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VisionAnalyzer analyzes images with a multimodal model.
|
||||||
|
type VisionAnalyzer interface {
|
||||||
|
AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
// ChatStorer persists chat sessions and messages.
|
// ChatStorer persists chat sessions and messages.
|
||||||
type ChatStorer interface {
|
type ChatStorer interface {
|
||||||
CreateSession(ctx context.Context, id, userID, title string) (*store.Session, error)
|
CreateSession(ctx context.Context, id, userID, title string) (*store.Session, error)
|
||||||
@@ -117,6 +122,7 @@ type Dependencies struct {
|
|||||||
Automation *AutomationStore
|
Automation *AutomationStore
|
||||||
AutomationEng AutomationEngine
|
AutomationEng AutomationEngine
|
||||||
Reminders *ReminderStore
|
Reminders *ReminderStore
|
||||||
|
Vision VisionAnalyzer
|
||||||
PluginList []map[string]string
|
PluginList []map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user