feat(ai-core): real vision image analysis
- llm: Adapter.Vision sends multimodal chat requests (text + image_url, data URLs supported) and logs the call as purpose 'vision' - gateway /images/analyze now accepts image_url or image_base64 and returns real analysis instead of a placeholder message - main wires the default LLM adapter as the vision analyzer - httptest coverage for the vision request/response shape
This commit is contained in:
1 parent
2104fc0c6c
commit
ac0c2b2c73
5 files changed
+117
-7
No files matched your search
@@ -260,16 +260,35 @@ func (r *Router) sessionExportHandler(w http.ResponseWriter, req *http.Request)
|
||||
|
||||
func (r *Router) imageAnalyzeHandler(w http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != "POST" { writeJSON(w, 405, M{"error": "POST required"}); return }
|
||||
if r.deps.Vision == nil { writeJSON(w, 503, M{"error": "vision not available"}); return }
|
||||
var body struct {
|
||||
ImageURL string `json:"image_url"`
|
||||
Prompt string `json:"prompt"`
|
||||
ImageURL string `json:"image_url"`
|
||||
ImageBase64 string `json:"image_base64"` // raw base64 or data URL
|
||||
Prompt string `json:"prompt"`
|
||||
}
|
||||
json.NewDecoder(req.Body).Decode(&body)
|
||||
// Stub: requires multimodal LLM (GPT-4o / Qwen-VL)
|
||||
writeJSON(w, 200, M{
|
||||
"analysis": "图片分析需要多模态大模型。当前为预留接口。请配置支持 vision 的 LLM provider。",
|
||||
"image_url": body.ImageURL, "prompt": body.Prompt,
|
||||
})
|
||||
|
||||
imageRef := body.ImageURL
|
||||
if imageRef == "" && body.ImageBase64 != "" {
|
||||
imageRef = body.ImageBase64
|
||||
if !strings.HasPrefix(imageRef, "data:") {
|
||||
imageRef = "data:image/jpeg;base64," + imageRef
|
||||
}
|
||||
}
|
||||
if imageRef == "" {
|
||||
writeJSON(w, 400, M{"error": "missing image_url or image_base64"})
|
||||
return
|
||||
}
|
||||
if body.Prompt == "" {
|
||||
body.Prompt = "描述这张图片的内容。"
|
||||
}
|
||||
|
||||
analysis, err := r.deps.Vision.AnalyzeImage(req.Context(), body.Prompt, imageRef)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, M{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, M{"analysis": analysis, "prompt": body.Prompt})
|
||||
}
|
||||
|
||||
func (r *Router) hostExecHandler(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
@@ -91,6 +91,11 @@ type AutomationEngine interface {
|
||||
Trigger(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
// VisionAnalyzer analyzes images with a multimodal model.
|
||||
type VisionAnalyzer interface {
|
||||
AnalyzeImage(ctx context.Context, prompt, imageRef string) (string, error)
|
||||
}
|
||||
|
||||
// ChatStorer persists chat sessions and messages.
|
||||
type ChatStorer interface {
|
||||
CreateSession(ctx context.Context, id, userID, title string) (*store.Session, error)
|
||||
@@ -117,6 +122,7 @@ type Dependencies struct {
|
||||
Automation *AutomationStore
|
||||
AutomationEng AutomationEngine
|
||||
Reminders *ReminderStore
|
||||
Vision VisionAnalyzer
|
||||
PluginList []map[string]string
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user