feat(multimodal): 视觉分层路由(多模态检测→主/视觉/OCR 工具)+ 语音 STT→TTS 降级链(异步)
This commit is contained in:
1 parent
02aae2bbb4
commit
0a936a3202
19 files changed
+2121
No files matched your search
@@ -0,0 +1,128 @@
|
||||
package vision
|
||||
|
||||
// Assessment is the model's own confidence in reading an image.
|
||||
type Assessment string
|
||||
|
||||
const (
|
||||
// AssessmentUnknown means the model reported no self-assessment.
|
||||
AssessmentUnknown Assessment = "unknown"
|
||||
// AssessmentGood means the model is confident in its read.
|
||||
AssessmentGood Assessment = "good"
|
||||
// AssessmentPoor means the model judged its recognition as poor.
|
||||
AssessmentPoor Assessment = "poor"
|
||||
)
|
||||
|
||||
// ModelInfo describes the main model's vision capability.
|
||||
type ModelInfo struct {
|
||||
// Name identifies the model.
|
||||
Name string
|
||||
// SupportsVision reports whether the model accepts image input.
|
||||
SupportsVision bool
|
||||
// SelfAssessment is the model's own confidence for the current image.
|
||||
// It drives the layer-2 tool route when SupportsVision is true.
|
||||
SelfAssessment Assessment
|
||||
}
|
||||
|
||||
// DecisionKind is the chosen layered route.
|
||||
type DecisionKind string
|
||||
|
||||
const (
|
||||
// KindMain routes the image to the main model with the image in context.
|
||||
KindMain DecisionKind = "main"
|
||||
// KindVisionTool routes to a dedicated vision tool, returning only the tool
|
||||
// result so the image never occupies main-model context.
|
||||
KindVisionTool DecisionKind = "vision_tool"
|
||||
// KindOCRTool routes to an OCR tool, returning only the extracted text.
|
||||
KindOCRTool DecisionKind = "ocr_tool"
|
||||
// KindUnsupported means no route can process the image.
|
||||
KindUnsupported DecisionKind = "unsupported"
|
||||
)
|
||||
|
||||
// Decision is the outcome of a routing call.
|
||||
type Decision struct {
|
||||
// Kind is the chosen route.
|
||||
Kind DecisionKind
|
||||
// Tool names the vision or OCR tool to invoke, set when Kind is a tool route.
|
||||
Tool string
|
||||
// Reason is a human-readable explanation for observability and logs.
|
||||
Reason string
|
||||
}
|
||||
|
||||
// RouteRequest carries routing hints from the caller on top of the model info.
|
||||
type RouteRequest struct {
|
||||
// TextPrimary reports that the user indicated the image is mainly text, so
|
||||
// an OCR tool is preferred over a general vision model.
|
||||
TextPrimary bool
|
||||
// MainSelfAssessedPoor is set when the main model has already judged its own
|
||||
// read of the image as poor, requesting a vision/OCR tool route.
|
||||
MainSelfAssessedPoor bool
|
||||
}
|
||||
|
||||
// RouterOption configures a Router.
|
||||
type RouterOption func(*Router)
|
||||
|
||||
// WithVisionTool binds the name of the registered vision-understanding tool.
|
||||
func WithVisionTool(name string) RouterOption {
|
||||
return func(r *Router) {
|
||||
if name != "" {
|
||||
r.visionTool = name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithOCRTool binds the name of the registered OCR tool.
|
||||
func WithOCRTool(name string) RouterOption {
|
||||
return func(r *Router) {
|
||||
if name != "" {
|
||||
r.ocrTool = name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Router implements the layered vision routing policy. It does no I/O; the
|
||||
// caller uses the returned Decision to dispatch the request.
|
||||
type Router struct {
|
||||
visionTool string
|
||||
ocrTool string
|
||||
}
|
||||
|
||||
// NewRouter builds a Router. Unknown (empty) tool names leave the corresponding
|
||||
// route unavailable, so the router degrades to KindMain or KindUnsupported.
|
||||
func NewRouter(opts ...RouterOption) *Router {
|
||||
r := &Router{}
|
||||
for _, o := range opts {
|
||||
o(r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Route picks a layered route for an image request given the target model info
|
||||
// and caller hints. The order is: the main model directly when it can see and
|
||||
// has not self-assessed poorly (layer 1); otherwise a vision or OCR tool whose
|
||||
// result replaces the image in context (layers 2 and 3); otherwise
|
||||
// KindUnsupported when nothing can process the image.
|
||||
func (r *Router) Route(target ModelInfo, req RouteRequest) Decision {
|
||||
poor := target.SelfAssessment == AssessmentPoor || req.MainSelfAssessedPoor
|
||||
// Layer 1: the main model accepts images and is confident, so use it first.
|
||||
if target.SupportsVision && !poor {
|
||||
return Decision{
|
||||
Kind: KindMain,
|
||||
Reason: "main model supports vision and is confident; use it directly",
|
||||
}
|
||||
}
|
||||
// OCR is preferred when the user says the image is mostly text.
|
||||
if req.TextPrimary && r.ocrTool != "" {
|
||||
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "image is mainly text; route to OCR"}
|
||||
}
|
||||
if r.visionTool != "" {
|
||||
return Decision{Kind: KindVisionTool, Tool: r.visionTool, Reason: "route to dedicated vision tool"}
|
||||
}
|
||||
if r.ocrTool != "" {
|
||||
return Decision{Kind: KindOCRTool, Tool: r.ocrTool, Reason: "route to OCR tool"}
|
||||
}
|
||||
// No tool is bound. A sighted main model remains the only option.
|
||||
if target.SupportsVision {
|
||||
return Decision{Kind: KindMain, Reason: "no vision/OCR tool bound; keep image with main model"}
|
||||
}
|
||||
return Decision{Kind: KindUnsupported, Reason: "main model cannot see and no vision/OCR tool is bound"}
|
||||
}
|
||||
Reference in new issue
Block a user