feat: admin 端点补齐 + 视觉真推理 + 语音链路 + wire.go 收口(A+B+第3步)
批次A admin 端点(原本 404 变 200 真实数据):
- GET /admin/llm:读 s.cfg.LLM 组 openai+anthropic 两 provider(enabled=当前 Provider、multimodal=配 VisionModel),APIKey 不进 wire 零泄露。
- GET /admin/automations:*automations.Service.List()(reminders+event rules+cron schedules,cron 加只读登记)。
- GET /admin/users:cfg.AdminUser + *auth.APIKey.List()(密钥掩码);无用户存储,恒先返回当前 admin 账号。
- GET /admin/backthink:*backthink.Backthink.List(ctx,50)。均只读、owner/admin RBAC、{data} 信封。
批次B 视觉真推理 + 语音:
- rest.go chatRequest 加 Image?;流式 publishChatStream 带 image;userChatMessages 构造 ImageMessage(model.go:71)。
- loop/vision.go:ImageResolver 可注入(MVP 先看图再对话);run.go Run 开头 resolveImages,无 resolver 透传直交主模型。
- vision/resolver.go:MessageResolver 调 vision.Router.Route——KindMain 保留图片交主模型多模态、KindVisionTool/OCRTool 经 dispatcher.Invoke 折叠成 [图片内容] 文本、KindUnsupported 记日志保留原图不崩。
- gateway/voice.go:VoiceService 接口 + SetVoice 注入点 + handleVoiceInput(未注入回 error 不崩)+ runVoiceTurn 异步 STT→loop chat→TTS→定向 voice_output 帧。
- ws_hub/ws_route:voice_input 接 handleVoiceInput;前端 ws.ts decodeBase64/playVoiceAudio/case voice_output。
第3步 wire.go 收口(唯一碰 wire.go 的一步):
- Deps 补传 Automations:a.automation / Backthink:a.backthink / Users:auth.NewAPIKey(a.cfg.Auth)(*auth.APIKey 原生满足 UserStore,避免 admin 重复的复合类型)。
- gw.SetVoice(a.voice);loop.Options.Vision = vision.NewMessageResolver(router, info, dispatcher.Invoke)(router=NewRouter(WithVisionTool/WithOCRTool)、info=cfg.LLM.VisionModel)。
实机验证:四 admin 端点 200 真实数据(llm 双 provider 脱敏、users=admin,yeij0942,owner);WS 握手正常。go build/vet/test 全绿 + 前端 tsc。wire.go 557 行。
This commit is contained in:
1 parent
648292d035
commit
518bd0a230
21 files changed
+976
-38
No files matched your search
@@ -97,6 +97,7 @@ type Loop struct {
|
||||
maxContextTokens int
|
||||
compactor Compactor
|
||||
transformContext func(context.Context, []model.Message, int) []model.Message
|
||||
imageResolver ImageResolver
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
@@ -141,6 +142,7 @@ func New(opts Options) *Loop {
|
||||
maxContextTokens: opts.MaxContextTokens,
|
||||
compactor: compactor,
|
||||
transformContext: opts.TransformContext,
|
||||
imageResolver: opts.Vision,
|
||||
log: sourcedLog(opts.Log),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
func TestRun_ImageResolverFoldsToText(t *testing.T) {
|
||||
t.Parallel()
|
||||
ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}}
|
||||
a := newTestAgent(t, ad, nil)
|
||||
l := New(Options{
|
||||
Agent: a,
|
||||
MaxRounds: 8,
|
||||
Vision: ImageResolveFunc(func(_ context.Context, _ model.Message) (model.Message, error) {
|
||||
return model.TextMessage(model.RoleUser, "图中是一只猫"), nil
|
||||
}),
|
||||
})
|
||||
|
||||
in := []model.Message{model.ImageMessage(model.RoleUser, "", "data:image/png;base64,AAAA")}
|
||||
if _, err := l.Run(context.Background(), "sess-1", in); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if len(ad.reqs) == 0 {
|
||||
t.Fatalf("no model request recorded")
|
||||
}
|
||||
var userSeen bool
|
||||
for _, m := range ad.reqs[0].Messages {
|
||||
if m.Role == model.RoleUser {
|
||||
userSeen = true
|
||||
if messageHasImage(m) {
|
||||
t.Fatalf("image still reached the model: %+v", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !userSeen {
|
||||
t.Fatalf("no user message seen by the model")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_ImageKeptWithoutResolver(t *testing.T) {
|
||||
t.Parallel()
|
||||
ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}}
|
||||
a := newTestAgent(t, ad, nil)
|
||||
l := New(Options{Agent: a, MaxRounds: 8})
|
||||
|
||||
in := []model.Message{model.ImageMessage(model.RoleUser, "look", "data:image/png;base64,AAAA")}
|
||||
if _, err := l.Run(context.Background(), "sess-1", in); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if len(ad.reqs) == 0 {
|
||||
t.Fatalf("no model request recorded")
|
||||
}
|
||||
var sawImage bool
|
||||
for _, m := range ad.reqs[0].Messages {
|
||||
if m.Role == model.RoleUser && messageHasImage(m) {
|
||||
sawImage = true
|
||||
}
|
||||
}
|
||||
if !sawImage {
|
||||
t.Fatalf("expected the image part to pass through without a resolver")
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,13 @@ type Options struct {
|
||||
// an LLM-backed or custom summarizer. It is only invoked when
|
||||
// MaxContextTokens is positive.
|
||||
Compactor Compactor
|
||||
// Vision, when non-nil, rewrites an image-bearing user message at the start
|
||||
// of Run so the main model can reason over the image (design/06 §0). The
|
||||
// concrete resolver (vision.MessageResolver) decides via the vision Router
|
||||
// whether to keep the image for a vision-capable main model or fold it into a
|
||||
// text description from the vision/OCR tool ("先看图再对话"). Nil keeps the
|
||||
// image content as-is, passing it through to a vision-capable main model.
|
||||
Vision ImageResolver
|
||||
// TransformContext, when non-nil, is called immediately before every model
|
||||
// step with the assembled message list (after memory injection and any
|
||||
// compaction) and the current round number (1-based). It may project, filter
|
||||
|
||||
@@ -31,7 +31,7 @@ func (l *Loop) Run(ctx context.Context, sessionID string, messages []model.Messa
|
||||
return nil, fmt.Errorf("loop: run: %w", ErrNilAgent)
|
||||
}
|
||||
|
||||
msgs := messages
|
||||
msgs := l.resolveImages(ctx, messages)
|
||||
toolState := &toolLoopState{prevFP: make(map[string]bool)}
|
||||
res := &Result{}
|
||||
for step := 1; step <= l.maxRounds; step++ {
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
|
||||
)
|
||||
|
||||
// ImageResolver is the loop's seam to the multimodal vision layer (design/06
|
||||
// §0). It is called once at the start of Run for every message that carries an
|
||||
// image content part, and may rewrite that message so the main model can reason
|
||||
// over the image. The concrete resolver lives in the vision package and decides,
|
||||
// via the vision Router, whether to keep the image for a vision-capable main
|
||||
// model or fold it into a text description produced by the vision/OCR tool ("先
|
||||
// 看图再对话"). Nil leaves the image content untouched so it flows straight to a
|
||||
// vision-capable main model.
|
||||
type ImageResolver interface {
|
||||
// Resolve rewrites msg, which carries at least one image content part. It
|
||||
// returns the (possibly rewritten) message. An error leaves the original
|
||||
// message in place so the turn still proceeds.
|
||||
Resolve(ctx context.Context, msg model.Message) (model.Message, error)
|
||||
}
|
||||
|
||||
// ImageResolveFunc adapts a function to the ImageResolver interface.
|
||||
type ImageResolveFunc func(ctx context.Context, msg model.Message) (model.Message, error)
|
||||
|
||||
// Resolve implements ImageResolver.
|
||||
func (f ImageResolveFunc) Resolve(ctx context.Context, msg model.Message) (model.Message, error) {
|
||||
return f(ctx, msg)
|
||||
}
|
||||
|
||||
// resolveImages rewrites every image-bearing message in messages using the
|
||||
// injected resolver. It returns the input slice unchanged when no resolver is
|
||||
// configured, no image is present, or a message fails to resolve (the failure is
|
||||
// logged and the original message is kept so the turn still runs). A fresh slice
|
||||
// is returned when any message changed.
|
||||
func (l *Loop) resolveImages(ctx context.Context, messages []model.Message) []model.Message {
|
||||
if l.imageResolver == nil {
|
||||
return messages
|
||||
}
|
||||
out := make([]model.Message, len(messages))
|
||||
changed := false
|
||||
for i, m := range messages {
|
||||
if !messageHasImage(m) {
|
||||
out[i] = m
|
||||
continue
|
||||
}
|
||||
resolved, err := l.imageResolver.Resolve(ctx, m)
|
||||
if err != nil {
|
||||
l.log.Warn("image resolve", "error", err)
|
||||
out[i] = m
|
||||
continue
|
||||
}
|
||||
out[i] = resolved
|
||||
changed = true
|
||||
}
|
||||
if !changed {
|
||||
return messages
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// messageHasImage reports whether any content part of msg is an image part.
|
||||
func messageHasImage(m model.Message) bool {
|
||||
for _, part := range m.Content {
|
||||
if part.Type == model.ContentTypeImage {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in new issue
Block a user