feat: admin 端点补齐 + 视觉真推理 + 语音链路 + wire.go 收口(A+B+第3步)
CI / Backend (build / vet / test / cross-compile) (push) Has been cancelled
CI / Frontend (tsc + vite build) (push) Has been cancelled

批次A admin 端点(原本 404 变 200 真实数据):
- GET /admin/llm:读 s.cfg.LLM 组 openai+anthropic 两 provider(enabled=当前 Provider、multimodal=配 VisionModel),APIKey 不进 wire 零泄露。
- GET /admin/automations:*automations.Service.List()(reminders+event rules+cron schedules,cron 加只读登记)。
- GET /admin/users:cfg.AdminUser + *auth.APIKey.List()(密钥掩码);无用户存储,恒先返回当前 admin 账号。
- GET /admin/backthink:*backthink.Backthink.List(ctx,50)。均只读、owner/admin RBAC、{data} 信封。

批次B 视觉真推理 + 语音:
- rest.go chatRequest 加 Image?;流式 publishChatStream 带 image;userChatMessages 构造 ImageMessage(model.go:71)。
- loop/vision.go:ImageResolver 可注入(MVP 先看图再对话);run.go Run 开头 resolveImages,无 resolver 透传直交主模型。
- vision/resolver.go:MessageResolver 调 vision.Router.Route——KindMain 保留图片交主模型多模态、KindVisionTool/OCRTool 经 dispatcher.Invoke 折叠成 [图片内容] 文本、KindUnsupported 记日志保留原图不崩。
- gateway/voice.go:VoiceService 接口 + SetVoice 注入点 + handleVoiceInput(未注入回 error 不崩)+ runVoiceTurn 异步 STT→loop chat→TTS→定向 voice_output 帧。
- ws_hub/ws_route:voice_input 接 handleVoiceInput;前端 ws.ts decodeBase64/playVoiceAudio/case voice_output。

第3步 wire.go 收口(唯一碰 wire.go 的一步):
- Deps 补传 Automations:a.automation / Backthink:a.backthink / Users:auth.NewAPIKey(a.cfg.Auth)(*auth.APIKey 原生满足 UserStore,避免 admin 重复的复合类型)。
- gw.SetVoice(a.voice);loop.Options.Vision = vision.NewMessageResolver(router, info, dispatcher.Invoke)(router=NewRouter(WithVisionTool/WithOCRTool)、info=cfg.LLM.VisionModel)。

实机验证:四 admin 端点 200 真实数据(llm 双 provider 脱敏、users=admin,yeij0942,owner);WS 握手正常。go build/vet/test 全绿 + 前端 tsc。wire.go 557 行。
This commit is contained in:
Aska committed 2026-09-06 18:54:05 +08:00
1 parent 648292d035
commit 518bd0a230
21 files changed
+976 -38

No files matched your search

+23
View File
@@ -71,3 +71,26 @@ func (k *APIKey) Revoke(key string) {
defer k.mu.Unlock()
delete(k.keys, key)
}
// List returns a summary of every currently registered API key for the admin
// panel. Key material is never returned; each identifier is masked so the panel
// can tell keys apart without exposing the secret. The in-memory store keeps no
// creation time, so CreatedAt is nil for every entry.
func (k *APIKey) List() []UserSummary {
k.mu.RLock()
defer k.mu.RUnlock()
out := make([]UserSummary, 0, len(k.keys))
for key, role := range k.keys {
out = append(out, UserSummary{ID: maskKey(key), Username: "apikey", Role: role})
}
return out
}
// maskKey renders a key's leading and trailing groups so the panel can identify
// an entry without revealing the secret. Short keys are fully hidden.
func maskKey(key string) string {
if len(key) <= 8 {
return "••••"
}
return key[:4] + "••••" + key[len(key)-4:]
}
+12
View File
@@ -12,6 +12,7 @@ package auth
import (
"errors"
"fmt"
"time"
)
// Role is an authorization role in the owner>admin>viewer hierarchy.
@@ -91,3 +92,14 @@ func (r Role) rank() int {
func HasRole(role, required Role) bool {
return role.rank() >= required.rank()
}
// UserSummary is the read-only shape of an account or API key visible to the
// admin panel (GET /api/v1/admin/users). CreatedAt is nil when the source does
// not track a creation time (the configured admin account and in-memory keys
// both omit it).
type UserSummary struct {
ID string
Username string
Role Role
CreatedAt *time.Time
}
+2
View File
@@ -97,6 +97,7 @@ type Loop struct {
maxContextTokens int
compactor Compactor
transformContext func(context.Context, []model.Message, int) []model.Message
imageResolver ImageResolver
log *slog.Logger
}
@@ -141,6 +142,7 @@ func New(opts Options) *Loop {
maxContextTokens: opts.MaxContextTokens,
compactor: compactor,
transformContext: opts.TransformContext,
imageResolver: opts.Vision,
log: sourcedLog(opts.Log),
}
}
+65
View File
@@ -0,0 +1,65 @@
package loop
import (
"context"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
func TestRun_ImageResolverFoldsToText(t *testing.T) {
t.Parallel()
ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}}
a := newTestAgent(t, ad, nil)
l := New(Options{
Agent: a,
MaxRounds: 8,
Vision: ImageResolveFunc(func(_ context.Context, _ model.Message) (model.Message, error) {
return model.TextMessage(model.RoleUser, "图中是一只猫"), nil
}),
})
in := []model.Message{model.ImageMessage(model.RoleUser, "", "data:image/png;base64,AAAA")}
if _, err := l.Run(context.Background(), "sess-1", in); err != nil {
t.Fatalf("Run: %v", err)
}
if len(ad.reqs) == 0 {
t.Fatalf("no model request recorded")
}
var userSeen bool
for _, m := range ad.reqs[0].Messages {
if m.Role == model.RoleUser {
userSeen = true
if messageHasImage(m) {
t.Fatalf("image still reached the model: %+v", m)
}
}
}
if !userSeen {
t.Fatalf("no user message seen by the model")
}
}
func TestRun_ImageKeptWithoutResolver(t *testing.T) {
t.Parallel()
ad := &mockAdapter{chat: []mgResp{{resp: model.Response{Model: "m", Content: "Done."}}}}
a := newTestAgent(t, ad, nil)
l := New(Options{Agent: a, MaxRounds: 8})
in := []model.Message{model.ImageMessage(model.RoleUser, "look", "data:image/png;base64,AAAA")}
if _, err := l.Run(context.Background(), "sess-1", in); err != nil {
t.Fatalf("Run: %v", err)
}
if len(ad.reqs) == 0 {
t.Fatalf("no model request recorded")
}
var sawImage bool
for _, m := range ad.reqs[0].Messages {
if m.Role == model.RoleUser && messageHasImage(m) {
sawImage = true
}
}
if !sawImage {
t.Fatalf("expected the image part to pass through without a resolver")
}
}
+7
View File
@@ -77,6 +77,13 @@ type Options struct {
// an LLM-backed or custom summarizer. It is only invoked when
// MaxContextTokens is positive.
Compactor Compactor
// Vision, when non-nil, rewrites an image-bearing user message at the start
// of Run so the main model can reason over the image (design/06 §0). The
// concrete resolver (vision.MessageResolver) decides via the vision Router
// whether to keep the image for a vision-capable main model or fold it into a
// text description from the vision/OCR tool ("先看图再对话"). Nil keeps the
// image content as-is, passing it through to a vision-capable main model.
Vision ImageResolver
// TransformContext, when non-nil, is called immediately before every model
// step with the assembled message list (after memory injection and any
// compaction) and the current round number (1-based). It may project, filter
+1 -1
View File
@@ -31,7 +31,7 @@ func (l *Loop) Run(ctx context.Context, sessionID string, messages []model.Messa
return nil, fmt.Errorf("loop: run: %w", ErrNilAgent)
}
msgs := messages
msgs := l.resolveImages(ctx, messages)
toolState := &toolLoopState{prevFP: make(map[string]bool)}
res := &Result{}
for step := 1; step <= l.maxRounds; step++ {
+71
View File
@@ -0,0 +1,71 @@
package loop
import (
"context"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
// ImageResolver is the loop's seam to the multimodal vision layer (design/06
// §0). It is called once at the start of Run for every message that carries an
// image content part, and may rewrite that message so the main model can reason
// over the image. The concrete resolver lives in the vision package and decides,
// via the vision Router, whether to keep the image for a vision-capable main
// model or fold it into a text description produced by the vision/OCR tool ("先
// 看图再对话"). Nil leaves the image content untouched so it flows straight to a
// vision-capable main model.
type ImageResolver interface {
// Resolve rewrites msg, which carries at least one image content part. It
// returns the (possibly rewritten) message. An error leaves the original
// message in place so the turn still proceeds.
Resolve(ctx context.Context, msg model.Message) (model.Message, error)
}
// ImageResolveFunc adapts a function to the ImageResolver interface.
type ImageResolveFunc func(ctx context.Context, msg model.Message) (model.Message, error)
// Resolve implements ImageResolver.
func (f ImageResolveFunc) Resolve(ctx context.Context, msg model.Message) (model.Message, error) {
return f(ctx, msg)
}
// resolveImages rewrites every image-bearing message in messages using the
// injected resolver. It returns the input slice unchanged when no resolver is
// configured, no image is present, or a message fails to resolve (the failure is
// logged and the original message is kept so the turn still runs). A fresh slice
// is returned when any message changed.
func (l *Loop) resolveImages(ctx context.Context, messages []model.Message) []model.Message {
if l.imageResolver == nil {
return messages
}
out := make([]model.Message, len(messages))
changed := false
for i, m := range messages {
if !messageHasImage(m) {
out[i] = m
continue
}
resolved, err := l.imageResolver.Resolve(ctx, m)
if err != nil {
l.log.Warn("image resolve", "error", err)
out[i] = m
continue
}
out[i] = resolved
changed = true
}
if !changed {
return messages
}
return out
}
// messageHasImage reports whether any content part of msg is an image part.
func messageHasImage(m model.Message) bool {
for _, part := range m.Content {
if part.Type == model.ContentTypeImage {
return true
}
}
return false
}
@@ -0,0 +1,123 @@
package vision
import (
"context"
"fmt"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
// ModelInfoProvider supplies the main model's vision capability so the resolver
// can route an image the way the vision Router decides (design/06 §0). It is
// called per image; the same provider is shared across a single resolution.
type ModelInfoProvider func() ModelInfo
// ToolInvoker invokes a registered tool (the vision or OCR tool) by name with
// the given arguments. It is satisfied by *tool.Dispatcher.Invoke.
type ToolInvoker func(ctx context.Context, name string, args map[string]any) (any, error)
// MessageResolver is the loop's concrete ImageResolver. For each image part in a
// user message it asks the vision Router how to route it: when the main model
// supports vision and is confident, the image part is kept so the main model sees
// the image; otherwise the vision/OCR tool is invoked and its text result replaces
// the image part, so a text-only main model can reason over the image ("先看图再
// 对话"). It is safe for concurrent use.
type MessageResolver struct {
router *Router
info ModelInfoProvider
invoke ToolInvoker
}
// NewMessageResolver builds a MessageResolver. router decides the layer; info
// reports the main model's vision capability; invoke runs a registered vision/OCR
// tool. A nil invoke leaves any tool route failed (the image is then kept), so a
// resolver without tools degrades to the main-model path.
func NewMessageResolver(router *Router, info ModelInfoProvider, invoke ToolInvoker) *MessageResolver {
return &MessageResolver{
router: router,
info: info,
invoke: invoke,
}
}
// Resolve rewrites msg so every image part is either kept (main model sees it) or
// replaced by a text description produced by the vision/OCR tool. Text parts are
// preserved in order. It errors only when an image cannot be routed or a tool
// invocation fails; the caller (loop) then keeps the original message.
func (r *MessageResolver) Resolve(ctx context.Context, msg model.Message) (model.Message, error) {
if r.router == nil {
return msg, nil
}
out := make([]model.Content, 0, len(msg.Content))
for _, part := range msg.Content {
if part.Type != model.ContentTypeImage {
out = append(out, part)
continue
}
replacement, err := r.resolveImage(ctx, part.ImageURL)
if err != nil {
return model.Message{}, err
}
out = append(out, replacement)
}
return model.Message{Role: msg.Role, Content: out}, nil
}
// resolveImage picks a route and either keeps the image (KindMain) or folds it
// into a text description via the vision/OCR tool.
func (r *MessageResolver) resolveImage(ctx context.Context, imageURL string) (model.Content, error) {
target := ModelInfo{}
if r.info != nil {
target = r.info()
}
decision := r.router.Route(target, RouteRequest{})
switch decision.Kind {
case KindMain:
// A vision-capable main model sees the image directly.
return model.Content{Type: model.ContentTypeImage, ImageURL: imageURL}, nil
case KindVisionTool:
return r.invokeTool(ctx, decision.Tool, map[string]any{
"image": imageURL,
"prompt": "Describe this image in detail.",
}, "description")
case KindOCRTool:
return r.invokeTool(ctx, decision.Tool, map[string]any{
"image": imageURL,
}, "text")
default:
return model.Content{}, fmt.Errorf("image route: %w: %s", ErrUnsupportedSource, decision.Reason)
}
}
// invokeTool runs a registered vision/OCR tool and turns its text result into a
// text content part. field is the result map key ("description" or "text").
func (r *MessageResolver) invokeTool(ctx context.Context, toolName string, args map[string]any, field string) (model.Content, error) {
if r.invoke == nil {
return model.Content{}, fmt.Errorf("%s: %w", toolName, ErrNotConfigured)
}
result, err := r.invoke(ctx, toolName, args)
if err != nil {
return model.Content{}, fmt.Errorf("image tool %s: %w", toolName, err)
}
return model.Content{Type: model.ContentTypeText, Text: descriptionFragment(fieldText(result, field))}, nil
}
// fieldText extracts a string field from a tool result map, tolerating the
// map[string]any shape returned by the vision/OCR tools.
func fieldText(result any, field string) string {
m, ok := result.(map[string]any)
if !ok {
return ""
}
s, _ := m[field].(string)
return s
}
// descriptionFragment labels a tool-produced description so the model can tell it
// refers to the picture the user attached.
func descriptionFragment(desc string) string {
if desc == "" {
return "[图片内容] 无法识别该图片"
}
return "[图片内容] " + desc
}
@@ -0,0 +1,94 @@
package vision
import (
"context"
"testing"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
)
func TestMessageResolver_FoldsImageToDescription(t *testing.T) {
t.Parallel()
rt := NewRouter(WithVisionTool("vision"), WithOCRTool("ocr"))
invoked := ""
res := NewMessageResolver(rt, func() ModelInfo {
return ModelInfo{Name: "text-model", SupportsVision: false}
}, func(_ context.Context, name string, args map[string]any) (any, error) {
invoked = name
return map[string]any{"description": "一只橘色的猫"}, nil
})
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "看到了吗", "data:image/png;base64,AAAA"))
if err != nil {
t.Fatalf("Resolve: %v", err)
}
if invoked != "vision" {
t.Fatalf("invoked tool = %q, want vision", invoked)
}
var text string
for _, p := range msg.Content {
if p.Type == model.ContentTypeText {
text += p.Text
}
if p.Type == model.ContentTypeImage {
t.Fatalf("image part still present after tool route: %+v", p)
}
}
if text == "" || text == "看到了吗" {
t.Fatalf("expected a description fragment, got %q", text)
}
}
func TestMessageResolver_KeepsImageForVisionModel(t *testing.T) {
t.Parallel()
rt := NewRouter(WithVisionTool("vision"), WithOCRTool("ocr"))
res := NewMessageResolver(rt, func() ModelInfo {
return ModelInfo{Name: "vision-model", SupportsVision: true}
}, func(_ context.Context, _ string, _ map[string]any) (any, error) {
t.Fatalf("tool should not be invoked when the main model is sighted")
return nil, nil
})
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", "data:image/png;base64,AAAA"))
if err != nil {
t.Fatalf("Resolve: %v", err)
}
if !messageHasImagePart(msg) {
t.Fatalf("expected the image part to be kept for a vision-capable main model")
}
}
func TestMessageResolver_OCRToolForText(t *testing.T) {
t.Parallel()
rt := NewRouter(WithVisionTool("vision"), WithOCRTool("ocr"))
invoked := ""
res := NewMessageResolver(rt, func() ModelInfo {
return ModelInfo{Name: "text-model", SupportsVision: false}
}, func(_ context.Context, name string, _ map[string]any) (any, error) {
invoked = name
return map[string]any{"text": "HELLO"}, nil
})
// Force the text-primary hint by using a router route request path: this is
// exercised directly through Resolve which passes no hint, so the vision tool
// is preferred; assert at least a text part replaces the image.
msg, err := res.Resolve(context.Background(), model.ImageMessage(model.RoleUser, "x", "data:image/png;base64,AAAA"))
if err != nil {
t.Fatalf("Resolve: %v", err)
}
if invoked != "vision" {
t.Fatalf("invoked tool = %q, want vision (router prefers vision when no text hint)", invoked)
}
if messageHasImagePart(msg) {
t.Fatalf("expected the image to be replaced by text")
}
}
func messageHasImagePart(m model.Message) bool {
for _, p := range m.Content {
if p.Type == model.ContentTypeImage {
return true
}
}
return false
}
@@ -140,6 +140,20 @@ func (d Deps) pipeline() Pipeline {
return nil
}
// AutomationRecord is the read-only summary of one automation or reminder,
// returned by Service.List for the admin panel. Kind is "cron", "event" or
// "reminder"; Cron carries the cron expression for the cron kind; LastRun and
// NextRun are zero times when the value is unknown (typically before Start).
type AutomationRecord struct {
ID string
Name string
Kind string
Cron string
Enabled bool
LastRun time.Time
NextRun time.Time
}
// Result is the structured outcome of an executed automation, published to
// SubjectResult. Upper layers subscribe to it to react or push notifications.
type Result struct {
+49
View File
@@ -22,6 +22,7 @@ type Service struct {
mu sync.RWMutex
cronRules map[string]ScheduleID // rule id -> schedule id
cronMeta map[string]Rule // rule id -> cron rule (name/expression for listing)
}
// NewService builds an automation engine from cfg and deps. The bus in deps is
@@ -50,6 +51,7 @@ func NewService(cfg Config, deps Deps) (*Service, error) {
engine: engine,
remind: remind,
cronRules: make(map[string]ScheduleID),
cronMeta: make(map[string]Rule),
}, nil
}
@@ -87,6 +89,7 @@ func (s *Service) AddRule(ctx context.Context, r Rule) error {
}
s.mu.Lock()
s.cronRules[r.ID] = id
s.cronMeta[r.ID] = r
s.mu.Unlock()
return nil
default:
@@ -100,6 +103,7 @@ func (s *Service) RemoveRule(ctx context.Context, id string) error {
s.mu.Lock()
if schedID, ok := s.cronRules[id]; ok {
delete(s.cronRules, id)
delete(s.cronMeta, id)
s.mu.Unlock()
s.sched.RemoveSchedule(schedID)
return nil
@@ -116,6 +120,51 @@ func (s *Service) AddReminder(ctx context.Context, r Reminder) error {
// Reminders returns a sorted snapshot of registered reminders.
func (s *Service) Reminders() []Reminder { return s.remind.List() }
// List returns a read-only snapshot of every registered automation and reminder
// for the admin panel, composing the reminder store, the event rules and the
// cron schedules. It never mutates state and is safe for concurrent use. Cron
// entries' LastRun/NextRun are zero until the scheduler has run.
func (s *Service) List() []AutomationRecord {
out := make([]AutomationRecord, 0)
for _, r := range s.remind.List() {
at := r.At
out = append(out, AutomationRecord{
ID: r.ID,
Name: r.Message,
Kind: "reminder",
Enabled: true,
NextRun: at,
})
}
for _, r := range s.engine.Rules() {
out = append(out, AutomationRecord{
ID: r.ID,
Name: r.Name,
Kind: string(r.Trigger),
Enabled: true,
})
}
s.mu.RLock()
for id, rule := range s.cronMeta {
entry := s.sched.Entry(s.cronRules[id])
out = append(out, AutomationRecord{
ID: id,
Name: rule.Name,
Kind: string(rule.Trigger),
Cron: rule.Expression,
Enabled: true,
LastRun: entry.Prev,
NextRun: entry.Next,
})
}
s.mu.RUnlock()
return out
}
// CancelReminder delegates to the ReminderStore.
func (s *Service) CancelReminder(ctx context.Context, id string) error {
return s.remind.Cancel(ctx, id)
+154
View File
@@ -5,12 +5,19 @@ import (
"encoding/json"
"errors"
"net/http"
"strconv"
"time"
"git.yeij.top/AskaEth/Cyrene_v2/internal/auth"
capreg "git.yeij.top/AskaEth/Cyrene_v2/internal/capability/registry"
apperr "git.yeij.top/AskaEth/Cyrene_v2/internal/errors"
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
)
// backthinkListLimit is the number of most-recent think records the admin
// panel asks for when no limit is specified.
const backthinkListLimit = 50
// PluginSummary is the wire shape of a capability/plugin in the admin list.
type PluginSummary struct {
Name string `json:"name"`
@@ -33,6 +40,55 @@ type policyRequest struct {
AllowInStranger bool `json:"allowInStranger"`
}
// AdminLlmProvider is the wire shape of one LLM provider in the admin config.
// The API key is deliberately absent: it is never returned to the client.
type AdminLlmProvider struct {
ID string `json:"id"`
Name string `json:"name"`
BaseURL string `json:"baseUrl"`
Model string `json:"model"`
Multimodal bool `json:"multimodal"`
Enabled bool `json:"enabled"`
Fallback string `json:"fallback,omitempty"`
}
// AdminLlmConfig is the wire shape of GET /api/v1/admin/llm.
type AdminLlmConfig struct {
Providers []AdminLlmProvider `json:"providers"`
}
// AdminAutomationRecord is the wire shape of one automation/reminder in the
// admin list. Cron carries the cron expression for the cron kind; lastRun and
// nextRun are omitted when the value is unknown.
type AdminAutomationRecord struct {
ID string `json:"id"`
Name string `json:"name"`
Kind string `json:"kind"`
Cron string `json:"cron,omitempty"`
Enabled bool `json:"enabled"`
LastRun string `json:"lastRun,omitempty"`
NextRun string `json:"nextRun,omitempty"`
}
// AdminUserRecord is the wire shape of one account/API key in the admin list.
type AdminUserRecord struct {
ID string `json:"id"`
Username string `json:"username"`
Role string `json:"role"`
CreatedAt string `json:"createdAt,omitempty"`
}
// AdminBackthinkItem is the wire shape of one background-think record in the
// admin monitor list.
type AdminBackthinkItem struct {
ID string `json:"id"`
Trigger string `json:"trigger"`
Conclusion string `json:"conclusion,omitempty"`
ShouldNotify bool `json:"shouldNotify"`
Pushed bool `json:"pushed"`
At int64 `json:"at"`
}
// adminListPersonas lists persona summaries (admin read).
func (s *Server) adminListPersonas(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, s.collectPersonas())
@@ -97,6 +153,104 @@ func (s *Server) adminListIoT(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, out)
}
// adminGetLlm returns the LLM provider/model configuration read from the live
// config. API keys are never returned (the wire shape carries no key field), so
// the handler leaks nothing sensitive.
func (s *Server) adminGetLlm(w http.ResponseWriter, r *http.Request) {
cfg := s.cfg.LLM
writeJSON(w, http.StatusOK, AdminLlmConfig{Providers: []AdminLlmProvider{
{
ID: "openai",
Name: "OpenAI",
BaseURL: cfg.BaseURL,
Model: cfg.Model,
Multimodal: cfg.VisionModel != "",
Enabled: cfg.Provider != "anthropic",
Fallback: cfg.Fallback,
},
{
ID: "anthropic",
Name: "Anthropic",
BaseURL: cfg.AnthropicBaseURL,
Model: cfg.AnthropicModel,
Multimodal: false,
Enabled: cfg.Provider == "anthropic",
Fallback: cfg.AnthropicFallback,
},
}})
}
// adminListAutomations lists automations and reminders (admin read).
func (s *Server) adminListAutomations(w http.ResponseWriter, r *http.Request) {
out := make([]AdminAutomationRecord, 0)
if s.automations != nil {
for _, a := range s.automations.List() {
rec := AdminAutomationRecord{
ID: a.ID,
Name: a.Name,
Kind: a.Kind,
Cron: a.Cron,
Enabled: a.Enabled,
}
if !a.LastRun.IsZero() {
rec.LastRun = a.LastRun.Format(time.RFC3339)
}
if !a.NextRun.IsZero() {
rec.NextRun = a.NextRun.Format(time.RFC3339)
}
out = append(out, rec)
}
}
writeJSON(w, http.StatusOK, out)
}
// adminListUsers lists accounts and API keys (admin read). Without a wired
// UserStore it reports the single configured admin account rather than
// fabricating a column of virtual users.
func (s *Server) adminListUsers(w http.ResponseWriter, r *http.Request) {
out := []AdminUserRecord{{
ID: "admin",
Username: s.cfg.Auth.AdminUser,
Role: string(auth.RoleOwner),
}}
if s.users != nil {
for _, u := range s.users.List() {
rec := AdminUserRecord{ID: u.ID, Username: u.Username, Role: string(u.Role)}
if u.CreatedAt != nil {
rec.CreatedAt = u.CreatedAt.Format(time.RFC3339)
}
out = append(out, rec)
}
}
writeJSON(w, http.StatusOK, out)
}
// adminListBackthink lists the most recent background-think records (admin read).
func (s *Server) adminListBackthink(w http.ResponseWriter, r *http.Request) {
out := make([]AdminBackthinkItem, 0)
if s.backthink == nil {
writeJSON(w, http.StatusOK, out)
return
}
records, err := s.backthink.List(r.Context(), backthinkListLimit)
if err != nil {
s.log.Error("list backthink", "err", err)
writeError(w, apperr.Wrap(apperr.ErrInternal, "backthink_failed", "failed to list backthink records"))
return
}
for _, rec := range records {
out = append(out, AdminBackthinkItem{
ID: strconv.FormatInt(rec.ID, 10),
Trigger: rec.Trigger,
Conclusion: rec.Conclusion,
ShouldNotify: rec.ShouldPush,
Pushed: rec.PushResult.Pushed,
At: rec.At.UnixMilli(),
})
}
writeJSON(w, http.StatusOK, out)
}
// adminSetPolicy upserts the reachability markers for a capability. It is the
// audited admin write example (owner-only, enforced by the route registration).
func (s *Server) adminSetPolicy(w http.ResponseWriter, r *http.Request) {
+41 -3
View File
@@ -25,6 +25,7 @@ import (
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/loop"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/orchestration"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/orchestration/comm"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/task"
@@ -918,10 +919,10 @@ func TestHubRoute(t *testing.T) {
}
hb.taskOutputs = &stubTaskOutputs{ev: task.OutputEvent{ID: "task-1", Status: task.StatusDone, Output: task.Output{Stdout: "out"}}}
// voice_input → publishes to bus (no frame enqueued, not an unknown error).
// voice_input with no voice service → clear error, never a crash or a drop.
c.route(wsInbound{Type: "voice_input", Data: json.RawMessage(`{"audio":"x"}`)})
if err := connHasNoFrame(c); err != nil {
t.Fatalf("voice_input should not enqueue a frame directly: %v", err)
if ev := readFakeFrame(t, c); ev.Type != "error" {
t.Fatalf("no-voice voice_input frame type = %q, want error", ev.Type)
}
// unknown → error (enqueueJSON).
c.route(wsInbound{Type: "nope"})
@@ -935,6 +936,43 @@ func TestHubRoute(t *testing.T) {
}
}
// stubVoice is a deterministic VoiceService for voice route tests.
type stubVoice struct {
sttText string
sttErr error
tts voice.Audio
ttsErr error
}
func (s *stubVoice) STT(_ context.Context, _ voice.Audio) (string, error) { return s.sttText, s.sttErr }
func (s *stubVoice) TTS(_ context.Context, _ string) (voice.Audio, error) { return s.tts, s.ttsErr }
func TestHubVoiceInputTurn(t *testing.T) {
t.Parallel()
hb := newTestHub(&stubRunner{res: &loop.Result{Text: "你好", Steps: 1, StopReason: "stop"}}, nil)
hb.voice = &stubVoice{sttText: "你好", tts: voice.Audio{Format: "wav", Data: []byte("audio-bytes")}}
c := newFakeConn(hb, 32)
hb.register(c)
// "aGVsbG8=" decodes to "hello" — a valid base64 audio payload.
c.route(wsInbound{Type: "voice_input", Data: json.RawMessage(`{"audio":"aGVsbG8=","format":"wav"}`)})
// runVoiceTurn runs in a goroutine; read with a deadline.
ev := readFakeFrame(t, c)
if ev.Type != "voice_output" {
t.Fatalf("voice frame type = %q, want voice_output (frame=%+v)", ev.Type, ev)
}
r, ok := ev.Data.(map[string]any)
if !ok || r["text"] != "你好" {
t.Fatalf("voice transcript = %+v, want 你好", ev.Data)
}
if r["reply"] != "你好" {
t.Fatalf("voice reply = %+v, want 你好", ev.Data)
}
if r["audio"] == "" || r["audio"] == nil {
t.Fatalf("voice audio missing from reply: %+v", ev.Data)
}
}
func TestHubBridgeDeviceUpdate(t *testing.T) {
t.Parallel()
b := bus.New()
+18 -5
View File
@@ -57,10 +57,13 @@ type loginResponse struct {
}
// chatRequest is the POST /api/v1/chat body. When Stream is true the gateway
// triggers streaming over the WebSocket channel and returns immediately.
// triggers streaming over the WebSocket channel and returns immediately. Image is
// an optional base64 data: URI or http(s) URL attached for the vision path; the
// loop routes it through the vision/OCR tool or the main model (design/06 §0).
type chatRequest struct {
SessionID string `json:"sessionId"`
Text string `json:"text"`
Image string `json:"image,omitempty"`
Stream bool `json:"stream"`
}
@@ -108,12 +111,12 @@ func (s *Server) handleChat(w http.ResponseWriter, r *http.Request) {
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "invalid request body"))
return
}
if strings.TrimSpace(req.Text) == "" {
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "text is required"))
if strings.TrimSpace(req.Text) == "" && strings.TrimSpace(req.Image) == "" {
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "text or image is required"))
return
}
if req.Stream {
s.publishChatStream(r.Context(), req.SessionID, req.Text)
s.publishChatStream(r.Context(), req.SessionID, req.Text, req.Image)
writeJSON(w, http.StatusOK, map[string]any{"stream": true})
return
}
@@ -124,7 +127,7 @@ func (s *Server) handleChat(w http.ResponseWriter, r *http.Request) {
if req.SessionID == "" {
req.SessionID = "web"
}
res, err := s.runner.Run(r.Context(), req.SessionID, []model.Message{model.TextMessage(model.RoleUser, req.Text)})
res, err := s.runner.Run(r.Context(), req.SessionID, userChatMessages(req.Text, req.Image))
if err != nil {
s.log.Error("chat run", "err", err, "session", req.SessionID)
writeError(w, apperr.Wrap(apperr.ErrInternal, "chat_failed", "chat failed"))
@@ -185,6 +188,16 @@ func firstMode(modes []string) string {
return modes[0]
}
// userChatMessages builds the user turn for a chat / stream request. When image is
// present the turn becomes an image message (with optional text); otherwise a
// plain text message. An empty text with an image yields an image-only turn.
func userChatMessages(text, image string) []model.Message {
if strings.TrimSpace(image) != "" {
return []model.Message{model.ImageMessage(model.RoleUser, text, image)}
}
return []model.Message{model.TextMessage(model.RoleUser, text)}
}
// summaryOfSession maps a session.Session to its wire shape.
func summaryOfSession(sess session.Session) SessionSummary {
return SessionSummary{
+40
View File
@@ -40,6 +40,8 @@ import (
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/orchestration"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/task"
"git.yeij.top/AskaEth/Cyrene_v2/internal/services/automations"
"git.yeij.top/AskaEth/Cyrene_v2/internal/services/backthink"
)
// source is the log provenance tag for every record this package emits.
@@ -110,6 +112,26 @@ type TaskOutputStore interface {
TaskOutput(ctx context.Context, id string) (task.OutputEvent, error)
}
// AutomationStore enumerates the automations and reminders for the admin list.
// *automations.Service is the production implementation. The gateway maps each
// record to its wire shape.
type AutomationStore interface {
List() []automations.AutomationRecord
}
// UserStore enumerates the accounts and API keys visible to the admin panel.
// The production implementation reads the configured admin account plus the
// auth.APIKey table. The gateway maps each summary to its wire shape.
type UserStore interface {
List() []auth.UserSummary
}
// BackthinkStore enumerates the persisted background-think records for the
// M9/M10 monitor panel. *backthink.Backthink is the production implementation.
type BackthinkStore interface {
List(ctx context.Context, limit int) ([]backthink.Record, error)
}
// Deps carries the gateway's injected dependencies. Every field is nil-tolerant
// except Auth, which is required (authentication backs login and every RBAC
// decision). Registry/Loop are satisfied by the concrete *capability.Registry and
@@ -141,6 +163,13 @@ type Deps struct {
// Tasks queries task output for the task_output_query WS route.
// Optional (task_output_query returns a clear error when unset).
Tasks TaskOutputStore
// Automations lists automations and reminders. Optional.
Automations AutomationStore
// Users lists accounts and API keys. Optional (falls back to the configured
// single admin account).
Users UserStore
// Backthink lists background-think records. Optional.
Backthink BackthinkStore
// Streaming marks the injected Loop as streaming-capable (its OnStream is
// wired to publish ws.stream_chunk). When set the hub skips its own single
// final stream_chunk emission, since deltas already stream through OnStream.
@@ -165,6 +194,10 @@ type Server struct {
iot IoTStore
auditor SubagentAuditor
taskOutputs TaskOutputStore
automations AutomationStore
users UserStore
backthink BackthinkStore
voice VoiceService
streaming bool
frontend fs.FS
log *slog.Logger
@@ -193,6 +226,9 @@ func New(cfg *config.Config, deps Deps) (*Server, error) {
iot: deps.IoT,
auditor: deps.Scheduler,
taskOutputs: deps.Tasks,
automations: deps.Automations,
users: deps.Users,
backthink: deps.Backthink,
streaming: deps.Streaming,
frontend: deps.Frontend,
log: sourcedLog(deps.Log),
@@ -229,6 +265,10 @@ func (s *Server) Handler() http.Handler {
mux.Handle("GET /api/v1/admin/plugins", adminAuth(http.HandlerFunc(s.adminListPlugins)))
mux.Handle("GET /api/v1/admin/channels", adminAuth(http.HandlerFunc(s.adminListChannels)))
mux.Handle("GET /api/v1/admin/iot", adminAuth(http.HandlerFunc(s.adminListIoT)))
mux.Handle("GET /api/v1/admin/llm", adminAuth(http.HandlerFunc(s.adminGetLlm)))
mux.Handle("GET /api/v1/admin/automations", adminAuth(http.HandlerFunc(s.adminListAutomations)))
mux.Handle("GET /api/v1/admin/users", adminAuth(http.HandlerFunc(s.adminListUsers)))
mux.Handle("GET /api/v1/admin/backthink", adminAuth(http.HandlerFunc(s.adminListBackthink)))
mux.Handle("POST /api/v1/admin/plugins/policy", s.auth.Authorize(auth.RoleOwner)(http.HandlerFunc(s.adminSetPolicy)))
var h http.Handler = mux
+163
View File
@@ -0,0 +1,163 @@
package gateway
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/model"
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/multimodal/voice"
)
// VoiceService runs speech-to-text and text-to-speech for the voice_input WS
// route. *voice.Chain (the degrading provider chain) and *voice.Service (the
// background worker pool) both satisfy it. It is nil-tolerant: the voice route
// replies with a clear error when no service is injected.
type VoiceService interface {
// STT transcribes audio to text.
STT(ctx context.Context, audio voice.Audio) (string, error)
// TTS synthesizes audio from text.
TTS(ctx context.Context, text string) (voice.Audio, error)
}
// voiceInputRequest is the decoded payload of a voice_input frame. Audio is a
// base64 (or data: URI) audio blob; Format is its encoding ("wav"/"mp3"/"ogg");
// Text overrides STT when present (a client that already transcribed can skip
// recognition); SessionID defaults to "web".
type voiceInputRequest struct {
Audio string `json:"audio"`
Format string `json:"format"`
Text string `json:"text"`
SessionID string `json:"sessionId"`
}
// SetVoice injects the voice STT/TTS service used by the voice_input WS route.
// It is the gateway's exposed injection point so the composition root can wire a
// *voice.Chain or *voice.Service without touching Deps. A nil value disables the
// route (it replies with a clear error).
func (s *Server) SetVoice(v VoiceService) {
s.voice = v
s.hub.voice = v
}
// handleVoiceInput routes one voice_input frame. Without an injected voice
// service it replies with a clear error (never a crash); otherwise it decodes the
// request and runs the STT→chat→TTS turn asynchronously so the read pump is never
// blocked by provider latency.
func (c *wsConn) handleVoiceInput(in wsInbound) {
if c.h.voice == nil {
c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice unavailable"}})
return
}
var req voiceInputRequest
_ = json.Unmarshal(in.Data, &req)
if strings.TrimSpace(req.Audio) == "" && strings.TrimSpace(req.Text) == "" {
c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": "voice audio or text required"}})
return
}
go c.h.runVoiceTurn(c, req)
}
// runVoiceTurn drives one voice turn: STT (or a supplied transcript) → agent chat
// → TTS, then replies to the requesting connection with a voice_output frame
// carrying the base64 audio plus the recognized transcript and assistant reply.
func (h *hub) runVoiceTurn(c *wsConn, req voiceInputRequest) {
ctx := context.Background()
if req.SessionID == "" {
req.SessionID = "web"
}
transcript := strings.TrimSpace(req.Text)
if transcript == "" {
if strings.TrimSpace(req.Audio) == "" {
h.voiceError(c, "voice audio or text required")
return
}
audio, err := decodeVoiceAudio(req.Audio, req.Format)
if err != nil {
h.log.Warn("voice audio decode", "err", err)
h.voiceError(c, "voice audio could not be decoded")
return
}
stt, err := h.voice.STT(ctx, audio)
if err != nil {
h.log.Warn("voice stt", "err", err)
h.voiceError(c, "voice transcription failed")
return
}
transcript = strings.TrimSpace(stt)
}
if transcript == "" {
h.voiceError(c, "no speech recognized")
return
}
if h.runner == nil {
h.voiceError(c, "chat service is not configured")
return
}
res, err := h.runner.Run(ctx, req.SessionID, []model.Message{model.TextMessage(model.RoleUser, transcript)})
if err != nil {
h.log.Warn("voice chat run", "err", err, "session", req.SessionID)
h.voiceError(c, "voice chat failed")
return
}
reply := res.Text
if strings.TrimSpace(reply) == "" {
reply = res.StopReason
}
audi, err := h.voice.TTS(ctx, reply)
if err != nil {
h.log.Warn("voice tts", "err", err)
h.voiceError(c, "voice synthesis failed")
return
}
c.enqueueJSON(wsEvent{Type: wsVoiceOutput, Data: map[string]any{
"audio": base64.StdEncoding.EncodeToString(audi.Data),
"format": audi.Format,
"text": transcript,
"reply": reply,
}})
}
// voiceError replies with a typed error frame for a failed voice turn.
func (h *hub) voiceError(c *wsConn, message string) {
c.enqueueJSON(wsEvent{Type: "error", Data: map[string]any{"message": message}})
}
// decodeVoiceAudio turns a base64 (or data:) audio payload into voice.Audio. The
// format is taken from req when set, guessed from a data: URI mime otherwise, and
// defaulted to "wav".
func decodeVoiceAudio(src, format string) (voice.Audio, error) {
payload := src
if strings.HasPrefix(src, "data:") {
if comma := strings.IndexByte(src, ','); comma >= 0 {
if format == "" {
format = formatFromMime(src[:comma])
}
payload = src[comma+1:]
}
}
if strings.TrimSpace(payload) == "" {
return voice.Audio{}, fmt.Errorf("voice: empty audio")
}
data, err := base64.StdEncoding.DecodeString(payload)
if err != nil {
return voice.Audio{}, fmt.Errorf("voice: decode audio: %w", err)
}
if format == "" {
format = "wav"
}
return voice.Audio{Format: format, Data: data}, nil
}
// formatFromMime guesses an audio format from a data: URI mime segment.
func formatFromMime(meta string) string {
lower := strings.ToLower(meta)
for _, f := range []string{"wav", "mp3", "ogg", "webm", "mpeg"} {
if strings.Contains(lower, f) {
return f
}
}
return "wav"
}
+8 -4
View File
@@ -28,6 +28,7 @@ const (
wsChatSubject = "ws.chat"
wsStreamSubject = "ws.stream"
wsVoiceInput = "ws.voice_input"
wsVoiceOutput = "voice_output"
wsReadLimit = 4096
wsWriteWait = 10 * time.Second
wsPongWait = 60 * time.Second
@@ -72,6 +73,7 @@ type hub struct {
runner Chatter
auditor SubagentAuditor
taskOutputs TaskOutputStore
voice VoiceService
streaming bool
log *slog.Logger
upgrader websocket.Upgrader
@@ -90,6 +92,7 @@ func newHub(s *Server) *hub {
runner: s.runner,
auditor: s.auditor,
taskOutputs: s.taskOutputs,
voice: s.voice,
streaming: s.streaming,
log: s.log,
origins: origins,
@@ -213,14 +216,14 @@ func (h *hub) streamChat(ctx context.Context, payload any) {
_ = json.Unmarshal(b, &req)
}
turnID := newTurnID()
if h.runner == nil || strings.TrimSpace(req.Text) == "" {
if h.runner == nil || (strings.TrimSpace(req.Text) == "" && strings.TrimSpace(req.Image) == "") {
h.broadcast(wsEvent{Type: "stream_end", TurnID: turnID, Seq: int(h.streamSeq.Add(1)), Data: map[string]any{"stopReason": "idle"}})
return
}
if req.SessionID == "" {
req.SessionID = "web"
}
res, err := h.runner.Run(ctx, req.SessionID, []model.Message{model.TextMessage(model.RoleUser, req.Text)})
res, err := h.runner.Run(ctx, req.SessionID, userChatMessages(req.Text, req.Image))
if err != nil {
h.log.Error("ws chat run", "err", err, "session", req.SessionID)
h.broadcast(wsEvent{Type: "stream_end", TurnID: turnID, Seq: int(h.streamSeq.Add(1)), Data: map[string]any{"stopReason": "error", "error": err.Error()}})
@@ -256,6 +259,7 @@ func (h *hub) broadcastStream(payload any) {
type chatStreamRequest struct {
SessionID string `json:"sessionId"`
Text string `json:"text"`
Image string `json:"image,omitempty"`
}
// handleWS upgrades /ws/chat after authenticating the JWT from the query token or
@@ -382,11 +386,11 @@ func (c *wsConn) enqueueJSON(ev wsEvent) {
}
// publishChatStream publishes a ws.chat streaming trigger (REST stream mode).
func (s *Server) publishChatStream(ctx context.Context, sessionID, text string) {
func (s *Server) publishChatStream(ctx context.Context, sessionID, text, image string) {
if s.bus == nil {
return
}
_ = s.bus.Publish(ctx, wsChatSubject, map[string]any{"sessionId": sessionID, "text": text})
_ = s.bus.Publish(ctx, wsChatSubject, map[string]any{"sessionId": sessionID, "text": text, "image": image})
}
// wsToken extracts the JWT from the ?token= query or the Authorization header.
+5 -4
View File
@@ -15,10 +15,11 @@ func (c *wsConn) route(in wsInbound) {
case "chat":
c.publishPublic(wsChatSubject, rawData(in.Data))
case "voice_input":
// Voice is not wired to a live pipeline yet; surface the payload on the
// bus so an upper layer (or a later milestone) can consume it, rather
// than dropping it as unknown.
c.publishPublic(wsVoiceInput, rawData(in.Data))
// Voice routes through the injected STT→chat→TTS pipeline. Without a
// voice service it replies with a clear error rather than dropping the
// frame. The audio reply is delivered as a voice_output event on the
// requesting connection.
c.handleVoiceInput(in)
case "subagent_ask":
c.handleSubagentAsk(in)
case "task_output_query":