feat: admin 端点补齐 + 视觉真推理 + 语音链路 + wire.go 收口(A+B+第3步)
批次A admin 端点(原本 404 变 200 真实数据):
- GET /admin/llm:读 s.cfg.LLM 组 openai+anthropic 两 provider(enabled=当前 Provider、multimodal=配 VisionModel),APIKey 不进 wire 零泄露。
- GET /admin/automations:*automations.Service.List()(reminders+event rules+cron schedules,cron 加只读登记)。
- GET /admin/users:cfg.AdminUser + *auth.APIKey.List()(密钥掩码);无用户存储,恒先返回当前 admin 账号。
- GET /admin/backthink:*backthink.Backthink.List(ctx,50)。均只读、owner/admin RBAC、{data} 信封。
批次B 视觉真推理 + 语音:
- rest.go chatRequest 加 Image?;流式 publishChatStream 带 image;userChatMessages 构造 ImageMessage(model.go:71)。
- loop/vision.go:ImageResolver 可注入(MVP 先看图再对话);run.go Run 开头 resolveImages,无 resolver 透传直交主模型。
- vision/resolver.go:MessageResolver 调 vision.Router.Route——KindMain 保留图片交主模型多模态、KindVisionTool/OCRTool 经 dispatcher.Invoke 折叠成 [图片内容] 文本、KindUnsupported 记日志保留原图不崩。
- gateway/voice.go:VoiceService 接口 + SetVoice 注入点 + handleVoiceInput(未注入回 error 不崩)+ runVoiceTurn 异步 STT→loop chat→TTS→定向 voice_output 帧。
- ws_hub/ws_route:voice_input 接 handleVoiceInput;前端 ws.ts decodeBase64/playVoiceAudio/case voice_output。
第3步 wire.go 收口(唯一碰 wire.go 的一步):
- Deps 补传 Automations:a.automation / Backthink:a.backthink / Users:auth.NewAPIKey(a.cfg.Auth)(*auth.APIKey 原生满足 UserStore,避免 admin 重复的复合类型)。
- gw.SetVoice(a.voice);loop.Options.Vision = vision.NewMessageResolver(router, info, dispatcher.Invoke)(router=NewRouter(WithVisionTool/WithOCRTool)、info=cfg.LLM.VisionModel)。
实机验证:四 admin 端点 200 真实数据(llm 双 provider 脱敏、users=admin,yeij0942,owner);WS 握手正常。go build/vet/test 全绿 + 前端 tsc。wire.go 557 行。
This commit is contained in:
1 parent
648292d035
commit
518bd0a230
21 files changed
+976
-38
No files matched your search
@@ -37,6 +37,34 @@ function toRecord(v: unknown): Record<string, unknown> | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Decode a base64 string into bytes (used for the voice audio reply). */
|
||||
function decodeBase64(b64: string): Uint8Array {
|
||||
const bin = atob(b64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Best-effort playback of a voice_output audio payload (blocked by autoplay
|
||||
* policy, so sound may be silent until the user interacts). */
|
||||
function playVoiceAudio(payload: unknown): void {
|
||||
const r = toRecord(payload);
|
||||
if (!r || typeof r.audio !== "string" || !r.audio) return;
|
||||
try {
|
||||
const bytes = decodeBase64(r.audio);
|
||||
const format = typeof r.format === "string" && r.format ? r.format : "wav";
|
||||
const buf = new ArrayBuffer(bytes.byteLength);
|
||||
new Uint8Array(buf).set(bytes);
|
||||
const blob = new Blob([buf], { type: `audio/${format}` });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const audio = new Audio(url);
|
||||
void audio.play().catch(() => {});
|
||||
audio.onended = () => URL.revokeObjectURL(url);
|
||||
} catch {
|
||||
// Ignore malformed audio payloads.
|
||||
}
|
||||
}
|
||||
|
||||
function parseTool(v: unknown): ToolUsed | null {
|
||||
const r = toRecord(v);
|
||||
if (!r) return null;
|
||||
@@ -253,6 +281,18 @@ function dispatch(ev: { type: string; data: unknown }): void {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "voice_output": {
|
||||
// Voice turn reply: the backend sends back the synthesized audio plus the
|
||||
// recognized transcript and the assistant reply. Playback is best-effort
|
||||
// (autoplay may be blocked); the transcript/reply is surfaced as a log line.
|
||||
playVoiceAudio(ev.data);
|
||||
if (r && typeof r.reply === "string") {
|
||||
s.addDeviceUpdate(`语音回复:${r.reply}`);
|
||||
} else if (r && typeof r.text === "string") {
|
||||
s.addDeviceUpdate(`语音识别:${r.text}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
// Unknown future event: keep it in the context log so no frame is lost.
|
||||
const text = typeof ev.data === "string" ? ev.data : JSON.stringify(ev.data);
|
||||
|
||||
Reference in new issue
Block a user