feat(web): 用户侧体验补齐——思维链/工具卡数据源 + 图片附件 + 语音上行 + home/vision/voice 真表现
- 打通 thinking/tool_call 运行时数据源:openai 发 thinking_*(reasoning_content)/tool_call_* (delta.tool_calls 累积重新组装 Args 非 JSON 串);anthropic_stream 发 thinking_*(content_block thinking start/delta/stop)。ws_bridge 已映射 thinking.*/tool_call.*,前端渲染活。 - 前端消息模型 + 附件:ChatMessage 加 image?;ChatInput 加附件按钮(file→base64)、预览/移除、 image-aware onSend(text,image);VisionMessageBody 升级图像卡+识别 chip;VoiceMessageBody 升级通话控制卡(live2D预留/波形/静音/挂断);HomeMessageBody 读真实 deviceUpdates(去掉 sampleDevices 猜)。 - 激活 voice_input:ChatShell onVoice 不再 no-op → sendUp voice_input + 切 voice mode。 - addTool 改按工具名 upsert(修复 tool_call.start/end 重复卡);ws_mode_switch 应用 agent 驱动 mode;addUserMessage/beginAssistant 带 image/mode;chatStream/useChat.send 带 image。 - 新增 image/phone 图标。 已知限制(标注):图片仅前端收集(后端 chatRequest 无 image 字段未接真推理);流式工具卡无真实 工具名(StreamEvent 只带 ID,回退 tool_<id前缀>);mode 由 agent 驱动但后端未逐消息下发 mode (mode_switch 回显+默认当前 mode 兜底);voice 为 UI+voice_input 上行(真实通话后置)。 go build/vet/test + 前端 tsc 绿。
This commit is contained in:
1 parent
b0bd93b15c
commit
2248ddce15
14 files changed
+576
-102
No files matched your search
@@ -154,11 +154,18 @@ export function chatSync(
|
||||
});
|
||||
}
|
||||
|
||||
/** POST /api/v1/chat with stream:true — triggers streaming over the WS channel. */
|
||||
export function chatStream(sessionId: string, text: string): Promise<{ stream: true }> {
|
||||
/** POST /api/v1/chat with stream:true — triggers streaming over the WS channel.
|
||||
* `image` (a base64 data: URI or URL) is attached for the vision path; the
|
||||
* gateway's chat body currently has no image field, so an unconfigured backend
|
||||
* simply ignores it. */
|
||||
export function chatStream(
|
||||
sessionId: string,
|
||||
text: string,
|
||||
image?: string
|
||||
): Promise<{ stream: true }> {
|
||||
return request<{ stream: true }>("/chat", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ sessionId, text, stream: true }),
|
||||
body: JSON.stringify({ sessionId, text, image, stream: true }),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,11 @@ export interface ChatMessage {
|
||||
thinking?: string;
|
||||
mood?: string;
|
||||
tools?: ToolUsed[];
|
||||
/** Optional image attached to the turn (base64 data: URI or http(s) URL). The
|
||||
* user bubble renders it as a thumb; vision-mode assistant bodies render it
|
||||
* as an image card. Absent for legacy persisted messages, so renders fall back
|
||||
* to plain text. */
|
||||
image?: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useCallback, useState } from "react";
|
||||
import { chatStream } from "./api";
|
||||
import { connectWs, isWsOpen } from "./ws";
|
||||
import { useAppStore } from "../store/useAppStore";
|
||||
import { useModeStore } from "../store/useModeStore";
|
||||
import { now } from "./util";
|
||||
|
||||
/** Drives one user turn: push the user bubble, blanket a streaming assistant
|
||||
@@ -11,9 +12,10 @@ import { now } from "./util";
|
||||
*
|
||||
* The behavior mode is NOT chosen by the user at send time — the agent / backend
|
||||
* hot-switches it (design/01 §3), and each message carries its own mode, so the
|
||||
* bubble renders per that message's mode. */
|
||||
* bubble renders per that message's mode. Until the backend advertises a
|
||||
* per-message mode, a new assistant bubble defaults to the current UI mode. */
|
||||
export function useSendChat(): {
|
||||
send: (text: string) => Promise<void>;
|
||||
send: (text: string, image?: string) => Promise<void>;
|
||||
sending: boolean;
|
||||
} {
|
||||
const [sending, setSending] = useState(false);
|
||||
@@ -24,19 +26,23 @@ export function useSendChat(): {
|
||||
const appendChunk = useAppStore((s) => s.appendChunk);
|
||||
const addTool = useAppStore((s) => s.addTool);
|
||||
const appendTaskLog = useAppStore((s) => s.appendTaskLog);
|
||||
const mode = useModeStore((s) => s.mode);
|
||||
|
||||
const send = useCallback(
|
||||
async (text: string) => {
|
||||
async (text: string, image?: string) => {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed || sending) return;
|
||||
if ((!trimmed && !image) || sending) return;
|
||||
setSending(true);
|
||||
if (!isWsOpen()) connectWs();
|
||||
|
||||
addUserMessage(trimmed);
|
||||
beginAssistant();
|
||||
// The gateway's chat body still requires a non-empty text, so a pure image
|
||||
// turn (vision) falls back to a default prompt to keep the trigger valid.
|
||||
const prompt = trimmed || (image ? "分析这张图片" : "");
|
||||
addUserMessage(trimmed, undefined, image);
|
||||
beginAssistant(mode);
|
||||
|
||||
try {
|
||||
await chatStream(sessionId, trimmed);
|
||||
await chatStream(sessionId, prompt, image);
|
||||
} catch (err: unknown) {
|
||||
// Backend loop not wired or gateway unreachable: show the failure inline
|
||||
// rather than leaving a permanently-streaming bubble.
|
||||
@@ -49,7 +55,7 @@ export function useSendChat(): {
|
||||
setSending(false);
|
||||
}
|
||||
},
|
||||
[sending, sessionId, addUserMessage, beginAssistant, endStream, appendChunk, addTool, appendTaskLog]
|
||||
[sending, sessionId, mode, addUserMessage, beginAssistant, endStream, appendChunk, addTool, appendTaskLog]
|
||||
);
|
||||
|
||||
return { send, sending };
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useAppStore } from "../store/useAppStore";
|
||||
import { useModeStore } from "../store/useModeStore";
|
||||
import { getAuthToken } from "./api";
|
||||
import type { AgentInfo, FlowGraph, ToolUsed, WsUpEvent } from "./types";
|
||||
import { MODES, type AgentInfo, type FlowGraph, type Mode, type ToolUsed, type WsUpEvent } from "./types";
|
||||
import { now, uid } from "./util";
|
||||
|
||||
/**
|
||||
@@ -244,6 +244,15 @@ function dispatch(ev: { type: string; data: unknown }): void {
|
||||
// handshake acknowledgement; nothing to persist yet.
|
||||
break;
|
||||
}
|
||||
case "mode_switch": {
|
||||
// The agent drives the behavior mode (design/01 §3). Apply it to the shell
|
||||
// so newly begun message bubbles render per that mode.
|
||||
if (r && typeof r.mode === "string") {
|
||||
const m = r.mode as Mode;
|
||||
if (MODES.includes(m)) useModeStore.getState().setMode(m);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
// Unknown future event: keep it in the context log so no frame is lost.
|
||||
const text = typeof ev.data === "string" ? ev.data : JSON.stringify(ev.data);
|
||||
|
||||
Reference in new issue
Block a user