feat(web): 用户侧体验补齐——思维链/工具卡数据源 + 图片附件 + 语音上行 + home/vision/voice 真表现
- 打通 thinking/tool_call 运行时数据源:openai 发 thinking_*(reasoning_content)/tool_call_* (delta.tool_calls 累积重新组装 Args 非 JSON 串);anthropic_stream 发 thinking_*(content_block thinking start/delta/stop)。ws_bridge 已映射 thinking.*/tool_call.*,前端渲染活。 - 前端消息模型 + 附件:ChatMessage 加 image?;ChatInput 加附件按钮(file→base64)、预览/移除、 image-aware onSend(text,image);VisionMessageBody 升级图像卡+识别 chip;VoiceMessageBody 升级通话控制卡(live2D预留/波形/静音/挂断);HomeMessageBody 读真实 deviceUpdates(去掉 sampleDevices 猜)。 - 激活 voice_input:ChatShell onVoice 不再 no-op → sendUp voice_input + 切 voice mode。 - addTool 改按工具名 upsert(修复 tool_call.start/end 重复卡);ws_mode_switch 应用 agent 驱动 mode;addUserMessage/beginAssistant 带 image/mode;chatStream/useChat.send 带 image。 - 新增 image/phone 图标。 已知限制(标注):图片仅前端收集(后端 chatRequest 无 image 字段未接真推理);流式工具卡无真实 工具名(StreamEvent 只带 ID,回退 tool_<id前缀>);mode 由 agent 驱动但后端未逐消息下发 mode (mode_switch 回显+默认当前 mode 兜底);voice 为 UI+voice_input 上行(真实通话后置)。 go build/vet/test + 前端 tsc 绿。
This commit is contained in:
1 parent
b0bd93b15c
commit
2248ddce15
14 files changed
+576
-102
No files matched your search
@@ -29,9 +29,10 @@ type anthropicStartBlock struct {
|
||||
|
||||
// anthropicDelta is the incremental payload inside a content_block_delta event.
|
||||
type anthropicDelta struct {
|
||||
Type string `json:"type"` // text_delta | input_json_delta
|
||||
Type string `json:"type"` // text_delta | input_json_delta | thinking_delta
|
||||
Text string `json:"text,omitempty"`
|
||||
PartialJSON string `json:"partial_json,omitempty"`
|
||||
Thinking string `json:"thinking,omitempty"`
|
||||
}
|
||||
|
||||
// anthropicStreamError is the payload of an error event.
|
||||
@@ -141,6 +142,13 @@ func applyAnthropicEvent(
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindToolCallStart, Index: *evt.Index, ID: evt.Content.ID})
|
||||
}
|
||||
case "thinking":
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingStart, Index: *evt.Index})
|
||||
}
|
||||
if evt.Content.Text != "" && onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingDelta, Delta: evt.Content.Text, Index: *evt.Index})
|
||||
}
|
||||
}
|
||||
if _, ok := blocks[*evt.Index]; !ok {
|
||||
*order = append(*order, *evt.Index)
|
||||
@@ -166,6 +174,10 @@ func applyAnthropicEvent(
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindToolCallDelta, Delta: evt.Delta.PartialJSON, Index: *evt.Index, ID: b.id})
|
||||
}
|
||||
case "thinking_delta":
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingDelta, Delta: evt.Delta.Thinking, Index: *evt.Index})
|
||||
}
|
||||
}
|
||||
case "content_block_stop":
|
||||
if evt.Index == nil {
|
||||
@@ -183,6 +195,10 @@ func applyAnthropicEvent(
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindToolCallEnd, Index: *evt.Index, ID: b.id})
|
||||
}
|
||||
} else if b.typ == "thinking" {
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingEnd, Index: *evt.Index})
|
||||
}
|
||||
}
|
||||
case "message_delta":
|
||||
if evt.Usage != nil {
|
||||
|
||||
@@ -436,6 +436,63 @@ func TestAnthropicChat_ChatStream_NoMessageStop(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicChat_ChatStream_Thinking(t *testing.T) {
|
||||
t.Parallel()
|
||||
sse := strings.Join([]string{
|
||||
`data: {"type":"message_start","message":{"model":"claude-test","usage":{"input_tokens":5}}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me"}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" think"}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_stop","index":0}`,
|
||||
``,
|
||||
`data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Hel"}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"lo"}}`,
|
||||
``,
|
||||
`data: {"type":"content_block_stop","index":1}`,
|
||||
``,
|
||||
`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}`,
|
||||
``,
|
||||
`data: {"type":"message_stop"}`,
|
||||
``,
|
||||
}, "\n")
|
||||
srv := newAnthropicServer(t, http.StatusOK, "text/event-stream", sse, nil)
|
||||
ad := NewAnthropicChat(config.LLMConfig{AnthropicBaseURL: srv.URL, AnthropicModel: "claude-test"})
|
||||
|
||||
var events []StreamEvent
|
||||
resp, err := ad.ChatStream(context.Background(),
|
||||
Request{Messages: []Message{TextMessage(RoleUser, "hi")}, Model: "claude-test"},
|
||||
func(ev StreamEvent) { events = append(events, ev) })
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream error: %v", err)
|
||||
}
|
||||
if resp.Content != "Hello" {
|
||||
t.Errorf("resp.Content = %q, want Hello (thinking must not leak into body content)", resp.Content)
|
||||
}
|
||||
if resp.TotalTokens != 12 {
|
||||
t.Errorf("resp.TotalTokens = %d, want 12 (5 input + 7 output)", resp.TotalTokens)
|
||||
}
|
||||
wantEvents := []StreamEvent{
|
||||
{Kind: StreamKindThinkingStart, Index: 0},
|
||||
{Kind: StreamKindThinkingDelta, Index: 0, Delta: "Let me"},
|
||||
{Kind: StreamKindThinkingDelta, Index: 0, Delta: " think"},
|
||||
{Kind: StreamKindThinkingEnd, Index: 0},
|
||||
{Kind: StreamKindTextStart, Index: 1},
|
||||
{Kind: StreamKindTextDelta, Index: 1, Delta: "Hel"},
|
||||
{Kind: StreamKindTextDelta, Index: 1, Delta: "lo"},
|
||||
{Kind: StreamKindTextEnd, Index: 1},
|
||||
}
|
||||
if !reflect.DeepEqual(events, wantEvents) {
|
||||
t.Errorf("events = %+v, want %+v", events, wantEvents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicChat_ToolResult(t *testing.T) {
|
||||
t.Parallel()
|
||||
blk := AnthropicToolResult("toolu_1", `{"ok":true}`, true)
|
||||
|
||||
@@ -90,10 +90,49 @@ type openAIResponseMessage struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// openAIDelta is the incremental payload inside a stream chunk.
|
||||
// openAIDelta is the incremental payload inside a stream chunk. Reasoning content
|
||||
// (OpenAI o-series / DeepSeek / DashScope) arrives in ReasoningContent; streaming
|
||||
// function calls arrive in ToolCalls, accumulated per fragment index.
|
||||
type openAIDelta struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
ReasoningContent string `json:"reasoning_content"`
|
||||
ToolCalls []openAIToolCallDelta `json:"tool_calls"`
|
||||
}
|
||||
|
||||
// openAIToolCallDelta is one streaming function-call fragment. OpenAI emits a
|
||||
// sequence of these sharing the same Index: the ID and Function.Name usually
|
||||
// arrive once (in the first fragment) while Function.Arguments streams across
|
||||
// fragments.
|
||||
type openAIToolCallDelta struct {
|
||||
Index int `json:"index"`
|
||||
ID string `json:"id"`
|
||||
Function openAIToolCallFn `json:"function"`
|
||||
}
|
||||
|
||||
// openAIToolCallFn is the function part of a streaming tool-call fragment.
|
||||
type openAIToolCallFn struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
}
|
||||
|
||||
// openAIToolAcc accumulates the partial state of one streaming tool call (keyed
|
||||
// by the provider's tool_calls[].index) so its arguments re-assemble across the
|
||||
// delta fragments. started guards a single ToolCallStart emission per block.
|
||||
type openAIToolAcc struct {
|
||||
id string
|
||||
name string
|
||||
args strings.Builder
|
||||
started bool
|
||||
}
|
||||
|
||||
// idOrName resolves the block identifier used to correlate start/delta/end: the
|
||||
// provider tool-call id when present, else the function name.
|
||||
func (a *openAIToolAcc) idOrName() string {
|
||||
if a.id != "" {
|
||||
return a.id
|
||||
}
|
||||
return a.name
|
||||
}
|
||||
|
||||
// openAIUsage reports token consumption.
|
||||
@@ -378,16 +417,30 @@ func responseFromWire(wire *openAIChatResponse) Response {
|
||||
return out
|
||||
}
|
||||
|
||||
// scanSSE incrementally decodes stream chunks, accumulates the full content
|
||||
// scanSSE incrementally decodes stream chunks, accumulates the full body text
|
||||
// with a strings.Builder (never reading the whole body at once) and captures
|
||||
// usage from the final chunk. Each text delta is delivered as a text_delta
|
||||
// StreamEvent with the body-text segment bracketed by text_start/text_end.
|
||||
// usage from the final chunk. It splits each delta into its segment and brackets
|
||||
// them: reasoning_content as thinking_start/delta/end, tool_calls fragments as
|
||||
// tool_call_start/delta/end, and body text as text_start/delta/end. The
|
||||
// accumulated tool-call arguments re-assemble across fragments into a structured
|
||||
// Args map, and are attached to the returned Response alongside the text.
|
||||
func scanSSE(r io.Reader, onEvent func(StreamEvent)) (Response, error) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 0, sseBufInit), maxSSELineBytes)
|
||||
agg := Response{}
|
||||
var b strings.Builder
|
||||
started := false
|
||||
var text strings.Builder
|
||||
textStarted := false
|
||||
thinkingStarted := false
|
||||
toolCalls := map[int]*openAIToolAcc{}
|
||||
var toolOrder []int
|
||||
|
||||
endThinking := func() {
|
||||
if onEvent != nil && thinkingStarted {
|
||||
thinkingStarted = false
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingEnd})
|
||||
}
|
||||
}
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || !strings.HasPrefix(line, sseDataPrefix) {
|
||||
@@ -409,26 +462,100 @@ func scanSSE(r io.Reader, onEvent func(StreamEvent)) (Response, error) {
|
||||
if len(chunk.Choices) == 0 || chunk.Choices[0].Delta == nil {
|
||||
continue
|
||||
}
|
||||
delta := chunk.Choices[0].Delta.Content
|
||||
if delta == "" {
|
||||
delta := chunk.Choices[0].Delta
|
||||
|
||||
// Reasoning / thinking segment (OpenAI o-series, DeepSeek, DashScope).
|
||||
if delta.ReasoningContent != "" {
|
||||
if onEvent != nil && !thinkingStarted {
|
||||
thinkingStarted = true
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingStart})
|
||||
}
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingDelta, Delta: delta.ReasoningContent})
|
||||
}
|
||||
continue
|
||||
}
|
||||
b.WriteString(delta)
|
||||
|
||||
// Streaming tool-call fragments.
|
||||
if len(delta.ToolCalls) > 0 {
|
||||
endThinking()
|
||||
for _, tc := range delta.ToolCalls {
|
||||
acc := toolCalls[tc.Index]
|
||||
if acc == nil {
|
||||
acc = &openAIToolAcc{}
|
||||
toolCalls[tc.Index] = acc
|
||||
toolOrder = append(toolOrder, tc.Index)
|
||||
}
|
||||
if acc.id == "" {
|
||||
acc.id = tc.ID
|
||||
}
|
||||
if acc.name == "" {
|
||||
acc.name = tc.Function.Name
|
||||
}
|
||||
if onEvent != nil && !acc.started && acc.idOrName() != "" {
|
||||
acc.started = true
|
||||
onEvent(StreamEvent{Kind: StreamKindToolCallStart, Index: tc.Index, ID: acc.idOrName()})
|
||||
}
|
||||
if tc.Function.Arguments != "" {
|
||||
acc.args.WriteString(tc.Function.Arguments)
|
||||
if onEvent != nil {
|
||||
onEvent(StreamEvent{Kind: StreamKindToolCallDelta, Index: tc.Index, ID: acc.idOrName(), Delta: tc.Function.Arguments})
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Body-text segment.
|
||||
if delta.Content == "" {
|
||||
continue
|
||||
}
|
||||
endThinking()
|
||||
text.WriteString(delta.Content)
|
||||
if onEvent != nil {
|
||||
if !started {
|
||||
started = true
|
||||
if !textStarted {
|
||||
textStarted = true
|
||||
onEvent(StreamEvent{Kind: StreamKindTextStart})
|
||||
}
|
||||
onEvent(StreamEvent{Kind: StreamKindTextDelta, Delta: delta})
|
||||
onEvent(StreamEvent{Kind: StreamKindTextDelta, Delta: delta.Content})
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return Response{}, fmt.Errorf("model: %w: read stream: %v", ErrUpstream, err)
|
||||
}
|
||||
if onEvent != nil && started {
|
||||
onEvent(StreamEvent{Kind: StreamKindTextEnd})
|
||||
|
||||
if onEvent != nil {
|
||||
if thinkingStarted {
|
||||
onEvent(StreamEvent{Kind: StreamKindThinkingEnd})
|
||||
}
|
||||
if textStarted {
|
||||
onEvent(StreamEvent{Kind: StreamKindTextEnd})
|
||||
}
|
||||
// OpenAI has no per-block stop event, so each tool_call block is closed
|
||||
// when the stream terminates.
|
||||
for _, idx := range toolOrder {
|
||||
acc := toolCalls[idx]
|
||||
if acc == nil || !acc.started {
|
||||
continue
|
||||
}
|
||||
onEvent(StreamEvent{Kind: StreamKindToolCallEnd, Index: idx, ID: acc.idOrName()})
|
||||
}
|
||||
}
|
||||
agg.Content = b.String()
|
||||
|
||||
for _, idx := range toolOrder {
|
||||
acc := toolCalls[idx]
|
||||
if acc == nil {
|
||||
continue
|
||||
}
|
||||
args := map[string]any{}
|
||||
if acc.args.Len() > 0 {
|
||||
if err := json.Unmarshal([]byte(acc.args.String()), &args); err != nil {
|
||||
args = map[string]any{}
|
||||
}
|
||||
}
|
||||
agg.ToolCalls = append(agg.ToolCalls, ToolCall{Name: acc.name, Args: args})
|
||||
}
|
||||
agg.Content = text.String()
|
||||
agg.Message = Message{Role: RoleAssistant, Content: []Content{{Type: ContentTypeText, Text: agg.Content}}}
|
||||
return agg, nil
|
||||
}
|
||||
@@ -260,3 +260,95 @@ func TestOpenAIChat_Chat_NoModel(t *testing.T) {
|
||||
t.Fatalf("err = %v, want ErrNoModel", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIChat_ChatStream_ReasoningAndText(t *testing.T) {
|
||||
t.Parallel()
|
||||
sse := strings.Join([]string{
|
||||
`data: {"model":"gpt-test","choices":[{"index":0,"delta":{"role":"assistant","reasoning_content":"Let me th"}}]}`,
|
||||
``,
|
||||
`data: {"choices":[{"delta":{"reasoning_content":"ink about it"}}]}`,
|
||||
``,
|
||||
`data: {"choices":[{"delta":{"content":"Hel"}}]}`,
|
||||
``,
|
||||
`data: {"choices":[{"delta":{"content":"lo"}}]}`,
|
||||
``,
|
||||
`data: {"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}`,
|
||||
``,
|
||||
`data: [DONE]`,
|
||||
``,
|
||||
}, "\n")
|
||||
srv := newCompletionServer(t, http.StatusOK, "text/event-stream", sse, nil)
|
||||
ad := NewOpenAIChat(config.LLMConfig{BaseURL: srv.URL, Model: "gpt-test"})
|
||||
|
||||
var events []StreamEvent
|
||||
resp, err := ad.ChatStream(context.Background(),
|
||||
Request{Messages: []Message{TextMessage(RoleUser, "hi")}, Model: "gpt-test"},
|
||||
func(ev StreamEvent) { events = append(events, ev) })
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream error: %v", err)
|
||||
}
|
||||
if resp.Content != "Hello" {
|
||||
t.Errorf("resp.Content = %q, want Hello (reasoning must not leak into body content)", resp.Content)
|
||||
}
|
||||
wantEvents := []StreamEvent{
|
||||
{Kind: StreamKindThinkingStart},
|
||||
{Kind: StreamKindThinkingDelta, Delta: "Let me th"},
|
||||
{Kind: StreamKindThinkingDelta, Delta: "ink about it"},
|
||||
{Kind: StreamKindThinkingEnd},
|
||||
{Kind: StreamKindTextStart},
|
||||
{Kind: StreamKindTextDelta, Delta: "Hel"},
|
||||
{Kind: StreamKindTextDelta, Delta: "lo"},
|
||||
{Kind: StreamKindTextEnd},
|
||||
}
|
||||
if !reflect.DeepEqual(events, wantEvents) {
|
||||
t.Errorf("events = %+v, want %+v", events, wantEvents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIChat_ChatStream_ToolCall(t *testing.T) {
|
||||
t.Parallel()
|
||||
sse := strings.Join([]string{
|
||||
`data: {"model":"gpt-test","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_1","function":{"name":"lookup","arguments":"{\"loc"}}]}}]}`,
|
||||
``,
|
||||
`data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ation\":\"Paris\"}"}}]}}]}`,
|
||||
``,
|
||||
`data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}`,
|
||||
``,
|
||||
`data: {"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":6,"total_tokens":11}}`,
|
||||
``,
|
||||
`data: [DONE]`,
|
||||
``,
|
||||
}, "\n")
|
||||
srv := newCompletionServer(t, http.StatusOK, "text/event-stream", sse, nil)
|
||||
ad := NewOpenAIChat(config.LLMConfig{BaseURL: srv.URL, Model: "gpt-test"})
|
||||
|
||||
var events []StreamEvent
|
||||
resp, err := ad.ChatStream(context.Background(),
|
||||
Request{Messages: []Message{TextMessage(RoleUser, "hi")}, Model: "gpt-test"},
|
||||
func(ev StreamEvent) { events = append(events, ev) })
|
||||
if err != nil {
|
||||
t.Fatalf("ChatStream error: %v", err)
|
||||
}
|
||||
if len(resp.ToolCalls) != 1 {
|
||||
t.Fatalf("resp.ToolCalls = %+v, want 1", resp.ToolCalls)
|
||||
}
|
||||
tc := resp.ToolCalls[0]
|
||||
if tc.Name != "lookup" {
|
||||
t.Errorf("tool call name = %q, want lookup", tc.Name)
|
||||
}
|
||||
if tc.Args["location"] != "Paris" {
|
||||
t.Errorf("tool call args.location = %v, want Paris (reassembled from argument fragments)", tc.Args["location"])
|
||||
}
|
||||
if resp.Content != "" {
|
||||
t.Errorf("resp.Content = %q, want empty (tool-only response)", resp.Content)
|
||||
}
|
||||
wantEvents := []StreamEvent{
|
||||
{Kind: StreamKindToolCallStart, Index: 0, ID: "call_1"},
|
||||
{Kind: StreamKindToolCallDelta, Index: 0, ID: "call_1", Delta: `{"loc`},
|
||||
{Kind: StreamKindToolCallDelta, Index: 0, ID: "call_1", Delta: `ation":"Paris"}`},
|
||||
{Kind: StreamKindToolCallEnd, Index: 0, ID: "call_1"},
|
||||
}
|
||||
if !reflect.DeepEqual(events, wantEvents) {
|
||||
t.Errorf("events = %+v, want %+v", events, wantEvents)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user