package adapter import ( "encoding/json" "fmt" "sort" "strings" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/trace" ) // utf8BOM 是 UTF-8 的字节序标记。对端(审查模型网关)会把它原样回带,而 // encoding/json 见到它报 `invalid character 'ï' looking for beginning of value` —— // 实测 9 天 1,909 次审查里 1,288 次(67%)因此解析失败,每一次都白烧一次模型调用 // 并把「每窗口一次」的节流打穿(见 DevDocs/audit/群聊排队与回旧话-2026-10-24.md)。 const utf8BOM = "\ufeff" // reviewExcerptLen 是解析失败时附带在错误里的模型输出长度(**rune**)。它只含模型 // 返回的编号数组(数字/方括号/逗号/围栏),足以定位「又换了什么壳」,不至于把 // 消息内容带进日志。 // // 🔴 单位是 rune 不是字节:审查模型回的中文/全角标点每个字 3 字节,按 40 **字节** // 截会把其中一个字切成半个 —— 错误串变成非法 UTF-8,终端显示 U+FFFD、JSON 客户端 // 各自乱码。带上限出镜是刻意的(失败原因对排障有用),但上限必须以字符计。 const reviewExcerptLen = 40 // ParseReviewIndices decodes the keep-index array a review model returned, rising // above the shapes real gateways produce instead of requiring a bare JSON array: // // - a UTF-8 BOM prefix (`\ufeff`) — the actual cause of the 67% failure rate; // - leading/trailing whitespace and newlines; // - a Markdown code fence (` ```json … ``` ` or ` ``` … ``` `); // - surrounding prose ("好的,保留以下编号:[0, 2]") — the outermost `[ … ]` // span is used. // // The decoded indices are validated the same way the previous strict parser did: // out-of-range and duplicate values are dropped and the rest sorted ascending, so a // hallucinated number can never evict a real message. An empty or absent array is // NOT an error here — the caller decides; a missing/non-array answer is. func ParseReviewIndices(s string, bound int) ([]int, error) { body, ok := reviewArraySpan(s) if !ok { return nil, fmt.Errorf("no JSON array in reviewer output %q", reviewExcerpt(s)) } var idx []int if err := json.Unmarshal([]byte(body), &idx); err != nil { return nil, fmt.Errorf("decode reviewer indices %q: %w", reviewExcerpt(body), err) } seen := make(map[int]bool, len(idx)) out := make([]int, 0, len(idx)) for _, i := range idx { if i >= 0 && i < bound && !seen[i] { seen[i] = true out = append(out, i) } } sort.Ints(out) return out, nil } // reviewArraySpan strips the decorations a model puts around its answer (BOM, // whitespace, code fence, prose) and returns the outermost JSON array span. func reviewArraySpan(s string) (string, bool) { s = strings.TrimSpace(strings.TrimPrefix(s, utf8BOM)) s = strings.TrimSpace(unwrapCodeFence(s)) start := strings.IndexByte(s, '[') end := strings.LastIndexByte(s, ']') if start < 0 || end < start { return "", false } return s[start : end+1], true } // unwrapCodeFence drops a leading ``` fence line and the closing ``` so the fenced // body is left for the array scan. Anything without a leading fence is returned // unchanged; a fence with no body is returned as-is and fails the array scan. func unwrapCodeFence(s string) string { if !strings.HasPrefix(s, "```") { return s } nl := strings.IndexByte(s, '\n') if nl < 0 { return s } s = s[nl+1:] if close := strings.LastIndex(s, "```"); close >= 0 { s = s[:close] } return s } // reviewExcerpt 把模型输出裁成一段可以进错误串的摘要。两步都不是可选的: // // - trace.RedactTextLimited 先清洗凭据形状(审查网关偶有把 Authorization 回显进 // 输出的),再按 **rune** 截到 reviewExcerptLen —— 按字节截会把多字节字符切成 // 半个,错误串因此变成非法 UTF-8; // - 压平空白:一条诊断只占一行,也挡掉换行注入。 func reviewExcerpt(s string) string { return trace.RedactTextLimited(strings.Join(strings.Fields(s), " "), reviewExcerptLen) }