feat(io): web-fetch 直连→浏览器渲染降级档 + fetchedBy 如实标注(Z5 机制)
CI / Backend (build / vet / test / cross-compile) (push) Has been cancelled
CI / Frontend (tsc + vite build) (push) Has been cancelled

- 降级链:直连失败 / 判定为动态页 ⇒ 浏览器渲染取正文;返回体带 fetchedBy=direct|browser,
  两条档的失败各自留痕(directError/browserError/browserSource/dynamicSuspect),**绝不静默替换**
- 被 SSRF 拒绝的目标**不换路**(红线);渲染器缺位时如实报双失败
- 3 个变异自检(去掉动态页判据 / 去掉 SSRF 短路 / 掩盖双失败)
⚠️ **生产接不通**:internal/capability/browser 没有「取正文」入口(不在本片文件面)⇒
   开关默认关、不假装可用;解封形状已在报告里给出(加 RenderText ≈40 行)
This commit is contained in:
AskaEth committed 2026-10-07 09:34:47 +08:00
1 parent 20f82c9679
commit c2e413ec38
6 files changed
+687 -41

No files matched your search

+32 -38
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"log/slog"
"mime"
"net/http"
"strings"
@@ -21,6 +20,9 @@ type webFetchTool struct {
allowedHosts []string
maxBytes int64
log *slog.Logger
// renderer 是「浏览器渲染取正文」这一档(design/28 §1 M5 / §8 B5)。
// nil = 关着(默认,见 WithBrowserFallback)。
renderer BodyRenderer
}
func newWebFetchTool(t *tools) *webFetchTool {
@@ -29,6 +31,7 @@ func newWebFetchTool(t *tools) *webFetchTool {
allowedHosts: t.allowedHosts,
maxBytes: t.maxFetchBytes,
log: t.log,
renderer: t.bodyRenderer,
}
}
@@ -53,6 +56,10 @@ func (w *webFetchTool) Parameters() (map[string]any, error) {
}
// Execute implements sdk.Tool.
//
// 取正文走**降级链**(design/28 §1 M5 / §8 B5):直连 fetch →(失败 / 判定为动态页)
// ⇒ 浏览器渲染。结果里的 `fetchedBy` 是**如实标注**(direct / browser);两条档的
// 失败原文各自留在 directError / browserError 里,绝不静默替换、绝不掩盖。
func (w *webFetchTool) Execute(ctx context.Context, args any) (any, error) {
m, ok := args.(map[string]any)
if !ok {
@@ -62,48 +69,35 @@ func (w *webFetchTool) Execute(ctx context.Context, args any) (any, error) {
if strings.TrimSpace(rawURL) == "" {
return nil, fmt.Errorf("io: web-fetch: url is required")
}
if _, err := validateRequestURL(ctx, rawURL, w.allowedHosts); err != nil {
return nil, err
}
if w.client == nil {
return nil, fmt.Errorf("io: web-fetch: no underlying client")
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, fmt.Errorf("io: web-fetch: build request: %w", err)
}
resp, err := w.client.Do(req)
if err != nil {
return nil, fmt.Errorf("io: web-fetch: do: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("io: web-fetch: url returned status %d", resp.StatusCode)
}
mt := "text/plain"
if rawCT := resp.Header.Get("Content-Type"); rawCT != "" {
var err error
mt, _, err = mime.ParseMediaType(rawCT)
if err != nil || !isTextContentType(mt) {
return nil, fmt.Errorf("io: web-fetch: unexpected content type %q", rawCT)
}
}
max := w.maxBytes
if v, ok := argInt(m, "max_bytes"); ok && v > 0 {
max = int64(v)
}
data, truncated, err := readLimited(resp.Body, max)
body, err := w.fetchBody(ctx, rawURL, max)
if err != nil {
return nil, fmt.Errorf("io: web-fetch: read: %w", err)
return nil, err
}
text := string(data)
if isHTML(mt) {
text = htmlToText(text)
}
return map[string]any{
out := map[string]any{
"url": rawURL,
"content_type": mt,
"text": text,
"truncated": truncated,
}, nil
"content_type": body.contentType,
"text": body.text,
"truncated": body.truncated,
// fetchedBy:正文**实际**来自哪条路。它是 M5 的硬要求(不许静默替换)——
// 见到 browser 就意味着直连失败过、或被判成了动态页。
"fetchedBy": body.fetchedBy,
}
// 以下字段只在**确有其事**时出现(有内容的失败/来源才写,不塞空串)。
if body.directError != "" {
out["directError"] = body.directError
}
if body.browserSource != "" {
out["browserSource"] = body.browserSource
}
if body.browserError != "" {
out["browserError"] = body.browserError
}
if body.dynamicSuspect {
out["dynamicSuspect"] = true
}
return out, nil
}