feat(io): web-fetch 直连→浏览器渲染降级档 + fetchedBy 如实标注(Z5 机制)
- 降级链:直连失败 / 判定为动态页 ⇒ 浏览器渲染取正文;返回体带 fetchedBy=direct|browser,
两条档的失败各自留痕(directError/browserError/browserSource/dynamicSuspect),**绝不静默替换**
- 被 SSRF 拒绝的目标**不换路**(红线);渲染器缺位时如实报双失败
- 3 个变异自检(去掉动态页判据 / 去掉 SSRF 短路 / 掩盖双失败)
⚠️ **生产接不通**:internal/capability/browser 没有「取正文」入口(不在本片文件面)⇒
开关默认关、不假装可用;解封形状已在报告里给出(加 RenderText ≈40 行)
This commit is contained in:
1 parent
20f82c9679
commit
c2e413ec38
6 files changed
+687
-41
No files matched your search
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
@@ -21,6 +20,9 @@ type webFetchTool struct {
|
||||
allowedHosts []string
|
||||
maxBytes int64
|
||||
log *slog.Logger
|
||||
// renderer 是「浏览器渲染取正文」这一档(design/28 §1 M5 / §8 B5)。
|
||||
// nil = 关着(默认,见 WithBrowserFallback)。
|
||||
renderer BodyRenderer
|
||||
}
|
||||
|
||||
func newWebFetchTool(t *tools) *webFetchTool {
|
||||
@@ -29,6 +31,7 @@ func newWebFetchTool(t *tools) *webFetchTool {
|
||||
allowedHosts: t.allowedHosts,
|
||||
maxBytes: t.maxFetchBytes,
|
||||
log: t.log,
|
||||
renderer: t.bodyRenderer,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +56,10 @@ func (w *webFetchTool) Parameters() (map[string]any, error) {
|
||||
}
|
||||
|
||||
// Execute implements sdk.Tool.
|
||||
//
|
||||
// 取正文走**降级链**(design/28 §1 M5 / §8 B5):直连 fetch →(失败 / 判定为动态页)
|
||||
// ⇒ 浏览器渲染。结果里的 `fetchedBy` 是**如实标注**(direct / browser);两条档的
|
||||
// 失败原文各自留在 directError / browserError 里,绝不静默替换、绝不掩盖。
|
||||
func (w *webFetchTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
m, ok := args.(map[string]any)
|
||||
if !ok {
|
||||
@@ -62,48 +69,35 @@ func (w *webFetchTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
if strings.TrimSpace(rawURL) == "" {
|
||||
return nil, fmt.Errorf("io: web-fetch: url is required")
|
||||
}
|
||||
if _, err := validateRequestURL(ctx, rawURL, w.allowedHosts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if w.client == nil {
|
||||
return nil, fmt.Errorf("io: web-fetch: no underlying client")
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("io: web-fetch: build request: %w", err)
|
||||
}
|
||||
resp, err := w.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("io: web-fetch: do: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("io: web-fetch: url returned status %d", resp.StatusCode)
|
||||
}
|
||||
mt := "text/plain"
|
||||
if rawCT := resp.Header.Get("Content-Type"); rawCT != "" {
|
||||
var err error
|
||||
mt, _, err = mime.ParseMediaType(rawCT)
|
||||
if err != nil || !isTextContentType(mt) {
|
||||
return nil, fmt.Errorf("io: web-fetch: unexpected content type %q", rawCT)
|
||||
}
|
||||
}
|
||||
max := w.maxBytes
|
||||
if v, ok := argInt(m, "max_bytes"); ok && v > 0 {
|
||||
max = int64(v)
|
||||
}
|
||||
data, truncated, err := readLimited(resp.Body, max)
|
||||
body, err := w.fetchBody(ctx, rawURL, max)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("io: web-fetch: read: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
text := string(data)
|
||||
if isHTML(mt) {
|
||||
text = htmlToText(text)
|
||||
}
|
||||
return map[string]any{
|
||||
out := map[string]any{
|
||||
"url": rawURL,
|
||||
"content_type": mt,
|
||||
"text": text,
|
||||
"truncated": truncated,
|
||||
}, nil
|
||||
"content_type": body.contentType,
|
||||
"text": body.text,
|
||||
"truncated": body.truncated,
|
||||
// fetchedBy:正文**实际**来自哪条路。它是 M5 的硬要求(不许静默替换)——
|
||||
// 见到 browser 就意味着直连失败过、或被判成了动态页。
|
||||
"fetchedBy": body.fetchedBy,
|
||||
}
|
||||
// 以下字段只在**确有其事**时出现(有内容的失败/来源才写,不塞空串)。
|
||||
if body.directError != "" {
|
||||
out["directError"] = body.directError
|
||||
}
|
||||
if body.browserSource != "" {
|
||||
out["browserSource"] = body.browserSource
|
||||
}
|
||||
if body.browserError != "" {
|
||||
out["browserError"] = body.browserError
|
||||
}
|
||||
if body.dynamicSuspect {
|
||||
out["dynamicSuspect"] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in new issue
Block a user