feat(search): web-search 接线本机 SearXNG 兜底 + 显式开关(她不再拿一把用不了的锤子)
主人报的故障:她调 web-search 必失败(no search endpoint configured)。根因两条:组合根从未注入 WithSearchEndpoint + 配置层根本没有搜索配置项。 - internal/config/websearch.go:WEB_SEARCH_ENDPOINT(默认本机兜底 http://127.0.0.1:38471)+ WEB_SEARCH_TIMEOUT;ResolveEndpoint() 三态裁决(可用/显式关闭/非法),非法值启动即报错 - cmd/cyrene/wire_io_tools.go:组合根唯一的 io 选项装配点;端点不可用 ⇒ WithoutSearch() + 带 source 的 WARN(只记键名与原因,不打印 URL 值) - plugins/tools/io:新增 WithoutSearch()(**根本不进注册面** ⇒ 模型看不见)+ WithSearchTimeout();协议解析逻辑一字未动 - 隐蔽坑(额外查出并修):SearXNG 对 /?q= 回 308 → /search,重定向那一跳走**共享 SSRF 白名单** ⇒ 只注入端点照样被拦。改为**只给 web-search 一件**加宽自己的客户端与白名单,共享白名单与 http/web-fetch 客户端一字未动(有测试钉住它仍拒绝本机网关地址) - .env.example 新增 2 键 + 注释说明「空串/off/none = 不注册」 - 真机 e2e 两次绿(SearXNG 返回 9 条);变异:不注入端点 / 关闭态仍注册 / 默认值改错 / 不并入端点主机 / 重定向用共享白名单 ⇒ 5 条全红
This commit is contained in:
1 parent
8e56163418
commit
8e6c77ad10
11 files changed
+1137
-20
No files matched your search
@@ -20,7 +20,13 @@
|
||||
// - http: GET/POST only, timeouts, response truncation and SSRF protection
|
||||
// (loopback/private/link-local addresses and localhost are refused unless
|
||||
// the host is on an explicit allowlist).
|
||||
// - web-search: requires a configured search endpoint, otherwise a clear error.
|
||||
// - web-search: queries the configured SearXNG-compatible endpoint. When no
|
||||
// usable endpoint is configured the composition root passes WithoutSearch, so
|
||||
// the tool is not registered at all — the model never sees a tool that could
|
||||
// only error. The endpoint itself is operator configuration (the model only
|
||||
// supplies the query), hence its host is trusted by this tool alone: a
|
||||
// loopback fallback endpoint stays reachable without widening the shared
|
||||
// http/web-fetch allowlist.
|
||||
// - web-fetch: size- and content-type-limited readable text extraction.
|
||||
// - knowledge: RAG lookup over the runtime memory retriever; a clear message
|
||||
// when no knowledge base is configured.
|
||||
|
||||
+84
-16
@@ -48,6 +48,13 @@ type tools struct {
|
||||
maxResults int
|
||||
maxList int
|
||||
log *slog.Logger
|
||||
|
||||
// searchDisabled 让 web-search **根本不进注册面**(切片 WS1-A 的诚实性要求):
|
||||
// 拿不到可用端点时,不该给模型一把拎起来必报错的锤子。见 WithoutSearch。
|
||||
searchDisabled bool
|
||||
// searchTimeout 是 web-search 单次请求的超时覆盖(0 = 沿用本套工具的
|
||||
// httpTimeout,也就是 io 包的既有超时 —— 这里刻意不另造默认值)。
|
||||
searchTimeout time.Duration
|
||||
}
|
||||
|
||||
// Option customises the tool set built by New.
|
||||
@@ -81,10 +88,37 @@ func WithAllowedHosts(hosts ...string) Option {
|
||||
|
||||
// WithSearchEndpoint sets the web-search endpoint (a SearXNG-compatible URL).
|
||||
// WithSearchEndpoint("") clears it so web-search reports a clear error.
|
||||
//
|
||||
// 语义与既往逐字不变。**注意**:置空只是「端点为空」,工具仍在注册面里;
|
||||
// 要「模型根本看不见它」请用 WithoutSearch(组合根在端点显式关闭 / 非法时的用法)。
|
||||
func WithSearchEndpoint(u string) Option {
|
||||
return func(t *tools) { t.searchURL = u }
|
||||
}
|
||||
|
||||
// WithoutSearch removes the web-search tool from the built set entirely: it is
|
||||
// not registered, so the model neither sees it nor wastes a round on it. This is
|
||||
// the honest state for "no usable search endpoint" (WS1-A): a registered tool
|
||||
// that can only ever error is worse than an absent one.
|
||||
//
|
||||
// 它只影响 web-search,其余工具照常构建。
|
||||
func WithoutSearch() Option {
|
||||
return func(t *tools) { t.searchDisabled = true }
|
||||
}
|
||||
|
||||
// WithSearchTimeout overrides the timeout of a single web-search call. Zero or
|
||||
// negative means "use the set's existing HTTP timeout" — the io package does not
|
||||
// define a second default for search (project rule §10: one default per knob).
|
||||
//
|
||||
// 它只约束 web-search 这一次请求:与 WithHTTPTimeout(整套工具共用的客户端
|
||||
// 超时,会同时改到 http / web-fetch)不是同一件事。
|
||||
func WithSearchTimeout(d time.Duration) Option {
|
||||
return func(t *tools) {
|
||||
if d > 0 {
|
||||
t.searchTimeout = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithKnowledgeRetriever wires the knowledge tool to a runtime memory retriever.
|
||||
// WithKnowledgeRetriever(nil) makes knowledge report "no knowledge base".
|
||||
func WithKnowledgeRetriever(r KnowledgeRetriever) Option {
|
||||
@@ -150,6 +184,9 @@ func WithLogger(l *slog.Logger) Option {
|
||||
// New builds the full configured tool set over cfg and opts. It returns an error
|
||||
// only when cfg is nil. A nil retriever or empty search endpoint is allowed: the
|
||||
// corresponding tool reports a clear runtime error rather than failing startup.
|
||||
//
|
||||
// 组合根在「拿不到可用搜索端点」时应传 WithoutSearch,让 web-search **不进注册面**
|
||||
// (模型看不见它),而不是塞一个空端点进去等着每次调用报错(切片 WS1-A)。
|
||||
func New(cfg *config.Config, opts ...Option) ([]sdk.Tool, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("io: new: nil config")
|
||||
@@ -203,33 +240,64 @@ func Register(reg sdk.Registrar) error {
|
||||
// build assembles the tools sharing this set's client, limits and logger.
|
||||
func (t *tools) build() []sdk.Tool {
|
||||
file := newFileTool(t)
|
||||
return []sdk.Tool{
|
||||
set := []sdk.Tool{
|
||||
file,
|
||||
newCropTool(t, file),
|
||||
newHTTPTool(t),
|
||||
newWebSearchTool(t),
|
||||
}
|
||||
// web-search 只有在**确实有可用端点**时才进注册面:端点显式关闭或非法时由
|
||||
// 组合根传 WithoutSearch,模型便根本看不见它。顺序与既往一致(http 之后、
|
||||
// web-fetch 之前),启用态下工具表的形状逐字不变。
|
||||
if !t.searchDisabled {
|
||||
set = append(set, newWebSearchTool(t))
|
||||
}
|
||||
return append(set,
|
||||
newWebFetchTool(t),
|
||||
newKnowledgeTool(t),
|
||||
newDocumentReadTool(t),
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// newHTTPClient builds a client with a unified timeout and an SSRF-guarded
|
||||
// redirect policy that refuses to follow into private/loopback targets.
|
||||
func newHTTPClient(t *tools) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: t.httpTimeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("io: too many redirects")
|
||||
}
|
||||
if req.URL == nil {
|
||||
return ErrSSRF
|
||||
}
|
||||
if _, err := validateRequestURL(req.Context(), req.URL.String(), t.allowedHosts); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Timeout: t.httpTimeout,
|
||||
CheckRedirect: redirectGuard(t.allowedHosts),
|
||||
}
|
||||
}
|
||||
|
||||
// redirectGuard 造 SSRF 重定向守卫:最多跳 maxRedirects 次,且每次跳转的目标都要
|
||||
// 过同一份判定(跳去内网、回环一样要拦)。判定所用的白名单是参数给的这一份,
|
||||
// 因此调用方能让它按**自己**加宽后的名单判定,而不牵动共享白名单。
|
||||
func redirectGuard(allowedHosts []string) func(*http.Request, []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("io: too many redirects")
|
||||
}
|
||||
if req.URL == nil {
|
||||
return ErrSSRF
|
||||
}
|
||||
if _, err := validateRequestURL(req.Context(), req.URL.String(), allowedHosts); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// clientFor 返回「白名单为 allowedHosts」时某个工具该用的客户端。
|
||||
//
|
||||
// allowedHosts 与共享白名单**长度相同**即代表没有加宽(withSearchEndpointHost 只会
|
||||
// 追加、绝不改写已有条目),此时直接复用共享客户端;确有加宽时返回一份**副本**,
|
||||
// 把重定向守卫换成按加宽名单判定的那一份 —— 副本共享同一个 Transport,连接池不变。
|
||||
//
|
||||
// 这条分工的意义:让 web-search 认自己配置里的端点主机(含本机兜底的回环地址),
|
||||
// 同时**不**污染 http / web-fetch 共用的那份白名单与那个客户端。
|
||||
func (t *tools) clientFor(allowedHosts []string) *http.Client {
|
||||
if len(allowedHosts) == len(t.allowedHosts) {
|
||||
return t.client
|
||||
}
|
||||
client := *t.client
|
||||
client.CheckRedirect = redirectGuard(allowedHosts)
|
||||
return &client
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
||||
)
|
||||
@@ -24,16 +25,29 @@ type webSearchTool struct {
|
||||
maxBytes int64
|
||||
maxResults int
|
||||
log *slog.Logger
|
||||
// timeout 是本次搜索的超时覆盖(0 = 沿用共享客户端自己的超时)。它只约束
|
||||
// web-search 一次请求,不牵动 http / web-fetch 共用的那个客户端(WS1-A)。
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func newWebSearchTool(t *tools) *webSearchTool {
|
||||
// 端点来自**运维配置**,不是模型输入(模型的 query 只会进查询串,主机名恒为
|
||||
// 配置里那一个):因此把它并入本工具**自己**的 SSRF 白名单。否则本机兜底端点
|
||||
// http://127.0.0.1:38471 会被 SSRF 守卫按回环地址拒掉——那正是「端点配了却
|
||||
// 仍然用不了」的另一种形态(SearXNG 还会 308 跳到 /search,重定向那一跳同样
|
||||
// 要过守卫,所以加宽后的名单也必须交给客户端,见 tools.clientFor)。
|
||||
//
|
||||
// 刻意只并入 web-search:http / web-fetch 的 URL 由模型给出,它们共用
|
||||
// t.allowedHosts,把回环塞进**那份**名单会让模型能伸手去摸本机服务。
|
||||
allowedHosts := withSearchEndpointHost(t.allowedHosts, t.searchURL)
|
||||
return &webSearchTool{
|
||||
client: t.client,
|
||||
client: t.clientFor(allowedHosts),
|
||||
endpoint: t.searchURL,
|
||||
allowedHosts: t.allowedHosts,
|
||||
allowedHosts: allowedHosts,
|
||||
maxBytes: t.maxHTTPBytes,
|
||||
maxResults: t.maxResults,
|
||||
log: t.log,
|
||||
timeout: t.searchTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +84,12 @@ func (s *webSearchTool) Execute(ctx context.Context, args any) (any, error) {
|
||||
if strings.TrimSpace(s.endpoint) == "" {
|
||||
return nil, fmt.Errorf("io: web-search: no search endpoint configured (set a SearXNG or custom URL)")
|
||||
}
|
||||
// 每次调用各自的超时预算:ctx 在此派生,取消随本次调用一起结束。
|
||||
if s.timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, s.timeout)
|
||||
defer cancel()
|
||||
}
|
||||
target, err := buildSearchURL(s.endpoint, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -135,6 +155,33 @@ func buildSearchURL(endpoint, query string) (string, error) {
|
||||
return endpoint + sep + "q=" + q + "&format=json", nil
|
||||
}
|
||||
|
||||
// withSearchEndpointHost 把**配置里给定的**搜索端点主机并入 web-search 自己的
|
||||
// SSRF 白名单(返回新切片,不改动入参切片——组合根可能把它交给别的工具)。
|
||||
// 端点解析不出主机、或主机已在名单里时原样返回。
|
||||
//
|
||||
// 理由见 newWebSearchTool:本机兜底端点必然是回环地址(127.0.0.1:38471),而
|
||||
// SSRF 守卫默认拒绝回环;端点是运维配置而非模型输入,因此对它放行**不会**给模型
|
||||
// 打开新的目标面——工具能去的主机恒等于配置里的那一个。
|
||||
func withSearchEndpointHost(allowed []string, endpoint string) []string {
|
||||
host := searchEndpointHost(endpoint)
|
||||
if host == "" || isHostAllowed(host, allowed) {
|
||||
return allowed
|
||||
}
|
||||
out := make([]string, 0, len(allowed)+1)
|
||||
out = append(out, allowed...)
|
||||
return append(out, host)
|
||||
}
|
||||
|
||||
// searchEndpointHost 取出搜索端点的主机名(去端口、去 IPv6 方括号);解析不出来
|
||||
// 或没有主机时返回空串。
|
||||
func searchEndpointHost(endpoint string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(endpoint))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return hostOnly(u.Host)
|
||||
}
|
||||
|
||||
// searchResult is one entry of a SearXNG-compatible results payload.
|
||||
type searchResult struct {
|
||||
Title string `json:"title"`
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
package io
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/config"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
||||
)
|
||||
|
||||
// 本文件是切片 WS1-A 的 io 侧契约:**拿不到可用搜索端点时,web-search 不进注册面**,
|
||||
// 以及「配置里给定的端点确实被用上」——后者包含本机兜底端点(回环地址)必须真能
|
||||
// 走通 SSRF 守卫,否则「端点配了却仍然用不了」换个形态又回来了。
|
||||
|
||||
// hasTool 报告工具表里有没有某个名字的工具。
|
||||
func hasTool(set []sdk.Tool, name string) bool {
|
||||
for _, tl := range set {
|
||||
if tl.Name() == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// searxngFallbackURL 是本机兜底 SearXNG 的地址(与 config 的 WEB_SEARCH_ENDPOINT
|
||||
// 默认值同源;那个默认值由 cmd/cyrene 的接线测试钉住)。它只用于真机 e2e。
|
||||
const searxngFallbackURL = "http://127.0.0.1:38471"
|
||||
|
||||
// TestNew_WithoutSearchOmitsTheTool 钉住「关闭态 = 模型看不见这把锤子」:
|
||||
// WithoutSearch 之后注册面里没有 web-search,其余六个工具一个不少、顺序不变。
|
||||
func TestNew_WithoutSearchOmitsTheTool(t *testing.T) {
|
||||
t.Parallel()
|
||||
set := buildTools(t, &config.Config{}, WithoutSearch())
|
||||
if hasTool(set, "web-search") {
|
||||
t.Fatal("WithoutSearch 之后注册面里仍有 web-search —— 模型会看见一把必报错的锤子")
|
||||
}
|
||||
for _, name := range []string{"file", "image-crop", "http", "web-fetch", "knowledge", "document-read"} {
|
||||
if !hasTool(set, name) {
|
||||
t.Errorf("WithoutSearch 不得影响其它工具,但 %q 不见了", name)
|
||||
}
|
||||
}
|
||||
if len(set) != 6 {
|
||||
t.Fatalf("工具数 = %d, want 6", len(set))
|
||||
}
|
||||
|
||||
// 对照:不给该选项时仍然注册(既有调用方行为不变)。
|
||||
if plain := buildTools(t, &config.Config{}); !hasTool(plain, "web-search") {
|
||||
t.Fatal("未传 WithoutSearch 时 web-search 必须照旧注册(后向兼容)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNew_WithoutSearchWinsOverEndpoint 钉住两个选项同时出现时的裁决:
|
||||
// 关闭优先 —— 否则「端点为空 + 关闭」的组合会退化成既注册又报错。
|
||||
func TestNew_WithoutSearchWinsOverEndpoint(t *testing.T) {
|
||||
t.Parallel()
|
||||
set := buildTools(t, &config.Config{}, WithSearchEndpoint("http://127.0.0.1:38471"), WithoutSearch())
|
||||
if hasTool(set, "web-search") {
|
||||
t.Fatal("WithoutSearch 必须压过 WithSearchEndpoint:注册面里不得出现 web-search")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebSearch_ConfiguredEndpointIsUsed 是「配置值真被透传」的 io 侧断言:
|
||||
// 断言的不是「没报错」,而是**请求真的打到了配置里那个地址**(含 q / format=json)。
|
||||
func TestWebSearch_ConfiguredEndpointIsUsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
const query = "昔涟"
|
||||
got := make(chan *http.Request, 1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got <- r
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"results":[{"title":"A","url":"http://a.example","content":"alpha"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ws := webSearchToolForTest(t,
|
||||
WithSearchEndpoint(srv.URL+"/search"),
|
||||
WithHTTPClient(srv.Client()),
|
||||
)
|
||||
if ws.endpoint != srv.URL+"/search" {
|
||||
t.Fatalf("工具持有的端点 = %q, want %q(配置值没被透传)", ws.endpoint, srv.URL+"/search")
|
||||
}
|
||||
res, err := ws.Execute(context.Background(), map[string]any{"query": query})
|
||||
if err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
select {
|
||||
case r := <-got:
|
||||
if r.URL.Path != "/search" {
|
||||
t.Errorf("请求路径 = %q, want /search", r.URL.Path)
|
||||
}
|
||||
if r.URL.Query().Get("q") != query {
|
||||
t.Errorf("q = %q, want %q", r.URL.Query().Get("q"), query)
|
||||
}
|
||||
if r.URL.Query().Get("format") != "json" {
|
||||
t.Errorf("format = %q, want json", r.URL.Query().Get("format"))
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("配置里的端点一个请求都没收到 —— 端点没被真正使用")
|
||||
}
|
||||
if m := res.(map[string]any); m["count"].(int) != 1 {
|
||||
t.Fatalf("count = %v, want 1", m["count"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebSearch_LoopbackEndpointNeedsNoAllowlist 钉住本机兜底端点(回环地址)必须
|
||||
// **开箱可用**:不额外传 WithAllowedHosts 也能查询成功。少了这条,默认端点
|
||||
// http://127.0.0.1:38471 会被 SSRF 守卫拒掉,等于换个姿势又给了把用不了的锤子。
|
||||
func TestWebSearch_LoopbackEndpointNeedsNoAllowlist(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"results":[{"title":"A","url":"http://a.example","content":"alpha"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ws := webSearchToolForTest(t,
|
||||
WithSearchEndpoint(srv.URL),
|
||||
WithHTTPClient(srv.Client()),
|
||||
)
|
||||
res, err := ws.Execute(context.Background(), map[string]any{"query": "cyrene"})
|
||||
if err != nil {
|
||||
t.Fatalf("回环端点必须开箱可用,却报错: %v", err)
|
||||
}
|
||||
if m := res.(map[string]any); m["count"].(int) != 1 {
|
||||
t.Fatalf("count = %v, want 1", m["count"])
|
||||
}
|
||||
if !isHostAllowed(serverHost(srv), ws.allowedHosts) {
|
||||
t.Fatalf("端点主机不在本工具的 SSRF 白名单里: %v", ws.allowedHosts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebSearch_DoesNotWidenSharedAllowlist 钉住上一条的**边界**:给 web-search 放行
|
||||
// 它自己的端点主机,绝不许顺带把回环地址塞进 http / web-fetch 共用的那份白名单
|
||||
// —— 那会让模型伸手去摸本机服务(含后端自己的 :10942)。
|
||||
func TestWebSearch_DoesNotWidenSharedAllowlist(t *testing.T) {
|
||||
t.Parallel()
|
||||
set := buildTools(t, &config.Config{}, WithSearchEndpoint(searxngFallbackURL))
|
||||
ht := findTool(t, set, "http").(*httpTool)
|
||||
if isHostAllowed("127.0.0.1", ht.allowedHosts) {
|
||||
t.Fatalf("http 工具的 SSRF 白名单被搜索端点污染了: %v", ht.allowedHosts)
|
||||
}
|
||||
_, err := ht.Execute(context.Background(), map[string]any{"url": "http://127.0.0.1:10942/api/v1/health"})
|
||||
if !errors.Is(err, ErrSSRF) {
|
||||
t.Fatalf("http 工具仍必须拒绝回环地址,err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebSearch_Redirects 用本机兜底 SearXNG 的真实形状(308 → /search)钉住重定向:
|
||||
//
|
||||
// - 跳到**端点自己**的相对路径必须被跟随(SearXNG 收到 "/?q=…" 就是 308 到
|
||||
// "/search?q=…";少了这条,默认端点仍是「配了却用不了」);
|
||||
// - 跳到**别的**受保护地址(私网/回环)必须仍被拦下 —— 放行的是「配置里那一个
|
||||
// 端点主机」,不是「所有重定向」。
|
||||
func TestWebSearch_Redirects(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("跟随端点自身的相对跳转", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/search" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"results":[{"title":"A","url":"http://a.example","content":"alpha"}]}`))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/search?"+r.URL.RawQuery)
|
||||
w.WriteHeader(http.StatusPermanentRedirect)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ws := webSearchToolForTest(t, WithSearchEndpoint(srv.URL))
|
||||
res, err := ws.Execute(context.Background(), map[string]any{"query": "cyrene"})
|
||||
if err != nil {
|
||||
t.Fatalf("相对重定向必须被跟随: %v", err)
|
||||
}
|
||||
if m := res.(map[string]any); m["count"].(int) != 1 {
|
||||
t.Fatalf("count = %v, want 1", m["count"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("拒绝跳到私网地址", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Location", "http://10.255.255.1/search")
|
||||
w.WriteHeader(http.StatusPermanentRedirect)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ws := webSearchToolForTest(t, WithSearchEndpoint(srv.URL))
|
||||
if _, err := ws.Execute(context.Background(), map[string]any{"query": "cyrene"}); !errors.Is(err, ErrSSRF) {
|
||||
t.Fatalf("跳到私网地址必须被 SSRF 守卫拦下,err = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestWebSearch_TimeoutIsPerCallAndLocal 钉住超时旋钮是**每次搜索自己**的预算:
|
||||
// 它生效(慢端点被掐断),且**不改**任何客户端自身的超时(http / web-fetch 共用
|
||||
// 的那个客户端因此不受影响)。
|
||||
func TestWebSearch_TimeoutIsPerCallAndLocal(t *testing.T) {
|
||||
t.Parallel()
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"results":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
defer close(release)
|
||||
|
||||
set := buildTools(t, &config.Config{},
|
||||
WithSearchEndpoint(srv.URL),
|
||||
WithSearchTimeout(50*time.Millisecond),
|
||||
)
|
||||
ws := findTool(t, set, "web-search").(*webSearchTool)
|
||||
if ws.timeout != 50*time.Millisecond {
|
||||
t.Fatalf("web-search 超时 = %v, want 50ms", ws.timeout)
|
||||
}
|
||||
if _, err := ws.Execute(context.Background(), map[string]any{"query": "cyrene"}); err == nil {
|
||||
t.Fatal("50ms 预算下慢端点必须被掐断")
|
||||
}
|
||||
// 两个客户端自身的超时都还是 io 包的既有默认值:搜索超时是「按次预算」,不是
|
||||
// 客户端超时——否则改 WEB_SEARCH_TIMEOUT 会顺手改掉 http/web-fetch 的行为。
|
||||
ht := findTool(t, set, "http").(*httpTool)
|
||||
if ht.client.Timeout != defaultHTTPTimeout {
|
||||
t.Fatalf("http 客户端超时 = %v, want %v(WEB_SEARCH_TIMEOUT 不得改到别的工具)",
|
||||
ht.client.Timeout, defaultHTTPTimeout)
|
||||
}
|
||||
if ws.client.Timeout != defaultHTTPTimeout {
|
||||
t.Fatalf("web-search 客户端超时 = %v, want %v(超时应是每次调用的预算)",
|
||||
ws.client.Timeout, defaultHTTPTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchEndpointHost 覆盖端点主机抽取的边角:空、非法、带端口、IPv6、已命中白名单。
|
||||
func TestSearchEndpointHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
want string
|
||||
}{
|
||||
{"带端口", "http://127.0.0.1:38471", "127.0.0.1"},
|
||||
{"带路径", "https://searx.example.com/search?format=json", "searx.example.com"},
|
||||
{"IPv6", "http://[::1]:8080", "::1"},
|
||||
{"空串", "", ""},
|
||||
{"无主机", "http:///search", ""},
|
||||
{"非法", "http://%zz", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := searchEndpointHost(tc.endpoint); got != tc.want {
|
||||
t.Fatalf("searchEndpointHost(%q) = %q, want %q", tc.endpoint, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 已命中白名单时不重复添加;空主机时原样返回(不改动调用方的切片)。
|
||||
base := []string{"example.com"}
|
||||
if out := withSearchEndpointHost(base, "https://example.com"); len(out) != 1 {
|
||||
t.Fatalf("已在白名单里的主机不得重复添加: %v", out)
|
||||
}
|
||||
if out := withSearchEndpointHost(base, "not a url"); len(out) != 1 || out[0] != "example.com" {
|
||||
t.Fatalf("解析不出主机时必须原样返回: %v", out)
|
||||
}
|
||||
out := withSearchEndpointHost(base, "http://127.0.0.1:38471")
|
||||
if len(out) != 2 || out[0] != "example.com" || out[1] != "127.0.0.1" {
|
||||
t.Fatalf("应返回追加主机后的新切片: %v", out)
|
||||
}
|
||||
if len(base) != 1 {
|
||||
t.Fatalf("不得就地修改入参切片: %v", base)
|
||||
}
|
||||
}
|
||||
|
||||
// searxngResultCount 用**裸 HTTP**(与工具实现无关)问兜底服务同一个查询,返回它此刻
|
||||
// 真能产出的结果条数;服务不可达/应答不可解析时返回 -1。
|
||||
//
|
||||
// 它是真机 e2e 的**前置探针**:上层引擎被限流或 CAPTCHA 时兜底服务会「200 但 0 条」,
|
||||
// 那时 0 条结果不是本仓的缺陷(2026-10-05 实测:sogou 被 Suspended: CAPTCHA、bing
|
||||
// 静默返回空)—— 探针让这种外部降级表现为 skip 而不是误报红,同时保住断言强度:
|
||||
// 探针报 >0 而工具报 0,就是本仓的错。
|
||||
func searxngResultCount(t *testing.T, endpoint, query string) int {
|
||||
t.Helper()
|
||||
client := &http.Client{Timeout: 25 * time.Second}
|
||||
resp, err := client.Get(endpoint + "/search?q=" + url.QueryEscape(query) + "&format=json")
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return -1
|
||||
}
|
||||
var payload struct {
|
||||
Results []searchResult `json:"results"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
|
||||
return -1
|
||||
}
|
||||
return len(payload.Results)
|
||||
}
|
||||
|
||||
// TestWebSearch_RealSearXNG_EndToEnd 是**真机**端到端:直接调工具实现打本机兜底
|
||||
// SearXNG(scripts/searxng/settings.yml 的配套实例)。它只读外部服务,不碰后端进程、
|
||||
// 不碰仓库根产物。兜底服务没起、或它的上游引擎被限流到产不出结果时 **skip**
|
||||
// (外部降级不该判成代码红;探针与工具的查询逐字相同,探针有结果而工具没有 = 真红)。
|
||||
func TestWebSearch_RealSearXNG_EndToEnd(t *testing.T) {
|
||||
t.Parallel()
|
||||
const query = "昔涟 崩坏星穹铁道"
|
||||
upstream := searxngResultCount(t, searxngFallbackURL, query)
|
||||
if upstream < 0 {
|
||||
t.Skipf("本机兜底 SearXNG 不可达(%s),跳过真机 e2e", searxngFallbackURL)
|
||||
}
|
||||
if upstream == 0 {
|
||||
t.Skipf("兜底 SearXNG 当前产不出结果(上游引擎被限流/CAPTCHA),跳过真机 e2e")
|
||||
}
|
||||
|
||||
ws := webSearchToolForTest(t, WithSearchEndpoint(searxngFallbackURL))
|
||||
res, err := ws.Execute(context.Background(), map[string]any{"query": query})
|
||||
if err != nil {
|
||||
t.Fatalf("真机查询失败: %v", err)
|
||||
}
|
||||
m := res.(map[string]any)
|
||||
results := m["results"].([]searchResult)
|
||||
if len(results) == 0 {
|
||||
t.Fatalf("裸 HTTP 探针拿到 %d 条、工具却拿到 0 条 —— 这不再是外部降级: %v", upstream, m)
|
||||
}
|
||||
t.Logf("真机 e2e:端点 %s 返回 %d 条结果,首条标题=%q url=%q",
|
||||
searxngFallbackURL, len(results), results[0].Title, results[0].URL)
|
||||
if strings.TrimSpace(results[0].URL) == "" {
|
||||
t.Fatalf("首条结果没有 url: %+v", results[0])
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user