package webfetch import ( "context" "fmt" "io" "net/http" "regexp" "strings" "time" "git.yeij.top/AskaEth/Cyrene_v2/internal/plugin" ) type Plugin struct{ client *http.Client } func New() *Plugin { return &Plugin{client: &http.Client{Timeout: 20 * time.Second}} } func (p *Plugin) Name() string { return "web-fetch" } func (p *Plugin) Version() string { return "1.0.0" } func (p *Plugin) Description() string { return "网页抓取" } func (p *Plugin) Init(_ context.Context, _ plugin.PluginConfig) error { return nil } func (p *Plugin) Close() error { return nil } func (p *Plugin) Enabled() bool { return true } func (p *Plugin) Tools() []plugin.ToolDefinition { return []plugin.ToolDefinition{{ Name: "web_fetch", Description: "Fetch a web page and return its text content (HTML stripped, truncated).", Parameters: map[string]any{ "type": "object", "properties": map[string]any{ "url": map[string]string{"type": "string", "description": "Page URL (http/https)"}, }, "required": []string{"url"}, }, }} } var ( reScript = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) reTags = regexp.MustCompile(`(?s)<[^>]+>`) reBlank = regexp.MustCompile(`\n\s*\n+`) maxBody = 1 << 20 // 1MB maxContent = 4000 // runes returned to the LLM ) func (p *Plugin) Execute(ctx context.Context, _ string, params map[string]any) (string, error) { rawURL, _ := params["url"].(string) if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") { return "", fmt.Errorf("url must start with http:// or https://") } req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil) if err != nil { return "", err } req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Cyrene/2.0)") resp, err := p.client.Do(req) if err != nil { return "", fmt.Errorf("fetch: %w", err) } defer resp.Body.Close() if resp.StatusCode != 200 { return "", fmt.Errorf("fetch API %d", resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxBody))) if err != nil { return "", fmt.Errorf("read body: %w", err) } text := htmlToText(string(body)) if runes := []rune(text); len(runes) > maxContent { text = string(runes[:maxContent]) + "\n…(内容过长已截断)" } if text == "" { return "页面无可用文本内容", nil } return text, nil } // htmlToText strips scripts/styles/tags and collapses whitespace. func htmlToText(html string) string { html = reScript.ReplaceAllString(html, " ") html = reTags.ReplaceAllString(html, "\n") html = reBlank.ReplaceAllString(html, "\n") var lines []string for _, line := range strings.Split(html, "\n") { if s := strings.TrimSpace(line); s != "" { lines = append(lines, s) } } return strings.Join(lines, "\n") }