Aska
9d040fde0b
- web-search: real SearXNG /search?format=json query, top-5 formatted results (was a 'not configured' stub) - web-fetch: fetch page, strip scripts/styles/tags, truncate to 4000 runes for the LLM context - http: generic request tool (GET/POST/PUT/DELETE/PATCH) with scheme and method validation, status + truncated body - file: sandboxed read/write/list inside the data directory with path escape rejection; split into file_read/file_write/file_list - all tool schemas converted to proper JSON Schema; httptest/tempdir tests for each plugin
97 lines
2.8 KiB
Go
97 lines
2.8 KiB
Go
package webfetch
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.yeij.top/AskaEth/Cyrene_v2/internal/plugin"
|
|
)
|
|
|
|
type Plugin struct{ client *http.Client }
|
|
|
|
func New() *Plugin { return &Plugin{client: &http.Client{Timeout: 20 * time.Second}} }
|
|
|
|
func (p *Plugin) Name() string { return "web-fetch" }
|
|
func (p *Plugin) Version() string { return "1.0.0" }
|
|
func (p *Plugin) Description() string { return "网页抓取" }
|
|
func (p *Plugin) Init(_ context.Context, _ plugin.PluginConfig) error { return nil }
|
|
func (p *Plugin) Close() error { return nil }
|
|
func (p *Plugin) Enabled() bool { return true }
|
|
|
|
func (p *Plugin) Tools() []plugin.ToolDefinition {
|
|
return []plugin.ToolDefinition{{
|
|
Name: "web_fetch",
|
|
Description: "Fetch a web page and return its text content (HTML stripped, truncated).",
|
|
Parameters: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"url": map[string]string{"type": "string", "description": "Page URL (http/https)"},
|
|
},
|
|
"required": []string{"url"},
|
|
},
|
|
}}
|
|
}
|
|
|
|
var (
|
|
reScript = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?</(script|style|noscript)>`)
|
|
reTags = regexp.MustCompile(`(?s)<[^>]+>`)
|
|
reBlank = regexp.MustCompile(`\n\s*\n+`)
|
|
maxBody = 1 << 20 // 1MB
|
|
maxContent = 4000 // runes returned to the LLM
|
|
)
|
|
|
|
func (p *Plugin) Execute(ctx context.Context, _ string, params map[string]any) (string, error) {
|
|
rawURL, _ := params["url"].(string)
|
|
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
|
|
return "", fmt.Errorf("url must start with http:// or https://")
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Cyrene/2.0)")
|
|
|
|
resp, err := p.client.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("fetch: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != 200 {
|
|
return "", fmt.Errorf("fetch API %d", resp.StatusCode)
|
|
}
|
|
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxBody)))
|
|
if err != nil {
|
|
return "", fmt.Errorf("read body: %w", err)
|
|
}
|
|
|
|
text := htmlToText(string(body))
|
|
if runes := []rune(text); len(runes) > maxContent {
|
|
text = string(runes[:maxContent]) + "\n…（内容过长已截断）"
|
|
}
|
|
if text == "" {
|
|
return "页面无可用文本内容", nil
|
|
}
|
|
return text, nil
|
|
}
|
|
|
|
// htmlToText strips scripts/styles/tags and collapses whitespace.
|
|
func htmlToText(html string) string {
|
|
html = reScript.ReplaceAllString(html, " ")
|
|
html = reTags.ReplaceAllString(html, "\n")
|
|
html = reBlank.ReplaceAllString(html, "\n")
|
|
var lines []string
|
|
for _, line := range strings.Split(html, "\n") {
|
|
if s := strings.TrimSpace(line); s != "" {
|
|
lines = append(lines, s)
|
|
}
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|