feat(plugins): implement web-search, web-fetch, http and file tools
- web-search: real SearXNG /search?format=json query, top-5 formatted results (was a 'not configured' stub) - web-fetch: fetch page, strip scripts/styles/tags, truncate to 4000 runes for the LLM context - http: generic request tool (GET/POST/PUT/DELETE/PATCH) with scheme and method validation, status + truncated body - file: sandboxed read/write/list inside the data directory with path escape rejection; split into file_read/file_write/file_list - all tool schemas converted to proper JSON Schema; httptest/tempdir tests for each plugin
This commit is contained in:
1 parent
d80710d85a
commit
9d040fde0b
8 files changed
+599
-44
No files matched your search
@@ -1,19 +1,96 @@
|
||||
package webfetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/plugin"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/plugin"
|
||||
)
|
||||
type Plugin struct{}
|
||||
func New() *Plugin { return &Plugin{} }
|
||||
func (p *Plugin) Name() string { return "web-fetch" }
|
||||
func (p *Plugin) Version() string { return "1.0.0" }
|
||||
|
||||
type Plugin struct{ client *http.Client }
|
||||
|
||||
func New() *Plugin { return &Plugin{client: &http.Client{Timeout: 20 * time.Second}} }
|
||||
|
||||
func (p *Plugin) Name() string { return "web-fetch" }
|
||||
func (p *Plugin) Version() string { return "1.0.0" }
|
||||
func (p *Plugin) Description() string { return "网页抓取" }
|
||||
func (p *Plugin) Init(_ context.Context, _ plugin.PluginConfig) error { return nil }
|
||||
func (p *Plugin) Close() error { return nil }
|
||||
func (p *Plugin) Enabled() bool { return true }
|
||||
func (p *Plugin) Close() error { return nil }
|
||||
func (p *Plugin) Enabled() bool { return true }
|
||||
|
||||
func (p *Plugin) Tools() []plugin.ToolDefinition {
|
||||
return []plugin.ToolDefinition{{Name:"web_fetch", Description:"获取网页内容", Parameters:map[string]any{"url": map[string]string{"type":"string"}}}}
|
||||
return []plugin.ToolDefinition{{
|
||||
Name: "web_fetch",
|
||||
Description: "Fetch a web page and return its text content (HTML stripped, truncated).",
|
||||
Parameters: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"url": map[string]string{"type": "string", "description": "Page URL (http/https)"},
|
||||
},
|
||||
"required": []string{"url"},
|
||||
},
|
||||
}}
|
||||
}
|
||||
func (p *Plugin) Execute(_ context.Context, _ string, _ map[string]any) (string, error) {
|
||||
return "[抓取] 功能占位", nil
|
||||
|
||||
var (
|
||||
reScript = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?</(script|style|noscript)>`)
|
||||
reTags = regexp.MustCompile(`(?s)<[^>]+>`)
|
||||
reBlank = regexp.MustCompile(`\n\s*\n+`)
|
||||
maxBody = 1 << 20 // 1MB
|
||||
maxContent = 4000 // runes returned to the LLM
|
||||
)
|
||||
|
||||
func (p *Plugin) Execute(ctx context.Context, _ string, params map[string]any) (string, error) {
|
||||
rawURL, _ := params["url"].(string)
|
||||
if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
|
||||
return "", fmt.Errorf("url must start with http:// or https://")
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Cyrene/2.0)")
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetch: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("fetch API %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxBody)))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
|
||||
text := htmlToText(string(body))
|
||||
if runes := []rune(text); len(runes) > maxContent {
|
||||
text = string(runes[:maxContent]) + "\n…(内容过长已截断)"
|
||||
}
|
||||
if text == "" {
|
||||
return "页面无可用文本内容", nil
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// htmlToText strips scripts/styles/tags and collapses whitespace.
|
||||
func htmlToText(html string) string {
|
||||
html = reScript.ReplaceAllString(html, " ")
|
||||
html = reTags.ReplaceAllString(html, "\n")
|
||||
html = reBlank.ReplaceAllString(html, "\n")
|
||||
var lines []string
|
||||
for _, line := range strings.Split(html, "\n") {
|
||||
if s := strings.TrimSpace(line); s != "" {
|
||||
lines = append(lines, s)
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
Reference in new issue
Block a user