110 lines
2.9 KiB
Go
110 lines
2.9 KiB
Go
package io
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"mime"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk"
|
|
)
|
|
|
|
// Compile-time assertion that webFetchTool implements sdk.Tool.
|
|
var _ sdk.Tool = (*webFetchTool)(nil)
|
|
|
|
// webFetchTool fetches a URL over HTTP and returns its readable text, limited by
|
|
// size and content type, extracting text from HTML when appropriate.
|
|
type webFetchTool struct {
|
|
client *http.Client
|
|
allowedHosts []string
|
|
maxBytes int64
|
|
log *slog.Logger
|
|
}
|
|
|
|
func newWebFetchTool(t *tools) *webFetchTool {
|
|
return &webFetchTool{
|
|
client: t.client,
|
|
allowedHosts: t.allowedHosts,
|
|
maxBytes: t.maxFetchBytes,
|
|
log: t.log,
|
|
}
|
|
}
|
|
|
|
// Name implements sdk.Tool.
|
|
func (w *webFetchTool) Name() string { return "web-fetch" }
|
|
|
|
// Description implements sdk.Tool.
|
|
func (w *webFetchTool) Description() string {
|
|
return "Fetch a URL and return its readable text, limited by size and content type."
|
|
}
|
|
|
|
// Parameters implements sdk.Tool.
|
|
func (w *webFetchTool) Parameters() (map[string]any, error) {
|
|
return map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"url": map[string]any{"type": "string", "description": "Absolute http(s) URL."},
|
|
"max_bytes": map[string]any{"type": "integer", "description": "Response size cap in bytes."},
|
|
},
|
|
"required": []any{"url"},
|
|
}, nil
|
|
}
|
|
|
|
// Execute implements sdk.Tool.
|
|
func (w *webFetchTool) Execute(ctx context.Context, args any) (any, error) {
|
|
m, ok := args.(map[string]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("io: web-fetch: args must be an object")
|
|
}
|
|
rawURL := argString(m, "url")
|
|
if strings.TrimSpace(rawURL) == "" {
|
|
return nil, fmt.Errorf("io: web-fetch: url is required")
|
|
}
|
|
if _, err := validateRequestURL(ctx, rawURL, w.allowedHosts); err != nil {
|
|
return nil, err
|
|
}
|
|
if w.client == nil {
|
|
return nil, fmt.Errorf("io: web-fetch: no underlying client")
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("io: web-fetch: build request: %w", err)
|
|
}
|
|
resp, err := w.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("io: web-fetch: do: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("io: web-fetch: url returned status %d", resp.StatusCode)
|
|
}
|
|
mt := "text/plain"
|
|
if rawCT := resp.Header.Get("Content-Type"); rawCT != "" {
|
|
var err error
|
|
mt, _, err = mime.ParseMediaType(rawCT)
|
|
if err != nil || !isTextContentType(mt) {
|
|
return nil, fmt.Errorf("io: web-fetch: unexpected content type %q", rawCT)
|
|
}
|
|
}
|
|
max := w.maxBytes
|
|
if v, ok := argInt(m, "max_bytes"); ok && v > 0 {
|
|
max = int64(v)
|
|
}
|
|
data, truncated, err := readLimited(resp.Body, max)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("io: web-fetch: read: %w", err)
|
|
}
|
|
text := string(data)
|
|
if isHTML(mt) {
|
|
text = htmlToText(text)
|
|
}
|
|
return map[string]any{
|
|
"url": rawURL,
|
|
"content_type": mt,
|
|
"text": text,
|
|
"truncated": truncated,
|
|
}, nil
|
|
}
|