package io import ( "context" "fmt" "log/slog" "mime" "net/http" "strings" "git.yeij.top/AskaEth/Cyrene_v2/pkg/plugins/sdk" ) // Compile-time assertion that webFetchTool implements sdk.Tool. var _ sdk.Tool = (*webFetchTool)(nil) // webFetchTool fetches a URL over HTTP and returns its readable text, limited by // size and content type, extracting text from HTML when appropriate. type webFetchTool struct { client *http.Client allowedHosts []string maxBytes int64 log *slog.Logger } func newWebFetchTool(t *tools) *webFetchTool { return &webFetchTool{ client: t.client, allowedHosts: t.allowedHosts, maxBytes: t.maxFetchBytes, log: t.log, } } // Name implements sdk.Tool. func (w *webFetchTool) Name() string { return "web-fetch" } // Description implements sdk.Tool. func (w *webFetchTool) Description() string { return "Fetch a URL and return its readable text, limited by size and content type." } // Parameters implements sdk.Tool. func (w *webFetchTool) Parameters() (map[string]any, error) { return map[string]any{ "type": "object", "properties": map[string]any{ "url": map[string]any{"type": "string", "description": "Absolute http(s) URL."}, "max_bytes": map[string]any{"type": "integer", "description": "Response size cap in bytes."}, }, "required": []any{"url"}, }, nil } // Execute implements sdk.Tool. func (w *webFetchTool) Execute(ctx context.Context, args any) (any, error) { m, ok := args.(map[string]any) if !ok { return nil, fmt.Errorf("io: web-fetch: args must be an object") } rawURL := argString(m, "url") if strings.TrimSpace(rawURL) == "" { return nil, fmt.Errorf("io: web-fetch: url is required") } if _, err := validateRequestURL(ctx, rawURL, w.allowedHosts); err != nil { return nil, err } if w.client == nil { return nil, fmt.Errorf("io: web-fetch: no underlying client") } req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return nil, fmt.Errorf("io: web-fetch: build request: %w", err) } resp, err := w.client.Do(req) if err != nil { return nil, fmt.Errorf("io: web-fetch: do: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("io: web-fetch: url returned status %d", resp.StatusCode) } mt := "text/plain" if rawCT := resp.Header.Get("Content-Type"); rawCT != "" { var err error mt, _, err = mime.ParseMediaType(rawCT) if err != nil || !isTextContentType(mt) { return nil, fmt.Errorf("io: web-fetch: unexpected content type %q", rawCT) } } max := w.maxBytes if v, ok := argInt(m, "max_bytes"); ok && v > 0 { max = int64(v) } data, truncated, err := readLimited(resp.Body, max) if err != nil { return nil, fmt.Errorf("io: web-fetch: read: %w", err) } text := string(data) if isHTML(mt) { text = htmlToText(text) } return map[string]any{ "url": rawURL, "content_type": mt, "text": text, "truncated": truncated, }, nil }