feat(tools/io): 内置 IO/网络工具（file(路径安全)/http(SSRF防护)/web-search/web-fetch/knowledge）+ 大小截断

plugins/tools/io/util.go

@@ -0,0 +1,133 @@
+package io
+
+import (
+	"fmt"
+	"io"
+	"mime"
+	"strings"
+)
+
+// readLimited reads up to max bytes from r, returning the payload and whether it
+// was truncated. A non-positive max falls back to the http response cap.
+func readLimited(r io.Reader, max int64) ([]byte, bool, error) {
+	if max <= 0 {
+		max = defaultMaxHTTPBytes
+	}
+	data, err := io.ReadAll(io.LimitReader(r, max+1))
+	if err != nil {
+		return nil, false, err
+	}
+	truncated := int64(len(data)) > max
+	if truncated {
+		data = data[:max]
+	}
+	return data, truncated, nil
+}
+
+// validateContentType parses a caller-supplied Content-Type, returning the
+// normalized media type. An empty value is allowed and yields "".
+func validateContentType(ct string) (string, error) {
+	if strings.TrimSpace(ct) == "" {
+		return "", nil
+	}
+	mt, _, err := mime.ParseMediaType(ct)
+	if err != nil {
+		return "", fmt.Errorf("io: invalid content-type %q: %w", ct, err)
+	}
+	return mt, nil
+}
+
+// isTextContentType reports whether a media type is acceptable to fetch as text.
+// An empty (unset) media type is accepted so a server that omits Content-Type is
+// treated as text; binary media types are refused.
+func isTextContentType(mt string) bool {
+	mt = strings.ToLower(strings.TrimSpace(mt))
+	if mt == "" {
+		return true
+	}
+	if strings.HasPrefix(mt, "text/") {
+		return true
+	}
+	switch mt {
+	case "application/json", "application/ld+json", "application/xml",
+		"application/xhtml+xml", "application/rss+xml", "application/atom+xml",
+		"application/x-www-form-urlencoded", "application/javascript",
+		"application/x-javascript", "image/svg+xml", "application/graphql":
+		return true
+	default:
+		return false
+	}
+}
+
+// isHTML reports whether a media type should be converted from HTML to text.
+func isHTML(mt string) bool {
+	mt = strings.ToLower(strings.TrimSpace(mt))
+	return mt == "text/html" || mt == "application/xhtml+xml"
+}
+
+// htmlToText strips script/style/comment blocks and tags from an HTML document,
+// decoding a few common entities and collapsing whitespace for readable text.
+func htmlToText(src string) string {
+	s := dropBlocks(src)
+	var out strings.Builder
+	inTag := false
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case c == '<':
+			inTag = true
+		case c == '>':
+			inTag = false
+		case !inTag:
+			out.WriteByte(c)
+		}
+	}
+	return normalizeSpace(htmlUnescape(out.String()))
+}
+
+// dropBlocks removes <script>, <style> and HTML comment regions.
+func dropBlocks(s string) string {
+	for _, tag := range []string{"script", "style"} {
+		for {
+			start := strings.Index(s, "<"+tag)
+			if start < 0 {
+				break
+			}
+			end := strings.Index(s, "</"+tag+">")
+			if end < 0 {
+				s = s[:start]
+				break
+			}
+			s = s[:start] + s[end+len(tag)+3:]
+		}
+	}
+	for {
+		start := strings.Index(s, "<!--")
+		if start < 0 {
+			break
+		}
+		end := strings.Index(s, "-->")
+		if end < 0 {
+			s = s[:start]
+			break
+		}
+		s = s[:start] + s[end+3:]
+	}
+	return s
+}
+
+// htmlUnescape replaces the small set of entities common in extracted text.
+func htmlUnescape(s string) string {
+	return strings.NewReplacer(
+		"&amp;", "&",
+		"&lt;", "<",
+		"&gt;", ">",
+		"&quot;", "\"",
+		"&#39;", "'",
+		"&nbsp;", " ",
+	).Replace(s)
+}
+
+// normalizeSpace collapses runs of whitespace to single spaces and trims ends.
+func normalizeSpace(s string) string {
+	return strings.Join(strings.Fields(s), " ")
+}