134 lines
3.2 KiB
Go
134 lines
3.2 KiB
Go
package io
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"strings"
|
|
)
|
|
|
|
// readLimited reads up to max bytes from r, returning the payload and whether it
|
|
// was truncated. A non-positive max falls back to the http response cap.
|
|
func readLimited(r io.Reader, max int64) ([]byte, bool, error) {
|
|
if max <= 0 {
|
|
max = defaultMaxHTTPBytes
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(r, max+1))
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
truncated := int64(len(data)) > max
|
|
if truncated {
|
|
data = data[:max]
|
|
}
|
|
return data, truncated, nil
|
|
}
|
|
|
|
// validateContentType parses a caller-supplied Content-Type, returning the
|
|
// normalized media type. An empty value is allowed and yields "".
|
|
func validateContentType(ct string) (string, error) {
|
|
if strings.TrimSpace(ct) == "" {
|
|
return "", nil
|
|
}
|
|
mt, _, err := mime.ParseMediaType(ct)
|
|
if err != nil {
|
|
return "", fmt.Errorf("io: invalid content-type %q: %w", ct, err)
|
|
}
|
|
return mt, nil
|
|
}
|
|
|
|
// isTextContentType reports whether a media type is acceptable to fetch as text.
|
|
// An empty (unset) media type is accepted so a server that omits Content-Type is
|
|
// treated as text; binary media types are refused.
|
|
func isTextContentType(mt string) bool {
|
|
mt = strings.ToLower(strings.TrimSpace(mt))
|
|
if mt == "" {
|
|
return true
|
|
}
|
|
if strings.HasPrefix(mt, "text/") {
|
|
return true
|
|
}
|
|
switch mt {
|
|
case "application/json", "application/ld+json", "application/xml",
|
|
"application/xhtml+xml", "application/rss+xml", "application/atom+xml",
|
|
"application/x-www-form-urlencoded", "application/javascript",
|
|
"application/x-javascript", "image/svg+xml", "application/graphql":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// isHTML reports whether a media type should be converted from HTML to text.
|
|
func isHTML(mt string) bool {
|
|
mt = strings.ToLower(strings.TrimSpace(mt))
|
|
return mt == "text/html" || mt == "application/xhtml+xml"
|
|
}
|
|
|
|
// htmlToText strips script/style/comment blocks and tags from an HTML document,
|
|
// decoding a few common entities and collapsing whitespace for readable text.
|
|
func htmlToText(src string) string {
|
|
s := dropBlocks(src)
|
|
var out strings.Builder
|
|
inTag := false
|
|
for i := 0; i < len(s); i++ {
|
|
switch c := s[i]; {
|
|
case c == '<':
|
|
inTag = true
|
|
case c == '>':
|
|
inTag = false
|
|
case !inTag:
|
|
out.WriteByte(c)
|
|
}
|
|
}
|
|
return normalizeSpace(htmlUnescape(out.String()))
|
|
}
|
|
|
|
// dropBlocks removes <script>, <style> and HTML comment regions.
|
|
func dropBlocks(s string) string {
|
|
for _, tag := range []string{"script", "style"} {
|
|
for {
|
|
start := strings.Index(s, "<"+tag)
|
|
if start < 0 {
|
|
break
|
|
}
|
|
end := strings.Index(s, "</"+tag+">")
|
|
if end < 0 {
|
|
s = s[:start]
|
|
break
|
|
}
|
|
s = s[:start] + s[end+len(tag)+3:]
|
|
}
|
|
}
|
|
for {
|
|
start := strings.Index(s, "<!--")
|
|
if start < 0 {
|
|
break
|
|
}
|
|
end := strings.Index(s, "-->")
|
|
if end < 0 {
|
|
s = s[:start]
|
|
break
|
|
}
|
|
s = s[:start] + s[end+3:]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// htmlUnescape replaces the small set of entities common in extracted text.
|
|
func htmlUnescape(s string) string {
|
|
return strings.NewReplacer(
|
|
"&", "&",
|
|
"<", "<",
|
|
">", ">",
|
|
""", "\"",
|
|
"'", "'",
|
|
" ", " ",
|
|
).Replace(s)
|
|
}
|
|
|
|
// normalizeSpace collapses runs of whitespace to single spaces and trims ends.
|
|
func normalizeSpace(s string) string {
|
|
return strings.Join(strings.Fields(s), " ")
|
|
}
|