jbergner
8dc4de3eae
All checks were successful
release-tag / release-image (push) Successful in 2m22s
320 lines
10 KiB
Go
320 lines
10 KiB
Go
package research
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
const (
|
|
defaultFetchBytes = int64(2 << 20)
|
|
defaultFetchChars = 14000
|
|
)
|
|
|
|
type FetchOptions struct {
|
|
MaxBytes int64
|
|
MaxChars int
|
|
Timeout time.Duration
|
|
AllowPrivate bool
|
|
}
|
|
|
|
type FetchDiagnostic struct {
|
|
URL string `json:"url"`
|
|
FinalURL string `json:"final_url,omitempty"`
|
|
HTTPStatus int `json:"http_status,omitempty"`
|
|
ContentType string `json:"content_type,omitempty"`
|
|
Bytes int `json:"bytes,omitempty"`
|
|
Characters int `json:"characters,omitempty"`
|
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
|
ErrorKind string `json:"error_kind,omitempty"`
|
|
}
|
|
|
|
type FetchedPage struct {
|
|
URL string
|
|
Title string
|
|
Content string
|
|
ContentType string
|
|
}
|
|
|
|
func (c *Client) FetchPage(ctx context.Context, rawURL string, options FetchOptions) (FetchedPage, FetchDiagnostic, error) {
|
|
started := time.Now()
|
|
diagnostic := FetchDiagnostic{URL: strings.TrimSpace(rawURL)}
|
|
finish := func(page FetchedPage, err error) (FetchedPage, FetchDiagnostic, error) {
|
|
diagnostic.DurationMS = time.Since(started).Milliseconds()
|
|
if page.URL != "" {
|
|
diagnostic.FinalURL = page.URL
|
|
}
|
|
if page.ContentType != "" {
|
|
diagnostic.ContentType = page.ContentType
|
|
}
|
|
diagnostic.Characters = len([]rune(page.Content))
|
|
if err != nil && diagnostic.ErrorKind == "" {
|
|
diagnostic.ErrorKind = classifyFetchError(err)
|
|
}
|
|
return page, diagnostic, err
|
|
}
|
|
|
|
if options.MaxBytes <= 0 {
|
|
options.MaxBytes = defaultFetchBytes
|
|
}
|
|
if options.MaxChars <= 0 {
|
|
options.MaxChars = defaultFetchChars
|
|
}
|
|
if options.Timeout <= 0 {
|
|
options.Timeout = 20 * time.Second
|
|
}
|
|
parsed, err := validateFetchURL(ctx, rawURL, options.AllowPrivate)
|
|
if err != nil {
|
|
diagnostic.ErrorKind = "unsafe_url"
|
|
return finish(FetchedPage{}, err)
|
|
}
|
|
|
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
|
transport.Proxy = nil
|
|
transport.DialContext = safeDialContext(options.AllowPrivate)
|
|
transport.MaxIdleConns = 4
|
|
transport.MaxIdleConnsPerHost = 2
|
|
transport.IdleConnTimeout = 20 * time.Second
|
|
client := &http.Client{
|
|
Transport: transport,
|
|
Timeout: options.Timeout,
|
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
|
if len(via) >= 5 {
|
|
return errors.New("zu viele Weiterleitungen")
|
|
}
|
|
_, err := validateFetchURL(req.Context(), req.URL.String(), options.AllowPrivate)
|
|
return err
|
|
},
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
|
if err != nil {
|
|
diagnostic.ErrorKind = "request_build"
|
|
return finish(FetchedPage{}, err)
|
|
}
|
|
req.Header.Set("Accept", "text/html, text/plain;q=0.9, application/xhtml+xml;q=0.8, application/pdf;q=0.2")
|
|
req.Header.Set("Accept-Language", "de-DE,de;q=0.9,en;q=0.8")
|
|
req.Header.Set("User-Agent", "glpi-neural-brain-research/1.0")
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return finish(FetchedPage{}, fmt.Errorf("Webquelle %s konnte nicht geladen werden: %w", parsed.Hostname(), err))
|
|
}
|
|
defer resp.Body.Close()
|
|
diagnostic.HTTPStatus = resp.StatusCode
|
|
diagnostic.ContentType = strings.TrimSpace(resp.Header.Get("Content-Type"))
|
|
if resp.StatusCode/100 != 2 {
|
|
diagnostic.ErrorKind = "http_status"
|
|
return finish(FetchedPage{}, fmt.Errorf("Webquelle lieferte HTTP %d (%s)", resp.StatusCode, http.StatusText(resp.StatusCode)))
|
|
}
|
|
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, options.MaxBytes+1))
|
|
if err != nil {
|
|
diagnostic.ErrorKind = "response_read"
|
|
return finish(FetchedPage{}, fmt.Errorf("Webquelle konnte nicht gelesen werden: %w", err))
|
|
}
|
|
if int64(len(body)) > options.MaxBytes {
|
|
diagnostic.ErrorKind = "response_too_large"
|
|
return finish(FetchedPage{}, fmt.Errorf("Webquelle überschreitet das Größenlimit von %d Bytes", options.MaxBytes))
|
|
}
|
|
diagnostic.Bytes = len(body)
|
|
contentType := strings.ToLower(strings.TrimSpace(strings.Split(diagnostic.ContentType, ";")[0]))
|
|
finalURL := resp.Request.URL.String()
|
|
page := FetchedPage{URL: finalURL, ContentType: contentType}
|
|
switch {
|
|
case contentType == "application/pdf" || strings.HasSuffix(strings.ToLower(resp.Request.URL.Path), ".pdf"):
|
|
diagnostic.ErrorKind = "pdf_unsupported"
|
|
return finish(FetchedPage{}, errors.New("PDF-Quelle erkannt; PDF-Textextraktion ist in diesem Build nicht aktiviert"))
|
|
case contentType == "text/plain" || contentType == "text/markdown" || contentType == "application/json":
|
|
page.Content = normalizeExtractedText(string(body), options.MaxChars)
|
|
case contentType == "" || contentType == "text/html" || contentType == "application/xhtml+xml":
|
|
page.Title, page.Content = extractHTMLText(string(body), options.MaxChars)
|
|
default:
|
|
diagnostic.ErrorKind = "unsupported_content_type"
|
|
return finish(FetchedPage{}, fmt.Errorf("nicht unterstützter Content-Type %q", diagnostic.ContentType))
|
|
}
|
|
if page.Title == "" {
|
|
page.Title = strings.TrimSpace(resp.Request.URL.Hostname())
|
|
}
|
|
if len([]rune(page.Content)) < 160 {
|
|
diagnostic.ErrorKind = "content_too_short"
|
|
return finish(FetchedPage{}, errors.New("Webquelle enthält nach der Textextraktion zu wenig verwertbaren Inhalt"))
|
|
}
|
|
return finish(page, nil)
|
|
}
|
|
|
|
func validateFetchURL(ctx context.Context, rawURL string, allowPrivate bool) (*url.URL, error) {
|
|
u, err := url.Parse(strings.TrimSpace(rawURL))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ungültige URL: %w", err)
|
|
}
|
|
if u.Scheme != "http" && u.Scheme != "https" {
|
|
return nil, errors.New("Webquellen müssen HTTP oder HTTPS verwenden")
|
|
}
|
|
if u.Hostname() == "" {
|
|
return nil, errors.New("Webquelle enthält keinen Host")
|
|
}
|
|
if u.User != nil {
|
|
return nil, errors.New("Webquellen mit Zugangsdaten in der URL sind nicht erlaubt")
|
|
}
|
|
if allowPrivate {
|
|
return u, nil
|
|
}
|
|
host := strings.ToLower(strings.TrimSuffix(u.Hostname(), "."))
|
|
if host == "localhost" || strings.HasSuffix(host, ".localhost") {
|
|
return nil, errors.New("lokale Webadressen sind als Recherchequelle nicht erlaubt")
|
|
}
|
|
ips, err := resolveHost(ctx, host)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Host %q konnte nicht aufgelöst werden: %w", host, err)
|
|
}
|
|
for _, ip := range ips {
|
|
if !isPublicIP(ip) {
|
|
return nil, fmt.Errorf("Host %q verweist auf eine private oder lokale Adresse", host)
|
|
}
|
|
}
|
|
return u, nil
|
|
}
|
|
|
|
func safeDialContext(allowPrivate bool) func(context.Context, string, string) (net.Conn, error) {
|
|
dialer := &net.Dialer{Timeout: 12 * time.Second, KeepAlive: 20 * time.Second}
|
|
return func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
host, port, err := net.SplitHostPort(address)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if allowPrivate {
|
|
return dialer.DialContext(ctx, network, address)
|
|
}
|
|
ips, err := resolveHost(ctx, host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var lastErr error
|
|
for _, ip := range ips {
|
|
if !isPublicIP(ip) {
|
|
continue
|
|
}
|
|
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
|
if err == nil {
|
|
return conn, nil
|
|
}
|
|
lastErr = err
|
|
}
|
|
if lastErr != nil {
|
|
return nil, lastErr
|
|
}
|
|
return nil, errors.New("Host besitzt keine zulässige öffentliche IP-Adresse")
|
|
}
|
|
}
|
|
|
|
func resolveHost(ctx context.Context, host string) ([]net.IP, error) {
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
return []net.IP{ip}, nil
|
|
}
|
|
values, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]net.IP, 0, len(values))
|
|
for _, value := range values {
|
|
out = append(out, value.IP)
|
|
}
|
|
if len(out) == 0 {
|
|
return nil, errors.New("keine IP-Adresse gefunden")
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func isPublicIP(ip net.IP) bool {
|
|
return ip != nil && ip.IsGlobalUnicast() && !ip.IsPrivate() && !ip.IsLoopback() && !ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() && !ip.IsUnspecified()
|
|
}
|
|
|
|
var (
|
|
titlePattern = regexp.MustCompile(`(?is)<title\b[^>]*>(.*?)</title>`)
|
|
mainPattern = regexp.MustCompile(`(?is)<main\b[^>]*>(.*?)</main\s*>|<article\b[^>]*>(.*?)</article\s*>`)
|
|
discardBlockPattern = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>|<svg\b[^>]*>.*?</svg\s*>|<canvas\b[^>]*>.*?</canvas\s*>|<form\b[^>]*>.*?</form\s*>|<nav\b[^>]*>.*?</nav\s*>|<footer\b[^>]*>.*?</footer\s*>|<header\b[^>]*>.*?</header\s*>|<aside\b[^>]*>.*?</aside\s*>`)
|
|
commentPattern = regexp.MustCompile(`(?is)<!--.*?-->`)
|
|
blockTagPattern = regexp.MustCompile(`(?is)</?(p|div|section|article|main|h[1-6]|li|ul|ol|table|tr|td|th|pre|blockquote|br|hr)\b[^>]*>`)
|
|
tagPattern = regexp.MustCompile(`(?is)<[^>]+>`)
|
|
spacePattern = regexp.MustCompile(`[\t\f\v ]+`)
|
|
newlinePattern = regexp.MustCompile(`\n{3,}`)
|
|
)
|
|
|
|
func extractHTMLText(raw string, maxChars int) (string, string) {
|
|
title := ""
|
|
if match := titlePattern.FindStringSubmatch(raw); len(match) == 2 {
|
|
title = normalizeExtractedText(tagPattern.ReplaceAllString(html.UnescapeString(match[1]), " "), 300)
|
|
}
|
|
text := raw
|
|
if match := mainPattern.FindStringSubmatch(raw); len(match) >= 3 {
|
|
if strings.TrimSpace(match[1]) != "" {
|
|
text = match[1]
|
|
} else if strings.TrimSpace(match[2]) != "" {
|
|
text = match[2]
|
|
}
|
|
}
|
|
text = commentPattern.ReplaceAllString(text, " ")
|
|
text = discardBlockPattern.ReplaceAllString(text, " ")
|
|
text = blockTagPattern.ReplaceAllString(text, "\n")
|
|
text = tagPattern.ReplaceAllString(text, " ")
|
|
text = html.UnescapeString(text)
|
|
return title, normalizeExtractedText(text, maxChars)
|
|
}
|
|
|
|
func normalizeExtractedText(value string, maxChars int) string {
|
|
if !utf8.ValidString(value) {
|
|
value = strings.ToValidUTF8(value, " ")
|
|
}
|
|
value = strings.ReplaceAll(value, "\r\n", "\n")
|
|
value = strings.ReplaceAll(value, "\r", "\n")
|
|
lines := strings.Split(value, "\n")
|
|
out := make([]string, 0, len(lines))
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(spacePattern.ReplaceAllString(line, " "))
|
|
if line == "" {
|
|
if len(out) > 0 && out[len(out)-1] != "" {
|
|
out = append(out, "")
|
|
}
|
|
continue
|
|
}
|
|
printable := 0
|
|
for _, r := range line {
|
|
if unicode.IsLetter(r) || unicode.IsNumber(r) || unicode.IsPunct(r) || unicode.IsSymbol(r) {
|
|
printable++
|
|
}
|
|
}
|
|
if printable < 2 {
|
|
continue
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
text := strings.TrimSpace(newlinePattern.ReplaceAllString(strings.Join(out, "\n"), "\n\n"))
|
|
if maxChars > 0 {
|
|
runes := []rune(text)
|
|
if len(runes) > maxChars {
|
|
text = strings.TrimSpace(string(runes[:maxChars]))
|
|
}
|
|
}
|
|
return text
|
|
}
|
|
|
|
func classifyFetchError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
return "timeout"
|
|
}
|
|
return classifyError(err)
|
|
}
|