322 lines
8.7 KiB
Go
322 lines
8.7 KiB
Go
package research
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
)
|
|
|
|
const maxResponseBytes = 8 << 20
|
|
|
|
type Diagnostic struct {
|
|
Configured bool `json:"configured"`
|
|
BaseURL string `json:"base_url,omitempty"`
|
|
OK bool `json:"ok"`
|
|
CheckedAt time.Time `json:"checked_at,omitempty"`
|
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
|
HTTPStatus int `json:"http_status,omitempty"`
|
|
ContentType string `json:"content_type,omitempty"`
|
|
ResultCount int `json:"result_count,omitempty"`
|
|
ErrorKind string `json:"error_kind,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type Client struct {
|
|
BaseURL string
|
|
HTTP *http.Client
|
|
|
|
mu sync.RWMutex
|
|
last Diagnostic
|
|
}
|
|
|
|
func New(base string) *Client {
|
|
base = strings.TrimSpace(base)
|
|
return &Client{
|
|
BaseURL: strings.TrimRight(base, "/"),
|
|
HTTP: &http.Client{Timeout: 45 * time.Second},
|
|
last: Diagnostic{
|
|
Configured: base != "",
|
|
BaseURL: displayBaseURL(base),
|
|
},
|
|
}
|
|
}
|
|
|
|
func (c *Client) Status() Diagnostic {
|
|
if c == nil {
|
|
return Diagnostic{}
|
|
}
|
|
c.mu.RLock()
|
|
out := c.last
|
|
c.mu.RUnlock()
|
|
if out.BaseURL == "" {
|
|
out.BaseURL = displayBaseURL(c.BaseURL)
|
|
}
|
|
out.Configured = strings.TrimSpace(c.BaseURL) != ""
|
|
return out
|
|
}
|
|
|
|
func (c *Client) Search(ctx context.Context, q string, limit int) ([]model.ResearchResult, error) {
|
|
results, _, err := c.SearchDetailed(ctx, q, limit)
|
|
return results, err
|
|
}
|
|
|
|
func (c *Client) SearchDetailed(ctx context.Context, q string, limit int) ([]model.ResearchResult, Diagnostic, error) {
|
|
return c.SearchDetailedLanguage(ctx, q, limit, "de-DE")
|
|
}
|
|
|
|
func (c *Client) SearchDetailedLanguage(ctx context.Context, q string, limit int, language string) ([]model.ResearchResult, Diagnostic, error) {
|
|
started := time.Now()
|
|
diagnostic := Diagnostic{
|
|
Configured: c != nil && strings.TrimSpace(c.BaseURL) != "",
|
|
CheckedAt: started.UTC(),
|
|
}
|
|
if c != nil {
|
|
diagnostic.BaseURL = displayBaseURL(c.BaseURL)
|
|
}
|
|
finish := func(results []model.ResearchResult, err error) ([]model.ResearchResult, Diagnostic, error) {
|
|
diagnostic.DurationMS = time.Since(started).Milliseconds()
|
|
diagnostic.ResultCount = len(results)
|
|
diagnostic.OK = err == nil
|
|
if err != nil {
|
|
diagnostic.Error = err.Error()
|
|
if diagnostic.ErrorKind == "" {
|
|
diagnostic.ErrorKind = classifyError(err)
|
|
}
|
|
}
|
|
if c != nil {
|
|
c.mu.Lock()
|
|
c.last = diagnostic
|
|
c.mu.Unlock()
|
|
}
|
|
return results, diagnostic, err
|
|
}
|
|
|
|
if c == nil || strings.TrimSpace(c.BaseURL) == "" {
|
|
diagnostic.ErrorKind = "not_configured"
|
|
return finish(nil, errors.New("SearXNG ist nicht konfiguriert: SEARXNG_URL ist leer"))
|
|
}
|
|
q = strings.TrimSpace(q)
|
|
if q == "" {
|
|
diagnostic.ErrorKind = "invalid_query"
|
|
return finish(nil, errors.New("SearXNG-Suchanfrage ist leer"))
|
|
}
|
|
|
|
u, err := searchURL(c.BaseURL, q, language)
|
|
if err != nil {
|
|
diagnostic.ErrorKind = "invalid_url"
|
|
return finish(nil, fmt.Errorf("ungültige SEARXNG_URL %q: %w", diagnostic.BaseURL, err))
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
if err != nil {
|
|
diagnostic.ErrorKind = "request_build"
|
|
return finish(nil, fmt.Errorf("SearXNG-Anfrage konnte nicht erstellt werden: %w", err))
|
|
}
|
|
req.Header.Set("Accept", "application/json")
|
|
req.Header.Set("Accept-Language", "de-DE,de;q=0.9,en;q=0.7")
|
|
req.Header.Set("User-Agent", "glpi-neural-brain/1.0")
|
|
|
|
client := c.HTTP
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 45 * time.Second}
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
diagnostic.ErrorKind = classifyError(err)
|
|
return finish(nil, fmt.Errorf("SearXNG unter %s ist nicht erreichbar: %w%s", endpointForError(u), err, networkHint(u, diagnostic.ErrorKind)))
|
|
}
|
|
defer resp.Body.Close()
|
|
diagnostic.HTTPStatus = resp.StatusCode
|
|
diagnostic.ContentType = strings.TrimSpace(resp.Header.Get("Content-Type"))
|
|
|
|
body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1))
|
|
if readErr != nil {
|
|
diagnostic.ErrorKind = "response_read"
|
|
return finish(nil, fmt.Errorf("SearXNG-Antwort konnte nicht gelesen werden: %w", readErr))
|
|
}
|
|
if len(body) > maxResponseBytes {
|
|
diagnostic.ErrorKind = "response_too_large"
|
|
return finish(nil, fmt.Errorf("SearXNG-Antwort überschreitet %d MiB", maxResponseBytes>>20))
|
|
}
|
|
if resp.StatusCode/100 != 2 {
|
|
diagnostic.ErrorKind = "http_status"
|
|
return finish(nil, fmt.Errorf("SearXNG lieferte HTTP %d (%s)%s%s", resp.StatusCode, http.StatusText(resp.StatusCode), bodySuffix(body), httpStatusHint(resp.StatusCode)))
|
|
}
|
|
|
|
var env struct {
|
|
Results []struct {
|
|
Title string `json:"title"`
|
|
URL string `json:"url"`
|
|
Content string `json:"content"`
|
|
} `json:"results"`
|
|
}
|
|
if err := json.Unmarshal(body, &env); err != nil {
|
|
diagnostic.ErrorKind = "invalid_json"
|
|
return finish(nil, fmt.Errorf("SearXNG lieferte kein gültiges JSON (Content-Type %q): %w%s; in settings.yml muss search.formats den Wert json enthalten", diagnostic.ContentType, err, bodySuffix(body)))
|
|
}
|
|
|
|
out := make([]model.ResearchResult, 0, len(env.Results))
|
|
for _, r := range env.Results {
|
|
if strings.TrimSpace(r.URL) == "" {
|
|
continue
|
|
}
|
|
snippet := strings.TrimSpace(r.Content)
|
|
out = append(out, model.ResearchResult{Title: strings.TrimSpace(r.Title), URL: strings.TrimSpace(r.URL), Snippet: snippet, Content: snippet, Language: normalizeLanguage(language)})
|
|
if limit > 0 && len(out) >= limit {
|
|
break
|
|
}
|
|
}
|
|
return finish(out, nil)
|
|
}
|
|
|
|
func searchURL(base, query, language string) (*url.URL, error) {
|
|
u, err := url.Parse(strings.TrimSpace(base))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if u.Scheme != "http" && u.Scheme != "https" {
|
|
return nil, fmt.Errorf("Schema muss http oder https sein")
|
|
}
|
|
if u.Host == "" {
|
|
return nil, fmt.Errorf("Host fehlt")
|
|
}
|
|
path := strings.TrimRight(u.Path, "/")
|
|
if !strings.HasSuffix(strings.ToLower(path), "/search") {
|
|
path += "/search"
|
|
}
|
|
if path == "" {
|
|
path = "/search"
|
|
}
|
|
u.Path = path
|
|
values := u.Query()
|
|
values.Set("format", "json")
|
|
values.Set("language", normalizeLanguage(language))
|
|
values.Set("q", query)
|
|
u.RawQuery = values.Encode()
|
|
return u, nil
|
|
}
|
|
|
|
func normalizeLanguage(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "de-DE"
|
|
}
|
|
switch strings.ToLower(value) {
|
|
case "de", "de-de":
|
|
return "de-DE"
|
|
case "en", "en-us", "en-gb":
|
|
return "en-US"
|
|
default:
|
|
return value
|
|
}
|
|
}
|
|
|
|
func displayBaseURL(raw string) string {
|
|
u, err := url.Parse(strings.TrimSpace(raw))
|
|
if err != nil {
|
|
return strings.TrimSpace(raw)
|
|
}
|
|
u.User = nil
|
|
u.RawQuery = ""
|
|
u.Fragment = ""
|
|
return strings.TrimRight(u.String(), "/")
|
|
}
|
|
|
|
func endpointForError(u *url.URL) string {
|
|
if u == nil {
|
|
return "unbekannter Adresse"
|
|
}
|
|
clean := *u
|
|
clean.User = nil
|
|
clean.RawQuery = ""
|
|
clean.Fragment = ""
|
|
return clean.String()
|
|
}
|
|
|
|
func classifyError(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
return "timeout"
|
|
}
|
|
if errors.Is(err, context.Canceled) {
|
|
return "canceled"
|
|
}
|
|
text := strings.ToLower(err.Error())
|
|
switch {
|
|
case strings.Contains(text, "connection refused"):
|
|
return "connection_refused"
|
|
case strings.Contains(text, "no such host"):
|
|
return "dns"
|
|
case strings.Contains(text, "certificate") || strings.Contains(text, "tls"):
|
|
return "tls"
|
|
}
|
|
var dnsErr *net.DNSError
|
|
if errors.As(err, &dnsErr) {
|
|
return "dns"
|
|
}
|
|
var netErr net.Error
|
|
if errors.As(err, &netErr) {
|
|
if netErr.Timeout() {
|
|
return "timeout"
|
|
}
|
|
return "network"
|
|
}
|
|
return "request"
|
|
}
|
|
|
|
func networkHint(u *url.URL, kind string) string {
|
|
if u == nil {
|
|
return ""
|
|
}
|
|
host := strings.ToLower(u.Hostname())
|
|
if host == "localhost" || host == "127.0.0.1" || host == "::1" {
|
|
return "; Hinweis: Innerhalb des Brain-Containers verweist localhost auf den Brain-Container. Verwende den Docker-Service-Namen, zum Beispiel http://searxng:8080, oder host.docker.internal"
|
|
}
|
|
switch kind {
|
|
case "dns":
|
|
return "; prüfe Docker-Netzwerk und Service-Namen"
|
|
case "connection_refused", "network":
|
|
return "; am Zielhost ist auf diesem Port möglicherweise kein SearXNG-Dienst erreichbar"
|
|
case "tls":
|
|
return "; prüfe Zertifikat und HTTPS-Konfiguration"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func httpStatusHint(status int) string {
|
|
switch status {
|
|
case http.StatusForbidden:
|
|
return "; prüfe SearXNG-Limiter/Zugriffsregeln und ob das JSON-Format erlaubt ist"
|
|
case http.StatusNotFound:
|
|
return "; prüfe die SearXNG-Basis-URL und einen möglichen Reverse-Proxy-Pfad"
|
|
case http.StatusTooManyRequests:
|
|
return "; SearXNG oder ein vorgeschalteter Proxy begrenzt die Anfrage"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func bodySuffix(body []byte) string {
|
|
text := strings.Join(strings.Fields(string(body)), " ")
|
|
if text == "" {
|
|
return ""
|
|
}
|
|
const max = 320
|
|
if len(text) > max {
|
|
text = text[:max] + "…"
|
|
}
|
|
return ": " + text
|
|
}
|