92 lines
3.7 KiB
Go
92 lines
3.7 KiB
Go
package sourceagent
|
|
|
|
import (
|
|
"context"
|
|
"math/bits"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/research"
|
|
)
|
|
|
|
func TestValidateTaskSupportsPollerTypes(t *testing.T) {
|
|
for _, typ := range []string{"rss", "atom", "sitemap", "web"} {
|
|
task, err := validateTask(Task{ID: "security", AgentID: "a", Name: "Security", Type: typ, URL: "https://example.org/feed", Enabled: true, PollInterval: "2h", MaxItems: 25})
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", typ, err)
|
|
}
|
|
if task.Type != typ || task.MaxItems != 25 {
|
|
t.Fatalf("unexpected task: %+v", task)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestValidateTaskRejectsUnsafeShape(t *testing.T) {
|
|
if _, err := validateTask(Task{Type: "ftp", URL: "https://example.org", PollInterval: "1h"}); err == nil {
|
|
t.Fatal("expected unsupported type")
|
|
}
|
|
if _, err := validateTask(Task{Type: "rss", URL: "ftp://example.org/feed", PollInterval: "1h"}); err == nil {
|
|
t.Fatal("expected URL validation error")
|
|
}
|
|
if _, err := validateTask(Task{Type: "rss", URL: "https://example.org/feed", PollInterval: "1m"}); err == nil {
|
|
t.Fatal("expected interval validation error")
|
|
}
|
|
if _, err := validateTask(Task{Type: "rss", URL: "https://user:pass@example.org/feed", PollInterval: "1h"}); err == nil {
|
|
t.Fatal("expected userinfo URL validation error")
|
|
}
|
|
}
|
|
|
|
func TestNormalizeDocumentBuildsContentHash(t *testing.T) {
|
|
d, err := normalizeDocument(Document{URL: "https://example.org/a", Title: "A", Text: "Dies ist ein ausreichend langer Dokumenttext mit sicherheitsrelevanten Informationen und mehreren Details für die Verarbeitung."})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.ContentSHA256 == "" || d.CanonicalURL != "https://example.org/a" || d.ExternalID != "https://example.org/a" {
|
|
t.Fatalf("unexpected normalized document: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSimhashKeepsRelatedTextCloser(t *testing.T) {
|
|
base := simhash64("Windows Backup Repository Ransomware Schutz immutable storage MFA")
|
|
related := simhash64("Ransomware Schutz für Windows Backup Repository mit MFA und immutable Storage")
|
|
unrelated := simhash64("Kaffee Bohnen Espresso Maschine Mahlgrad Temperatur")
|
|
if bits.OnesCount64(base^related) >= bits.OnesCount64(base^unrelated) {
|
|
t.Fatalf("related text should hash closer")
|
|
}
|
|
}
|
|
|
|
func TestFetchRawBlocksPrivateSourceByDefault(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("<rss/>")) }))
|
|
defer srv.Close()
|
|
r := &Runner{cfg: RunnerConfig{HTTPTimeout: time.Second}, sourceHTTP: research.NewSafeHTTPClient(false, time.Second)}
|
|
if _, _, _, _, err := r.fetchRaw(context.Background(), srv.URL, 1024); err == nil {
|
|
t.Fatal("expected private/loopback source URL to be blocked")
|
|
}
|
|
}
|
|
|
|
func TestSitemapIndexCollectsChildURLs(t *testing.T) {
|
|
mux := http.NewServeMux()
|
|
var base string
|
|
mux.HandleFunc("/index.xml", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/xml")
|
|
_, _ = w.Write([]byte(`<sitemapindex><sitemap><loc>` + base + `/child.xml</loc></sitemap></sitemapindex>`))
|
|
})
|
|
mux.HandleFunc("/child.xml", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/xml")
|
|
_, _ = w.Write([]byte(`<urlset><url><loc>` + base + `/article-1</loc><lastmod>2026-08-07T12:00:00Z</lastmod></url></urlset>`))
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
defer srv.Close()
|
|
base = srv.URL
|
|
r := &Runner{cfg: RunnerConfig{HTTPTimeout: time.Second, AllowPrivate: true}, sourceHTTP: research.NewSafeHTTPClient(true, time.Second)}
|
|
items, err := r.collectSitemapItems(context.Background(), srv.URL+"/index.xml", 10, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(items) != 1 || items[0].Link != srv.URL+"/article-1" {
|
|
t.Fatalf("unexpected sitemap items: %+v", items)
|
|
}
|
|
}
|