217 lines
8.3 KiB
Go
217 lines
8.3 KiB
Go
package engine
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"math"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
"github.com/local/glpi-neural-brain/internal/ollama"
|
|
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
|
)
|
|
|
|
const SourceInboxClassifierVersion = 2
|
|
|
|
type sourceInboxClassification struct {
|
|
Status string
|
|
Priority float64
|
|
Semantic float64
|
|
Freshness float64
|
|
TaskContext float64
|
|
EventSignal float64
|
|
Reason string
|
|
MatchedNodeID string
|
|
}
|
|
|
|
func (e *Engine) SetSourceInbox(store *sourceagent.Store) { e.SourceInbox = store }
|
|
|
|
func (e *Engine) evidenceAcquisitionEnabled() bool {
|
|
return (e.SourceInbox != nil && e.Cfg.SourceInboxEnabled) || e.ResearchEnabledForRuntime()
|
|
}
|
|
|
|
func (e *Engine) sourceInboxLoop(ctx context.Context) {
|
|
// Classify once immediately so newly ingested or migration-requeued documents
|
|
// do not wait a full interval after startup.
|
|
e.processSourceInbox(ctx)
|
|
ticker := time.NewTicker(e.Cfg.SourceInboxInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
e.processSourceInbox(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Engine) processSourceInbox(ctx context.Context) {
|
|
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
|
|
return
|
|
}
|
|
items, err := e.SourceInbox.ClaimInbox(ctx, e.Cfg.SourceInboxBatchSize)
|
|
if err != nil {
|
|
slog.Warn("source inbox claim failed", "error", err)
|
|
return
|
|
}
|
|
if len(items) == 0 {
|
|
return
|
|
}
|
|
texts := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
text := item.Document.Title + "\n" + strings.Join(item.Document.Categories, " · ") + "\n" + item.Document.Text
|
|
if len([]rune(text)) > 12000 {
|
|
text = string([]rune(text)[:12000])
|
|
}
|
|
texts = append(texts, text)
|
|
}
|
|
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 4*time.Minute)
|
|
vectors, embedErr := e.Ollama.Embed(cctx, texts)
|
|
cancel()
|
|
if embedErr != nil || len(vectors) != len(items) {
|
|
for _, item := range items {
|
|
_ = e.SourceInbox.ReleaseInbox(ctx, item.ID, fmt.Sprint(embedErr))
|
|
}
|
|
return
|
|
}
|
|
candidateCount := 0
|
|
for i, item := range items {
|
|
hits, stats := e.similarKnowledge(vectors[i], 1, e.effectiveLearningFilter(), 0)
|
|
if len(hits) == 0 {
|
|
_ = e.SourceInbox.ReleaseInbox(ctx, item.ID, "knowledge vectors are not ready")
|
|
continue
|
|
}
|
|
classification := e.classifySourceInboxItem(item, hits[0].Score, hits[0].NodeID, time.Now().UTC())
|
|
if classification.Status == "candidate" {
|
|
candidateCount++
|
|
}
|
|
meta := make(map[string]any, len(item.Metadata)+12)
|
|
for key, value := range item.Metadata {
|
|
meta[key] = value
|
|
}
|
|
meta["processing_mode"] = e.RuntimeSettings().ProcessingMode
|
|
meta["exact_comparisons"] = stats.ExactComparisons
|
|
meta["coarse_comparisons"] = stats.CoarseComparisons
|
|
meta["candidate_pool"] = stats.CandidatePool
|
|
meta["classification_version"] = SourceInboxClassifierVersion
|
|
meta["classification_reason"] = classification.Reason
|
|
meta["semantic_similarity"] = classification.Semantic
|
|
meta["priority_score"] = classification.Priority
|
|
meta["freshness_score"] = classification.Freshness
|
|
meta["task_context_score"] = classification.TaskContext
|
|
meta["event_signal_score"] = classification.EventSignal
|
|
_ = e.SourceInbox.CompleteClassification(ctx, item.ID, classification.Status, classification.Priority, classification.MatchedNodeID, meta)
|
|
}
|
|
if e.Broker != nil {
|
|
e.Broker.Publish(model.Activity{Type: "source.inbox.classified", Source: "brain", Phase: "source-inbox", Message: fmt.Sprintf("Source-Inbox: %d Dokumente geprüft · %d als Wissenskandidaten vorgemerkt", len(items), candidateCount), Strength: .36, Metadata: map[string]any{"documents": len(items), "candidates": candidateCount, "minimum_similarity": e.Cfg.SourceInboxMinSimilarity, "minimum_priority": e.Cfg.SourceInboxMinPriority, "novelty_floor": e.Cfg.SourceInboxNoveltyFloor, "classifier_version": SourceInboxClassifierVersion}})
|
|
}
|
|
}
|
|
|
|
func (e *Engine) classifySourceInboxItem(item sourceagent.InboxDocument, semantic float64, matchedNodeID string, now time.Time) sourceInboxClassification {
|
|
freshness := sourceInboxFreshness(item, now)
|
|
taskContext := sourceInboxTaskContext(item)
|
|
eventSignal := sourceInboxEventSignal(item)
|
|
priority := clampInboxScore(0.65*semantic + 0.15*taskContext + 0.10*freshness + 0.10*eventSignal)
|
|
status := "archived"
|
|
reason := "zu geringe Nähe zur bestehenden Wissensbasis und keine ausreichend starken Aktualitäts-/Quellensignale"
|
|
if semantic >= e.Cfg.SourceInboxMinSimilarity {
|
|
status = "candidate"
|
|
reason = "direkte semantische Nähe zur bestehenden Wissensbasis"
|
|
} else if semantic >= e.Cfg.SourceInboxNoveltyFloor && priority >= e.Cfg.SourceInboxMinPriority {
|
|
status = "candidate"
|
|
reason = "neues, ausreichend KB-nahes Material mit zusätzlichem Quellen-, Aktualitäts- oder Advisory-Signal"
|
|
}
|
|
return sourceInboxClassification{Status: status, Priority: priority, Semantic: semantic, Freshness: freshness, TaskContext: taskContext, EventSignal: eventSignal, Reason: reason, MatchedNodeID: matchedNodeID}
|
|
}
|
|
|
|
func sourceInboxFreshness(item sourceagent.InboxDocument, now time.Time) float64 {
|
|
t := item.Document.PublishedAt
|
|
if t.IsZero() {
|
|
t = item.Document.DiscoveredAt
|
|
}
|
|
if t.IsZero() {
|
|
t = item.ReceivedAt
|
|
}
|
|
if t.IsZero() {
|
|
return .25
|
|
}
|
|
age := now.Sub(t)
|
|
if age < 0 {
|
|
age = 0
|
|
}
|
|
switch {
|
|
case age <= 24*time.Hour:
|
|
return 1
|
|
case age <= 7*24*time.Hour:
|
|
return .85
|
|
case age <= 30*24*time.Hour:
|
|
return .55
|
|
case age <= 90*24*time.Hour:
|
|
return .25
|
|
default:
|
|
return .10
|
|
}
|
|
}
|
|
|
|
func sourceInboxTaskContext(item sourceagent.InboxDocument) float64 {
|
|
contextText := strings.ToLower(strings.Join(append(append([]string{}, item.Document.Categories...), item.Document.SourceName, item.Document.SourceBaseURL), " "))
|
|
for _, marker := range []string{"security", "sicherheit", "advisory", "alert", "vulnerability", "vulnerabil", "cve", "incident", "patch", "release", "threat", "bedroh", "exploit"} {
|
|
if strings.Contains(contextText, marker) {
|
|
return 1
|
|
}
|
|
}
|
|
if len(item.Document.Categories) > 0 {
|
|
return .65
|
|
}
|
|
if strings.TrimSpace(item.Document.SourceName) != "" {
|
|
return .40
|
|
}
|
|
return .20
|
|
}
|
|
|
|
func sourceInboxEventSignal(item sourceagent.InboxDocument) float64 {
|
|
text := strings.ToLower(item.Document.Title + " " + strings.Join(item.Document.Categories, " ") + " " + item.Document.SourceName)
|
|
for _, marker := range []string{"0-day", "0day", "zero-day", "cve-", "kritisch", "critical", "actively exploited", "aktiv ausgenutzt", "backdoor", "authentifizierung umgehen", "authentication bypass", "remote code", "rce", "schadcode", "malware", "kompromitt"} {
|
|
if strings.Contains(text, marker) {
|
|
return 1
|
|
}
|
|
}
|
|
for _, marker := range []string{"angreifer", "attack", "sicherheitslücke", "sicherheitsleck", "vulnerability", "security update", "sicherheitsupdate", "patch", "update", "exploit", "datenleck", "security", "firewall", "advisory"} {
|
|
if strings.Contains(text, marker) {
|
|
return .75
|
|
}
|
|
}
|
|
return .20
|
|
}
|
|
|
|
func clampInboxScore(v float64) float64 { return math.Max(0, math.Min(1, v)) }
|
|
|
|
func (e *Engine) sourceInboxResearch(ctx context.Context, query string, limit int, freshnessSensitive bool) []model.ResearchResult {
|
|
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled || limit < 1 {
|
|
return nil
|
|
}
|
|
maxAge := time.Duration(0)
|
|
if freshnessSensitive {
|
|
maxAge = e.Cfg.SourceInboxFreshMaxAge
|
|
}
|
|
items, err := e.SourceInbox.SearchCandidates(ctx, query, limit, maxAge)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
out := make([]model.ResearchResult, 0, len(items))
|
|
for _, item := range items {
|
|
if item.QueryScore < .18 {
|
|
continue
|
|
}
|
|
d := item.Document
|
|
out = append(out, model.ResearchResult{Title: d.Title, URL: d.CanonicalURL, Snippet: clamp(d.Text, 1000), Content: d.Text, ContentType: d.ContentType, Query: query, Language: d.Language, Fetched: true, Relevant: true, Relevance: item.QueryScore, SourceQuality: "source_inbox", SourceQualityScore: .68, AssessmentReason: "Vorab durch Source-Agent gesammelt und vom Brain als thematisch passend zur Knowledgebase klassifiziert."})
|
|
}
|
|
if len(out) > 0 && e.Broker != nil {
|
|
e.Broker.Publish(model.Activity{Type: "article.research.inbox", Source: "brain", Phase: "knowledge-research-routing", Message: fmt.Sprintf("Source-Inbox liefert %d bereits gecrawlte Kandidaten vor SearXNG", len(out)), Strength: .56, Metadata: map[string]any{"query": query, "results": len(out), "freshness_sensitive": freshnessSensitive}})
|
|
}
|
|
return out
|
|
}
|