121 lines
4.3 KiB
Go
121 lines
4.3 KiB
Go
package engine
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
"github.com/local/glpi-neural-brain/internal/ollama"
|
|
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
|
)
|
|
|
|
func (e *Engine) SetSourceInbox(store *sourceagent.Store) { e.SourceInbox = store }
|
|
|
|
func (e *Engine) evidenceAcquisitionEnabled() bool {
|
|
return (e.SourceInbox != nil && e.Cfg.SourceInboxEnabled) || e.ResearchEnabledForRuntime()
|
|
}
|
|
|
|
func (e *Engine) sourceInboxLoop(ctx context.Context) {
|
|
ticker := time.NewTicker(e.Cfg.SourceInboxInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
e.processSourceInbox(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Engine) processSourceInbox(ctx context.Context) {
|
|
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
|
|
return
|
|
}
|
|
items, err := e.SourceInbox.ClaimInbox(ctx, e.Cfg.SourceInboxBatchSize)
|
|
if err != nil {
|
|
slog.Warn("source inbox claim failed", "error", err)
|
|
return
|
|
}
|
|
if len(items) == 0 {
|
|
return
|
|
}
|
|
texts := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
text := item.Document.Title + "\n" + strings.Join(item.Document.Categories, " · ") + "\n" + item.Document.Text
|
|
if len([]rune(text)) > 12000 {
|
|
text = string([]rune(text)[:12000])
|
|
}
|
|
texts = append(texts, text)
|
|
}
|
|
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 4*time.Minute)
|
|
vectors, embedErr := e.Ollama.Embed(cctx, texts)
|
|
cancel()
|
|
if embedErr != nil || len(vectors) != len(items) {
|
|
for _, item := range items {
|
|
_ = e.SourceInbox.ReleaseInbox(ctx, item.ID, fmt.Sprint(embedErr))
|
|
}
|
|
return
|
|
}
|
|
candidateCount := 0
|
|
for i, item := range items {
|
|
hits, stats := e.similarKnowledge(vectors[i], 1, e.effectiveLearningFilter(), 0)
|
|
if len(hits) == 0 {
|
|
_ = e.SourceInbox.ReleaseInbox(ctx, item.ID, "knowledge vectors are not ready")
|
|
continue
|
|
}
|
|
relevance := 0.0
|
|
matched := ""
|
|
if len(hits) > 0 {
|
|
relevance = hits[0].Score
|
|
matched = hits[0].NodeID
|
|
}
|
|
status := "archived"
|
|
if relevance >= e.Cfg.SourceInboxMinSimilarity {
|
|
status = "candidate"
|
|
candidateCount++
|
|
}
|
|
meta := make(map[string]any, len(item.Metadata)+4)
|
|
for key, value := range item.Metadata {
|
|
meta[key] = value
|
|
}
|
|
meta["processing_mode"] = e.RuntimeSettings().ProcessingMode
|
|
meta["exact_comparisons"] = stats.ExactComparisons
|
|
meta["coarse_comparisons"] = stats.CoarseComparisons
|
|
meta["candidate_pool"] = stats.CandidatePool
|
|
_ = e.SourceInbox.CompleteClassification(ctx, item.ID, status, relevance, matched, meta)
|
|
}
|
|
if e.Broker != nil {
|
|
e.Broker.Publish(model.Activity{Type: "source.inbox.classified", Source: "brain", Phase: "source-inbox", Message: fmt.Sprintf("Source-Inbox: %d Dokumente geprüft · %d als Wissenskandidaten vorgemerkt", len(items), candidateCount), Strength: .36, Metadata: map[string]any{"documents": len(items), "candidates": candidateCount, "minimum_similarity": e.Cfg.SourceInboxMinSimilarity}})
|
|
}
|
|
}
|
|
|
|
func (e *Engine) sourceInboxResearch(ctx context.Context, query string, limit int, freshnessSensitive bool) []model.ResearchResult {
|
|
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled || limit < 1 {
|
|
return nil
|
|
}
|
|
maxAge := time.Duration(0)
|
|
if freshnessSensitive {
|
|
maxAge = e.Cfg.SourceInboxFreshMaxAge
|
|
}
|
|
items, err := e.SourceInbox.SearchCandidates(ctx, query, limit, maxAge)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
out := make([]model.ResearchResult, 0, len(items))
|
|
for _, item := range items {
|
|
if item.QueryScore < .18 {
|
|
continue
|
|
}
|
|
d := item.Document
|
|
out = append(out, model.ResearchResult{Title: d.Title, URL: d.CanonicalURL, Snippet: clamp(d.Text, 1000), Content: d.Text, ContentType: d.ContentType, Query: query, Language: d.Language, Fetched: true, Relevant: true, Relevance: item.QueryScore, SourceQuality: "source_inbox", SourceQualityScore: .68, AssessmentReason: "Vorab durch Source-Agent gesammelt und vom Brain als thematisch passend zur Knowledgebase klassifiziert."})
|
|
}
|
|
if len(out) > 0 && e.Broker != nil {
|
|
e.Broker.Publish(model.Activity{Type: "article.research.inbox", Source: "brain", Phase: "knowledge-research-routing", Message: fmt.Sprintf("Source-Inbox liefert %d bereits gecrawlte Kandidaten vor SearXNG", len(out)), Strength: .56, Metadata: map[string]any{"query": query, "results": len(out), "freshness_sensitive": freshnessSensitive}})
|
|
}
|
|
return out
|
|
}
|