Ch1
This commit is contained in:
+67
-11
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
"github.com/local/glpi-neural-brain/internal/persist"
|
||||
"github.com/local/glpi-neural-brain/internal/research"
|
||||
"github.com/local/glpi-neural-brain/internal/workqueue"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -76,6 +77,9 @@ type Engine struct {
|
||||
runtimePath string
|
||||
researchEvidenceMu sync.RWMutex
|
||||
researchEvidenceCache map[string]researchEvidenceRecord
|
||||
sharedWork *workqueue.Limiter
|
||||
researchDedupeMu sync.Mutex
|
||||
researchDedupe map[string]*researchDedupeEntry
|
||||
interactiveInflight atomic.Int64
|
||||
autonomousWake chan struct{}
|
||||
autonomousScanRequests chan string
|
||||
@@ -134,13 +138,13 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
cfg.ArticleMaxResearchQueries = 6
|
||||
}
|
||||
if cfg.ArticleResearchResults < 1 {
|
||||
cfg.ArticleResearchResults = 8
|
||||
cfg.ArticleResearchResults = 12
|
||||
}
|
||||
if cfg.ArticleResearchRounds < 1 {
|
||||
cfg.ArticleResearchRounds = 3
|
||||
}
|
||||
if cfg.ArticleResearchFetchResults < 1 {
|
||||
cfg.ArticleResearchFetchResults = 4
|
||||
cfg.ArticleResearchFetchResults = 6
|
||||
}
|
||||
if cfg.ArticleResearchFetchResults > cfg.ArticleResearchResults {
|
||||
cfg.ArticleResearchFetchResults = cfg.ArticleResearchResults
|
||||
@@ -149,16 +153,16 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
cfg.ArticleResearchExplorationResults = cfg.ArticleResearchFetchResults
|
||||
}
|
||||
if cfg.ArticleResearchPrefetchMinRelevance <= 0 {
|
||||
cfg.ArticleResearchPrefetchMinRelevance = .35
|
||||
cfg.ArticleResearchPrefetchMinRelevance = .25
|
||||
}
|
||||
if cfg.ArticleResearchMinRelevance <= 0 {
|
||||
cfg.ArticleResearchMinRelevance = .65
|
||||
cfg.ArticleResearchMinRelevance = .55
|
||||
}
|
||||
if cfg.ArticleResearchPrefetchMinRelevance > cfg.ArticleResearchMinRelevance {
|
||||
cfg.ArticleResearchPrefetchMinRelevance = cfg.ArticleResearchMinRelevance
|
||||
}
|
||||
if cfg.ArticleResearchMinQuality <= 0 {
|
||||
cfg.ArticleResearchMinQuality = .45
|
||||
cfg.ArticleResearchMinQuality = .35
|
||||
}
|
||||
if cfg.ArticleResearchPageMaxBytes < 1 {
|
||||
cfg.ArticleResearchPageMaxBytes = 2 << 20
|
||||
@@ -169,6 +173,21 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
if cfg.ArticleResearchFetchTimeout < time.Second {
|
||||
cfg.ArticleResearchFetchTimeout = 20 * time.Second
|
||||
}
|
||||
if strings.TrimSpace(cfg.ArticleLanguage) == "" {
|
||||
cfg.ArticleLanguage = "de-DE"
|
||||
}
|
||||
if cfg.ResearchDedupeThreshold <= 0 {
|
||||
cfg.ResearchDedupeThreshold = .92
|
||||
}
|
||||
if cfg.ResearchDedupeTTL <= 0 {
|
||||
cfg.ResearchDedupeTTL = 45 * time.Minute
|
||||
}
|
||||
if cfg.ResearchOllamaMaxInflight < 1 {
|
||||
cfg.ResearchOllamaMaxInflight = 2
|
||||
}
|
||||
if cfg.ResearchOllamaQueueSize < 1 {
|
||||
cfg.ResearchOllamaQueueSize = 64
|
||||
}
|
||||
if cfg.AutonomousResearchInterval < time.Minute {
|
||||
cfg.AutonomousResearchInterval = 30 * time.Minute
|
||||
}
|
||||
@@ -229,8 +248,10 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
FailoverAttempts: cfg.OllamaFailoverAttempts, RequireSameModelDigest: cfg.OllamaRequireSameDigest,
|
||||
RequireEmbeddingModel: cfg.OllamaRequireEmbeddingModel,
|
||||
}, cfg.ChatModel, cfg.EmbeddingModel)
|
||||
sharedWork := workqueue.New(cfg.ResearchOllamaMaxInflight, cfg.ResearchOllamaQueueSize)
|
||||
pool.SetSharedLimiter(sharedWork)
|
||||
persistence := persist.New(g, b, cfg.PersistInterval)
|
||||
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs}, enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}}
|
||||
e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs}, enrichRequests: make(chan string, 1), autonomousWake: make(chan struct{}, 1), autonomousScanRequests: make(chan string, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}, sharedWork: sharedWork, researchDedupe: map[string]*researchDedupeEntry{}}
|
||||
e.loadRuntimeSettings()
|
||||
if cfg.SearXNGURL != "" {
|
||||
e.Research = research.New(cfg.SearXNGURL)
|
||||
@@ -240,7 +261,7 @@ func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
|
||||
e.GLPIKB = ingest.NewGLPIKBSyncer(ingest.GLPIKBConfig{Enabled: true, Path: cfg.GLPIKBPath, Filter: cfg.GLPIKBFilter, Limit: cfg.GLPIKBLimit, SyncInterval: cfg.GLPIKBSyncInterval, Source: cfg.GLPIKBSource, CachePath: filepath.Join(cfg.DataDir, "glpi-kb-cache.json"), ShouldSync: e.LearningEnabled}, client, g, b, persistence)
|
||||
}
|
||||
if b != nil {
|
||||
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "graph_version": g.Version()}})
|
||||
b.Publish(model.Activity{Type: "system.started", Source: "brain", Phase: "startup", Message: "Neural Brain wurde gestartet; das Analyseprotokoll zeichnet Läufe und Graphänderungen auf", Strength: .3, Metadata: map[string]any{"chat_model": cfg.ChatModel, "embedding_model": cfg.EmbeddingModel, "article_language": cfg.ArticleLanguage, "research_ollama_max_inflight": cfg.ResearchOllamaMaxInflight, "research_ollama_queue_size": cfg.ResearchOllamaQueueSize, "graph_version": g.Version()}})
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -770,23 +791,57 @@ func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome,
|
||||
}
|
||||
|
||||
var researchResults []model.ResearchResult
|
||||
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
|
||||
decision.ResearchQuery = sanitizeSearchQuerySiteFilters(decision.ResearchQuery)
|
||||
if strings.TrimSpace(decision.ResearchQuery) == "" {
|
||||
decision.NeedsResearch = false
|
||||
}
|
||||
}
|
||||
if decision.NeedsResearch && e.ResearchEnabledForRuntime() && strings.TrimSpace(decision.ResearchQuery) != "" {
|
||||
researchID := newResearchRunID("relation-research", decision.ResearchQuery)
|
||||
researchStarted := time.Now()
|
||||
startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "animation_min_ms": 2000}
|
||||
e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · SearXNG durchsucht externe Quellen", Strength: .9, Metadata: startMetadata})
|
||||
results, diagnostic, err := e.Research.SearchDetailed(ctx, decision.ResearchQuery, 4)
|
||||
if err != nil {
|
||||
|
||||
lease, reused, dedupeErr := e.beginResearchIntent(ctx, "relation", decision.ResearchQuery)
|
||||
var results []model.ResearchResult
|
||||
var diagnostic research.Diagnostic
|
||||
var researchErr error
|
||||
if dedupeErr != nil {
|
||||
researchErr = dedupeErr
|
||||
} else if !lease.owner {
|
||||
results = cloneResearchResults(reused)
|
||||
e.Broker.Publish(model.Activity{Type: "research.deduplicated", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Semantisch gleiche Relationsrecherche wurde wiederverwendet · %d Treffer", len(results)), Strength: .76, Metadata: mergeResearchMetadata(startMetadata, map[string]any{"similarity": lease.similarity, "reused_results": len(results), "dedupe_threshold": e.Cfg.ResearchDedupeThreshold})})
|
||||
} else {
|
||||
// Relation research intentionally considers more than the old four
|
||||
// snippets. The full article pipeline remains the final quality gate.
|
||||
relationResultLimit := 8
|
||||
if e.Cfg.ArticleResearchResults > relationResultLimit {
|
||||
relationResultLimit = e.Cfg.ArticleResearchResults
|
||||
}
|
||||
if relationResultLimit > 12 {
|
||||
relationResultLimit = 12
|
||||
}
|
||||
researchErr = e.withSharedResearchWork(ctx, "searxng.relation_search", func() error {
|
||||
var searchErr error
|
||||
results, diagnostic, searchErr = e.Research.SearchDetailed(ctx, decision.ResearchQuery, relationResultLimit)
|
||||
return searchErr
|
||||
})
|
||||
e.completeResearchIntent(lease, results, researchErr)
|
||||
}
|
||||
|
||||
if researchErr != nil {
|
||||
metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
|
||||
metadata["error"] = err.Error()
|
||||
metadata["error"] = researchErr.Error()
|
||||
metadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
|
||||
slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", err)
|
||||
slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", researchErr)
|
||||
e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
|
||||
} else {
|
||||
allowedResults := e.filterResearchEvidenceForThinking(results, unique(append(append([]string{}, a.Categories...), b.Categories...)))
|
||||
resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, decision.ResearchQuery, allowedResults, time.Since(researchStarted)), researchDiagnosticMetadata(diagnostic))
|
||||
resultMetadata["unfiltered_result_count"] = len(results)
|
||||
resultMetadata["source_filter_rejected_count"] = len(results) - len(allowedResults)
|
||||
resultMetadata["deduplicated"] = !lease.owner
|
||||
message := fmt.Sprintf("SearXNG hat %d durch den Thinking-Filter erlaubte Webquellen geliefert", len(allowedResults))
|
||||
if len(allowedResults) == 0 {
|
||||
message = "SearXNG-Treffer lagen außerhalb des wirksamen Thinking-Quellenfilters"
|
||||
@@ -874,6 +929,7 @@ func (e *Engine) Status() map[string]any {
|
||||
"article_research_min_relevance": e.Cfg.ArticleResearchMinRelevance, "article_research_min_quality": e.Cfg.ArticleResearchMinQuality,
|
||||
"article_research_page_max_bytes": e.Cfg.ArticleResearchPageMaxBytes, "article_research_page_max_chars": e.Cfg.ArticleResearchPageMaxChars,
|
||||
"article_research_fetch_timeout": e.Cfg.ArticleResearchFetchTimeout.String(), "article_research_allow_private": e.Cfg.ArticleResearchAllowPrivate,
|
||||
"article_language": e.Cfg.ArticleLanguage, "research_dedupe": e.researchDedupeStatus(),
|
||||
"enrich_interval": e.Cfg.EnrichInterval.String(),
|
||||
"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
|
||||
"research_enabled": e.ResearchEnabledForRuntime(), "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
|
||||
|
||||
Reference in New Issue
Block a user