glpi-neural-brain/internal/engine/engine.go

package engine

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"log/slog"
	"math"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode"

	"github.com/local/glpi-neural-brain/internal/activity"
	"github.com/local/glpi-neural-brain/internal/config"
	"github.com/local/glpi-neural-brain/internal/glpi"
	"github.com/local/glpi-neural-brain/internal/graph"
	"github.com/local/glpi-neural-brain/internal/ingest"
	"github.com/local/glpi-neural-brain/internal/model"
	"github.com/local/glpi-neural-brain/internal/ollama"
	"github.com/local/glpi-neural-brain/internal/persist"
	"github.com/local/glpi-neural-brain/internal/research"
)

var (
	ErrNoCandidate      = errors.New("no enrichment candidate")
	ErrLearningDisabled = errors.New("learning is disabled")
	ErrThinkingDisabled = errors.New("thinking is disabled")
)

type EnrichOutcome struct {
	Result          string
	Candidate       bool
	Created         bool
	RelationCreated bool
	ArticleCreated  bool
	ArticleSkipped  bool
	Rejected        bool
	Comparisons     int
}

type Engine struct {
	Cfg         config.Config
	Graph       *graph.Store
	Broker      *activity.Broker
	Ollama      *ollama.Client
	Research    *research.Client
	Scanner     *ingest.KnowledgeScanner
	GLPIKB      *ingest.GLPIKBSyncer
	Persistence *persist.Coordinator

	mu                    sync.Mutex
	stateMu               sync.RWMutex
	lastScan              time.Time
	lastEnrich            time.Time
	lastAttempt           time.Time
	nextEnrich            time.Time
	ollamaOK              bool
	enrichRunning         bool
	enrichTrigger         string
	enrichResult          string
	enrichError           string
	enrichCycles          uint64
	enrichCreated         uint64
	enrichRejected        uint64
	relationsCreated      uint64
	articlesCreated       uint64
	articlesSkipped       uint64
	enrichRequests        chan string
	runtimeMu             sync.RWMutex
	runtime               RuntimeSettings
	runtimePath           string
	researchEvidenceMu    sync.RWMutex
	researchEvidenceCache map[string]researchEvidenceRecord
}

func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine {
	if strings.TrimSpace(cfg.GLPIKBSource) == "" {
		cfg.GLPIKBSource = "GLPI Knowledge Base"
	}
	if !cfg.RuntimeDefaultsConfigured {
		cfg.LearningEnabled = true
		cfg.ThinkingEnabled = true
		cfg.DefaultView = "neural"
	}
	if cfg.EnrichBatchSize < 1 {
		cfg.EnrichBatchSize = 1
	}
	if cfg.EnrichAnchors < 1 {
		cfg.EnrichAnchors = 48
	}
	if cfg.ArticleMinSources < 2 {
		cfg.ArticleMinSources = 3
	}
	if cfg.ArticleMaxSources < cfg.ArticleMinSources {
		cfg.ArticleMaxSources = 8
	}
	if cfg.ArticleMinProductionRatio == 0 {
		cfg.ArticleMinProductionRatio = .70
	}
	if cfg.ArticleMaxGenerationDepth < 1 {
		cfg.ArticleMaxGenerationDepth = 2
	}
	if cfg.ArticleMinConfidence == 0 {
		cfg.ArticleMinConfidence = .74
	}
	if cfg.ArticleMinTextChars < 1 {
		cfg.ArticleMinTextChars = 180
	}
	if cfg.ArticleMinAnswerChars < 1 {
		cfg.ArticleMinAnswerChars = 420
	}
	if cfg.ArticleMaxResearchQueries < 1 {
		cfg.ArticleMaxResearchQueries = 6
	}
	if cfg.ArticleResearchResults < 1 {
		cfg.ArticleResearchResults = 8
	}
	if cfg.ArticleResearchRounds < 1 {
		cfg.ArticleResearchRounds = 3
	}
	if cfg.ArticleResearchFetchResults < 1 {
		cfg.ArticleResearchFetchResults = 4
	}
	if cfg.ArticleResearchFetchResults > cfg.ArticleResearchResults {
		cfg.ArticleResearchFetchResults = cfg.ArticleResearchResults
	}
	if cfg.ArticleResearchMinRelevance <= 0 {
		cfg.ArticleResearchMinRelevance = .65
	}
	if cfg.ArticleResearchMinQuality <= 0 {
		cfg.ArticleResearchMinQuality = .45
	}
	if cfg.ArticleResearchPageMaxBytes < 1 {
		cfg.ArticleResearchPageMaxBytes = 2 << 20
	}
	if cfg.ArticleResearchPageMaxChars < 1 {
		cfg.ArticleResearchPageMaxChars = 14000
	}
	if cfg.ArticleResearchFetchTimeout < time.Second {
		cfg.ArticleResearchFetchTimeout = 20 * time.Second
	}
	ollamaURLs := append([]string(nil), cfg.OllamaURLs...)
	if len(ollamaURLs) == 0 && strings.TrimSpace(cfg.OllamaURL) != "" {
		ollamaURLs = []string{cfg.OllamaURL}
	}
	nodes := make([]ollama.NodeConfig, 0, len(ollamaURLs))
	for i, rawURL := range ollamaURLs {
		name := fmt.Sprintf("ollama-%d", i+1)
		if i < len(cfg.OllamaNodeNames) && strings.TrimSpace(cfg.OllamaNodeNames[i]) != "" {
			name = strings.TrimSpace(cfg.OllamaNodeNames[i])
		}
		weight := 1
		if i < len(cfg.OllamaNodeWeights) && cfg.OllamaNodeWeights[i] > 0 {
			weight = cfg.OllamaNodeWeights[i]
		}
		nodes = append(nodes, ollama.NodeConfig{Name: name, URL: rawURL, Weight: weight})
	}
	clearedVectors := g.ConfigureEmbeddingModel(cfg.EmbeddingModel)
	if clearedVectors > 0 && b != nil {
		b.Publish(model.Activity{Type: "embedding.model_changed", Source: "brain", Phase: "learning", Message: fmt.Sprintf("Embedding-Modell geändert · %d Vektoren werden neu gelernt", clearedVectors), Strength: .65, Metadata: map[string]any{"embedding_model": cfg.EmbeddingModel, "cleared_vectors": clearedVectors}})
	}
	pool := ollama.NewPool(ollama.PoolConfig{
		Nodes: nodes, RoutingMode: cfg.OllamaRoutingMode, NodeMaxInflight: cfg.OllamaNodeMaxInflight,
		HealthInterval: cfg.OllamaHealthInterval, FailureCooldown: cfg.OllamaFailureCooldown,
		RequestTimeout: cfg.OllamaRequestTimeout, FailoverEnabled: cfg.OllamaFailoverEnabled,
		FailoverAttempts: cfg.OllamaFailoverAttempts, RequireSameModelDigest: cfg.OllamaRequireSameDigest,
		RequireEmbeddingModel: cfg.OllamaRequireEmbeddingModel,
	}, cfg.ChatModel, cfg.EmbeddingModel)
	persistence := persist.New(g, b, cfg.PersistInterval)
	e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: pool, Persistence: persistence, Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs}, enrichRequests: make(chan string, 1), runtimePath: filepath.Join(cfg.DataDir, "runtime-settings.json"), researchEvidenceCache: map[string]researchEvidenceRecord{}}
	e.loadRuntimeSettings()
	if cfg.SearXNGURL != "" {
		e.Research = research.New(cfg.SearXNGURL)
	}
	if cfg.GLPIKBEnabled {
		client := glpi.New(cfg.GLPIURL, cfg.GLPIAPIVersion, cfg.GLPIClientID, cfg.GLPIClientSecret, cfg.GLPIUsername, cfg.GLPIPassword, cfg.GLPITimeout)
		e.GLPIKB = ingest.NewGLPIKBSyncer(ingest.GLPIKBConfig{Enabled: true, Path: cfg.GLPIKBPath, Filter: cfg.GLPIKBFilter, Limit: cfg.GLPIKBLimit, SyncInterval: cfg.GLPIKBSyncInterval, Source: cfg.GLPIKBSource, CachePath: filepath.Join(cfg.DataDir, "glpi-kb-cache.json"), ShouldSync: e.LearningEnabled}, client, g, b, persistence)
	}
	return e
}
func (e *Engine) Start(ctx context.Context) {
	e.Persistence.Start(ctx)
	e.Ollama.Start(ctx)
	if e.GLPIKB != nil {
		e.GLPIKB.Start(ctx)
	}
	go func() {
		if e.LearningEnabled() {
			if err := e.Scan(ctx); err != nil && !errors.Is(err, ErrLearningDisabled) {
				slog.Error("initial brain scan failed", "error", err)
			}
		}
		ticker := time.NewTicker(e.Cfg.ScanInterval)
		defer ticker.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				if !e.LearningEnabled() {
					continue
				}
				if err := e.Scan(ctx); err != nil && !errors.Is(err, ErrLearningDisabled) {
					slog.Error("brain scan failed", "error", err)
				}
			}
		}
	}()

	go e.enrichmentWorker(ctx)
	if e.Cfg.AutoEnrich {
		go e.enrichmentScheduler(ctx)
	}
	go e.idle(ctx)
}

func (e *Engine) enrichmentScheduler(ctx context.Context) {
	firstDelay := 12 * time.Second
	e.setNextEnrich(time.Now().Add(firstDelay))
	timer := time.NewTimer(firstDelay)
	defer timer.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-timer.C:
			if !e.RequestEnrich("automatic") {
				slog.Debug("automatic enrichment already queued")
			}
			next := time.Now().Add(e.Cfg.EnrichInterval)
			e.setNextEnrich(next)
			timer.Reset(e.Cfg.EnrichInterval)
		}
	}
}

func (e *Engine) enrichmentWorker(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		case trigger := <-e.enrichRequests:
			e.runEnrichmentCycle(ctx, trigger)
		}
	}
}

func (e *Engine) RequestEnrich(trigger string) bool {
	if !e.ThinkingEnabled() {
		e.stateMu.Lock()
		e.enrichResult = "disabled"
		e.enrichError = ErrThinkingDisabled.Error()
		e.stateMu.Unlock()
		return false
	}
	if strings.TrimSpace(trigger) == "" {
		trigger = "manual"
	}
	e.stateMu.Lock()
	if e.enrichRunning || e.enrichResult == "queued" {
		e.stateMu.Unlock()
		return false
	}
	e.enrichResult = "queued"
	e.enrichTrigger = trigger
	e.enrichError = ""
	e.stateMu.Unlock()

	select {
	case e.enrichRequests <- trigger:
		e.Broker.Publish(model.Activity{Type: "think.queued", Source: "brain", Phase: "queue", Message: "AI-THINK-Zyklus wurde eingeplant", Strength: .38, Metadata: map[string]any{"trigger": trigger, "batch_size": e.Cfg.EnrichBatchSize}})
		return true
	default:
		e.stateMu.Lock()
		if e.enrichResult == "queued" && e.enrichTrigger == trigger {
			e.enrichResult = "idle"
		}
		e.stateMu.Unlock()
		return false
	}
}

func (e *Engine) runEnrichmentCycle(ctx context.Context, trigger string) {
	started := time.Now()
	e.stateMu.Lock()
	e.enrichRunning = true
	e.enrichTrigger = trigger
	e.enrichResult = "running"
	e.enrichError = ""
	e.lastAttempt = started.UTC()
	e.enrichCycles++
	e.stateMu.Unlock()

	e.Broker.Publish(model.Activity{Type: "think.cycle.started", Source: "brain", Phase: "autonomous", Message: fmt.Sprintf("Autonomer AI-THINK-Zyklus startet · bis zu %d sequenzielle Prüfungen", e.Cfg.EnrichBatchSize), Strength: .72, Metadata: map[string]any{"trigger": trigger, "batch_size": e.Cfg.EnrichBatchSize, "anchors": e.Cfg.EnrichAnchors}})

	created, rejected, checked := 0, 0, 0
	relationsCreated, articlesCreated, articlesSkipped := 0, 0, 0
	result := "completed"
	var cycleErr error
	for step := 0; step < e.Cfg.EnrichBatchSize; step++ {
		if !e.ThinkingEnabled() {
			result = "disabled"
			break
		}
		outcome, err := e.enrichOne(ctx, trigger)
		if err != nil {
			cycleErr = err
			result = "failed"
			break
		}
		if !outcome.Candidate {
			if checked == 0 {
				result = "no_candidate"
			}
			break
		}
		checked++
		if outcome.Created {
			created++
		}
		if outcome.RelationCreated {
			relationsCreated++
		}
		if outcome.ArticleCreated {
			articlesCreated++
		}
		if outcome.ArticleSkipped {
			articlesSkipped++
		}
		if outcome.Rejected {
			rejected++
		}
		if step+1 < e.Cfg.EnrichBatchSize && e.Cfg.EnrichStepDelay > 0 {
			select {
			case <-ctx.Done():
				cycleErr = ctx.Err()
				result = "cancelled"
				step = e.Cfg.EnrichBatchSize
			case <-time.After(e.Cfg.EnrichStepDelay):
			}
		}
	}

	e.stateMu.Lock()
	e.enrichRunning = false
	e.enrichResult = result
	if cycleErr != nil {
		e.enrichError = cycleErr.Error()
	} else {
		e.enrichError = ""
	}
	e.enrichCreated += uint64(created)
	e.enrichRejected += uint64(rejected)
	e.relationsCreated += uint64(relationsCreated)
	e.articlesCreated += uint64(articlesCreated)
	e.articlesSkipped += uint64(articlesSkipped)
	e.stateMu.Unlock()

	metadata := map[string]any{"trigger": trigger, "checked": checked, "created": created, "relations_created": relationsCreated, "articles_created": articlesCreated, "articles_skipped": articlesSkipped, "rejected": rejected, "duration_ms": time.Since(started).Milliseconds(), "result": result}
	if cycleErr != nil {
		e.Broker.Publish(model.Activity{Type: "think.cycle.failed", Source: "brain", Phase: "autonomous", Message: "AI-THINK-Zyklus wurde mit Fehler beendet", Strength: .45, Metadata: metadata})
		slog.Warn("enrichment cycle failed", "trigger", trigger, "error", cycleErr)
		return
	}
	message := fmt.Sprintf("AI-THINK-Zyklus abgeschlossen · %d Relationen · %d Artikel · %d verworfen", relationsCreated, articlesCreated, rejected)
	if result == "no_candidate" {
		message = "AI-THINK hat im aktuell geprüften Graphbereich keinen Kandidaten oberhalb des Schwellwerts gefunden"
	}
	e.Broker.Publish(model.Activity{Type: "think.cycle.completed", Source: "brain", Phase: "autonomous", Message: message, Strength: .58, Metadata: metadata})
}

func (e *Engine) setNextEnrich(t time.Time) {
	e.stateMu.Lock()
	e.nextEnrich = t.UTC()
	e.stateMu.Unlock()
}

func (e *Engine) setOllamaOK(ok bool) {
	e.stateMu.Lock()
	e.ollamaOK = ok
	e.stateMu.Unlock()
}

func (e *Engine) isOllamaOK() bool {
	e.stateMu.RLock()
	ok := e.ollamaOK
	e.stateMu.RUnlock()
	return ok
}

func (e *Engine) idle(ctx context.Context) {
	ticker := time.NewTicker(7 * time.Second)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			n, ok := e.Graph.IdleNode(time.Now().Unix() / 7)
			if !ok {
				continue
			}
			e.Broker.Publish(model.Activity{Type: "brain.idle", Source: "brain", Phase: "idle", Message: "Leise Hintergrundaktivität", NodeIDs: []string{n.ID}, Strength: .18})
		}
	}
}
func (e *Engine) configureEmbeddingDigest() int {
	for _, node := range e.Ollama.NodeStatuses() {
		if node.Healthy && node.Compatible && node.EmbeddingModel && strings.TrimSpace(node.EmbeddingDigest) != "" {
			return e.Graph.ConfigureEmbeddingIdentity(e.Cfg.EmbeddingModel, node.EmbeddingDigest)
		}
	}
	return 0
}

func (e *Engine) Scan(ctx context.Context) error {
	if !e.LearningEnabled() {
		return ErrLearningDisabled
	}
	e.mu.Lock()
	defer e.mu.Unlock()
	beforeVersion := e.Graph.Version()
	count, err := e.Scanner.Scan()
	if err != nil {
		return err
	}
	pendingEmbeddings := len(e.Graph.NodesForEmbeddingScoped(e.effectiveLearningFilter()))
	if e.Graph.Version() != beforeVersion || pendingEmbeddings > 0 {
		e.Broker.Publish(model.Activity{Type: "scan.started", Source: "brain", Phase: "ingest", Message: "Neue oder geänderte Wissenselemente werden verarbeitet", Strength: .45, Metadata: map[string]any{"pending_embeddings": pendingEmbeddings}})
	}
	pingCtx, pingCancel := context.WithTimeout(ctx, 3*time.Second)
	pingErr := e.Ollama.Ping(pingCtx)
	pingCancel()
	if pingErr != nil {
		slog.Warn("Ollama unavailable; using deterministic local fallback", "error", pingErr)
		e.ensureFallbackEmbeddings()
		e.setOllamaOK(false)
	} else {
		if cleared := e.configureEmbeddingDigest(); cleared > 0 {
			e.Broker.Publish(model.Activity{Type: "embedding.identity_changed", Source: "brain", Phase: "learning", Message: fmt.Sprintf("Embedding-Digest geändert · %d Vektoren werden neu gelernt", cleared), Strength: .7, Metadata: map[string]any{"embedding_model": e.Cfg.EmbeddingModel, "cleared_vectors": cleared}})
		}
		// Local fallback vectors use 256 dimensions. Once Ollama becomes available,
		// discard those placeholders and replace them with real model embeddings.
		e.Graph.ClearVectorsByDimension(256)
		if err := e.ensureEmbeddings(ctx); err != nil {
			slog.Warn("Ollama embeddings failed; using deterministic local fallback", "error", err)
			e.ensureFallbackEmbeddings()
			e.setOllamaOK(false)
		} else {
			e.setOllamaOK(true)
		}
	}
	e.stateMu.Lock()
	e.lastScan = time.Now().UTC()
	e.stateMu.Unlock()
	if e.Graph.Version() != beforeVersion {
		nodes, edges, _ := e.Graph.Counts()
		e.Broker.Publish(model.Activity{Type: "graph.updated", Source: "brain", Phase: "indexed", Message: fmt.Sprintf("%d Wissenselemente · %d Nodes · %d Edges", count, nodes, edges), Strength: .55, Metadata: map[string]any{"nodes": nodes, "edges": edges, "knowledge_elements": count}})
	}
	return nil
}
func (e *Engine) ensureEmbeddings(ctx context.Context) error {
	pending := e.Graph.NodesForEmbeddingScoped(e.effectiveLearningFilter())
	if len(pending) == 0 {
		return nil
	}
	for start := 0; start < len(pending); start += 16 {
		end := start + 16
		if end > len(pending) {
			end = len(pending)
		}
		texts := make([]string, 0, end-start)
		for _, n := range pending[start:end] {
			texts = append(texts, embeddingText(n))
		}
		cctx, cancel := context.WithTimeout(ctx, 4*time.Minute)
		vecs, err := e.Ollama.Embed(cctx, texts)
		cancel()
		if err != nil {
			return err
		}
		for i, v := range vecs {
			e.Graph.SetVector(pending[start+i].ID, v)
		}
		ids := []string{}
		for _, n := range pending[start:end] {
			ids = append(ids, n.ID)
		}
		e.Broker.Publish(model.Activity{Type: "embedding.batch", Source: "ollama", Phase: "embedding", Message: fmt.Sprintf("EmbeddingGemma verarbeitet %d Elemente", len(ids)), NodeIDs: ids, Strength: .38, Metadata: map[string]any{"batch_count": len(ids), "model": e.Cfg.EmbeddingModel, "batch_start": start, "batch_total": len(pending)}})
	}
	return nil
}
func (e *Engine) ensureFallbackEmbeddings() {
	for _, n := range e.Graph.NodesForEmbeddingScoped(e.effectiveLearningFilter()) {
		e.Graph.SetVector(n.ID, hashEmbedding(embeddingText(n), 256))
	}
}
func embeddingText(n model.Node) string {
	return strings.TrimSpace(n.Label + "\n" + strings.Join(n.Categories, " · ") + "\n" + strings.Join(n.Keywords, " · ") + "\n" + n.Summary)
}
func hashEmbedding(s string, dims int) []float64 {
	v := make([]float64, dims)
	tokens := strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) })
	for _, t := range tokens {
		if t == "" {
			continue
		}
		h := sha256.Sum256([]byte(t))
		idx := (int(h[0])<<8 | int(h[1])) % dims
		sign := 1.0
		if h[2]&1 == 1 {
			sign = -1
		}
		v[idx] += sign * (1 + float64(h[3])/255)
	}
	var norm float64
	for _, x := range v {
		norm += x * x
	}
	if norm > 0 {
		norm = math.Sqrt(norm)
		for i := range v {
			v[i] /= norm
		}
	}
	return v
}

func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, error) {
	start := time.Now()
	q = strings.TrimSpace(q)
	if len([]rune(q)) < 2 {
		return model.QueryResponse{}, fmt.Errorf("query is too short")
	}
	e.Broker.Publish(model.Activity{Type: "query.started", Source: "ui", Phase: "perception", Query: q, Message: "Anfrage trifft im neuronalen Feld ein", Strength: 1})
	vecs, err := e.Ollama.Embed(ctx, []string{q})
	if err != nil || len(vecs) == 0 {
		vecs = [][]float64{hashEmbedding(q, 256)}
	}
	hits := e.Graph.SimilarFiltered(vecs[0], e.Cfg.TopK, e.effectiveLearningFilter())
	nodeIDs := make([]string, 0, len(hits))
	for i, h := range hits {
		nodeIDs = append(nodeIDs, h.NodeID)
		e.Broker.Publish(model.Activity{Type: "node.activated", Source: "brain", Phase: "retrieval", Query: q, NodeIDs: []string{h.NodeID}, Message: fmt.Sprintf("Treffer %d · %.0f%% · %s", i+1, h.Score*100, h.Label), Strength: math.Max(.25, h.Score)})
		time.Sleep(55 * time.Millisecond)
	}
	edgeIDs := e.Graph.ConnectingEdges(nodeIDs)
	if len(edgeIDs) > 0 {
		e.Broker.Publish(model.Activity{Type: "edges.traversed", Source: "brain", Phase: "association", Query: q, NodeIDs: nodeIDs, EdgeIDs: edgeIDs, Message: fmt.Sprintf("%d Wissensverbindungen werden durchlaufen", len(edgeIDs)), Strength: .92})
	}
	answer := e.fallbackAnswer(q, hits)
	used := append([]string(nil), nodeIDs...)
	var uncertainties []string
	if e.isOllamaOK() && len(hits) > 0 {
		system := "Du beantwortest Fragen ausschließlich aus dem bereitgestellten Wissensgraphen. Markiere Unklarheiten offen. Gib valides JSON nach Schema zurück. used_node_ids dürfen nur IDs aus dem Kontext sein."
		user := e.answerContext(q, hits)
		var dec model.AnswerDecision
		if err := e.Ollama.ChatJSON(ctx, system, user, answerSchema(), &dec); err == nil && strings.TrimSpace(dec.Answer) != "" {
			answer = dec.Answer
			used = validIDs(dec.UsedNodeIDs, nodeIDs)
			uncertainties = dec.Uncertainties
		} else if err != nil {
			slog.Warn("structured answer failed; fallback used", "error", err)
		}
	}
	e.Broker.Publish(model.Activity{Type: "query.completed", Source: "brain", Phase: "synthesis", Query: q, NodeIDs: used, EdgeIDs: e.Graph.ConnectingEdges(used), Message: "Antwortsynthese abgeschlossen", Strength: 1, Metadata: map[string]any{"duration_ms": time.Since(start).Milliseconds(), "hit_count": len(hits), "used_nodes": len(used), "uncertainty_count": len(uncertainties)}})
	return model.QueryResponse{Query: q, Answer: answer, Hits: hits, UsedNodeIDs: used, Uncertainties: uncertainties, DurationMS: time.Since(start).Milliseconds()}, nil
}
func (e *Engine) answerContext(q string, hits []model.Hit) string {
	var b strings.Builder
	b.WriteString("FRAGE:\n" + q + "\n\nKONTEXT:\n")
	used := 0
	for _, h := range hits {
		n, ok := e.Graph.GetNode(h.NodeID)
		if !ok {
			continue
		}
		part := fmt.Sprintf("\nNODE_ID: %s\nTITEL: %s\nSTATUS: %s\nKATEGORIEN: %s\nINHALT: %s\n", n.ID, n.Label, n.Status, strings.Join(n.Categories, ", "), n.Summary)
		if used+len(part) > e.Cfg.MaxContextChars {
			break
		}
		b.WriteString(part)
		used += len(part)
	}
	return b.String()
}
func (e *Engine) fallbackAnswer(q string, hits []model.Hit) string {
	if len(hits) == 0 {
		return "Im aktuellen Wissensgraphen wurde kein belastbarer Zusammenhang gefunden."
	}
	var b strings.Builder
	b.WriteString("Die stärksten passenden Wissensbereiche sind: ")
	for i, h := range hits {
		if i >= 4 {
			break
		}
		if i > 0 {
			b.WriteString("; ")
		}
		b.WriteString(h.Label)
	}
	b.WriteString(". Die Visualisierung zeigt die zugehörigen Aktivierungspfade. Ohne erreichbares Qwen-Modell bleibt dies eine Retrieval-Zusammenfassung.")
	return b.String()
}

func (e *Engine) EnrichOne(ctx context.Context) error {
	if !e.ThinkingEnabled() {
		return ErrThinkingDisabled
	}
	outcome, err := e.enrichOne(ctx, "direct")
	if err != nil {
		return err
	}
	e.stateMu.Lock()
	if outcome.RelationCreated {
		e.relationsCreated++
		e.enrichCreated++
	}
	if outcome.ArticleCreated {
		e.articlesCreated++
	}
	if outcome.ArticleSkipped {
		e.articlesSkipped++
	}
	if outcome.Rejected {
		e.enrichRejected++
	}
	e.stateMu.Unlock()
	return nil
}

func (e *Engine) enrichOne(ctx context.Context, trigger string) (EnrichOutcome, error) {
	if !e.ThinkingEnabled() {
		return EnrichOutcome{Result: "disabled"}, ErrThinkingDisabled
	}
	e.mu.Lock()
	defer e.mu.Unlock()

	if !e.isOllamaOK() {
		pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
		err := e.Ollama.Ping(pingCtx)
		cancel()
		if err != nil {
			e.Broker.Publish(model.Activity{Type: "think.paused", Source: "brain", Phase: "waiting", Message: "AI-THINK wartet auf ein erreichbares Ollama/Qwen-Modell", Strength: .25, Metadata: map[string]any{"trigger": trigger, "error": err.Error()}})
			return EnrichOutcome{Result: "ollama_unavailable"}, fmt.Errorf("Ollama/Qwen is unavailable; no AI edge or AI-THINK draft was created: %w", err)
		}
		e.setOllamaOK(true)
	}

	a, b, sim, ok, comparisons := e.Graph.NextPairScopedDepth(e.Cfg.SimilarityThreshold, e.Cfg.EnrichAnchors, e.effectiveThinkingFilter(), e.Cfg.ArticleMaxGenerationDepth)
	if !ok {
		e.stateMu.Lock()
		e.lastAttempt = time.Now().UTC()
		e.stateMu.Unlock()
		e.Broker.Publish(model.Activity{Type: "think.no_candidate", Source: "brain", Phase: "candidate-search", Message: "Im aktuell geprüften Graphbereich wurde keine ungeprüfte Beziehung oberhalb des Ähnlichkeitsschwellwerts gefunden", Strength: .28, Metadata: map[string]any{"trigger": trigger, "threshold": e.Cfg.SimilarityThreshold, "anchors": e.Cfg.EnrichAnchors, "comparisons": comparisons}})
		return EnrichOutcome{Result: "no_candidate", Comparisons: comparisons}, nil
	}

	now := time.Now().UTC()
	e.stateMu.Lock()
	e.lastAttempt = now
	e.lastEnrich = now
	e.stateMu.Unlock()
	e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"trigger": trigger, "semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel, "candidate_comparisons": comparisons}})

	system := "Du führst ausschließlich eine Relationserkennung für einen Wissensgraphen durch. Analysiere zwei interne Wissenseinträge, erfinde keine Fakten und entscheide, ob eine belastbare Beziehung besteht. Schreibe keinen Artikel und keine technische Synthese. Wenn externe Fakten zur Relationsentscheidung fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück."
	var decision model.RelationDecision
	if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil {
		e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35, Metadata: map[string]any{"trigger": trigger, "error": err.Error(), "model": e.Cfg.ChatModel}})
		return EnrichOutcome{Result: "inference_failed", Candidate: true, Comparisons: comparisons}, fmt.Errorf("relation inference failed: %w", err)
	}

	var researchResults []model.ResearchResult
	if decision.NeedsResearch && e.Cfg.ResearchEnabled && e.Research != nil && strings.TrimSpace(decision.ResearchQuery) != "" {
		researchID := newResearchRunID("relation-research", decision.ResearchQuery)
		researchStarted := time.Now()
		startMetadata := map[string]any{"trigger": trigger, "research_id": researchID, "research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label, "animation_min_ms": 2000}
		e.Broker.Publish(model.Activity{Type: "research.started", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · SearXNG durchsucht externe Quellen", Strength: .9, Metadata: startMetadata})
		results, diagnostic, err := e.Research.SearchDetailed(ctx, decision.ResearchQuery, 4)
		if err != nil {
			metadata := mergeResearchMetadata(startMetadata, researchDiagnosticMetadata(diagnostic))
			metadata["error"] = err.Error()
			metadata["duration_ms"] = time.Since(researchStarted).Milliseconds()
			slog.Warn("research failed", "query", decision.ResearchQuery, "base_url", diagnostic.BaseURL, "kind", diagnostic.ErrorKind, "http_status", diagnostic.HTTPStatus, "duration_ms", diagnostic.DurationMS, "error", err)
			e.Broker.Publish(model.Activity{Type: "research.failed", Source: "searxng", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "SearXNG-Recherche ist fehlgeschlagen", Strength: .35, Metadata: metadata})
		} else {
			allowedResults := e.filterResearchEvidenceForThinking(results, unique(append(append([]string{}, a.Categories...), b.Categories...)))
			resultMetadata := mergeResearchMetadata(researchEventMetadata(trigger, researchID, decision.ResearchQuery, allowedResults, time.Since(researchStarted)), researchDiagnosticMetadata(diagnostic))
			resultMetadata["unfiltered_result_count"] = len(results)
			resultMetadata["source_filter_rejected_count"] = len(results) - len(allowedResults)
			message := fmt.Sprintf("SearXNG hat %d durch den Thinking-Filter erlaubte Webquellen geliefert", len(allowedResults))
			if len(allowedResults) == 0 {
				message = "SearXNG-Treffer lagen außerhalb des wirksamen Thinking-Quellenfilters"
			}
			e.Broker.Publish(model.Activity{Type: "research.results", Source: "searxng", Phase: "research-results", NodeIDs: []string{a.ID, b.ID}, Message: message, Strength: .92, Metadata: resultMetadata})
			if len(allowedResults) > 0 {
				researchResults = allowedResults
				refs := e.addResearch(a, b, allowedResults)
				ingestMetadata := mergeResearchMetadata(resultMetadata, map[string]any{"result_node_ids": refs.NodeIDs, "result_edge_ids": refs.EdgeIDs})
				e.Broker.Publish(model.Activity{Type: "research.ingested", Source: "searxng", Phase: "research-ingest", NodeIDs: append([]string{a.ID, b.ID}, refs.NodeIDs...), EdgeIDs: refs.EdgeIDs, Message: fmt.Sprintf("%d Webquellen wurden als neue Forschungs-Nodes in den Graphen übernommen", len(refs.NodeIDs)), Strength: 1, Metadata: ingestMetadata})
				var reviewed model.RelationDecision
				reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und der beigefügten Web-Suchergebnisse. Suchtreffer sind Hinweise, keine garantierten Fakten. Erfinde nichts, kennzeichne verbleibende Unsicherheit und gib ausschließlich JSON nach Schema zurück."
				if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, allowedResults), relationSchema(), &reviewed); err != nil {
					slog.Warn("research review failed; keeping pre-research decision", "error", err)
				} else {
					decision = reviewed
				}
			}
		}
	}

	status := "staging"
	if !decision.Related || decision.Confidence < e.Cfg.RelationThreshold {
		status = "rejected"
	}
	edge := model.Edge{
		Source: a.ID, Target: b.ID, Type: safeRelation(decision.RelationType), Origin: "ai-inference", Status: status,
		Confidence: decision.Confidence, Weight: math.Max(.2, decision.Confidence), Explanation: decision.Explanation,
		Evidence: []model.Evidence{{NodeID: a.ID, URI: a.URI, Excerpt: clamp(a.Summary, 220)}, {NodeID: b.ID, URI: b.URI, Excerpt: clamp(b.Summary, 220)}},
		Metadata: map[string]any{"semantic_similarity": sim, "model": e.Cfg.ChatModel, "research_result_count": len(researchResults), "trigger": trigger},
	}
	e.Graph.UpsertEdge(edge)
	edge.ID = graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin)
	outcome := EnrichOutcome{Result: status, Candidate: true, Comparisons: comparisons}
	if status == "staging" {
		outcome.Created = true
		outcome.RelationCreated = true
		e.Broker.Publish(model.Activity{Type: "think.relation.created", Source: "brain", Phase: "relation", NodeIDs: []string{a.ID, b.ID}, EdgeIDs: []string{edge.ID}, Message: "Belastbare Wissensrelation wurde als überprüfbare Graph-Edge übernommen", Strength: .86, Metadata: map[string]any{"trigger": trigger, "relation_type": safeRelation(decision.RelationType), "confidence": decision.Confidence, "semantic_similarity": sim, "research_result_count": len(researchResults), "topic_label": decision.TopicLabel}})
		articleOutcome, err := e.synthesizeKnowledgeArticle(ctx, trigger, []model.Node{a, b}, decision, researchResults)
		if err != nil {
			e.Broker.Publish(model.Activity{Type: "article.failed", Source: "brain", Phase: "knowledge-synthesis", NodeIDs: []string{a.ID, b.ID}, EdgeIDs: []string{edge.ID}, Message: "Die Relation bleibt erhalten, aber die Artikelsynthese ist fehlgeschlagen", Strength: .4, Metadata: map[string]any{"trigger": trigger, "error": err.Error()}})
			outcome.ArticleSkipped = true
		} else {
			outcome.ArticleCreated = articleOutcome.Created
			outcome.ArticleSkipped = articleOutcome.Skipped
		}
	} else {
		outcome.Rejected = true
		e.Broker.Publish(model.Activity{Type: "think.rejected", Source: "brain", Phase: "validation", NodeIDs: []string{a.ID, b.ID}, Message: "Ähnlichkeit geprüft, aber nicht als belastbare Edge übernommen", Strength: .42, Metadata: map[string]any{"trigger": trigger, "relation_type": safeRelation(decision.RelationType), "confidence": decision.Confidence, "semantic_similarity": sim, "explanation": decision.Explanation}})
	}
	return outcome, nil
}

func (e *Engine) addResearch(a, b model.Node, results []model.ResearchResult) researchGraphRefs {
	refs := researchGraphRefs{}
	for _, r := range results {
		id := graph.ID("external", r.URL)
		n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Categories: unique(append(append([]string{}, a.Categories...), b.Categories...)), Weight: .8, Metadata: map[string]any{"source": graph.SourceFromURL(r.URL), "query_pair": []string{a.ID, b.ID}}, UpdatedAt: time.Now().UTC()}
		e.Graph.UpsertNode(n)
		refs.NodeIDs = append(refs.NodeIDs, id)
		for _, targetID := range []string{a.ID, b.ID} {
			edge := model.Edge{Source: id, Target: targetID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: .55, Weight: .4}
			e.Graph.UpsertEdge(edge)
			refs.EdgeIDs = append(refs.EdgeIDs, graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin))
		}
	}
	return uniqueResearchRefs(refs)
}
func (e *Engine) Status() map[string]any {
	nodes, edges, version := e.Graph.Counts()
	e.stateMu.RLock()
	status := map[string]any{
		"ok": true, "nodes": nodes, "edges": edges, "version": version,
		"last_scan": e.lastScan, "last_enrich": e.lastEnrich, "last_enrich_attempt": e.lastAttempt,
		"next_enrich": e.nextEnrich, "ollama_ok": e.ollamaOK, "auto_enrich": e.Cfg.AutoEnrich,
		"enrich_running": e.enrichRunning, "enrich_trigger": e.enrichTrigger, "enrich_result": e.enrichResult,
		"enrich_error": e.enrichError, "enrich_cycles": e.enrichCycles, "enrich_created": e.enrichCreated,
		"enrich_rejected": e.enrichRejected, "relations_created": e.relationsCreated, "articles_created": e.articlesCreated,
		"articles_skipped": e.articlesSkipped, "article_synthesis_enabled": e.Cfg.ArticleSynthesisEnabled,
		"article_min_sources": e.Cfg.ArticleMinSources, "article_max_sources": e.Cfg.ArticleMaxSources,
		"article_min_production_ratio": e.Cfg.ArticleMinProductionRatio, "article_max_generation_depth": e.Cfg.ArticleMaxGenerationDepth,
		"article_max_research_queries": e.Cfg.ArticleMaxResearchQueries, "article_research_results": e.Cfg.ArticleResearchResults,
		"article_research_rounds": e.Cfg.ArticleResearchRounds, "article_research_fetch_results": e.Cfg.ArticleResearchFetchResults,
		"article_research_min_relevance": e.Cfg.ArticleResearchMinRelevance, "article_research_min_quality": e.Cfg.ArticleResearchMinQuality,
		"article_research_page_max_bytes": e.Cfg.ArticleResearchPageMaxBytes, "article_research_page_max_chars": e.Cfg.ArticleResearchPageMaxChars,
		"article_research_fetch_timeout": e.Cfg.ArticleResearchFetchTimeout.String(), "article_research_allow_private": e.Cfg.ArticleResearchAllowPrivate,
		"enrich_interval":   e.Cfg.EnrichInterval.String(),
		"enrich_batch_size": e.Cfg.EnrichBatchSize, "enrich_anchors": e.Cfg.EnrichAnchors,
		"research_enabled": e.Cfg.ResearchEnabled, "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel,
		"searxng":     e.ResearchStatus(),
		"ollama_pool": e.Ollama.PoolStatus(), "persistence": e.Persistence.Status(), "graph_storage": e.Graph.StorageStatus(),
		"runtime_settings": e.RuntimeSettingsView(),
	}
	if e.GLPIKB != nil {
		status["glpi_kb"] = e.GLPIKB.Status()
	} else {
		status["glpi_kb"] = ingest.GLPIKBStatus{Enabled: false}
	}
	e.stateMu.RUnlock()
	return status
}

func (e *Engine) Flush(ctx context.Context) error {
	return e.Persistence.Flush(ctx, "manual")
}

func (e *Engine) ExportGraph(ctx context.Context, destination string) error {
	if err := e.Persistence.Flush(ctx, "export"); err != nil {
		return err
	}
	return e.Graph.Export(ctx, destination)
}

func (e *Engine) SyncGLPIKB(ctx context.Context) error {
	if !e.LearningEnabled() {
		return ErrLearningDisabled
	}
	if e.GLPIKB == nil {
		return fmt.Errorf("GLPI knowledge-base integration is disabled")
	}
	return e.GLPIKB.Sync(ctx, "manual")
}

func relationContextWithResearch(a, b model.Node, sim float64, results []model.ResearchResult) string {
	var out strings.Builder
	out.WriteString(relationContext(a, b, sim))
	out.WriteString("\n\nWEB-SUCHERGEBNISSE (ungeprüfte Hinweise):\n")
	for i, r := range results {
		fmt.Fprintf(&out, "\n%d. %s\nURL: %s\nAuszug: %s\n", i+1, r.Title, r.URL, clamp(r.Content, 700))
	}
	return out.String()
}

func relationContext(a, b model.Node, sim float64) string {
	return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.ID, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.ID, b.Label, strings.Join(b.Categories, ", "), b.Summary)
}
func relationSchema() map[string]any {
	return map[string]any{"type": "object", "properties": map[string]any{"related": map[string]any{"type": "boolean"}, "relation_type": map[string]any{"type": "string", "enum": []string{"related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by"}}, "confidence": map[string]any{"type": "number", "minimum": 0, "maximum": 1}, "explanation": map[string]any{"type": "string"}, "needs_research": map[string]any{"type": "boolean"}, "research_query": map[string]any{"type": "string"}, "topic_label": map[string]any{"type": "string"}, "keywords": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"related", "relation_type", "confidence", "explanation", "needs_research", "research_query", "topic_label", "keywords"}}
}
func answerSchema() map[string]any {
	return map[string]any{"type": "object", "properties": map[string]any{"answer": map[string]any{"type": "string"}, "used_node_ids": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, "uncertainties": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"answer", "used_node_ids", "uncertainties"}}
}
func validIDs(in, allowed []string) []string {
	set := map[string]bool{}
	for _, x := range allowed {
		set[x] = true
	}
	var out []string
	for _, x := range in {
		if set[x] {
			out = append(out, x)
		}
	}
	if len(out) == 0 {
		return allowed
	}
	return unique(out)
}
func safeRelation(s string) string {
	switch s {
	case "related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by":
		return s
	default:
		return "related_to"
	}
}
func common(a, b []string) []string {
	set := map[string]string{}
	for _, x := range a {
		set[strings.ToLower(x)] = x
	}
	var out []string
	for _, x := range b {
		if v, ok := set[strings.ToLower(x)]; ok {
			out = append(out, v)
		}
	}
	return out
}
func first(in []string, n int) []string {
	if len(in) > n {
		return in[:n]
	}
	return in
}
func unique(in []string) []string {
	set := map[string]bool{}
	var out []string
	for _, x := range in {
		x = strings.TrimSpace(x)
		k := strings.ToLower(x)
		if x == "" || set[k] {
			continue
		}
		set[k] = true
		out = append(out, x)
	}
	sort.Strings(out)
	return out
}
func clamp(s string, n int) string {
	r := []rune(strings.TrimSpace(s))
	if len(r) <= n {
		return string(r)
	}
	return string(r[:n]) + "…"
}
func nonempty(a, b string) string {
	if strings.TrimSpace(a) != "" {
		return strings.TrimSpace(a)
	}
	return b
}