package engine import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "log/slog" "math" "os" "path/filepath" "sort" "strings" "sync" "time" "unicode" "github.com/local/glpi-neural-brain/internal/activity" "github.com/local/glpi-neural-brain/internal/config" "github.com/local/glpi-neural-brain/internal/graph" "github.com/local/glpi-neural-brain/internal/ingest" "github.com/local/glpi-neural-brain/internal/model" "github.com/local/glpi-neural-brain/internal/ollama" "github.com/local/glpi-neural-brain/internal/research" ) type Engine struct { Cfg config.Config Graph *graph.Store Broker *activity.Broker Ollama *ollama.Client Research *research.Client Scanner *ingest.KnowledgeScanner mu sync.Mutex lastScan time.Time lastEnrich time.Time ollamaOK bool } func New(cfg config.Config, g *graph.Store, b *activity.Broker) *Engine { e := &Engine{Cfg: cfg, Graph: g, Broker: b, Ollama: ollama.New(cfg.OllamaURL, cfg.ChatModel, cfg.EmbeddingModel), Scanner: &ingest.KnowledgeScanner{Graph: g, ProductionDirs: cfg.KnowledgeDirs, StagingDirs: cfg.StagingDirs}} if cfg.SearXNGURL != "" { e.Research = research.New(cfg.SearXNGURL) } return e } func (e *Engine) Start(ctx context.Context) { go func() { if err := e.Scan(ctx); err != nil { slog.Error("initial brain scan failed", "error", err) } ticker := time.NewTicker(e.Cfg.ScanInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: if err := e.Scan(ctx); err != nil { slog.Error("brain scan failed", "error", err) } } } }() if e.Cfg.AutoEnrich { go func() { timer := time.NewTimer(8 * time.Second) defer timer.Stop() for { select { case <-ctx.Done(): return case <-timer.C: if err := e.EnrichOne(ctx); err != nil { slog.Warn("automatic enrichment skipped", "error", err) } timer.Reset(e.Cfg.EnrichInterval) } } }() } go e.idle(ctx) } func (e *Engine) idle(ctx context.Context) { ticker := time.NewTicker(7 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: s := e.Graph.Snapshot() if len(s.Nodes) == 0 { continue } idx := int(time.Now().Unix()/7) % len(s.Nodes) n := s.Nodes[idx] e.Broker.Publish(model.Activity{Type: "brain.idle", Source: "brain", Phase: "idle", Message: "Leise Hintergrundaktivität", NodeIDs: []string{n.ID}, Strength: .18}) } } } func (e *Engine) Scan(ctx context.Context) error { e.mu.Lock() defer e.mu.Unlock() e.Broker.Publish(model.Activity{Type: "scan.started", Source: "brain", Phase: "ingest", Message: "Wissensräume werden synchronisiert", Strength: .45}) count, err := e.Scanner.Scan() if err != nil { return err } pingCtx, pingCancel := context.WithTimeout(ctx, 3*time.Second) pingErr := e.Ollama.Ping(pingCtx) pingCancel() if pingErr != nil { slog.Warn("Ollama unavailable; using deterministic local fallback", "error", pingErr) e.ensureFallbackEmbeddings() e.ollamaOK = false } else { // Local fallback vectors use 256 dimensions. Once Ollama becomes available, // discard those placeholders and replace them with real model embeddings. e.Graph.ClearVectorsByDimension(256) if err := e.ensureEmbeddings(ctx); err != nil { slog.Warn("Ollama embeddings failed; using deterministic local fallback", "error", err) e.ensureFallbackEmbeddings() e.ollamaOK = false } else { e.ollamaOK = true } } if err := e.Graph.Persist(); err != nil { return err } e.lastScan = time.Now().UTC() s := e.Graph.Snapshot() e.Broker.Publish(model.Activity{Type: "graph.updated", Source: "brain", Phase: "indexed", Message: fmt.Sprintf("%d Wissenselemente · %d Nodes · %d Edges", count, len(s.Nodes), len(s.Edges)), Strength: .55, Metadata: map[string]any{"nodes": len(s.Nodes), "edges": len(s.Edges), "knowledge_elements": count}}) return nil } func (e *Engine) ensureEmbeddings(ctx context.Context) error { pending := e.Graph.NodesForEmbedding() if len(pending) == 0 { return nil } for start := 0; start < len(pending); start += 16 { end := start + 16 if end > len(pending) { end = len(pending) } texts := make([]string, 0, end-start) for _, n := range pending[start:end] { texts = append(texts, embeddingText(n)) } cctx, cancel := context.WithTimeout(ctx, 4*time.Minute) vecs, err := e.Ollama.Embed(cctx, texts) cancel() if err != nil { return err } for i, v := range vecs { e.Graph.SetVector(pending[start+i].ID, v) } ids := []string{} for _, n := range pending[start:end] { ids = append(ids, n.ID) } e.Broker.Publish(model.Activity{Type: "embedding.batch", Source: "ollama", Phase: "embedding", Message: fmt.Sprintf("EmbeddingGemma verarbeitet %d Elemente", len(ids)), NodeIDs: ids, Strength: .38, Metadata: map[string]any{"batch_count": len(ids), "model": e.Cfg.EmbeddingModel, "batch_start": start, "batch_total": len(pending)}}) } return nil } func (e *Engine) ensureFallbackEmbeddings() { for _, n := range e.Graph.NodesForEmbedding() { e.Graph.SetVector(n.ID, hashEmbedding(embeddingText(n), 256)) } } func embeddingText(n model.Node) string { return strings.TrimSpace(n.Label + "\n" + strings.Join(n.Categories, " · ") + "\n" + strings.Join(n.Keywords, " · ") + "\n" + n.Summary) } func hashEmbedding(s string, dims int) []float64 { v := make([]float64, dims) tokens := strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) for _, t := range tokens { if t == "" { continue } h := sha256.Sum256([]byte(t)) idx := (int(h[0])<<8 | int(h[1])) % dims sign := 1.0 if h[2]&1 == 1 { sign = -1 } v[idx] += sign * (1 + float64(h[3])/255) } var norm float64 for _, x := range v { norm += x * x } if norm > 0 { norm = math.Sqrt(norm) for i := range v { v[i] /= norm } } return v } func (e *Engine) Query(ctx context.Context, q string) (model.QueryResponse, error) { start := time.Now() q = strings.TrimSpace(q) if len([]rune(q)) < 2 { return model.QueryResponse{}, fmt.Errorf("query is too short") } e.Broker.Publish(model.Activity{Type: "query.started", Source: "ui", Phase: "perception", Query: q, Message: "Anfrage trifft im neuronalen Feld ein", Strength: 1}) vecs, err := e.Ollama.Embed(ctx, []string{q}) if err != nil || len(vecs) == 0 { vecs = [][]float64{hashEmbedding(q, 256)} } hits := e.Graph.Similar(vecs[0], e.Cfg.TopK) nodeIDs := make([]string, 0, len(hits)) for i, h := range hits { nodeIDs = append(nodeIDs, h.NodeID) e.Broker.Publish(model.Activity{Type: "node.activated", Source: "brain", Phase: "retrieval", Query: q, NodeIDs: []string{h.NodeID}, Message: fmt.Sprintf("Treffer %d · %.0f%% · %s", i+1, h.Score*100, h.Label), Strength: math.Max(.25, h.Score)}) time.Sleep(55 * time.Millisecond) } edgeIDs := e.Graph.ConnectingEdges(nodeIDs) if len(edgeIDs) > 0 { e.Broker.Publish(model.Activity{Type: "edges.traversed", Source: "brain", Phase: "association", Query: q, NodeIDs: nodeIDs, EdgeIDs: edgeIDs, Message: fmt.Sprintf("%d Wissensverbindungen werden durchlaufen", len(edgeIDs)), Strength: .92}) } answer := e.fallbackAnswer(q, hits) used := append([]string(nil), nodeIDs...) var uncertainties []string if e.ollamaOK && len(hits) > 0 { system := "Du beantwortest Fragen ausschließlich aus dem bereitgestellten Wissensgraphen. Markiere Unklarheiten offen. Gib valides JSON nach Schema zurück. used_node_ids dürfen nur IDs aus dem Kontext sein." user := e.answerContext(q, hits) var dec model.AnswerDecision if err := e.Ollama.ChatJSON(ctx, system, user, answerSchema(), &dec); err == nil && strings.TrimSpace(dec.Answer) != "" { answer = dec.Answer used = validIDs(dec.UsedNodeIDs, nodeIDs) uncertainties = dec.Uncertainties } else if err != nil { slog.Warn("structured answer failed; fallback used", "error", err) } } e.Broker.Publish(model.Activity{Type: "query.completed", Source: "brain", Phase: "synthesis", Query: q, NodeIDs: used, EdgeIDs: e.Graph.ConnectingEdges(used), Message: "Antwortsynthese abgeschlossen", Strength: 1, Metadata: map[string]any{"duration_ms": time.Since(start).Milliseconds(), "hit_count": len(hits), "used_nodes": len(used), "uncertainty_count": len(uncertainties)}}) return model.QueryResponse{Query: q, Answer: answer, Hits: hits, UsedNodeIDs: used, Uncertainties: uncertainties, DurationMS: time.Since(start).Milliseconds()}, nil } func (e *Engine) answerContext(q string, hits []model.Hit) string { var b strings.Builder b.WriteString("FRAGE:\n" + q + "\n\nKONTEXT:\n") used := 0 for _, h := range hits { n, ok := e.Graph.GetNode(h.NodeID) if !ok { continue } part := fmt.Sprintf("\nNODE_ID: %s\nTITEL: %s\nSTATUS: %s\nKATEGORIEN: %s\nINHALT: %s\n", n.ID, n.Label, n.Status, strings.Join(n.Categories, ", "), n.Summary) if used+len(part) > e.Cfg.MaxContextChars { break } b.WriteString(part) used += len(part) } return b.String() } func (e *Engine) fallbackAnswer(q string, hits []model.Hit) string { if len(hits) == 0 { return "Im aktuellen Wissensgraphen wurde kein belastbarer Zusammenhang gefunden." } var b strings.Builder b.WriteString("Die stärksten passenden Wissensbereiche sind: ") for i, h := range hits { if i >= 4 { break } if i > 0 { b.WriteString("; ") } b.WriteString(h.Label) } b.WriteString(". Die Visualisierung zeigt die zugehörigen Aktivierungspfade. Ohne erreichbares Qwen-Modell bleibt dies eine Retrieval-Zusammenfassung.") return b.String() } func (e *Engine) EnrichOne(ctx context.Context) error { e.mu.Lock() defer e.mu.Unlock() if !e.ollamaOK { e.Broker.Publish(model.Activity{Type: "think.paused", Source: "brain", Phase: "waiting", Message: "AI-THINK wartet auf ein erreichbares Ollama/Qwen-Modell", Strength: .25}) return fmt.Errorf("Ollama/Qwen is unavailable; no AI edge or AI-THINK draft was created") } a, b, sim, ok := e.Graph.BestPair(e.Cfg.SimilarityThreshold) if !ok { return nil } e.lastEnrich = time.Now().UTC() e.Broker.Publish(model.Activity{Type: "think.started", Source: "brain", Phase: "association", NodeIDs: []string{a.ID, b.ID}, Message: fmt.Sprintf("Verwandtschaft wird geprüft · %.0f%% semantische Nähe", sim*100), Strength: .88, Metadata: map[string]any{"semantic_similarity": sim, "source_label": a.Label, "target_label": b.Label, "model": e.Cfg.ChatModel}}) system := "Analysiere zwei interne Wissenseinträge. Erfinde keine Fakten. Entscheide, ob eine belastbare Beziehung besteht. Wenn externe Fakten fehlen, setze needs_research=true. Gib ausschließlich JSON nach Schema zurück." var decision model.RelationDecision if err := e.Ollama.ChatJSON(ctx, system, relationContext(a, b, sim), relationSchema(), &decision); err != nil { e.Broker.Publish(model.Activity{Type: "think.failed", Source: "brain", Phase: "inference", NodeIDs: []string{a.ID, b.ID}, Message: "Qwen-Beziehungsanalyse ist fehlgeschlagen; es wurde nichts gespeichert", Strength: .35}) return fmt.Errorf("relation inference failed: %w", err) } var researchResults []model.ResearchResult if decision.NeedsResearch && e.Cfg.ResearchEnabled && e.Research != nil && strings.TrimSpace(decision.ResearchQuery) != "" { e.Broker.Publish(model.Activity{Type: "research.started", Source: "brain", Phase: "research", NodeIDs: []string{a.ID, b.ID}, Message: "Unklarheit erkannt · kontrollierte Webrecherche startet", Strength: .9, Metadata: map[string]any{"research_query": decision.ResearchQuery, "source_label": a.Label, "target_label": b.Label}}) results, err := e.Research.Search(ctx, decision.ResearchQuery, 4) if err != nil { slog.Warn("research failed", "error", err) } else if len(results) > 0 { researchResults = results e.addResearch(a, b, results) var reviewed model.RelationDecision reviewSystem := "Bewerte die Beziehung erneut anhand der zwei internen Wissenseinträge und der beigefügten Web-Suchergebnisse. Suchtreffer sind Hinweise, keine garantierten Fakten. Erfinde nichts, kennzeichne verbleibende Unsicherheit und gib ausschließlich JSON nach Schema zurück." if err := e.Ollama.ChatJSON(ctx, reviewSystem, relationContextWithResearch(a, b, sim, results), relationSchema(), &reviewed); err != nil { slog.Warn("research review failed; keeping pre-research decision", "error", err) } else { decision = reviewed } } } status := "staging" if !decision.Related || decision.Confidence < e.Cfg.RelationThreshold { status = "rejected" } edge := model.Edge{ Source: a.ID, Target: b.ID, Type: safeRelation(decision.RelationType), Origin: "ai-inference", Status: status, Confidence: decision.Confidence, Weight: math.Max(.2, decision.Confidence), Explanation: decision.Explanation, Evidence: []model.Evidence{{NodeID: a.ID, URI: a.URI, Excerpt: clamp(a.Summary, 220)}, {NodeID: b.ID, URI: b.URI, Excerpt: clamp(b.Summary, 220)}}, Metadata: map[string]any{"semantic_similarity": sim, "model": e.Cfg.ChatModel, "research_result_count": len(researchResults)}, } e.Graph.UpsertEdge(edge) edge.ID = graph.EdgeID(edge.Source, edge.Target, edge.Type, edge.Origin) if status == "staging" { path, err := e.writeAIThink(a, b, decision, researchResults) if err != nil { return err } e.Broker.Publish(model.Activity{Type: "think.created", Source: "brain", Phase: "staging", NodeIDs: []string{a.ID, b.ID}, EdgeIDs: []string{edge.ID}, Message: "Neuer AI-THINK-Beitrag wurde im Staging erzeugt", Strength: 1, Metadata: map[string]any{"path": path, "relation_type": safeRelation(decision.RelationType), "confidence": decision.Confidence, "semantic_similarity": sim, "research_result_count": len(researchResults), "title": decision.Title}}) } else { e.Broker.Publish(model.Activity{Type: "think.rejected", Source: "brain", Phase: "validation", NodeIDs: []string{a.ID, b.ID}, Message: "Ähnlichkeit geprüft, aber nicht als belastbare Edge übernommen", Strength: .42, Metadata: map[string]any{"relation_type": safeRelation(decision.RelationType), "confidence": decision.Confidence, "semantic_similarity": sim, "explanation": decision.Explanation}}) } return e.Graph.Persist() } func (e *Engine) addResearch(a, b model.Node, results []model.ResearchResult) { for _, r := range results { id := graph.ID("external", r.URL) n := model.Node{ID: id, Kind: "external", Label: r.Title, Summary: clamp(r.Content, 700), Status: "research", Origin: "research", ExternalID: r.URL, URI: r.URL, Weight: .8, Metadata: map[string]any{"query_pair": []string{a.ID, b.ID}}, UpdatedAt: time.Now().UTC()} e.Graph.UpsertNode(n) e.Graph.UpsertEdge(model.Edge{Source: id, Target: a.ID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: .55, Weight: .4}) e.Graph.UpsertEdge(model.Edge{Source: id, Target: b.ID, Type: "research_evidence", Origin: "research", Status: "staging", Confidence: .55, Weight: .4}) } } func (e *Engine) writeAIThink(a, b model.Node, d model.RelationDecision, results []model.ResearchResult) (string, error) { if len(e.Cfg.StagingDirs) == 0 { return "", fmt.Errorf("no BRAIN_STAGING_DIRS configured") } dir := e.Cfg.StagingDirs[0] if err := os.MkdirAll(dir, 0o750); err != nil { return "", err } pair := a.ID + "\x00" + b.ID sum := sha256.Sum256([]byte(pair)) short := strings.ToUpper(hex.EncodeToString(sum[:5])) now := time.Now().UTC() id := fmt.Sprintf("KB-AI-THINK-%s-%s", now.Format("20060102"), short) cats := unique(append([]string{"AI-THINK", "AI-Staging"}, common(a.Categories, b.Categories)...)) keywords := unique(append(append([]string{}, d.Keywords...), first(a.Keywords, 4)...)) keywords = unique(append(keywords, first(b.Keywords, 4)...)) var evidence []map[string]any for _, r := range results { evidence = append(evidence, map[string]any{"title": r.Title, "url": r.URL, "excerpt": clamp(r.Content, 300)}) } doc := map[string]any{"id": id, "title": nonempty(d.Title, "Zusammenhang: "+a.Label+" ↔ "+b.Label), "text": nonempty(d.Synthesis, d.Explanation), "answer": "Interne AI-THINK-Arbeitsnotiz. Vor produktiver Nutzung im Editor prüfen, korrigieren und freigeben.", "auto_reply": false, "min_score": 0.78, "categories": cats, "keywords": keywords, "source": "Neural Brain / " + e.Cfg.ChatModel + " (AI-THINK)", "source_uri": "brain://edge/" + short, "language": "de-DE", "communication_style": "formal", "ai_think": map[string]any{"status": "staging", "generated_at": now, "source_nodes": []string{a.ExternalID, b.ExternalID}, "source_node_ids": []string{a.ID, b.ID}, "relation_type": safeRelation(d.RelationType), "confidence": d.Confidence, "explanation": d.Explanation, "needs_research": d.NeedsResearch, "research_query": d.ResearchQuery, "research_evidence": evidence}} bts, err := json.MarshalIndent(doc, "", " ") if err != nil { return "", err } path := filepath.Join(dir, strings.ToLower(id)+".json") if _, err := os.Stat(path); err == nil { return path, nil } tmp := path + ".tmp" if err := os.WriteFile(tmp, append(bts, '\n'), 0o640); err != nil { return "", err } if err := os.Rename(tmp, path); err != nil { return "", err } return path, nil } func (e *Engine) Status() map[string]any { s := e.Graph.Snapshot() return map[string]any{"ok": true, "nodes": len(s.Nodes), "edges": len(s.Edges), "version": s.Version, "last_scan": e.lastScan, "last_enrich": e.lastEnrich, "ollama_ok": e.ollamaOK, "auto_enrich": e.Cfg.AutoEnrich, "research_enabled": e.Cfg.ResearchEnabled, "chat_model": e.Cfg.ChatModel, "embedding_model": e.Cfg.EmbeddingModel} } func relationContextWithResearch(a, b model.Node, sim float64, results []model.ResearchResult) string { var out strings.Builder out.WriteString(relationContext(a, b, sim)) out.WriteString("\n\nWEB-SUCHERGEBNISSE (ungeprüfte Hinweise):\n") for i, r := range results { fmt.Fprintf(&out, "\n%d. %s\nURL: %s\nAuszug: %s\n", i+1, r.Title, r.URL, clamp(r.Content, 700)) } return out.String() } func relationContext(a, b model.Node, sim float64) string { return fmt.Sprintf("SEMANTISCHE_NÄHE: %.4f\n\nA\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s\n\nB\nID: %s\nTitel: %s\nKategorien: %s\nInhalt: %s", sim, a.ID, a.Label, strings.Join(a.Categories, ", "), a.Summary, b.ID, b.Label, strings.Join(b.Categories, ", "), b.Summary) } func relationSchema() map[string]any { return map[string]any{"type": "object", "properties": map[string]any{"related": map[string]any{"type": "boolean"}, "relation_type": map[string]any{"type": "string", "enum": []string{"related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by"}}, "confidence": map[string]any{"type": "number", "minimum": 0, "maximum": 1}, "explanation": map[string]any{"type": "string"}, "needs_research": map[string]any{"type": "boolean"}, "research_query": map[string]any{"type": "string"}, "title": map[string]any{"type": "string"}, "synthesis": map[string]any{"type": "string"}, "keywords": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"related", "relation_type", "confidence", "explanation", "needs_research", "research_query", "title", "synthesis", "keywords"}} } func answerSchema() map[string]any { return map[string]any{"type": "object", "properties": map[string]any{"answer": map[string]any{"type": "string"}, "used_node_ids": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, "uncertainties": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}}, "required": []string{"answer", "used_node_ids", "uncertainties"}} } func validIDs(in, allowed []string) []string { set := map[string]bool{} for _, x := range allowed { set[x] = true } var out []string for _, x := range in { if set[x] { out = append(out, x) } } if len(out) == 0 { return allowed } return unique(out) } func safeRelation(s string) string { switch s { case "related_to", "depends_on", "supports", "contradicts", "extends", "same_topic", "caused_by": return s default: return "related_to" } } func common(a, b []string) []string { set := map[string]string{} for _, x := range a { set[strings.ToLower(x)] = x } var out []string for _, x := range b { if v, ok := set[strings.ToLower(x)]; ok { out = append(out, v) } } return out } func first(in []string, n int) []string { if len(in) > n { return in[:n] } return in } func unique(in []string) []string { set := map[string]bool{} var out []string for _, x := range in { x = strings.TrimSpace(x) k := strings.ToLower(x) if x == "" || set[k] { continue } set[k] = true out = append(out, x) } sort.Strings(out) return out } func clamp(s string, n int) string { r := []rune(strings.TrimSpace(s)) if len(r) <= n { return string(r) } return string(r[:n]) + "…" } func nonempty(a, b string) string { if strings.TrimSpace(a) != "" { return strings.TrimSpace(a) } return b }