package graph import ( "context" "encoding/json" "fmt" "sort" "strings" "sync/atomic" "time" "github.com/local/glpi-neural-brain/internal/model" ) // MutationStats are monotonic, process-local counters for material changes to // the live graph. They deliberately distinguish creation, updates and removal // so the analysis dashboard can explain a run even when the net graph size did // not change. type MutationStats struct { NodesCreated uint64 `json:"nodes_created"` NodesUpdated uint64 `json:"nodes_updated"` NodesDeleted uint64 `json:"nodes_deleted"` EdgesCreated uint64 `json:"edges_created"` EdgesUpdated uint64 `json:"edges_updated"` EdgesDeleted uint64 `json:"edges_deleted"` VectorsCreated uint64 `json:"vectors_created"` VectorsUpdated uint64 `json:"vectors_updated"` VectorsDeleted uint64 `json:"vectors_deleted"` } func (m MutationStats) Delta(previous MutationStats) MutationStats { return MutationStats{ NodesCreated: safeDelta(m.NodesCreated, previous.NodesCreated), NodesUpdated: safeDelta(m.NodesUpdated, previous.NodesUpdated), NodesDeleted: safeDelta(m.NodesDeleted, previous.NodesDeleted), EdgesCreated: safeDelta(m.EdgesCreated, previous.EdgesCreated), EdgesUpdated: safeDelta(m.EdgesUpdated, previous.EdgesUpdated), EdgesDeleted: safeDelta(m.EdgesDeleted, previous.EdgesDeleted), VectorsCreated: safeDelta(m.VectorsCreated, previous.VectorsCreated), VectorsUpdated: safeDelta(m.VectorsUpdated, previous.VectorsUpdated), VectorsDeleted: safeDelta(m.VectorsDeleted, previous.VectorsDeleted), } } func (m *MutationStats) Add(other MutationStats) { m.NodesCreated += other.NodesCreated m.NodesUpdated += other.NodesUpdated m.NodesDeleted += other.NodesDeleted m.EdgesCreated += other.EdgesCreated m.EdgesUpdated += other.EdgesUpdated m.EdgesDeleted += other.EdgesDeleted m.VectorsCreated += other.VectorsCreated m.VectorsUpdated += other.VectorsUpdated m.VectorsDeleted += other.VectorsDeleted } func (m MutationStats) Empty() bool { return m.NodesCreated+m.NodesUpdated+m.NodesDeleted+ m.EdgesCreated+m.EdgesUpdated+m.EdgesDeleted+ m.VectorsCreated+m.VectorsUpdated+m.VectorsDeleted == 0 } func safeDelta(current, previous uint64) uint64 { if current < previous { return current } return current - previous } type GraphChange struct { ID int64 `json:"id,omitempty"` EventID string `json:"event_id,omitempty"` Timestamp time.Time `json:"timestamp"` ProcessID string `json:"process_id"` GraphVersion uint64 `json:"graph_version"` EntityKind string `json:"entity_kind"` Action string `json:"action"` EntityID string `json:"entity_id"` Label string `json:"label,omitempty"` RelationType string `json:"relation_type,omitempty"` Origin string `json:"origin,omitempty"` Details map[string]any `json:"details,omitempty"` } type AnalysisPoint struct { ProcessID string `json:"process_id"` GraphVersion uint64 `json:"graph_version"` NodeCount int `json:"node_count"` EdgeCount int `json:"edge_count"` VectorCount int `json:"vector_count"` Mutations MutationStats `json:"mutations"` Delta MutationStats `json:"delta"` } type AnalysisEventRecord struct { Activity model.Activity `json:"activity"` Point AnalysisPoint `json:"point"` ChangeCount int `json:"change_count"` ChangesTruncated int `json:"changes_truncated,omitempty"` } type AnalysisTimelineBucket struct { Start time.Time `json:"start"` Events int `json:"events"` Successes int `json:"successes"` Warnings int `json:"warnings"` Failures int `json:"failures"` Comparisons int64 `json:"comparisons"` SearchResults int64 `json:"search_results"` Mutations MutationStats `json:"mutations"` } type AnalysisRun struct { ID string `json:"id"` Kind string `json:"kind"` Title string `json:"title"` Status string `json:"status"` Verdict string `json:"verdict"` Explanation string `json:"explanation"` StartedAt time.Time `json:"started_at"` CompletedAt time.Time `json:"completed_at,omitempty"` DurationMS int64 `json:"duration_ms"` Trigger string `json:"trigger,omitempty"` Outcome string `json:"outcome,omitempty"` EventCount int `json:"event_count"` Mutations MutationStats `json:"mutations"` NodeIDs []string `json:"node_ids,omitempty"` EdgeIDs []string `json:"edge_ids,omitempty"` Metrics map[string]any `json:"metrics,omitempty"` Events []AnalysisEventRecord `json:"events,omitempty"` } type DetailedGraphAnalysis struct { Summary model.GraphAnalysis `json:"summary"` NodeKinds map[string]int `json:"node_kinds"` NodeStatuses map[string]int `json:"node_statuses"` NodeOrigins map[string]int `json:"node_origins"` NodeSources map[string]int `json:"node_sources"` EdgeTypes map[string]int `json:"edge_types"` EdgeStatuses map[string]int `json:"edge_statuses"` EdgeOrigins map[string]int `json:"edge_origins"` VectorRows int `json:"vector_rows"` VectorEligibleNodes int `json:"vector_eligible_nodes"` VectorCoverage float64 `json:"vector_coverage"` EmbeddingDimensions map[string]int `json:"embedding_dimensions"` AverageAIConfidence float64 `json:"average_ai_confidence"` AverageAISimilarity float64 `json:"average_ai_similarity"` SimilarityEdgeCount int `json:"similarity_edge_count"` NewestNodes []model.Node `json:"newest_nodes"` NewestEdges []model.Edge `json:"newest_edges"` CurrentProcessChange MutationStats `json:"current_process_changes"` } type AnalysisAuditStatus struct { QueueDepth int `json:"queue_depth"` QueueCapacity int `json:"queue_capacity"` DroppedEvents uint64 `json:"dropped_events"` LastPersistedAt time.Time `json:"last_persisted_at,omitempty"` LastError string `json:"last_error,omitempty"` } type AnalysisHistory struct { GeneratedAt time.Time `json:"generated_at"` Since time.Time `json:"since"` Events []AnalysisEventRecord `json:"events"` Runs []AnalysisRun `json:"runs"` Timeline []AnalysisTimelineBucket `json:"timeline"` Changes []GraphChange `json:"changes"` Totals MutationStats `json:"totals"` EventCounts map[string]int `json:"event_counts"` StatusCounts map[string]int `json:"status_counts"` DroppedEvents uint64 `json:"dropped_events"` DroppedDetailedChanges uint64 `json:"dropped_detailed_changes"` RawEventCount int `json:"raw_event_count"` ChangeCount int `json:"change_count"` Audit AnalysisAuditStatus `json:"audit"` } type analysisRecord struct { activity model.Activity point AnalysisPoint changes []GraphChange changesTruncated int } var processCounter atomic.Uint64 func newProcessID() string { return fmt.Sprintf("process-%d-%d", time.Now().UTC().UnixNano(), processCounter.Add(1)) } func (s *Store) initAnalysisWriter() { s.analysisMu.Lock() defer s.analysisMu.Unlock() if s.analysisQueue != nil { return } if strings.TrimSpace(s.processID) == "" { s.processID = newProcessID() } s.analysisQueue = make(chan analysisRecord, 4096) s.analysisWG.Add(1) go func(queue <-chan analysisRecord) { defer s.analysisWG.Done() for { first, ok := <-queue if !ok { return } batch := []analysisRecord{first} timer := time.NewTimer(50 * time.Millisecond) collect: for len(batch) < 64 { select { case record, open := <-queue: if !open { if !timer.Stop() { <-timer.C } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) err := s.persistAnalysisBatch(ctx, batch) cancel() s.recordAnalysisPersistResult(err) return } batch = append(batch, record) case <-timer.C: break collect } } if !timer.Stop() { select { case <-timer.C: default: } } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) err := s.persistAnalysisBatch(ctx, batch) cancel() s.recordAnalysisPersistResult(err) } }(s.analysisQueue) } func (s *Store) recordAnalysisPersistResult(err error) { s.analysisMu.Lock() defer s.analysisMu.Unlock() if err != nil { s.analysisLastError = err.Error() return } s.analysisLastError = "" s.analysisLastPersisted = time.Now().UTC() } // RecordActivity captures a cheap O(1) graph checkpoint synchronously and // performs the SQLite write asynchronously. The expensive graph analysis is // only calculated when the dedicated dashboard is opened. func (s *Store) RecordActivity(activity model.Activity) { if s == nil || s.db == nil || activity.Type == "brain.idle" { return } if activity.Timestamp.IsZero() { activity.Timestamp = time.Now().UTC() } s.mu.Lock() nodes := len(s.nodes) edges := len(s.edges) point := AnalysisPoint{ ProcessID: s.processID, GraphVersion: s.version, NodeCount: nodes, EdgeCount: edges, VectorCount: len(s.vectors), Mutations: s.mutations, } changes := append([]GraphChange(nil), s.analysisPendingChanges...) changesTruncated := s.analysisPendingTruncated s.analysisPendingChanges = s.analysisPendingChanges[:0] s.analysisPendingTruncated = 0 s.mu.Unlock() s.analysisMu.Lock() if s.analysisQueue == nil { s.analysisMu.Unlock() return } point.Delta = point.Mutations.Delta(s.analysisLastMutations) s.analysisLastMutations = point.Mutations queued := true select { case s.analysisQueue <- analysisRecord{activity: activity, point: point, changes: changes, changesTruncated: changesTruncated}: default: s.analysisDropped++ queued = false } s.analysisMu.Unlock() if !queued && len(changes)+changesTruncated > 0 { s.mu.Lock() s.analysisChangesDropped += uint64(len(changes) + changesTruncated) s.mu.Unlock() } } func (s *Store) closeAnalysisWriter() { s.analysisMu.Lock() queue := s.analysisQueue if queue != nil { s.analysisQueue = nil close(queue) } s.analysisMu.Unlock() if queue != nil { s.analysisWG.Wait() } } func (s *Store) persistAnalysisRecord(ctx context.Context, record analysisRecord) error { return s.persistAnalysisBatch(ctx, []analysisRecord{record}) } func (s *Store) persistAnalysisBatch(ctx context.Context, records []analysisRecord) error { if len(records) == 0 { return nil } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() eventStatement, err := tx.PrepareContext(ctx, `INSERT OR REPLACE INTO analysis_events(id,type,source,phase,query,message,node_ids_json,edge_ids_json,strength,metadata_json,timestamp_ns,process_id) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } defer eventStatement.Close() pointStatement, err := tx.PrepareContext(ctx, `INSERT OR REPLACE INTO analysis_points(event_id,timestamp_ns,process_id,graph_version,node_count,edge_count,vector_count, node_created,node_updated,node_deleted,edge_created,edge_updated,edge_deleted,vector_created,vector_updated,vector_deleted, delta_node_created,delta_node_updated,delta_node_deleted,delta_edge_created,delta_edge_updated,delta_edge_deleted,delta_vector_created,delta_vector_updated,delta_vector_deleted, change_count,changes_truncated) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } defer pointStatement.Close() changeStatement, err := tx.PrepareContext(ctx, `INSERT INTO analysis_changes(event_id,timestamp_ns,process_id,graph_version,entity_kind,action,entity_id,label,relation_type,origin,details_json) VALUES(?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { return err } defer changeStatement.Close() for _, record := range records { nodeIDs, _ := json.Marshal(record.activity.NodeIDs) edgeIDs, _ := json.Marshal(record.activity.EdgeIDs) metadata, _ := json.Marshal(record.activity.Metadata) if _, err := eventStatement.ExecContext(ctx, record.activity.ID, record.activity.Type, record.activity.Source, record.activity.Phase, record.activity.Query, record.activity.Message, string(nodeIDs), string(edgeIDs), record.activity.Strength, string(metadata), record.activity.Timestamp.UnixNano(), record.point.ProcessID); err != nil { return err } m, d := record.point.Mutations, record.point.Delta if _, err := pointStatement.ExecContext(ctx, record.activity.ID, record.activity.Timestamp.UnixNano(), record.point.ProcessID, record.point.GraphVersion, record.point.NodeCount, record.point.EdgeCount, record.point.VectorCount, m.NodesCreated, m.NodesUpdated, m.NodesDeleted, m.EdgesCreated, m.EdgesUpdated, m.EdgesDeleted, m.VectorsCreated, m.VectorsUpdated, m.VectorsDeleted, d.NodesCreated, d.NodesUpdated, d.NodesDeleted, d.EdgesCreated, d.EdgesUpdated, d.EdgesDeleted, d.VectorsCreated, d.VectorsUpdated, d.VectorsDeleted, len(record.changes), record.changesTruncated); err != nil { return err } for _, change := range record.changes { details, _ := json.Marshal(change.Details) timestamp := change.Timestamp if timestamp.IsZero() { timestamp = record.activity.Timestamp } if _, err := changeStatement.ExecContext(ctx, record.activity.ID, timestamp.UnixNano(), change.ProcessID, change.GraphVersion, change.EntityKind, change.Action, change.EntityID, change.Label, change.RelationType, change.Origin, string(details)); err != nil { return err } } } if err := tx.Commit(); err != nil { return err } last := records[len(records)-1] if last.activity.Timestamp.Unix()%997 == 0 { cutoff := time.Now().UTC().Add(-90 * 24 * time.Hour).UnixNano() _, _ = s.db.ExecContext(ctx, `DELETE FROM analysis_events WHERE timestamp_ns 0 { return s.analysisDetailCache } summary := s.Analyze() detail := DetailedGraphAnalysis{ Summary: summary, NodeKinds: map[string]int{}, NodeStatuses: map[string]int{}, NodeOrigins: map[string]int{}, NodeSources: map[string]int{}, EdgeTypes: map[string]int{}, EdgeStatuses: map[string]int{}, EdgeOrigins: map[string]int{}, EmbeddingDimensions: map[string]int{}, } s.mu.RLock() for _, node := range s.nodes { detail.NodeKinds[nonemptyAnalysis(node.Kind, "unknown")]++ detail.NodeStatuses[nonemptyAnalysis(node.Status, "unspecified")]++ detail.NodeOrigins[nonemptyAnalysis(node.Origin, "unknown")]++ source := explicitNodeSource(node) if source == "" { source = "(ohne source)" } detail.NodeSources[source]++ if node.Kind == "knowledge" || node.Kind == "ai-think" || node.Kind == "external" { detail.VectorEligibleNodes++ } detail.NewestNodes = insertNewestNode(detail.NewestNodes, node, 12) } var confidenceSum, similaritySum float64 for _, edge := range s.edges { if edge.Status == "rejected" { continue } detail.EdgeTypes[nonemptyAnalysis(edge.Type, "unknown")]++ detail.EdgeStatuses[nonemptyAnalysis(edge.Status, "unspecified")]++ detail.EdgeOrigins[nonemptyAnalysis(edge.Origin, "unknown")]++ if edge.Origin == "ai-inference" { confidenceSum += edge.Confidence if similarity, ok := numericMetadata(edge.Metadata, "semantic_similarity"); ok { similaritySum += similarity detail.SimilarityEdgeCount++ } } detail.NewestEdges = insertNewestEdge(detail.NewestEdges, edge, 12) } if summary.AIEdges > 0 { detail.AverageAIConfidence = confidenceSum / float64(summary.AIEdges) } if detail.SimilarityEdgeCount > 0 { detail.AverageAISimilarity = similaritySum / float64(detail.SimilarityEdgeCount) } detail.VectorRows = len(s.vectors) if detail.VectorEligibleNodes > 0 { detail.VectorCoverage = float64(detail.VectorRows) / float64(detail.VectorEligibleNodes) } for _, vector := range s.vectors { detail.EmbeddingDimensions[fmt.Sprint(len(vector))]++ } detail.CurrentProcessChange = s.mutations cachedVersion := s.version s.mu.RUnlock() s.analysisDetailVersion = cachedVersion s.analysisDetailCache = detail return detail } func insertNewestNode(nodes []model.Node, node model.Node, limit int) []model.Node { index := sort.Search(len(nodes), func(i int) bool { return !nodes[i].UpdatedAt.After(node.UpdatedAt) }) if index >= limit { return nodes } nodes = append(nodes, model.Node{}) copy(nodes[index+1:], nodes[index:]) nodes[index] = node if len(nodes) > limit { nodes = nodes[:limit] } return nodes } func insertNewestEdge(edges []model.Edge, edge model.Edge, limit int) []model.Edge { index := sort.Search(len(edges), func(i int) bool { return !edges[i].UpdatedAt.After(edge.UpdatedAt) }) if index >= limit { return edges } edges = append(edges, model.Edge{}) copy(edges[index+1:], edges[index:]) edges[index] = edge if len(edges) > limit { edges = edges[:limit] } return edges } func explicitNodeSource(node model.Node) string { if node.Metadata == nil { return "" } value, ok := node.Metadata["source"] if !ok || value == nil { return "" } text := strings.TrimSpace(fmt.Sprint(value)) if text == "" || strings.EqualFold(text, "") || strings.EqualFold(text, "null") { return "" } return text } func nonemptyAnalysis(value, fallback string) string { if strings.TrimSpace(value) == "" { return fallback } return value } func numericMetadata(metadata map[string]any, key string) (float64, bool) { if metadata == nil { return 0, false } value, ok := metadata[key] if !ok { return 0, false } switch typed := value.(type) { case float64: return typed, true case float32: return float64(typed), true case int: return float64(typed), true case int64: return float64(typed), true case json.Number: parsed, err := typed.Float64() return parsed, err == nil default: return 0, false } } func (s *Store) AnalysisHistory(ctx context.Context, since time.Time, limit int) (AnalysisHistory, error) { if limit < 20 { limit = 200 } if limit > 1000 { limit = 1000 } if since.IsZero() { since = time.Now().UTC().Add(-24 * time.Hour) } fetchLimit := limit * 8 if fetchLimit < 2000 { fetchLimit = 2000 } if fetchLimit > 10000 { fetchLimit = 10000 } events, err := s.analysisEvents(ctx, since, fetchLimit) if err != nil { return AnalysisHistory{}, err } chronological := append([]AnalysisEventRecord(nil), events...) sort.Slice(chronological, func(i, j int) bool { return chronological[i].Activity.Timestamp.Before(chronological[j].Activity.Timestamp) }) runs := buildAnalysisRuns(chronological) sort.Slice(runs, func(i, j int) bool { return runs[i].StartedAt.After(runs[j].StartedAt) }) if len(runs) > limit { runs = runs[:limit] } if len(events) > limit { events = events[:limit] } changeLimit := limit * 10 if changeLimit < 500 { changeLimit = 500 } if changeLimit > 10000 { changeLimit = 10000 } changes, changeCount, changeErr := s.analysisChanges(ctx, since, changeLimit) if changeErr != nil { return AnalysisHistory{}, changeErr } totals, rawCount, persistedTruncated, counts, aggregateErr := s.analysisAggregate(ctx, since) if aggregateErr != nil { return AnalysisHistory{}, aggregateErr } statusCounts := map[string]int{} for _, event := range chronological { statusCounts[eventSeverity(event.Activity)]++ } timeline := buildTimeline(chronological, since) s.analysisMu.Lock() dropped := s.analysisDropped audit := AnalysisAuditStatus{DroppedEvents: dropped, LastPersistedAt: s.analysisLastPersisted, LastError: s.analysisLastError} if s.analysisQueue != nil { audit.QueueDepth = len(s.analysisQueue) audit.QueueCapacity = cap(s.analysisQueue) } s.analysisMu.Unlock() s.mu.RLock() droppedChanges := s.analysisChangesDropped s.mu.RUnlock() return AnalysisHistory{ GeneratedAt: time.Now().UTC(), Since: since.UTC(), Events: events, Runs: runs, Timeline: timeline, Changes: changes, Totals: totals, EventCounts: counts, StatusCounts: statusCounts, DroppedEvents: dropped, DroppedDetailedChanges: uint64(persistedTruncated) + droppedChanges, RawEventCount: rawCount, ChangeCount: changeCount, Audit: audit, }, nil } func (s *Store) analysisAggregate(ctx context.Context, since time.Time) (MutationStats, int, int, map[string]int, error) { var totals MutationStats var count, truncated int err := s.db.QueryRowContext(ctx, `SELECT COUNT(*), COALESCE(SUM(delta_node_created),0),COALESCE(SUM(delta_node_updated),0),COALESCE(SUM(delta_node_deleted),0), COALESCE(SUM(delta_edge_created),0),COALESCE(SUM(delta_edge_updated),0),COALESCE(SUM(delta_edge_deleted),0), COALESCE(SUM(delta_vector_created),0),COALESCE(SUM(delta_vector_updated),0),COALESCE(SUM(delta_vector_deleted),0), COALESCE(SUM(changes_truncated),0) FROM analysis_points WHERE timestamp_ns>=?`, since.UnixNano()).Scan(&count, &totals.NodesCreated, &totals.NodesUpdated, &totals.NodesDeleted, &totals.EdgesCreated, &totals.EdgesUpdated, &totals.EdgesDeleted, &totals.VectorsCreated, &totals.VectorsUpdated, &totals.VectorsDeleted, &truncated) if err != nil { return MutationStats{}, 0, 0, nil, err } rows, err := s.db.QueryContext(ctx, `SELECT type,COUNT(*) FROM analysis_events WHERE timestamp_ns>=? GROUP BY type`, since.UnixNano()) if err != nil { return MutationStats{}, 0, 0, nil, err } defer rows.Close() counts := map[string]int{} for rows.Next() { var eventType string var eventCount int if err := rows.Scan(&eventType, &eventCount); err != nil { return MutationStats{}, 0, 0, nil, err } counts[eventType] = eventCount } return totals, count, truncated, counts, rows.Err() } func (s *Store) analysisEvents(ctx context.Context, since time.Time, limit int) ([]AnalysisEventRecord, error) { rows, err := s.db.QueryContext(ctx, `SELECT e.id,e.type,e.source,e.phase,e.query,e.message,e.node_ids_json,e.edge_ids_json,e.strength,e.metadata_json,e.timestamp_ns, p.process_id,p.graph_version,p.node_count,p.edge_count,p.vector_count, p.node_created,p.node_updated,p.node_deleted,p.edge_created,p.edge_updated,p.edge_deleted,p.vector_created,p.vector_updated,p.vector_deleted, p.delta_node_created,p.delta_node_updated,p.delta_node_deleted,p.delta_edge_created,p.delta_edge_updated,p.delta_edge_deleted,p.delta_vector_created,p.delta_vector_updated,p.delta_vector_deleted, p.change_count,p.changes_truncated FROM analysis_events e JOIN analysis_points p ON p.event_id=e.id WHERE e.timestamp_ns>=? ORDER BY e.timestamp_ns DESC LIMIT ?`, since.UnixNano(), limit) if err != nil { return nil, err } defer rows.Close() out := []AnalysisEventRecord{} for rows.Next() { var record AnalysisEventRecord var nodeIDs, edgeIDs, metadata string var timestamp int64 m, d := &record.Point.Mutations, &record.Point.Delta if err := rows.Scan(&record.Activity.ID, &record.Activity.Type, &record.Activity.Source, &record.Activity.Phase, &record.Activity.Query, &record.Activity.Message, &nodeIDs, &edgeIDs, &record.Activity.Strength, &metadata, ×tamp, &record.Point.ProcessID, &record.Point.GraphVersion, &record.Point.NodeCount, &record.Point.EdgeCount, &record.Point.VectorCount, &m.NodesCreated, &m.NodesUpdated, &m.NodesDeleted, &m.EdgesCreated, &m.EdgesUpdated, &m.EdgesDeleted, &m.VectorsCreated, &m.VectorsUpdated, &m.VectorsDeleted, &d.NodesCreated, &d.NodesUpdated, &d.NodesDeleted, &d.EdgesCreated, &d.EdgesUpdated, &d.EdgesDeleted, &d.VectorsCreated, &d.VectorsUpdated, &d.VectorsDeleted, &record.ChangeCount, &record.ChangesTruncated); err != nil { return nil, err } record.Activity.Timestamp = time.Unix(0, timestamp).UTC() _ = decodeJSON(nodeIDs, &record.Activity.NodeIDs) _ = decodeJSON(edgeIDs, &record.Activity.EdgeIDs) _ = decodeJSON(metadata, &record.Activity.Metadata) if record.Activity.Metadata == nil { record.Activity.Metadata = map[string]any{} } out = append(out, record) } return out, rows.Err() } func (s *Store) analysisChanges(ctx context.Context, since time.Time, limit int) ([]GraphChange, int, error) { var total int if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM analysis_changes WHERE timestamp_ns>=?`, since.UnixNano()).Scan(&total); err != nil { return nil, 0, err } rows, err := s.db.QueryContext(ctx, `SELECT id,event_id,timestamp_ns,process_id,graph_version,entity_kind,action,entity_id,label,relation_type,origin,details_json FROM analysis_changes WHERE timestamp_ns>=? ORDER BY timestamp_ns DESC,id DESC LIMIT ?`, since.UnixNano(), limit) if err != nil { return nil, 0, err } defer rows.Close() out := []GraphChange{} for rows.Next() { var change GraphChange var timestamp int64 var details string if err := rows.Scan(&change.ID, &change.EventID, ×tamp, &change.ProcessID, &change.GraphVersion, &change.EntityKind, &change.Action, &change.EntityID, &change.Label, &change.RelationType, &change.Origin, &details); err != nil { return nil, 0, err } change.Timestamp = time.Unix(0, timestamp).UTC() _ = decodeJSON(details, &change.Details) if change.Details == nil { change.Details = map[string]any{} } out = append(out, change) } return out, total, rows.Err() } func buildTimeline(events []AnalysisEventRecord, since time.Time) []AnalysisTimelineBucket { duration := time.Since(since) bucket := time.Hour if duration <= 3*time.Hour { bucket = 10 * time.Minute } else if duration <= 12*time.Hour { bucket = 30 * time.Minute } else if duration > 72*time.Hour { bucket = 6 * time.Hour } buckets := map[int64]*AnalysisTimelineBucket{} for _, event := range events { start := event.Activity.Timestamp.Truncate(bucket) key := start.UnixNano() entry := buckets[key] if entry == nil { entry = &AnalysisTimelineBucket{Start: start} buckets[key] = entry } entry.Events++ entry.Mutations.Add(event.Point.Delta) switch eventSeverity(event.Activity) { case "success": entry.Successes++ case "warning": entry.Warnings++ case "error": entry.Failures++ } entry.Comparisons += int64(metadataNumber(event.Activity.Metadata, "candidate_comparisons", "comparisons")) entry.SearchResults += int64(metadataNumber(event.Activity.Metadata, "result_count", "research_search_results")) } keys := make([]int64, 0, len(buckets)) for key := range buckets { keys = append(keys, key) } sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) out := make([]AnalysisTimelineBucket, 0, len(keys)) for _, key := range keys { out = append(out, *buckets[key]) } return out } func buildAnalysisRuns(events []AnalysisEventRecord) []AnalysisRun { active := map[string]*AnalysisRun{} activeOrder := []string{} completed := []AnalysisRun{} standalone := []AnalysisRun{} removeActive := func(key string) { delete(active, key) for i := len(activeOrder) - 1; i >= 0; i-- { if activeOrder[i] == key { activeOrder = append(activeOrder[:i], activeOrder[i+1:]...) break } } } for _, event := range events { kind, key, phase := classifyRunEvent(event.Activity) if strings.HasPrefix(key, "latest:") { wanted := strings.TrimPrefix(key, "latest:") key = "" for i := len(activeOrder) - 1; i >= 0; i-- { candidate := active[activeOrder[i]] if candidate != nil && candidate.Kind == wanted { key = activeOrder[i] break } } } if phase == "start" { // A duplicated start event for the same native run ID must never add // another active-order entry. Keep the original start timestamp and // attach the duplicate as an event so diagnostics stay lossless. if existing := active[key]; existing != nil { appendRunEvent(existing, event) continue } run := newAnalysisRun(kind, key, event) active[key] = &run activeOrder = append(activeOrder, key) continue } if key != "" { if run := active[key]; run != nil { appendRunEvent(run, event) if phase == "end" { finalizeAnalysisRun(run, event.Activity) completed = append(completed, *run) removeActive(key) } continue } } // Nested article, research and embedding events belong to the most recent // active primary operation. This reflects how the engine actually runs: // one AI-THINK cycle and one autonomous task are serialized. if len(activeOrder) > 0 { key := activeOrder[len(activeOrder)-1] if run := active[key]; run != nil { appendRunEvent(run, event) continue } } if isMeaningfulStandalone(event.Activity) { run := newAnalysisRun(kindForStandalone(event.Activity), "standalone:"+event.Activity.ID, event) finalizeAnalysisRun(&run, event.Activity) standalone = append(standalone, run) } } for _, key := range activeOrder { if run := active[key]; run != nil { // Old relation-research histories (written before research.completed // existed) can contain research.results but no terminal event whenever // no source was ingested. Do not display those historical records as // running forever. A grace period avoids prematurely closing a live // relation review that has just received its SearXNG results. if run.Kind == "research" && time.Since(run.CompletedAt) > 2*time.Minute && analysisRunHasEventType(run, "research.results") { run.Status = "success" run.Verdict = "abgeschlossen (Legacy)" run.Explanation = "Der ältere Lauf enthält SearXNG-Ergebnisse, aber kein explizites research.completed. Der Abschluss wurde aus dem letzten Ergebnis-Event rekonstruiert." completed = append(completed, *run) continue } run.Status = "running" run.Verdict = "läuft" run.Explanation = "Der Lauf ist noch nicht abgeschlossen oder sein Abschluss liegt außerhalb des gewählten Zeitfensters." completed = append(completed, *run) } } return append(completed, standalone...) } func analysisRunHasEventType(run *AnalysisRun, typeName string) bool { if run == nil { return false } for _, event := range run.Events { if event.Activity.Type == typeName { return true } } return false } func newAnalysisRun(kind, key string, event AnalysisEventRecord) AnalysisRun { if kind == "" { kind = kindForStandalone(event.Activity) } run := AnalysisRun{ ID: key, Kind: kind, Title: runTitle(kind, event.Activity), Status: "running", Verdict: "läuft", StartedAt: event.Activity.Timestamp, Metrics: map[string]any{}, } appendRunEvent(&run, event) return run } func appendRunEvent(run *AnalysisRun, event AnalysisEventRecord) { run.EventCount++ run.Mutations.Add(event.Point.Delta) run.NodeIDs = uniqueStrings(append(run.NodeIDs, event.Activity.NodeIDs...)) run.EdgeIDs = uniqueStrings(append(run.EdgeIDs, event.Activity.EdgeIDs...)) if len(run.Events) < 80 { run.Events = append(run.Events, event) } if trigger := metadataString(event.Activity.Metadata, "trigger", "requested_by"); trigger != "" { run.Trigger = trigger } mergeRunMetrics(run.Metrics, event.Activity) if event.Activity.Timestamp.After(run.CompletedAt) { run.CompletedAt = event.Activity.Timestamp } } func finalizeAnalysisRun(run *AnalysisRun, terminal model.Activity) { run.CompletedAt = terminal.Timestamp run.DurationMS = run.CompletedAt.Sub(run.StartedAt).Milliseconds() if duration := int64(metadataNumber(terminal.Metadata, "duration_ms")); duration > 0 { run.DurationMS = duration } run.Outcome = metadataString(terminal.Metadata, "result", "outcome", "reason") run.Status = eventSeverity(terminal) outcome := strings.ToLower(strings.TrimSpace(run.Outcome)) if run.Status != "error" { switch outcome { case "no_candidate", "unchanged", "idle", "completed_no_change": run.Status = "neutral" case "skipped", "rejected", "no_useful_evidence", "deferred", "disabled": run.Status = "warning" } derived := verdictFromRun(run) if derived == "success" { run.Status = "success" } else if run.Status == "neutral" && derived == "warning" { run.Status = "warning" } } run.Verdict, run.Explanation = explainRun(run, terminal) } func classifyRunEvent(activity model.Activity) (kind, key, phase string) { typeName := activity.Type switch typeName { case "think.cycle.started": return "thinking", "thinking:" + activity.ID, "start" case "think.cycle.completed", "think.cycle.failed": return "thinking", latestSyntheticKey("thinking"), "end" case "learning.scan.started": return "learning", nonemptyAnalysis(metadataString(activity.Metadata, "run_id"), "learning:"+activity.ID), "start" case "learning.scan.completed", "learning.scan.failed": return "learning", nonemptyAnalysis(metadataString(activity.Metadata, "run_id"), latestSyntheticKey("learning")), "end" case "autonomous.research.task.started": return "autonomous-research", "autonomous:" + nonemptyAnalysis(metadataString(activity.Metadata, "task_id"), activity.ID), "start" case "autonomous.research.task.completed", "autonomous.research.task.failed", "autonomous.research.task.cancelled": return "autonomous-research", "autonomous:" + nonemptyAnalysis(metadataString(activity.Metadata, "task_id"), activity.ID), "end" case "query.started": return "query", "query:" + activity.ID, "start" case "query.completed", "query.failed": return "query", latestSyntheticKey("query"), "end" case "research.test.started": return "searxng-test", "research-test:" + activity.ID, "start" case "research.test.results", "research.test.failed": return "searxng-test", latestSyntheticKey("searxng-test"), "end" } // Only the root lifecycle events open or close a research run. Nested // operations such as article.research.fetch.started/completed are updates of // the parent run, not independent processes. Treating every *.started as a // new run caused the same research_id to be appended to activeOrder once per // fetched page and made a single query appear five to seven times as // "running" in the Analysis Center. if strings.HasPrefix(typeName, "research.") || strings.HasPrefix(typeName, "article.research.") { id := metadataString(activity.Metadata, "research_id") if id != "" { switch typeName { case "research.started", "article.research.started": return "research", "research:" + id, "start" case "research.completed", "research.failed", "article.research.completed", "article.research.failed": return "research", "research:" + id, "end" default: return "research", "research:" + id, "update" } } } return "", "", "" } // latestSyntheticKey is a marker resolved by buildAnalysisRuns. Terminal events // without a native run ID are attached to the most recent active run of the // same kind. func latestSyntheticKey(kind string) string { return "latest:" + kind } func kindForStandalone(activity model.Activity) string { switch { case strings.HasPrefix(activity.Type, "glpi.kb."): return "glpi-sync" case strings.HasPrefix(activity.Type, "persistence."): return "persistence" case strings.HasPrefix(activity.Type, "article."): return "article" case strings.HasPrefix(activity.Type, "embedding."): return "embedding" case strings.HasPrefix(activity.Type, "autonomous.research.scan."): return "opportunity-scan" case activity.Type == "agent.run": return "agent" default: return "activity" } } func isMeaningfulStandalone(activity model.Activity) bool { if activity.Type == "brain.idle" || activity.Type == "think.queued" { return false } return strings.Contains(activity.Type, "completed") || strings.Contains(activity.Type, "failed") || strings.Contains(activity.Type, "created") || strings.Contains(activity.Type, "synced") || strings.Contains(activity.Type, "flushed") || activity.Type == "agent.run" || activity.Type == "graph.updated" } func runTitle(kind string, activity model.Activity) string { switch kind { case "thinking": return "AI-THINK-Zyklus" case "learning": return "KB-Lernlauf" case "autonomous-research": return nonemptyAnalysis(metadataString(activity.Metadata, "topic"), nonemptyAnalysis(activity.Message, "Autonome Recherche")) case "query": return nonemptyAnalysis(activity.Query, "Wissensabfrage") case "research", "searxng-test": return nonemptyAnalysis(metadataString(activity.Metadata, "research_query", "query"), nonemptyAnalysis(activity.Query, "SearXNG-Recherche")) case "article": return nonemptyAnalysis(metadataString(activity.Metadata, "title"), "Artikelsynthese") case "glpi-sync": return "GLPI-KB-Synchronisierung" case "persistence": return "Persistenz-Flush" case "embedding": return "Embedding-Lauf" case "opportunity-scan": return "Autonome Wissenslückensuche" case "agent": return "Agent-Lauf" default: return nonemptyAnalysis(activity.Message, activity.Type) } } func eventSeverity(activity model.Activity) string { typeName := strings.ToLower(activity.Type) message := strings.ToLower(activity.Message) if strings.Contains(typeName, "failed") || strings.Contains(typeName, "error") || strings.Contains(message, "fehlgeschlagen") { return "error" } if strings.Contains(typeName, "skipped") || strings.Contains(typeName, "rejected") || strings.Contains(typeName, "no_candidate") || strings.Contains(typeName, "paused") || strings.Contains(typeName, "deferred") || strings.Contains(message, "übersprungen") { return "warning" } if strings.Contains(typeName, "completed") || strings.Contains(typeName, "created") || strings.Contains(typeName, "accepted") || strings.Contains(typeName, "ingested") || strings.Contains(typeName, "learned") || strings.Contains(typeName, "synced") || strings.Contains(typeName, "results") || strings.Contains(typeName, "flushed") { return "success" } return "neutral" } func verdictFromRun(run *AnalysisRun) string { if hasFailureEvent(run.Events) { return "error" } if !run.Mutations.Empty() { return "success" } if hasWarningEvent(run.Events) { return "warning" } return "neutral" } func explainRun(run *AnalysisRun, terminal model.Activity) (string, string) { status := run.Status if status == "error" { return "fehlgeschlagen", nonemptyAnalysis(metadataString(terminal.Metadata, "error"), nonemptyAnalysis(terminal.Message, "Der Lauf wurde mit einem Fehler beendet.")) } if run.Mutations.NodesCreated > 0 || run.Mutations.EdgesCreated > 0 || run.Mutations.VectorsCreated > 0 { parts := []string{} if run.Mutations.NodesCreated > 0 { parts = append(parts, fmt.Sprintf("%d neue Nodes", run.Mutations.NodesCreated)) } if run.Mutations.EdgesCreated > 0 { parts = append(parts, fmt.Sprintf("%d neue Edges", run.Mutations.EdgesCreated)) } if run.Mutations.VectorsCreated > 0 { parts = append(parts, fmt.Sprintf("%d neue Embeddings", run.Mutations.VectorsCreated)) } if run.Mutations.VectorsUpdated > 0 { parts = append(parts, fmt.Sprintf("%d neu berechnete Embeddings", run.Mutations.VectorsUpdated)) } return "positives Ergebnis", "Der Lauf hat den Wissenszustand materiell erweitert: " + strings.Join(parts, ", ") + "." } if run.Mutations.NodesDeleted+run.Mutations.EdgesDeleted+run.Mutations.VectorsDeleted > 0 { return "Bereinigung", fmt.Sprintf("Der Lauf hat veraltete Daten entfernt: %d Nodes, %d Edges und %d Embeddings.", run.Mutations.NodesDeleted, run.Mutations.EdgesDeleted, run.Mutations.VectorsDeleted) } if run.Mutations.NodesUpdated+run.Mutations.EdgesUpdated+run.Mutations.VectorsUpdated > 0 { return "aktualisiert", fmt.Sprintf("Bestehendes Wissen wurde verändert: %d Nodes, %d Edges und %d Embeddings wurden aktualisiert oder neu berechnet.", run.Mutations.NodesUpdated, run.Mutations.EdgesUpdated, run.Mutations.VectorsUpdated) } if status == "warning" { return "ohne Übernahme", nonemptyAnalysis(terminal.Message, "Der Lauf hat geprüft, aber wegen Qualitäts- oder Relevanzregeln nichts in den Graphen übernommen.") } return "ohne Graphänderung", nonemptyAnalysis(terminal.Message, "Der Lauf wurde beendet, hat aber keine Nodes, Edges oder Embeddings verändert.") } func hasFailureEvent(events []AnalysisEventRecord) bool { for _, event := range events { if eventSeverity(event.Activity) == "error" { return true } } return false } func hasWarningEvent(events []AnalysisEventRecord) bool { for _, event := range events { if eventSeverity(event.Activity) == "warning" { return true } } return false } func mergeRunMetrics(metrics map[string]any, activity model.Activity) { if metrics == nil { return } for _, keys := range [][]string{{"comparisons", "candidate_comparisons"}, {"exact_comparisons"}, {"coarse_comparisons"}, {"candidate_pool"}, {"checked"}, {"relations_created"}, {"articles_created"}, {"articles_skipped"}, {"research_search_results", "result_count"}, {"research_fetched", "pages_fetched"}, {"research_accepted", "evidence_count"}, {"research_rejected"}, {"queries_executed"}, {"batch_count"}, {"duration_ms"}} { name := keys[0] value := metadataNumber(activity.Metadata, keys...) if value == 0 { continue } previous, _ := metrics[name].(float64) metrics[name] = previous + value } if similarity := metadataNumber(activity.Metadata, "semantic_similarity"); similarity > 0 { metrics["last_semantic_similarity"] = similarity } if confidence := metadataNumber(activity.Metadata, "confidence"); confidence > 0 { metrics["last_confidence"] = confidence } if modelName := metadataString(activity.Metadata, "model"); modelName != "" { metrics["model"] = modelName } if mode := metadataString(activity.Metadata, "processing_mode"); mode != "" { metrics["processing_mode"] = mode } } func metadataNumber(metadata map[string]any, keys ...string) float64 { for _, key := range keys { if value, ok := numericMetadata(metadata, key); ok { return value } } return 0 } func metadataString(metadata map[string]any, keys ...string) string { for _, key := range keys { if metadata == nil { return "" } value, ok := metadata[key] if !ok || value == nil { continue } text := strings.TrimSpace(fmt.Sprint(value)) if text != "" && !strings.EqualFold(text, "") && !strings.EqualFold(text, "null") { return text } } return "" } func uniqueStrings(values []string) []string { seen := map[string]struct{}{} out := make([]string, 0, len(values)) for _, value := range values { value = strings.TrimSpace(value) if value == "" { continue } if _, exists := seen[value]; exists { continue } seen[value] = struct{}{} out = append(out, value) } return out } func (s *Store) countNodeCreatedLocked() { s.mutations.NodesCreated++ } func (s *Store) countNodeUpdatedLocked() { s.mutations.NodesUpdated++ } func (s *Store) countNodeDeletedLocked() { s.mutations.NodesDeleted++ } func (s *Store) countEdgeCreatedLocked() { s.mutations.EdgesCreated++ } func (s *Store) countEdgeUpdatedLocked() { s.mutations.EdgesUpdated++ } func (s *Store) countEdgeDeletedLocked() { s.mutations.EdgesDeleted++ } func (s *Store) countVectorCreatedLocked() { s.mutations.VectorsCreated++ } func (s *Store) countVectorUpdatedLocked() { s.mutations.VectorsUpdated++ } func (s *Store) countVectorDeletedLocked() { s.mutations.VectorsDeleted++ } const analysisDetailedChangeLimit = 2000 func (s *Store) recordChangeLocked(change GraphChange) { change.Timestamp = time.Now().UTC() change.ProcessID = s.processID change.GraphVersion = s.version if change.Details == nil { change.Details = map[string]any{} } if len(s.analysisPendingChanges) < analysisDetailedChangeLimit { s.analysisPendingChanges = append(s.analysisPendingChanges, change) return } s.analysisPendingTruncated++ } func nodeChange(node model.Node, action string) GraphChange { return GraphChange{EntityKind: "node", Action: action, EntityID: node.ID, Label: node.Label, Origin: node.Origin, Details: map[string]any{"kind": node.Kind, "status": node.Status, "source": explicitNodeSource(node)}} } func nodeUpdateChange(previous, current model.Node) GraphChange { change := nodeChange(current, "updated") change.Details["previous_label"] = previous.Label change.Details["previous_status"] = previous.Status change.Details["previous_source"] = explicitNodeSource(previous) return change } func edgeChange(edge model.Edge, action string) GraphChange { details := map[string]any{"source": edge.Source, "target": edge.Target, "status": edge.Status, "confidence": edge.Confidence} if similarity, ok := numericMetadata(edge.Metadata, "semantic_similarity"); ok { details["semantic_similarity"] = similarity } return GraphChange{EntityKind: "edge", Action: action, EntityID: edge.ID, RelationType: edge.Type, Origin: edge.Origin, Details: details} } func edgeUpdateChange(previous, current model.Edge) GraphChange { change := edgeChange(current, "updated") change.Details["previous_status"] = previous.Status change.Details["previous_confidence"] = previous.Confidence if similarity, ok := numericMetadata(previous.Metadata, "semantic_similarity"); ok { change.Details["previous_semantic_similarity"] = similarity } return change } func vectorRecalculatedChange(nodeID string, previousDimensions, dimensions int, label string) GraphChange { change := vectorChange(nodeID, "recalculated", dimensions, label) change.Details["previous_dimensions"] = previousDimensions return change } func vectorChange(nodeID, action string, dimensions int, label string) GraphChange { return GraphChange{EntityKind: "vector", Action: action, EntityID: nodeID, Label: label, Origin: "embedding", Details: map[string]any{"dimensions": dimensions}} }