glpi-neural-brain/internal/ingest/knowledge.go

package ingest

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/local/glpi-neural-brain/internal/graph"
	"github.com/local/glpi-neural-brain/internal/model"
)

type KnowledgeScanner struct {
	Graph                   *graph.Store
	ProductionDirs          []string
	StagingDirs             []string
	lastFingerprint         string
	lastManifestFingerprint string
	lastCount               int
	manifestInitialized     bool
	FullVerifyInterval      time.Duration
	lastFullVerifyAt        time.Time
}

type KnowledgeScanResult struct {
	Count           int
	Changed         bool
	FastPath        bool
	ManifestChanged bool
	FullVerify      bool
	Mutations       graph.MutationStats
}

func (s *KnowledgeScanner) Scan() (int, error) {
	result, err := s.ScanDetailed()
	return result.Count, err
}

// ScanDetailed first fingerprints the filesystem manifest (path, size and
// nanosecond mtime) without opening/parsing every JSON document. Only when that
// cheap manifest changed is the full knowledge model rebuilt. A content
// fingerprint is still calculated after parsing, so a touched-but-unchanged
// file does not mutate the graph.
func (s *KnowledgeScanner) ScanDetailed() (KnowledgeScanResult, error) {
	if s.Graph == nil {
		return KnowledgeScanResult{}, fmt.Errorf("graph store is nil")
	}
	manifest, err := knowledgeManifestFingerprint(s.ProductionDirs, s.StagingDirs)
	if err != nil {
		return KnowledgeScanResult{}, err
	}
	forceFullVerify := s.manifestInitialized && s.FullVerifyInterval > 0 && (s.lastFullVerifyAt.IsZero() || time.Since(s.lastFullVerifyAt) >= s.FullVerifyInterval)
	if s.manifestInitialized && manifest == s.lastManifestFingerprint && !forceFullVerify {
		return KnowledgeScanResult{Count: s.lastCount, FastPath: true}, nil
	}

	var nodes []model.Node
	var edges []model.Edge
	seen := map[string]int{}
	for _, root := range s.StagingDirs {
		n, e, err := scanDir(root, "knowledge-staging", "staging")
		if err != nil {
			return KnowledgeScanResult{}, err
		}
		for _, x := range n {
			if _, ok := seen[x.ID]; !ok {
				seen[x.ID] = len(nodes)
				nodes = append(nodes, x)
			}
		}
		edges = append(edges, e...)
	}
	// Production wins if the same document ID exists in both scopes.
	for _, root := range s.ProductionDirs {
		n, e, err := scanDir(root, "knowledge-production", "production")
		if err != nil {
			return KnowledgeScanResult{}, err
		}
		for _, x := range n {
			if idx, ok := seen[x.ID]; ok {
				nodes[idx] = x
			} else {
				seen[x.ID] = len(nodes)
				nodes = append(nodes, x)
			}
		}
		edges = append(edges, e...)
	}
	fingerprint, err := knowledgeFingerprint(nodes, edges)
	if err != nil {
		return KnowledgeScanResult{}, err
	}
	result := KnowledgeScanResult{Count: len(nodes), ManifestChanged: s.manifestInitialized && manifest != s.lastManifestFingerprint, FullVerify: forceFullVerify}
	if fingerprint != s.lastFingerprint {
		result.Mutations = s.Graph.ReplaceOriginsWithStats([]string{"knowledge-production", "knowledge-staging", "knowledge-taxonomy"}, nodes, edges)
		result.Changed = !result.Mutations.Empty()
		s.lastFingerprint = fingerprint
	}
	s.lastManifestFingerprint = manifest
	s.lastCount = len(nodes)
	s.manifestInitialized = true
	s.lastFullVerifyAt = time.Now().UTC()
	return result, nil
}

func knowledgeManifestFingerprint(productionDirs, stagingDirs []string) (string, error) {
	entries := make([]string, 0, 1024)
	addRoot := func(scope, root string) error {
		root = strings.TrimSpace(root)
		if root == "" {
			return nil
		}
		clean := filepath.Clean(root)
		info, err := os.Stat(clean)
		if err != nil {
			if os.IsNotExist(err) {
				entries = append(entries, scope+"\x00"+filepath.ToSlash(clean)+"\x00missing")
				return nil
			}
			return err
		}
		if !info.IsDir() {
			return fmt.Errorf("knowledge root is not a directory: %s", clean)
		}
		entries = append(entries, scope+"\x00"+filepath.ToSlash(clean)+"\x00dir")
		return filepath.WalkDir(clean, func(path string, d fs.DirEntry, walkErr error) error {
			if walkErr != nil {
				return walkErr
			}
			if d.IsDir() {
				base := strings.ToLower(d.Name())
				if strings.HasPrefix(base, ".") && path != clean {
					return filepath.SkipDir
				}
				return nil
			}
			if !strings.EqualFold(filepath.Ext(d.Name()), ".json") {
				return nil
			}
			fileInfo, err := d.Info()
			if err != nil {
				return err
			}
			rel, err := filepath.Rel(clean, path)
			if err != nil {
				return err
			}
			entries = append(entries, fmt.Sprintf("%s\x00%s\x00%d\x00%d", scope, filepath.ToSlash(rel), fileInfo.Size(), fileInfo.ModTime().UnixNano()))
			return nil
		})
	}
	for _, root := range stagingDirs {
		if err := addRoot("staging", root); err != nil {
			return "", err
		}
	}
	for _, root := range productionDirs {
		if err := addRoot("production", root); err != nil {
			return "", err
		}
	}
	sort.Strings(entries)
	h := sha256.New()
	for _, entry := range entries {
		_, _ = h.Write([]byte(entry))
		_, _ = h.Write([]byte{0})
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

func scanDir(root, origin, status string) ([]model.Node, []model.Edge, error) {
	if strings.TrimSpace(root) == "" {
		return nil, nil, nil
	}
	if _, err := os.Stat(root); err != nil {
		if os.IsNotExist(err) {
			return nil, nil, nil
		}
		return nil, nil, err
	}
	var docs []model.Node
	categoryNodes := map[string]model.Node{}
	keywordNodes := map[string]model.Node{}
	sourceNodes := map[string]model.Node{}
	var edges []model.Edge
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if d.IsDir() {
			base := strings.ToLower(d.Name())
			if strings.HasPrefix(base, ".") && path != root {
				return filepath.SkipDir
			}
			return nil
		}
		if !strings.EqualFold(filepath.Ext(d.Name()), ".json") {
			return nil
		}
		b, err := os.ReadFile(path)
		if err != nil {
			return err
		}
		var doc map[string]any
		if err := json.Unmarshal(b, &doc); err != nil {
			return fmt.Errorf("decode %s: %w", path, err)
		}
		externalID := firstString(doc, "id", "key")
		if externalID == "" {
			rel, _ := filepath.Rel(root, path)
			externalID = filepath.ToSlash(rel)
		}
		label := firstString(doc, "title", "name")
		if label == "" {
			label = externalID
		}
		categories := stringSlice(doc["categories"])
		keywords := stringSlice(doc["keywords"])
		source := firstString(doc, "source")
		uri := firstString(doc, "source_uri", "uri")
		text := joinNonEmpty(firstString(doc, "text", "problem", "description"), firstString(doc, "answer", "solution"))
		aiThink := containsFold(categories, "AI-THINK") || strings.Contains(strings.ToLower(source), "ai-think")
		kind := "knowledge"
		if aiThink {
			kind = "ai-think"
		}
		nodeID := graph.ID("knowledge", externalID)
		rel, _ := filepath.Rel(root, path)
		metadata := map[string]any{"path": filepath.ToSlash(rel), "source": source, "auto_reply": doc["auto_reply"], "min_score": doc["min_score"]}
		if aiMeta, ok := doc["ai_think"].(map[string]any); ok {
			for _, key := range []string{"subtype", "action", "target_article_id", "target_node_id", "generation_depth", "confidence", "source_node_ids", "productive_source_count", "ai_source_count", "production_ratio", "source_fingerprint", "synthesis_model", "review_model", "pipeline"} {
				if value, exists := aiMeta[key]; exists {
					metadata[key] = value
				}
			}
		}
		n := model.Node{
			ID: nodeID, Kind: kind, Label: label, Summary: clamp(text, 1400), Status: status, Origin: origin,
			ExternalID: externalID, URI: uri, Categories: categories, Keywords: keywords, Weight: 1.3,
			Metadata: metadata, UpdatedAt: time.Now().UTC(),
		}
		docs = append(docs, n)
		for _, cat := range categories {
			cat = strings.TrimSpace(cat)
			if cat == "" {
				continue
			}
			cid := graph.ID("category", strings.ToLower(cat))
			if _, ok := categoryNodes[cid]; !ok {
				categoryNodes[cid] = model.Node{ID: cid, Kind: "category", Label: cat, Origin: "knowledge-taxonomy", ExternalID: cat, Weight: 0.75, UpdatedAt: time.Now().UTC()}
			}
			edges = append(edges, model.Edge{Source: nodeID, Target: cid, Type: "categorized_as", Origin: origin, Status: "verified", Confidence: 1, Weight: .55})
		}
		for _, kw := range keywords {
			kw = strings.TrimSpace(kw)
			if kw == "" || len([]rune(kw)) < 3 {
				continue
			}
			kid := graph.ID("keyword", strings.ToLower(kw))
			if _, ok := keywordNodes[kid]; !ok {
				keywordNodes[kid] = model.Node{ID: kid, Kind: "concept", Label: kw, Origin: "knowledge-taxonomy", ExternalID: kw, Weight: .55, UpdatedAt: time.Now().UTC()}
			}
			edges = append(edges, model.Edge{Source: nodeID, Target: kid, Type: "mentions", Origin: origin, Status: "verified", Confidence: 1, Weight: .28})
		}
		if source != "" {
			sid := graph.ID("source", strings.ToLower(source))
			if _, ok := sourceNodes[sid]; !ok {
				sourceNodes[sid] = model.Node{ID: sid, Kind: "source", Label: source, Origin: "knowledge-taxonomy", ExternalID: source, Weight: .6, UpdatedAt: time.Now().UTC()}
			}
			edges = append(edges, model.Edge{Source: nodeID, Target: sid, Type: "derived_from", Origin: origin, Status: "verified", Confidence: 1, Weight: .35})
		}
		return nil
	})
	if err != nil {
		return nil, nil, err
	}
	for _, m := range []map[string]model.Node{categoryNodes, keywordNodes, sourceNodes} {
		for _, n := range m {
			docs = append(docs, n)
		}
	}
	sort.Slice(docs, func(i, j int) bool { return docs[i].ID < docs[j].ID })
	return docs, edges, nil
}

func knowledgeFingerprint(nodes []model.Node, edges []model.Edge) (string, error) {
	nodeCopies := append([]model.Node(nil), nodes...)
	edgeCopies := append([]model.Edge(nil), edges...)
	for i := range nodeCopies {
		nodeCopies[i].UpdatedAt = time.Time{}
		nodeCopies[i].X, nodeCopies[i].Y, nodeCopies[i].Z = 0, 0, 0
	}
	for i := range edgeCopies {
		edgeCopies[i].ID = ""
		edgeCopies[i].CreatedAt = time.Time{}
		edgeCopies[i].UpdatedAt = time.Time{}
	}
	sort.Slice(nodeCopies, func(i, j int) bool { return nodeCopies[i].ID < nodeCopies[j].ID })
	sort.Slice(edgeCopies, func(i, j int) bool {
		a := edgeCopies[i].Source + "\x00" + edgeCopies[i].Target + "\x00" + edgeCopies[i].Type + "\x00" + edgeCopies[i].Origin
		b := edgeCopies[j].Source + "\x00" + edgeCopies[j].Target + "\x00" + edgeCopies[j].Type + "\x00" + edgeCopies[j].Origin
		return a < b
	})
	data, err := json.Marshal(struct {
		Nodes []model.Node `json:"nodes"`
		Edges []model.Edge `json:"edges"`
	}{Nodes: nodeCopies, Edges: edgeCopies})
	if err != nil {
		return "", err
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:]), nil
}

func firstString(m map[string]any, keys ...string) string {
	for _, k := range keys {
		if v, ok := m[k]; ok {
			if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "<nil>" {
				return s
			}
		}
	}
	return ""
}
func stringSlice(v any) []string {
	var out []string
	switch x := v.(type) {
	case []any:
		for _, e := range x {
			if s := strings.TrimSpace(fmt.Sprint(e)); s != "" {
				out = append(out, s)
			}
		}
	case []string:
		out = append(out, x...)
	case string:
		for _, s := range strings.Split(x, ",") {
			if s = strings.TrimSpace(s); s != "" {
				out = append(out, s)
			}
		}
	}
	return unique(out)
}
func unique(in []string) []string {
	seen := map[string]bool{}
	var out []string
	for _, s := range in {
		k := strings.ToLower(strings.TrimSpace(s))
		if k == "" || seen[k] {
			continue
		}
		seen[k] = true
		out = append(out, strings.TrimSpace(s))
	}
	return out
}
func containsFold(in []string, want string) bool {
	for _, s := range in {
		if strings.EqualFold(strings.TrimSpace(s), want) {
			return true
		}
	}
	return false
}
func joinNonEmpty(parts ...string) string {
	var out []string
	for _, p := range parts {
		if p = strings.TrimSpace(p); p != "" {
			out = append(out, p)
		}
	}
	return strings.Join(out, "\n\n")
}
func clamp(s string, n int) string {
	r := []rune(strings.TrimSpace(s))
	if len(r) <= n {
		return string(r)
	}
	return string(r[:n]) + "…"
}