package ingest import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io/fs" "os" "path/filepath" "sort" "strings" "time" "github.com/local/glpi-neural-brain/internal/graph" "github.com/local/glpi-neural-brain/internal/model" ) type KnowledgeScanner struct { Graph *graph.Store ProductionDirs []string StagingDirs []string lastFingerprint string } func (s *KnowledgeScanner) Scan() (int, error) { if s.Graph == nil { return 0, fmt.Errorf("graph store is nil") } var nodes []model.Node var edges []model.Edge seen := map[string]int{} for _, root := range s.StagingDirs { n, e, err := scanDir(root, "knowledge-staging", "staging") if err != nil { return 0, err } for _, x := range n { if _, ok := seen[x.ID]; !ok { seen[x.ID] = len(nodes) nodes = append(nodes, x) } } edges = append(edges, e...) } // Production wins if the same document ID exists in both scopes. for _, root := range s.ProductionDirs { n, e, err := scanDir(root, "knowledge-production", "production") if err != nil { return 0, err } for _, x := range n { if idx, ok := seen[x.ID]; ok { nodes[idx] = x } else { seen[x.ID] = len(nodes) nodes = append(nodes, x) } } edges = append(edges, e...) } fingerprint, err := knowledgeFingerprint(nodes, edges) if err != nil { return 0, err } if fingerprint == s.lastFingerprint { return len(nodes), nil } s.Graph.ReplaceOrigins([]string{"knowledge-production", "knowledge-staging", "knowledge-taxonomy"}, nodes, edges) s.lastFingerprint = fingerprint return len(nodes), nil } func scanDir(root, origin, status string) ([]model.Node, []model.Edge, error) { if strings.TrimSpace(root) == "" { return nil, nil, nil } if _, err := os.Stat(root); err != nil { if os.IsNotExist(err) { return nil, nil, nil } return nil, nil, err } var docs []model.Node categoryNodes := map[string]model.Node{} keywordNodes := map[string]model.Node{} sourceNodes := map[string]model.Node{} var edges []model.Edge err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() { base := strings.ToLower(d.Name()) if strings.HasPrefix(base, ".") && path != root { return filepath.SkipDir } return nil } if !strings.EqualFold(filepath.Ext(d.Name()), ".json") { return nil } b, err := os.ReadFile(path) if err != nil { return err } var doc map[string]any if err := json.Unmarshal(b, &doc); err != nil { return fmt.Errorf("decode %s: %w", path, err) } externalID := firstString(doc, "id", "key") if externalID == "" { rel, _ := filepath.Rel(root, path) externalID = filepath.ToSlash(rel) } label := firstString(doc, "title", "name") if label == "" { label = externalID } categories := stringSlice(doc["categories"]) keywords := stringSlice(doc["keywords"]) source := firstString(doc, "source") uri := firstString(doc, "source_uri", "uri") text := joinNonEmpty(firstString(doc, "text", "problem", "description"), firstString(doc, "answer", "solution")) aiThink := containsFold(categories, "AI-THINK") || strings.Contains(strings.ToLower(source), "ai-think") kind := "knowledge" if aiThink { kind = "ai-think" } nodeID := graph.ID("knowledge", externalID) rel, _ := filepath.Rel(root, path) metadata := map[string]any{"path": filepath.ToSlash(rel), "source": source, "auto_reply": doc["auto_reply"], "min_score": doc["min_score"]} if aiMeta, ok := doc["ai_think"].(map[string]any); ok { for _, key := range []string{"subtype", "action", "target_article_id", "target_node_id", "generation_depth", "confidence", "source_node_ids", "productive_source_count", "ai_source_count", "production_ratio"} { if value, exists := aiMeta[key]; exists { metadata[key] = value } } } n := model.Node{ ID: nodeID, Kind: kind, Label: label, Summary: clamp(text, 1400), Status: status, Origin: origin, ExternalID: externalID, URI: uri, Categories: categories, Keywords: keywords, Weight: 1.3, Metadata: metadata, UpdatedAt: time.Now().UTC(), } docs = append(docs, n) for _, cat := range categories { cat = strings.TrimSpace(cat) if cat == "" { continue } cid := graph.ID("category", strings.ToLower(cat)) if _, ok := categoryNodes[cid]; !ok { categoryNodes[cid] = model.Node{ID: cid, Kind: "category", Label: cat, Origin: "knowledge-taxonomy", ExternalID: cat, Weight: 0.75, UpdatedAt: time.Now().UTC()} } edges = append(edges, model.Edge{Source: nodeID, Target: cid, Type: "categorized_as", Origin: origin, Status: "verified", Confidence: 1, Weight: .55}) } for _, kw := range keywords { kw = strings.TrimSpace(kw) if kw == "" || len([]rune(kw)) < 3 { continue } kid := graph.ID("keyword", strings.ToLower(kw)) if _, ok := keywordNodes[kid]; !ok { keywordNodes[kid] = model.Node{ID: kid, Kind: "concept", Label: kw, Origin: "knowledge-taxonomy", ExternalID: kw, Weight: .55, UpdatedAt: time.Now().UTC()} } edges = append(edges, model.Edge{Source: nodeID, Target: kid, Type: "mentions", Origin: origin, Status: "verified", Confidence: 1, Weight: .28}) } if source != "" { sid := graph.ID("source", strings.ToLower(source)) if _, ok := sourceNodes[sid]; !ok { sourceNodes[sid] = model.Node{ID: sid, Kind: "source", Label: source, Origin: "knowledge-taxonomy", ExternalID: source, Weight: .6, UpdatedAt: time.Now().UTC()} } edges = append(edges, model.Edge{Source: nodeID, Target: sid, Type: "derived_from", Origin: origin, Status: "verified", Confidence: 1, Weight: .35}) } return nil }) if err != nil { return nil, nil, err } for _, m := range []map[string]model.Node{categoryNodes, keywordNodes, sourceNodes} { for _, n := range m { docs = append(docs, n) } } sort.Slice(docs, func(i, j int) bool { return docs[i].ID < docs[j].ID }) return docs, edges, nil } func knowledgeFingerprint(nodes []model.Node, edges []model.Edge) (string, error) { nodeCopies := append([]model.Node(nil), nodes...) edgeCopies := append([]model.Edge(nil), edges...) for i := range nodeCopies { nodeCopies[i].UpdatedAt = time.Time{} nodeCopies[i].X, nodeCopies[i].Y, nodeCopies[i].Z = 0, 0, 0 } for i := range edgeCopies { edgeCopies[i].ID = "" edgeCopies[i].CreatedAt = time.Time{} edgeCopies[i].UpdatedAt = time.Time{} } sort.Slice(nodeCopies, func(i, j int) bool { return nodeCopies[i].ID < nodeCopies[j].ID }) sort.Slice(edgeCopies, func(i, j int) bool { a := edgeCopies[i].Source + "\x00" + edgeCopies[i].Target + "\x00" + edgeCopies[i].Type + "\x00" + edgeCopies[i].Origin b := edgeCopies[j].Source + "\x00" + edgeCopies[j].Target + "\x00" + edgeCopies[j].Type + "\x00" + edgeCopies[j].Origin return a < b }) data, err := json.Marshal(struct { Nodes []model.Node `json:"nodes"` Edges []model.Edge `json:"edges"` }{Nodes: nodeCopies, Edges: edgeCopies}) if err != nil { return "", err } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil } func firstString(m map[string]any, keys ...string) string { for _, k := range keys { if v, ok := m[k]; ok { if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "" { return s } } } return "" } func stringSlice(v any) []string { var out []string switch x := v.(type) { case []any: for _, e := range x { if s := strings.TrimSpace(fmt.Sprint(e)); s != "" { out = append(out, s) } } case []string: out = append(out, x...) case string: for _, s := range strings.Split(x, ",") { if s = strings.TrimSpace(s); s != "" { out = append(out, s) } } } return unique(out) } func unique(in []string) []string { seen := map[string]bool{} var out []string for _, s := range in { k := strings.ToLower(strings.TrimSpace(s)) if k == "" || seen[k] { continue } seen[k] = true out = append(out, strings.TrimSpace(s)) } return out } func containsFold(in []string, want string) bool { for _, s := range in { if strings.EqualFold(strings.TrimSpace(s), want) { return true } } return false } func joinNonEmpty(parts ...string) string { var out []string for _, p := range parts { if p = strings.TrimSpace(p); p != "" { out = append(out, p) } } return strings.Join(out, "\n\n") } func clamp(s string, n int) string { r := []rune(strings.TrimSpace(s)) if len(r) <= n { return string(r) } return string(r[:n]) + "…" }