285 lines
8.3 KiB
Go
285 lines
8.3 KiB
Go
package ingest
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/graph"
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
)
|
|
|
|
type KnowledgeScanner struct {
|
|
Graph *graph.Store
|
|
ProductionDirs []string
|
|
StagingDirs []string
|
|
lastFingerprint string
|
|
}
|
|
|
|
func (s *KnowledgeScanner) Scan() (int, error) {
|
|
if s.Graph == nil {
|
|
return 0, fmt.Errorf("graph store is nil")
|
|
}
|
|
var nodes []model.Node
|
|
var edges []model.Edge
|
|
seen := map[string]int{}
|
|
for _, root := range s.StagingDirs {
|
|
n, e, err := scanDir(root, "knowledge-staging", "staging")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for _, x := range n {
|
|
if _, ok := seen[x.ID]; !ok {
|
|
seen[x.ID] = len(nodes)
|
|
nodes = append(nodes, x)
|
|
}
|
|
}
|
|
edges = append(edges, e...)
|
|
}
|
|
// Production wins if the same document ID exists in both scopes.
|
|
for _, root := range s.ProductionDirs {
|
|
n, e, err := scanDir(root, "knowledge-production", "production")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for _, x := range n {
|
|
if idx, ok := seen[x.ID]; ok {
|
|
nodes[idx] = x
|
|
} else {
|
|
seen[x.ID] = len(nodes)
|
|
nodes = append(nodes, x)
|
|
}
|
|
}
|
|
edges = append(edges, e...)
|
|
}
|
|
fingerprint, err := knowledgeFingerprint(nodes, edges)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if fingerprint == s.lastFingerprint {
|
|
return len(nodes), nil
|
|
}
|
|
s.Graph.ReplaceOrigins([]string{"knowledge-production", "knowledge-staging", "knowledge-taxonomy"}, nodes, edges)
|
|
s.lastFingerprint = fingerprint
|
|
return len(nodes), nil
|
|
}
|
|
|
|
func scanDir(root, origin, status string) ([]model.Node, []model.Edge, error) {
|
|
if strings.TrimSpace(root) == "" {
|
|
return nil, nil, nil
|
|
}
|
|
if _, err := os.Stat(root); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil, nil
|
|
}
|
|
return nil, nil, err
|
|
}
|
|
var docs []model.Node
|
|
categoryNodes := map[string]model.Node{}
|
|
keywordNodes := map[string]model.Node{}
|
|
sourceNodes := map[string]model.Node{}
|
|
var edges []model.Edge
|
|
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if d.IsDir() {
|
|
base := strings.ToLower(d.Name())
|
|
if strings.HasPrefix(base, ".") && path != root {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
if !strings.EqualFold(filepath.Ext(d.Name()), ".json") {
|
|
return nil
|
|
}
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var doc map[string]any
|
|
if err := json.Unmarshal(b, &doc); err != nil {
|
|
return fmt.Errorf("decode %s: %w", path, err)
|
|
}
|
|
externalID := firstString(doc, "id", "key")
|
|
if externalID == "" {
|
|
rel, _ := filepath.Rel(root, path)
|
|
externalID = filepath.ToSlash(rel)
|
|
}
|
|
label := firstString(doc, "title", "name")
|
|
if label == "" {
|
|
label = externalID
|
|
}
|
|
categories := stringSlice(doc["categories"])
|
|
keywords := stringSlice(doc["keywords"])
|
|
source := firstString(doc, "source")
|
|
uri := firstString(doc, "source_uri", "uri")
|
|
text := joinNonEmpty(firstString(doc, "text", "problem", "description"), firstString(doc, "answer", "solution"))
|
|
aiThink := containsFold(categories, "AI-THINK") || strings.Contains(strings.ToLower(source), "ai-think")
|
|
kind := "knowledge"
|
|
if aiThink {
|
|
kind = "ai-think"
|
|
}
|
|
nodeID := graph.ID("knowledge", externalID)
|
|
rel, _ := filepath.Rel(root, path)
|
|
metadata := map[string]any{"path": filepath.ToSlash(rel), "source": source, "auto_reply": doc["auto_reply"], "min_score": doc["min_score"]}
|
|
if aiMeta, ok := doc["ai_think"].(map[string]any); ok {
|
|
for _, key := range []string{"subtype", "action", "target_article_id", "target_node_id", "generation_depth", "confidence", "source_node_ids", "productive_source_count", "ai_source_count", "production_ratio"} {
|
|
if value, exists := aiMeta[key]; exists {
|
|
metadata[key] = value
|
|
}
|
|
}
|
|
}
|
|
n := model.Node{
|
|
ID: nodeID, Kind: kind, Label: label, Summary: clamp(text, 1400), Status: status, Origin: origin,
|
|
ExternalID: externalID, URI: uri, Categories: categories, Keywords: keywords, Weight: 1.3,
|
|
Metadata: metadata, UpdatedAt: time.Now().UTC(),
|
|
}
|
|
docs = append(docs, n)
|
|
for _, cat := range categories {
|
|
cat = strings.TrimSpace(cat)
|
|
if cat == "" {
|
|
continue
|
|
}
|
|
cid := graph.ID("category", strings.ToLower(cat))
|
|
if _, ok := categoryNodes[cid]; !ok {
|
|
categoryNodes[cid] = model.Node{ID: cid, Kind: "category", Label: cat, Origin: "knowledge-taxonomy", ExternalID: cat, Weight: 0.75, UpdatedAt: time.Now().UTC()}
|
|
}
|
|
edges = append(edges, model.Edge{Source: nodeID, Target: cid, Type: "categorized_as", Origin: origin, Status: "verified", Confidence: 1, Weight: .55})
|
|
}
|
|
for _, kw := range keywords {
|
|
kw = strings.TrimSpace(kw)
|
|
if kw == "" || len([]rune(kw)) < 3 {
|
|
continue
|
|
}
|
|
kid := graph.ID("keyword", strings.ToLower(kw))
|
|
if _, ok := keywordNodes[kid]; !ok {
|
|
keywordNodes[kid] = model.Node{ID: kid, Kind: "concept", Label: kw, Origin: "knowledge-taxonomy", ExternalID: kw, Weight: .55, UpdatedAt: time.Now().UTC()}
|
|
}
|
|
edges = append(edges, model.Edge{Source: nodeID, Target: kid, Type: "mentions", Origin: origin, Status: "verified", Confidence: 1, Weight: .28})
|
|
}
|
|
if source != "" {
|
|
sid := graph.ID("source", strings.ToLower(source))
|
|
if _, ok := sourceNodes[sid]; !ok {
|
|
sourceNodes[sid] = model.Node{ID: sid, Kind: "source", Label: source, Origin: "knowledge-taxonomy", ExternalID: source, Weight: .6, UpdatedAt: time.Now().UTC()}
|
|
}
|
|
edges = append(edges, model.Edge{Source: nodeID, Target: sid, Type: "derived_from", Origin: origin, Status: "verified", Confidence: 1, Weight: .35})
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for _, m := range []map[string]model.Node{categoryNodes, keywordNodes, sourceNodes} {
|
|
for _, n := range m {
|
|
docs = append(docs, n)
|
|
}
|
|
}
|
|
sort.Slice(docs, func(i, j int) bool { return docs[i].ID < docs[j].ID })
|
|
return docs, edges, nil
|
|
}
|
|
|
|
func knowledgeFingerprint(nodes []model.Node, edges []model.Edge) (string, error) {
|
|
nodeCopies := append([]model.Node(nil), nodes...)
|
|
edgeCopies := append([]model.Edge(nil), edges...)
|
|
for i := range nodeCopies {
|
|
nodeCopies[i].UpdatedAt = time.Time{}
|
|
nodeCopies[i].X, nodeCopies[i].Y, nodeCopies[i].Z = 0, 0, 0
|
|
}
|
|
for i := range edgeCopies {
|
|
edgeCopies[i].ID = ""
|
|
edgeCopies[i].CreatedAt = time.Time{}
|
|
edgeCopies[i].UpdatedAt = time.Time{}
|
|
}
|
|
sort.Slice(nodeCopies, func(i, j int) bool { return nodeCopies[i].ID < nodeCopies[j].ID })
|
|
sort.Slice(edgeCopies, func(i, j int) bool {
|
|
a := edgeCopies[i].Source + "\x00" + edgeCopies[i].Target + "\x00" + edgeCopies[i].Type + "\x00" + edgeCopies[i].Origin
|
|
b := edgeCopies[j].Source + "\x00" + edgeCopies[j].Target + "\x00" + edgeCopies[j].Type + "\x00" + edgeCopies[j].Origin
|
|
return a < b
|
|
})
|
|
data, err := json.Marshal(struct {
|
|
Nodes []model.Node `json:"nodes"`
|
|
Edges []model.Edge `json:"edges"`
|
|
}{Nodes: nodeCopies, Edges: edgeCopies})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sum := sha256.Sum256(data)
|
|
return hex.EncodeToString(sum[:]), nil
|
|
}
|
|
|
|
func firstString(m map[string]any, keys ...string) string {
|
|
for _, k := range keys {
|
|
if v, ok := m[k]; ok {
|
|
if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "<nil>" {
|
|
return s
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
func stringSlice(v any) []string {
|
|
var out []string
|
|
switch x := v.(type) {
|
|
case []any:
|
|
for _, e := range x {
|
|
if s := strings.TrimSpace(fmt.Sprint(e)); s != "" {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
case []string:
|
|
out = append(out, x...)
|
|
case string:
|
|
for _, s := range strings.Split(x, ",") {
|
|
if s = strings.TrimSpace(s); s != "" {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
}
|
|
return unique(out)
|
|
}
|
|
func unique(in []string) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, s := range in {
|
|
k := strings.ToLower(strings.TrimSpace(s))
|
|
if k == "" || seen[k] {
|
|
continue
|
|
}
|
|
seen[k] = true
|
|
out = append(out, strings.TrimSpace(s))
|
|
}
|
|
return out
|
|
}
|
|
func containsFold(in []string, want string) bool {
|
|
for _, s := range in {
|
|
if strings.EqualFold(strings.TrimSpace(s), want) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func joinNonEmpty(parts ...string) string {
|
|
var out []string
|
|
for _, p := range parts {
|
|
if p = strings.TrimSpace(p); p != "" {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return strings.Join(out, "\n\n")
|
|
}
|
|
func clamp(s string, n int) string {
|
|
r := []rune(strings.TrimSpace(s))
|
|
if len(r) <= n {
|
|
return string(r)
|
|
}
|
|
return string(r[:n]) + "…"
|
|
}
|