573 lines
13 KiB
Go
573 lines
13 KiB
Go
package graph
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/local/glpi-neural-brain/internal/model"
|
|
)
|
|
|
|
type Store struct {
|
|
mu sync.RWMutex
|
|
nodes map[string]model.Node
|
|
edges map[string]model.Edge
|
|
vectors map[string][]float64
|
|
version uint64
|
|
path string
|
|
pairCursor int
|
|
}
|
|
|
|
type diskState struct {
|
|
Version uint64 `json:"version"`
|
|
Nodes []model.Node `json:"nodes"`
|
|
Edges []model.Edge `json:"edges"`
|
|
Vectors map[string][]float64 `json:"vectors,omitempty"`
|
|
}
|
|
|
|
func Open(dir string) (*Store, error) {
|
|
s := &Store{nodes: map[string]model.Node{}, edges: map[string]model.Edge{}, vectors: map[string][]float64{}, path: filepath.Join(dir, "graph-state.json")}
|
|
b, err := os.ReadFile(s.path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return s, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var d diskState
|
|
if err = json.Unmarshal(b, &d); err != nil {
|
|
return nil, err
|
|
}
|
|
s.version = d.Version
|
|
for _, n := range d.Nodes {
|
|
s.nodes[n.ID] = n
|
|
}
|
|
for _, e := range d.Edges {
|
|
s.edges[e.ID] = e
|
|
}
|
|
if d.Vectors != nil {
|
|
s.vectors = d.Vectors
|
|
}
|
|
return s, nil
|
|
}
|
|
func ID(parts ...string) string {
|
|
h := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
|
return hex.EncodeToString(h[:12])
|
|
}
|
|
func EdgeID(source, target, typ, origin string) string {
|
|
return ID("edge", source, target, typ, origin)
|
|
}
|
|
func (s *Store) UpsertNode(n model.Node) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if n.UpdatedAt.IsZero() {
|
|
n.UpdatedAt = time.Now().UTC()
|
|
}
|
|
if n.Weight == 0 {
|
|
n.Weight = 1
|
|
}
|
|
if n.X == 0 && n.Y == 0 && n.Z == 0 {
|
|
n.X, n.Y, n.Z = position(n.ID, n.Categories)
|
|
}
|
|
s.nodes[n.ID] = n
|
|
s.version++
|
|
}
|
|
func (s *Store) UpsertEdge(e model.Edge) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
now := time.Now().UTC()
|
|
if e.ID == "" {
|
|
e.ID = EdgeID(e.Source, e.Target, e.Type, e.Origin)
|
|
}
|
|
if e.CreatedAt.IsZero() {
|
|
if old, ok := s.edges[e.ID]; ok {
|
|
e.CreatedAt = old.CreatedAt
|
|
} else {
|
|
e.CreatedAt = now
|
|
}
|
|
}
|
|
e.UpdatedAt = now
|
|
if e.Weight == 0 {
|
|
e.Weight = 1
|
|
}
|
|
s.edges[e.ID] = e
|
|
s.version++
|
|
}
|
|
func (s *Store) HasEdgeBetween(a, b string) bool {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
for _, e := range s.edges {
|
|
if (e.Source == a && e.Target == b) || (e.Source == b && e.Target == a) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func (s *Store) GetNode(id string) (model.Node, bool) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
n, ok := s.nodes[id]
|
|
return n, ok
|
|
}
|
|
func (s *Store) LookupExternal(id string) (model.Node, bool) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
for _, n := range s.nodes {
|
|
if strings.EqualFold(n.ExternalID, id) {
|
|
return n, true
|
|
}
|
|
}
|
|
return model.Node{}, false
|
|
}
|
|
func (s *Store) SetVector(id string, v []float64) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.vectors[id] = append([]float64(nil), v...)
|
|
}
|
|
func (s *Store) Vector(id string) ([]float64, bool) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
v, ok := s.vectors[id]
|
|
return append([]float64(nil), v...), ok
|
|
}
|
|
|
|
func (s *Store) ClearVectorsByDimension(dim int) int {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
removed := 0
|
|
for id, v := range s.vectors {
|
|
if len(v) == dim {
|
|
delete(s.vectors, id)
|
|
removed++
|
|
}
|
|
}
|
|
if removed > 0 {
|
|
s.version++
|
|
}
|
|
return removed
|
|
}
|
|
|
|
func (s *Store) NodesForEmbedding() []model.Node {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
out := []model.Node{}
|
|
for _, n := range s.nodes {
|
|
if n.Kind != "knowledge" && n.Kind != "ai-think" && n.Kind != "external" {
|
|
continue
|
|
}
|
|
if _, ok := s.vectors[n.ID]; !ok {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
|
return out
|
|
}
|
|
func (s *Store) KnowledgeNodes() []model.Node {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
out := []model.Node{}
|
|
for _, n := range s.nodes {
|
|
if n.Kind == "knowledge" || n.Kind == "ai-think" {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
func (s *Store) ReplaceOrigins(origins []string, nodes []model.Node, edges []model.Edge) {
|
|
set := map[string]bool{}
|
|
for _, o := range origins {
|
|
set[o] = true
|
|
}
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
oldVectors := make(map[string][]float64)
|
|
oldFingerprints := make(map[string]string)
|
|
for id, n := range s.nodes {
|
|
if set[n.Origin] {
|
|
if v, ok := s.vectors[id]; ok {
|
|
oldVectors[id] = append([]float64(nil), v...)
|
|
oldFingerprints[id] = n.Label + "\x00" + n.Summary + "\x00" + strings.Join(n.Categories, "\x00") + "\x00" + strings.Join(n.Keywords, "\x00")
|
|
}
|
|
delete(s.nodes, id)
|
|
delete(s.vectors, id)
|
|
}
|
|
}
|
|
for id, e := range s.edges {
|
|
if set[e.Origin] {
|
|
delete(s.edges, id)
|
|
}
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, n := range nodes {
|
|
if n.UpdatedAt.IsZero() {
|
|
n.UpdatedAt = now
|
|
}
|
|
if n.Weight == 0 {
|
|
n.Weight = 1
|
|
}
|
|
if n.X == 0 && n.Y == 0 && n.Z == 0 {
|
|
n.X, n.Y, n.Z = position(n.ID, n.Categories)
|
|
}
|
|
s.nodes[n.ID] = n
|
|
fingerprint := n.Label + "\x00" + n.Summary + "\x00" + strings.Join(n.Categories, "\x00") + "\x00" + strings.Join(n.Keywords, "\x00")
|
|
if fingerprint == oldFingerprints[n.ID] {
|
|
if v, ok := oldVectors[n.ID]; ok {
|
|
s.vectors[n.ID] = append([]float64(nil), v...)
|
|
}
|
|
}
|
|
}
|
|
for _, e := range edges {
|
|
if e.ID == "" {
|
|
e.ID = EdgeID(e.Source, e.Target, e.Type, e.Origin)
|
|
}
|
|
if e.CreatedAt.IsZero() {
|
|
e.CreatedAt = now
|
|
}
|
|
e.UpdatedAt = now
|
|
if e.Weight == 0 {
|
|
e.Weight = 1
|
|
}
|
|
s.edges[e.ID] = e
|
|
}
|
|
for id, e := range s.edges {
|
|
if _, ok := s.nodes[e.Source]; !ok {
|
|
delete(s.edges, id)
|
|
continue
|
|
}
|
|
if _, ok := s.nodes[e.Target]; !ok {
|
|
delete(s.edges, id)
|
|
}
|
|
}
|
|
s.version++
|
|
}
|
|
func (s *Store) Snapshot() model.Snapshot {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
n := make([]model.Node, 0, len(s.nodes))
|
|
e := make([]model.Edge, 0, len(s.edges))
|
|
for _, x := range s.nodes {
|
|
n = append(n, x)
|
|
}
|
|
for _, x := range s.edges {
|
|
if x.Status == "rejected" {
|
|
continue
|
|
}
|
|
e = append(e, x)
|
|
}
|
|
sort.Slice(n, func(i, j int) bool { return n[i].ID < n[j].ID })
|
|
sort.Slice(e, func(i, j int) bool { return e[i].ID < e[j].ID })
|
|
return model.Snapshot{Version: s.version, Nodes: n, Edges: e, UpdatedAt: time.Now().UTC()}
|
|
}
|
|
func (s *Store) Persist() error {
|
|
s.mu.RLock()
|
|
d := diskState{Version: s.version, Vectors: map[string][]float64{}}
|
|
for _, n := range s.nodes {
|
|
d.Nodes = append(d.Nodes, n)
|
|
}
|
|
for _, e := range s.edges {
|
|
d.Edges = append(d.Edges, e)
|
|
}
|
|
for k, v := range s.vectors {
|
|
d.Vectors[k] = v
|
|
}
|
|
s.mu.RUnlock()
|
|
b, err := json.Marshal(d)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmp := s.path + ".tmp"
|
|
if err = os.WriteFile(tmp, b, 0o640); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, s.path)
|
|
}
|
|
func (s *Store) Similar(query []float64, limit int) []model.Hit {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
hits := []model.Hit{}
|
|
for id, v := range s.vectors {
|
|
n, ok := s.nodes[id]
|
|
if !ok || (n.Kind != "knowledge" && n.Kind != "ai-think" && n.Kind != "external") {
|
|
continue
|
|
}
|
|
score := cosine(query, v)
|
|
hits = append(hits, model.Hit{NodeID: id, Label: n.Label, Score: score, Kind: n.Kind, Status: n.Status})
|
|
}
|
|
sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
|
|
if limit > 0 && len(hits) > limit {
|
|
hits = hits[:limit]
|
|
}
|
|
return hits
|
|
}
|
|
|
|
// NextPair searches a bounded rotating window of anchor nodes instead of
|
|
// comparing the complete graph on every AI-THINK cycle. This keeps candidate
|
|
// selection responsive even for tens of thousands of knowledge nodes while the
|
|
// rotating cursor eventually visits the complete corpus.
|
|
func (s *Store) NextPair(min float64, anchorLimit int) (model.Node, model.Node, float64, bool, int) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
nodes := make([]model.Node, 0, len(s.nodes))
|
|
for _, n := range s.nodes {
|
|
if n.Kind != "knowledge" && n.Kind != "ai-think" {
|
|
continue
|
|
}
|
|
if v, ok := s.vectors[n.ID]; ok && len(v) > 0 {
|
|
nodes = append(nodes, n)
|
|
}
|
|
}
|
|
if len(nodes) < 2 {
|
|
return model.Node{}, model.Node{}, 0, false, 0
|
|
}
|
|
sort.Slice(nodes, func(i, j int) bool { return nodes[i].ID < nodes[j].ID })
|
|
if anchorLimit <= 0 || anchorLimit > len(nodes) {
|
|
anchorLimit = len(nodes)
|
|
}
|
|
|
|
blocked := make(map[string]struct{}, len(s.edges))
|
|
for _, e := range s.edges {
|
|
blocked[pairKey(e.Source, e.Target)] = struct{}{}
|
|
}
|
|
|
|
start := s.pairCursor % len(nodes)
|
|
best := -1.0
|
|
var a, b model.Node
|
|
comparisons := 0
|
|
for step := 0; step < anchorLimit; step++ {
|
|
i := (start + step) % len(nodes)
|
|
left := nodes[i]
|
|
lv := s.vectors[left.ID]
|
|
for j := 0; j < len(nodes); j++ {
|
|
if i == j {
|
|
continue
|
|
}
|
|
right := nodes[j]
|
|
if _, exists := blocked[pairKey(left.ID, right.ID)]; exists {
|
|
continue
|
|
}
|
|
rv := s.vectors[right.ID]
|
|
if len(lv) != len(rv) {
|
|
continue
|
|
}
|
|
comparisons++
|
|
score := cosine(lv, rv)
|
|
if score >= min && score > best {
|
|
best = score
|
|
a, b = left, right
|
|
}
|
|
}
|
|
}
|
|
s.pairCursor = (start + anchorLimit) % len(nodes)
|
|
return a, b, best, best >= 0, comparisons
|
|
}
|
|
|
|
func pairKey(a, b string) string {
|
|
if a > b {
|
|
a, b = b, a
|
|
}
|
|
return a + "\x00" + b
|
|
}
|
|
|
|
func (s *Store) BestPair(min float64) (model.Node, model.Node, float64, bool) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
nodes := []model.Node{}
|
|
for _, n := range s.nodes {
|
|
if n.Kind == "knowledge" || n.Kind == "ai-think" {
|
|
if _, ok := s.vectors[n.ID]; ok {
|
|
nodes = append(nodes, n)
|
|
}
|
|
}
|
|
}
|
|
best := -1.0
|
|
var a, b model.Node
|
|
for i := 0; i < len(nodes); i++ {
|
|
for j := i + 1; j < len(nodes); j++ {
|
|
if edgeBetweenLocked(s.edges, nodes[i].ID, nodes[j].ID) {
|
|
continue
|
|
}
|
|
score := cosine(s.vectors[nodes[i].ID], s.vectors[nodes[j].ID])
|
|
if score >= min && score > best {
|
|
best = score
|
|
a = nodes[i]
|
|
b = nodes[j]
|
|
}
|
|
}
|
|
}
|
|
return a, b, best, best >= 0
|
|
}
|
|
func (s *Store) ConnectingEdges(ids []string) []string {
|
|
set := map[string]bool{}
|
|
for _, id := range ids {
|
|
set[id] = true
|
|
}
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
var out []string
|
|
for id, e := range s.edges {
|
|
if set[e.Source] && set[e.Target] {
|
|
out = append(out, id)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
func edgeBetweenLocked(edges map[string]model.Edge, a, b string) bool {
|
|
for _, e := range edges {
|
|
if (e.Source == a && e.Target == b) || (e.Source == b && e.Target == a) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func cosine(a, b []float64) float64 {
|
|
if len(a) == 0 || len(a) != len(b) {
|
|
return 0
|
|
}
|
|
var dot, aa, bb float64
|
|
for i := range a {
|
|
dot += a[i] * b[i]
|
|
aa += a[i] * a[i]
|
|
bb += b[i] * b[i]
|
|
}
|
|
if aa == 0 || bb == 0 {
|
|
return 0
|
|
}
|
|
return dot / (math.Sqrt(aa) * math.Sqrt(bb))
|
|
}
|
|
func position(id string, cats []string) (float64, float64, float64) {
|
|
seed := sha256.Sum256([]byte(id + "\x00" + strings.Join(cats, "|")))
|
|
u := func(i int) float64 { return float64(int(seed[i%len(seed)])) / 255 }
|
|
side := -1.0
|
|
if seed[0]%2 == 0 {
|
|
side = 1
|
|
}
|
|
biasY, biasZ := 0.0, 0.0
|
|
if len(cats) > 0 {
|
|
h := sha256.Sum256([]byte(cats[0]))
|
|
biasY = (float64(h[0])/255 - .5) * .9
|
|
biasZ = (float64(h[1])/255 - .5) * .65
|
|
}
|
|
for i := 0; i < 16; i++ {
|
|
x := side * (0.08 + u(1+i)*0.72)
|
|
y := biasY*.32 + (u(2+i)-.5)*1.18
|
|
z := biasZ*.28 + (u(3+i)-.5)*.94
|
|
if insideBrainShape(x, y, z) {
|
|
return x, y, z
|
|
}
|
|
}
|
|
return side * .34, biasY * .22, biasZ * .2
|
|
}
|
|
|
|
func insideBrainShape(x, y, z float64) bool {
|
|
if math.Abs(x) < .045 && y > -.58 && y < .42 {
|
|
return false
|
|
}
|
|
if y < -.76 || y > .82 {
|
|
return false
|
|
}
|
|
taperY := y + math.Abs(z)*.10 - math.Max(0, math.Abs(x)-.58)*.18
|
|
lx := (x + .35) / .58
|
|
rx := (x - .35) / .58
|
|
ny := taperY / .76
|
|
nz := z / .58
|
|
left := lx*lx+ny*ny+nz*nz <= 1
|
|
right := rx*rx+ny*ny+nz*nz <= 1
|
|
return left || right
|
|
}
|
|
|
|
func (s *Store) Analyze() model.GraphAnalysis {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
analysis := model.GraphAnalysis{NodeCount: len(s.nodes)}
|
|
degree := make(map[string]int, len(s.nodes))
|
|
knowledgeLinked := make(map[string]bool)
|
|
parent := make(map[string]string, len(s.nodes))
|
|
for id, n := range s.nodes {
|
|
parent[id] = id
|
|
if n.Status == "staging" {
|
|
analysis.StagingNodes++
|
|
}
|
|
if n.Kind == "ai-think" {
|
|
analysis.AIThinkNodes++
|
|
}
|
|
if n.Kind == "external" {
|
|
analysis.ExternalNodes++
|
|
}
|
|
}
|
|
var find func(string) string
|
|
find = func(x string) string {
|
|
p := parent[x]
|
|
if p != x {
|
|
parent[x] = find(p)
|
|
}
|
|
return parent[x]
|
|
}
|
|
union := func(a, b string) {
|
|
ra, rb := find(a), find(b)
|
|
if ra != rb {
|
|
parent[rb] = ra
|
|
}
|
|
}
|
|
for _, e := range s.edges {
|
|
if e.Status == "rejected" {
|
|
continue
|
|
}
|
|
if _, ok := s.nodes[e.Source]; !ok {
|
|
continue
|
|
}
|
|
if _, ok := s.nodes[e.Target]; !ok {
|
|
continue
|
|
}
|
|
analysis.EdgeCount++
|
|
degree[e.Source]++
|
|
degree[e.Target]++
|
|
union(e.Source, e.Target)
|
|
if e.Origin == "ai-inference" {
|
|
analysis.AIEdges++
|
|
}
|
|
if e.Type == "contradicts" {
|
|
analysis.Contradictions++
|
|
}
|
|
a, b := s.nodes[e.Source], s.nodes[e.Target]
|
|
if (a.Kind == "knowledge" || a.Kind == "ai-think") && (b.Kind == "knowledge" || b.Kind == "ai-think" || b.Kind == "external") {
|
|
knowledgeLinked[a.ID] = true
|
|
if b.Kind != "external" {
|
|
knowledgeLinked[b.ID] = true
|
|
}
|
|
}
|
|
}
|
|
roots := map[string]bool{}
|
|
for id, n := range s.nodes {
|
|
roots[find(id)] = true
|
|
if (n.Kind == "knowledge" || n.Kind == "ai-think") && !knowledgeLinked[id] {
|
|
analysis.KnowledgeOrphans++
|
|
}
|
|
}
|
|
analysis.Components = len(roots)
|
|
hubs := make([]model.Hub, 0, len(degree))
|
|
for id, d := range degree {
|
|
n := s.nodes[id]
|
|
hubs = append(hubs, model.Hub{NodeID: id, Label: n.Label, Kind: n.Kind, Degree: d})
|
|
}
|
|
sort.Slice(hubs, func(i, j int) bool {
|
|
if hubs[i].Degree == hubs[j].Degree {
|
|
return hubs[i].Label < hubs[j].Label
|
|
}
|
|
return hubs[i].Degree > hubs[j].Degree
|
|
})
|
|
if len(hubs) > 8 {
|
|
hubs = hubs[:8]
|
|
}
|
|
analysis.TopHubs = hubs
|
|
return analysis
|
|
}
|