+24
-73
@@ -8,96 +8,47 @@ import (
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
UncategorizedFilter = "__uncategorized__"
|
||||
UnsourcedFilter = "__unsourced__"
|
||||
)
|
||||
|
||||
// NodeFilter limits knowledge-bearing nodes by category and logical source.
|
||||
// Categories and Sources are AND-combined; values inside each dimension use
|
||||
// OR semantics. Empty dimensions and "*" mean unrestricted. MatchNone is used
|
||||
// for an empty administrative/runtime intersection and deliberately matches no
|
||||
// node.
|
||||
// NodeFilter limits knowledge-bearing nodes by the exact value stored in the
|
||||
// KB document's source field. Values are compared case-sensitively after
|
||||
// trimming surrounding whitespace. An empty Sources list means unrestricted.
|
||||
// No category, origin, URI or taxonomy fallback participates in matching.
|
||||
type NodeFilter struct {
|
||||
Categories []string
|
||||
Sources []string
|
||||
MatchNone bool
|
||||
Sources []string
|
||||
}
|
||||
|
||||
func (f NodeFilter) Matches(node model.Node) bool {
|
||||
if f.MatchNone {
|
||||
return false
|
||||
}
|
||||
return matchesDimension(node.Categories, f.Categories, UncategorizedFilter) &&
|
||||
matchesDimension(nodeSources(node), f.Sources, UnsourcedFilter)
|
||||
}
|
||||
|
||||
func matchesDimension(values, filters []string, emptyToken string) bool {
|
||||
if len(filters) == 0 {
|
||||
if len(f.Sources) == 0 {
|
||||
return true
|
||||
}
|
||||
wanted := make(map[string]struct{}, len(filters))
|
||||
for _, filter := range filters {
|
||||
filter = strings.ToLower(strings.TrimSpace(filter))
|
||||
if filter != "" {
|
||||
wanted[filter] = struct{}{}
|
||||
source := strings.TrimSpace(NodeSource(node))
|
||||
for _, wanted := range f.Sources {
|
||||
wanted = strings.TrimSpace(wanted)
|
||||
if wanted == "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(wanted) == 0 {
|
||||
return true
|
||||
}
|
||||
if _, ok := wanted["*"]; ok {
|
||||
return true
|
||||
}
|
||||
clean := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
clean = append(clean, value)
|
||||
}
|
||||
}
|
||||
if len(clean) == 0 {
|
||||
_, ok := wanted[emptyToken]
|
||||
return ok
|
||||
}
|
||||
for _, value := range clean {
|
||||
if _, ok := wanted[strings.ToLower(value)]; ok {
|
||||
if source == wanted {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// NodeSource returns the logical source shown to users. Explicit source
|
||||
// metadata wins. Research nodes fall back to their host name; all other nodes
|
||||
// fall back to their technical origin.
|
||||
// NodeSource returns only the explicit source metadata copied from the KB
|
||||
// file's top-level source field (or assigned explicitly to generated research
|
||||
// evidence). Technical origins such as glpi-kb are deliberately ignored.
|
||||
func NodeSource(node model.Node) string {
|
||||
values := nodeSources(node)
|
||||
if len(values) == 0 {
|
||||
return ""
|
||||
}
|
||||
return values[0]
|
||||
return metadataText(node.Metadata, "source")
|
||||
}
|
||||
|
||||
func nodeSources(node model.Node) []string {
|
||||
if source := metadataText(node.Metadata, "source"); source != "" {
|
||||
return []string{source}
|
||||
// SourceFromURL derives the explicit source value used for learned web
|
||||
// evidence. It is kept separate from NodeSource so ordinary nodes never fall
|
||||
// back to their URI implicitly.
|
||||
func SourceFromURL(raw string) string {
|
||||
parsed, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if node.Kind == "source" && strings.TrimSpace(node.Label) != "" {
|
||||
return []string{strings.TrimSpace(node.Label)}
|
||||
}
|
||||
if node.Kind == "external" {
|
||||
for _, raw := range []string{node.URI, node.ExternalID} {
|
||||
parsed, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err == nil && strings.TrimSpace(parsed.Hostname()) != "" {
|
||||
return []string{strings.ToLower(strings.TrimSpace(parsed.Hostname()))}
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(node.Origin) != "" {
|
||||
return []string{strings.TrimSpace(node.Origin)}
|
||||
}
|
||||
return nil
|
||||
return strings.ToLower(strings.TrimSpace(parsed.Hostname()))
|
||||
}
|
||||
|
||||
func metadataText(metadata map[string]any, key string) string {
|
||||
|
||||
@@ -6,38 +6,41 @@ import (
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
)
|
||||
|
||||
func TestNodeFilterCombinesCategoryAndSource(t *testing.T) {
|
||||
func TestNodeFilterMatchesExactSourceOnly(t *testing.T) {
|
||||
node := model.Node{
|
||||
Kind: "knowledge",
|
||||
Categories: []string{"IT-Security", "Cloud"},
|
||||
Origin: "knowledge-production",
|
||||
Origin: "glpi-kb",
|
||||
Metadata: map[string]any{"source": "GLPI Knowledge Base"},
|
||||
}
|
||||
if !(NodeFilter{Categories: []string{"Cloud"}, Sources: []string{"glpi knowledge base"}}).Matches(node) {
|
||||
t.Fatal("matching category and source should be accepted case-insensitively")
|
||||
if !(NodeFilter{Sources: []string{"GLPI Knowledge Base"}}).Matches(node) {
|
||||
t.Fatal("exact source should match")
|
||||
}
|
||||
if (NodeFilter{Categories: []string{"Cloud"}, Sources: []string{"internal-kb"}}).Matches(node) {
|
||||
t.Fatal("source mismatch must reject even when category matches")
|
||||
if (NodeFilter{Sources: []string{"glpi knowledge base"}}).Matches(node) {
|
||||
t.Fatal("source matching must be case-sensitive and exact")
|
||||
}
|
||||
if (NodeFilter{Categories: []string{"Backup"}, Sources: []string{"GLPI Knowledge Base"}}).Matches(node) {
|
||||
t.Fatal("category mismatch must reject even when source matches")
|
||||
if (NodeFilter{Sources: []string{"glpi-kb"}}).Matches(node) {
|
||||
t.Fatal("technical origin must not be treated as source")
|
||||
}
|
||||
if (NodeFilter{MatchNone: true}).Matches(node) {
|
||||
t.Fatal("MatchNone must reject every node")
|
||||
if !(NodeFilter{}).Matches(node) {
|
||||
t.Fatal("empty source list must be unrestricted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFilterVirtualEmptyValuesAndResearchHost(t *testing.T) {
|
||||
unsourced := model.Node{Kind: "knowledge"}
|
||||
if !(NodeFilter{Categories: []string{UncategorizedFilter}, Sources: []string{UnsourcedFilter}}).Matches(unsourced) {
|
||||
t.Fatal("virtual empty category/source filters should match")
|
||||
func TestNodeFilterUnsourcedAndNoFallback(t *testing.T) {
|
||||
unsourced := model.Node{Kind: "knowledge", Origin: "internal-category", URI: "https://example.test/a"}
|
||||
if NodeSource(unsourced) != "" {
|
||||
t.Fatalf("origin or URI leaked into source: %q", NodeSource(unsourced))
|
||||
}
|
||||
research := model.Node{Kind: "external", Origin: "research", URI: "https://docs.example.org/guide", Categories: []string{"Cloud"}}
|
||||
if (NodeFilter{Sources: []string{"internal-category"}}).Matches(unsourced) {
|
||||
t.Fatal("node without source must not match a concrete source")
|
||||
}
|
||||
research := model.Node{Kind: "external", Origin: "research", URI: "https://docs.example.org/guide", Metadata: map[string]any{"source": SourceFromURL("https://docs.example.org/guide")}}
|
||||
if got := NodeSource(research); got != "docs.example.org" {
|
||||
t.Fatalf("unexpected research source %q", got)
|
||||
t.Fatalf("unexpected explicit research source %q", got)
|
||||
}
|
||||
if !(NodeFilter{Sources: []string{"DOCS.EXAMPLE.ORG"}}).Matches(research) {
|
||||
t.Fatal("research host should be source-filterable")
|
||||
if !(NodeFilter{Sources: []string{"docs.example.org"}}).Matches(research) {
|
||||
t.Fatal("explicit research source should be filterable")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +48,7 @@ func TestScopedEmbeddingRetrievalAndThinking(t *testing.T) {
|
||||
s := &Store{
|
||||
nodes: map[string]model.Node{
|
||||
"a": {ID: "a", Kind: "knowledge", Label: "A", Categories: []string{"Cloud"}, Metadata: map[string]any{"source": "internal-kb"}},
|
||||
"b": {ID: "b", Kind: "knowledge", Label: "B", Categories: []string{"Cloud"}, Metadata: map[string]any{"source": "internal-kb"}},
|
||||
"b": {ID: "b", Kind: "knowledge", Label: "B", Categories: []string{"Backup"}, Metadata: map[string]any{"source": "internal-kb"}},
|
||||
"c": {ID: "c", Kind: "knowledge", Label: "C", Categories: []string{"Cloud"}, Metadata: map[string]any{"source": "GLPI Knowledge Base"}},
|
||||
},
|
||||
edges: map[string]model.Edge{},
|
||||
@@ -57,14 +60,14 @@ func TestScopedEmbeddingRetrievalAndThinking(t *testing.T) {
|
||||
deletedEdges: map[string]uint64{},
|
||||
deletedVectors: map[string]uint64{},
|
||||
}
|
||||
filter := NodeFilter{Categories: []string{"Cloud"}, Sources: []string{"internal-kb"}}
|
||||
filter := NodeFilter{Sources: []string{"internal-kb"}}
|
||||
hits := s.SimilarFiltered([]float64{1, 0}, 10, filter)
|
||||
if len(hits) != 2 {
|
||||
t.Fatalf("retrieval escaped source filter: %+v", hits)
|
||||
t.Fatalf("retrieval escaped exact source filter: %+v", hits)
|
||||
}
|
||||
a, b, _, ok, _ := s.NextPairScopedDepth(.5, 8, filter, 0)
|
||||
if !ok || NodeSource(a) != "internal-kb" || NodeSource(b) != "internal-kb" {
|
||||
t.Fatalf("thinking escaped scoped filter: ok=%v a=%+v b=%+v", ok, a, b)
|
||||
t.Fatalf("thinking escaped source filter: ok=%v a=%+v b=%+v", ok, a, b)
|
||||
}
|
||||
if pending := s.NodesForEmbeddingScoped(NodeFilter{Sources: []string{"GLPI Knowledge Base"}}); len(pending) != 0 {
|
||||
t.Fatalf("nodes with existing vectors should not be pending: %+v", pending)
|
||||
|
||||
@@ -235,8 +235,8 @@ func (s *Store) NodesForEmbedding() []model.Node {
|
||||
return s.NodesForEmbeddingScoped(NodeFilter{})
|
||||
}
|
||||
|
||||
func (s *Store) NodesForEmbeddingFiltered(categories []string) []model.Node {
|
||||
return s.NodesForEmbeddingScoped(NodeFilter{Categories: categories})
|
||||
func (s *Store) NodesForEmbeddingFiltered(sources []string) []model.Node {
|
||||
return s.NodesForEmbeddingScoped(NodeFilter{Sources: sources})
|
||||
}
|
||||
|
||||
func (s *Store) NodesForEmbeddingScoped(filter NodeFilter) []model.Node {
|
||||
@@ -547,12 +547,12 @@ func (s *Store) NextPair(min float64, anchorLimit int) (model.Node, model.Node,
|
||||
return s.NextPairFiltered(min, anchorLimit, nil)
|
||||
}
|
||||
|
||||
func (s *Store) NextPairFiltered(min float64, anchorLimit int, categories []string) (model.Node, model.Node, float64, bool, int) {
|
||||
return s.NextPairFilteredDepth(min, anchorLimit, categories, 0)
|
||||
func (s *Store) NextPairFiltered(min float64, anchorLimit int, sources []string) (model.Node, model.Node, float64, bool, int) {
|
||||
return s.NextPairFilteredDepth(min, anchorLimit, sources, 0)
|
||||
}
|
||||
|
||||
func (s *Store) NextPairFilteredDepth(min float64, anchorLimit int, categories []string, maxAIDepth int) (model.Node, model.Node, float64, bool, int) {
|
||||
return s.NextPairScopedDepth(min, anchorLimit, NodeFilter{Categories: categories}, maxAIDepth)
|
||||
func (s *Store) NextPairFilteredDepth(min float64, anchorLimit int, sources []string, maxAIDepth int) (model.Node, model.Node, float64, bool, int) {
|
||||
return s.NextPairScopedDepth(min, anchorLimit, NodeFilter{Sources: sources}, maxAIDepth)
|
||||
}
|
||||
|
||||
func (s *Store) NextPairScopedDepth(min float64, anchorLimit int, filter NodeFilter, maxAIDepth int) (model.Node, model.Node, float64, bool, int) {
|
||||
|
||||
@@ -73,34 +73,30 @@ func TestNextPairUsesBoundedRotatingAnchors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCategoryFiltersLimitEmbeddingAndThinkingCandidates(t *testing.T) {
|
||||
func TestSourceFiltersLimitEmbeddingAndThinkingCandidates(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
nodes := []model.Node{
|
||||
{ID: "net-a", Kind: "knowledge", Label: "Net A", Categories: []string{"Netzwerk"}, Origin: "test"},
|
||||
{ID: "net-b", Kind: "knowledge", Label: "Net B", Categories: []string{"Netzwerk"}, Origin: "test"},
|
||||
{ID: "app-a", Kind: "knowledge", Label: "App A", Categories: []string{"Applikation"}, Origin: "test"},
|
||||
{ID: "internal-a", Kind: "knowledge", Label: "Internal A", Categories: []string{"Netzwerk"}, Origin: "test", Metadata: map[string]any{"source": "internal-category"}},
|
||||
{ID: "internal-b", Kind: "knowledge", Label: "Internal B", Categories: []string{"Applikation"}, Origin: "test", Metadata: map[string]any{"source": "internal-category"}},
|
||||
{ID: "glpi-a", Kind: "knowledge", Label: "GLPI A", Categories: []string{"Netzwerk"}, Origin: "glpi-kb", Metadata: map[string]any{"source": "GLPI Knowledge Base"}},
|
||||
{ID: "none", Kind: "knowledge", Label: "Ohne", Origin: "test"},
|
||||
}
|
||||
for _, node := range nodes {
|
||||
s.UpsertNode(node)
|
||||
}
|
||||
pending := s.NodesForEmbeddingFiltered([]string{"Netzwerk"})
|
||||
pending := s.NodesForEmbeddingFiltered([]string{"internal-category"})
|
||||
if len(pending) != 2 {
|
||||
t.Fatalf("expected two network embeddings, got %d", len(pending))
|
||||
}
|
||||
uncategorized := s.NodesForEmbeddingFiltered([]string{"__uncategorized__"})
|
||||
if len(uncategorized) != 1 || uncategorized[0].ID != "none" {
|
||||
t.Fatalf("unexpected uncategorized nodes: %#v", uncategorized)
|
||||
t.Fatalf("expected two internal source embeddings, got %d", len(pending))
|
||||
}
|
||||
for _, node := range nodes {
|
||||
s.SetVector(node.ID, []float64{1, .01})
|
||||
}
|
||||
a, b, _, ok, _ := s.NextPairFiltered(.5, 8, []string{"Netzwerk"})
|
||||
if !ok || a.Categories[0] != "Netzwerk" || b.Categories[0] != "Netzwerk" {
|
||||
t.Fatalf("thinking filter returned wrong pair: ok=%v a=%+v b=%+v", ok, a, b)
|
||||
a, b, _, ok, _ := s.NextPairFiltered(.5, 8, []string{"internal-category"})
|
||||
if !ok || NodeSource(a) != "internal-category" || NodeSource(b) != "internal-category" {
|
||||
t.Fatalf("thinking source filter returned wrong pair: ok=%v a=%+v b=%+v", ok, a, b)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user