jbergner
440423c5b6
All checks were successful
release-tag / release-image (push) Successful in 2m43s
275 lines
8.6 KiB
Go
275 lines
8.6 KiB
Go
package engine
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
|
||
"github.com/local/glpi-neural-brain/internal/model"
|
||
)
|
||
|
||
func clusterPendingArticleCandidates(values []*pendingArticleCandidate) [][]*pendingArticleCandidate {
|
||
clusters := make([][]*pendingArticleCandidate, 0)
|
||
for _, value := range values {
|
||
if value == nil || len(value.Seeds) == 0 {
|
||
continue
|
||
}
|
||
placed := false
|
||
for i := range clusters {
|
||
for _, existing := range clusters[i] {
|
||
if articleCandidatesBelongTogether(existing, value) {
|
||
clusters[i] = append(clusters[i], value)
|
||
placed = true
|
||
break
|
||
}
|
||
}
|
||
if placed {
|
||
break
|
||
}
|
||
}
|
||
if !placed {
|
||
clusters = append(clusters, []*pendingArticleCandidate{value})
|
||
}
|
||
}
|
||
return clusters
|
||
}
|
||
|
||
func articleCandidatesBelongTogether(a, b *pendingArticleCandidate) bool {
|
||
if a == nil || b == nil {
|
||
return false
|
||
}
|
||
// Article batching is deliberately stricter than semantic candidate search.
|
||
// Knowledge articles share a lot of security boilerplate (hardening, detection,
|
||
// forensics, incident response), so generic terms/categories must never be
|
||
// sufficient to merge distinct topics such as Water Leak Detection, BGP
|
||
// Prefix Filtering and Triple Extortion into one author context.
|
||
topicA := articleCandidateTopicTerms(a)
|
||
topicB := articleCandidateTopicTerms(b)
|
||
if len(topicA) == 0 || len(topicB) == 0 {
|
||
return false
|
||
}
|
||
intersection := boolSetIntersectionSize(topicA, topicB)
|
||
if intersection == 0 {
|
||
return false
|
||
}
|
||
jaccard := boolSetJaccard(topicA, topicB)
|
||
minSize := len(topicA)
|
||
if len(topicB) < minSize {
|
||
minSize = len(topicB)
|
||
}
|
||
containment := float64(intersection) / float64(minSize)
|
||
if jaccard >= .50 || containment >= .67 {
|
||
return true
|
||
}
|
||
|
||
// A shared seed is useful only when that seed is a real topical anchor for
|
||
// both relations. Merely sharing a broad/generic node must not override the
|
||
// topic guard. This keeps legitimate multi-perspective articles together
|
||
// while preventing one hub-like source from joining unrelated subjects.
|
||
shared := map[string]model.Node{}
|
||
for _, seed := range a.Seeds {
|
||
shared[seed.ID] = seed
|
||
}
|
||
for _, seed := range b.Seeds {
|
||
anchor, ok := shared[seed.ID]
|
||
if !ok {
|
||
continue
|
||
}
|
||
anchorTerms := articleTopicTermsFromText(anchor.Label)
|
||
if boolSetIntersectionSize(anchorTerms, topicA) > 0 && boolSetIntersectionSize(anchorTerms, topicB) > 0 {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
var articleTopicStopwords = map[string]bool{
|
||
"security": true, "sicherheit": true, "sicher": true, "sichere": true, "sicherheitsmassnahmen": true, "sicherheitsmaßnahmen": true,
|
||
"hardening": true, "haertung": true, "härtung": true, "haerten": true, "härten": true, "gestalten": true, "praeventiv": true, "präventiv": true,
|
||
"resilient": true, "ueberwachen": true, "überwachen": true, "erkennen": true, "untersuchen": true, "forensisch": true, "forensik": true,
|
||
"incident": true, "vorfall": true, "vorfaelle": true, "vorfälle": true, "eindaemmen": true, "eindämmen": true, "wiederherstellen": true,
|
||
"risiko": true, "risiken": true, "schutz": true, "massnahmen": true, "maßnahmen": true, "konfiguration": true,
|
||
"und": true, "oder": true, "der": true, "die": true, "das": true, "den": true, "des": true, "von": true, "bei": true, "mit": true, "fuer": true, "für": true,
|
||
}
|
||
|
||
func articleCandidateTopicTerms(value *pendingArticleCandidate) map[string]bool {
|
||
if value == nil {
|
||
return nil
|
||
}
|
||
parts := make([]string, 0, 1+len(value.Seeds))
|
||
if topic := strings.TrimSpace(value.Relation.TopicLabel); topic != "" {
|
||
parts = append(parts, topic)
|
||
}
|
||
if len(parts) == 0 {
|
||
for _, seed := range value.Seeds {
|
||
label := strings.TrimSpace(seed.Label)
|
||
if idx := strings.Index(label, " – "); idx > 0 {
|
||
label = label[:idx]
|
||
} else if idx := strings.Index(label, " - "); idx > 0 {
|
||
label = label[:idx]
|
||
}
|
||
parts = append(parts, label)
|
||
}
|
||
}
|
||
return articleTopicTermsFromText(strings.Join(parts, " "))
|
||
}
|
||
|
||
func articleTopicTermsFromText(text string) map[string]bool {
|
||
// Source titles in the KB commonly use a stable topical prefix followed by
|
||
// a templated perspective such as " – sicher planen und durchführen". The
|
||
// suffix must not participate in topic coherence: otherwise two unrelated
|
||
// articles can look related merely because they share the same template.
|
||
text = articleTopicCore(text)
|
||
terms := researchTerms(text)
|
||
for term := range terms {
|
||
if articleTopicStopwords[term] || len([]rune(term)) < 2 {
|
||
delete(terms, term)
|
||
}
|
||
}
|
||
return terms
|
||
}
|
||
|
||
func articleTopicCore(text string) string {
|
||
text = strings.TrimSpace(text)
|
||
for _, separator := range []string{" – ", " - "} {
|
||
if idx := strings.Index(text, separator); idx > 0 {
|
||
return strings.TrimSpace(text[:idx])
|
||
}
|
||
}
|
||
return text
|
||
}
|
||
|
||
func articleTopicSetsBelongTogether(a, b map[string]bool) bool {
|
||
if len(a) == 0 || len(b) == 0 {
|
||
return false
|
||
}
|
||
intersection := boolSetIntersectionSize(a, b)
|
||
if intersection == 0 {
|
||
return false
|
||
}
|
||
minSize := len(a)
|
||
if len(b) < minSize {
|
||
minSize = len(b)
|
||
}
|
||
containment := float64(intersection) / float64(minSize)
|
||
return boolSetJaccard(a, b) >= .50 || containment >= .67
|
||
}
|
||
|
||
func boolSetIntersectionSize(a, b map[string]bool) int {
|
||
n := 0
|
||
for key := range a {
|
||
if b[key] {
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
func boolSetJaccard(a, b map[string]bool) float64 {
|
||
if len(a) == 0 || len(b) == 0 {
|
||
return 0
|
||
}
|
||
intersection := 0
|
||
union := map[string]bool{}
|
||
for key := range a {
|
||
union[key] = true
|
||
if b[key] {
|
||
intersection++
|
||
}
|
||
}
|
||
for key := range b {
|
||
union[key] = true
|
||
}
|
||
if len(union) == 0 {
|
||
return 0
|
||
}
|
||
return float64(intersection) / float64(len(union))
|
||
}
|
||
|
||
func combinePendingArticleCluster(cluster []*pendingArticleCandidate) ([]model.Node, model.RelationDecision, []model.ResearchResult) {
|
||
seedByID := map[string]model.Node{}
|
||
keywords := []string{}
|
||
topics := []string{}
|
||
relationTypes := map[string]int{}
|
||
confidenceSum := 0.0
|
||
count := 0
|
||
researchResults := []model.ResearchResult{}
|
||
for _, item := range cluster {
|
||
if item == nil {
|
||
continue
|
||
}
|
||
for _, seed := range item.Seeds {
|
||
seedByID[seed.ID] = seed
|
||
}
|
||
keywords = append(keywords, item.Relation.Keywords...)
|
||
if topic := strings.TrimSpace(item.Relation.TopicLabel); topic != "" {
|
||
topics = append(topics, topic)
|
||
}
|
||
if rel := safeRelation(item.Relation.RelationType); rel != "" {
|
||
relationTypes[rel]++
|
||
}
|
||
confidenceSum += item.Relation.Confidence
|
||
count++
|
||
researchResults = append(researchResults, item.Research...)
|
||
}
|
||
seeds := make([]model.Node, 0, len(seedByID))
|
||
for _, seed := range seedByID {
|
||
seeds = append(seeds, seed)
|
||
}
|
||
topic := ""
|
||
if len(topics) > 0 {
|
||
topic = topics[0]
|
||
}
|
||
relationType := "same_topic"
|
||
maxCount := 0
|
||
for rel, n := range relationTypes {
|
||
if n > maxCount {
|
||
relationType, maxCount = rel, n
|
||
}
|
||
}
|
||
confidence := .8
|
||
if count > 0 {
|
||
confidence = confidenceSum / float64(count)
|
||
}
|
||
decision := model.RelationDecision{
|
||
Related: true,
|
||
RelationType: relationType,
|
||
Confidence: confidence,
|
||
TopicLabel: topic,
|
||
Keywords: unique(keywords),
|
||
Explanation: fmt.Sprintf("%d thematisch kompatible neue Relationen wurden für einen gemeinsamen Artikelauftrag gebündelt.", count),
|
||
}
|
||
return seeds, decision, uniqueResearchEvidence(researchResults)
|
||
}
|
||
|
||
func (e *Engine) synthesizePendingArticleClusters(ctx context.Context, trigger string, pending []*pendingArticleCandidate) (created, skipped int) {
|
||
clusters := clusterPendingArticleCandidates(pending)
|
||
for index, cluster := range clusters {
|
||
seeds, relation, researchResults := combinePendingArticleCluster(cluster)
|
||
if len(seeds) == 0 {
|
||
continue
|
||
}
|
||
topicLabels := make([]string, 0, len(cluster))
|
||
for _, candidate := range cluster {
|
||
if candidate != nil && strings.TrimSpace(candidate.Relation.TopicLabel) != "" {
|
||
topicLabels = append(topicLabels, strings.TrimSpace(candidate.Relation.TopicLabel))
|
||
}
|
||
}
|
||
e.Broker.Publish(model.Activity{Type: "article.cluster.started", Source: "brain", Phase: "knowledge-planning", NodeIDs: nodeIDsFromNodes(seeds), Message: fmt.Sprintf("%d Relation(en) werden als gemeinsamer Artikelauftrag verarbeitet", len(cluster)), Strength: .62, Metadata: map[string]any{"trigger": trigger, "cluster_index": index + 1, "relation_count": len(cluster), "seed_count": len(seeds), "processing_mode": "clustered", "topic_guard": "strict-v2", "topic_labels": unique(topicLabels)}})
|
||
outcome, err := e.synthesizeKnowledgeArticle(ctx, trigger, seeds, relation, researchResults)
|
||
if err != nil {
|
||
skipped++
|
||
// synthesizeKnowledgeArticle owns the terminal article.failed event and
|
||
// its native run_id. Do not publish a second orphan terminal here.
|
||
continue
|
||
}
|
||
if outcome.Created {
|
||
created++
|
||
}
|
||
if outcome.Skipped {
|
||
skipped++
|
||
}
|
||
}
|
||
return created, skipped
|
||
}
|