glpi-neural-brain/internal/engine/research_work.go

package engine

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/local/glpi-neural-brain/internal/model"
	"github.com/local/glpi-neural-brain/internal/ollama"
)

type researchIntentProfile struct {
	Primary  string
	Focus    []string
	Entities []string
}

type researchDedupeEntry struct {
	ID        string
	Kind      string
	Intent    string
	RawIntent string
	Profile   researchIntentProfile
	Vector    []float64
	Created   time.Time
	Finished  time.Time
	InFlight  bool
	Done      chan struct{}
	Results   []model.ResearchResult
}

type researchIntentLease struct {
	entry          *researchDedupeEntry
	owner          bool
	similarity     float64
	requestProfile researchIntentProfile
}

func (e *Engine) withSharedResearchWork(ctx context.Context, kind string, fn func() error) error {
	if e.sharedWork == nil {
		return fn()
	}
	release, err := e.sharedWork.AcquireKind(ctx, kind)
	if err != nil {
		if e.Broker != nil {
			e.Broker.Publish(model.Activity{Type: "work.queue.rejected", Source: "brain", Phase: "queue", Message: "Gemeinsame Research/Ollama-Queue ist ausgelastet", Strength: .3, Metadata: map[string]any{"kind": kind, "error": err.Error(), "queue": e.sharedWork.Status()}})
		}
		return err
	}
	defer release()
	return fn()
}

func (e *Engine) beginResearchIntent(ctx context.Context, kind, intent string) (researchIntentLease, []model.ResearchResult, error) {
	return e.beginResearchIntentMode(ctx, kind, intent, false)
}

// beginFreshResearchIntent intentionally bypasses semantic reuse. It is used when
// a semantically similar cached research run exists but its evidence fails the
// stricter target-question revalidation. Keeping the old cache entry is useful for
// its original intent while the new run becomes the newest exact candidate.
func (e *Engine) beginFreshResearchIntent(ctx context.Context, kind, intent string) (researchIntentLease, error) {
	lease, _, err := e.beginResearchIntentMode(ctx, kind, intent, true)
	return lease, err
}

func (e *Engine) beginResearchIntentMode(ctx context.Context, kind, rawIntent string, forceOwner bool) (researchIntentLease, []model.ResearchResult, error) {
	kind = strings.ToLower(strings.TrimSpace(kind))
	if kind == "" {
		kind = "evidence"
	}
	rawIntent = strings.TrimSpace(rawIntent)
	intent := normalizeResearchIntent(rawIntent)
	profile := classifyResearchIntent(rawIntent)
	if intent == "" {
		return researchIntentLease{owner: true, requestProfile: profile}, nil, nil
	}

	var vector []float64
	if e.Ollama != nil {
		// Embed the natural-language intent rather than the sorted normalization.
		// Word order and phrases such as "volatile evidence before reboot" carry
		// information that is lost by a bag-of-terms representation.
		vectors, err := e.Ollama.Embed(ollama.WithLowPriority(ctx), []string{rawIntent})
		if err == nil && len(vectors) == 1 {
			vector = vectors[0]
		}
	}

	now := time.Now().UTC()
	ttl := e.Cfg.ResearchDedupeTTL
	if ttl <= 0 {
		ttl = 45 * time.Minute
	}
	threshold := e.Cfg.ResearchDedupeThreshold
	if threshold <= 0 {
		threshold = .92
	}

	e.researchDedupeMu.Lock()
	if e.researchDedupe == nil {
		e.researchDedupe = map[string]*researchDedupeEntry{}
	}
	for id, entry := range e.researchDedupe {
		if !entry.InFlight && !entry.Finished.IsZero() && now.Sub(entry.Finished) > ttl {
			delete(e.researchDedupe, id)
		}
	}

	if !forceOwner {
		var best *researchDedupeEntry
		bestSimilarity := 0.0
		for _, entry := range e.researchDedupe {
			if entry.Kind != kind {
				continue
			}
			compatible, guardReason := researchIntentProfilesCompatible(profile, entry.Profile)
			if !compatible {
				e.researchDedupeGuardFiltered++
				switch guardReason {
				case "primary_intent_mismatch":
					e.researchDedupeGuardPrimaryMismatch++
				case "technical_focus_mismatch":
					e.researchDedupeGuardFocusMismatch++
				case "core_entity_mismatch":
					e.researchDedupeGuardEntityMismatch++
				}
				continue
			}
			similarity := researchIntentSimilarity(intent, vector, entry.Intent, entry.Vector)
			if similarity > bestSimilarity || (similarity == bestSimilarity && best != nil && entry.Created.After(best.Created)) {
				bestSimilarity = similarity
				best = entry
			}
		}
		if best != nil && bestSimilarity >= threshold {
			done := best.Done
			inFlight := best.InFlight
			e.researchDedupeMu.Unlock()
			if inFlight {
				select {
				case <-done:
				case <-ctx.Done():
					return researchIntentLease{}, nil, ctx.Err()
				}
			}
			e.researchDedupeMu.Lock()
			current, stillCached := e.researchDedupe[best.ID]
			if !stillCached {
				e.researchDedupeMu.Unlock()
				// The owner failed and removed its cache entry. Retry as a new
				// contender instead of treating a failed duplicate as an empty
				// successful research result.
				return e.beginResearchIntentMode(ctx, kind, rawIntent, false)
			}
			results := cloneResearchResults(current.Results)
			e.researchDedupeMu.Unlock()
			return researchIntentLease{entry: current, owner: false, similarity: bestSimilarity, requestProfile: profile}, results, nil
		}
	}

	id := newResearchRunID("research-intent", rawIntent+"\x00"+now.Format(time.RFC3339Nano))
	entry := &researchDedupeEntry{ID: id, Kind: kind, Intent: intent, RawIntent: rawIntent, Profile: profile, Vector: append([]float64(nil), vector...), Created: now, InFlight: true, Done: make(chan struct{})}
	e.researchDedupe[id] = entry
	e.researchDedupeMu.Unlock()
	return researchIntentLease{entry: entry, owner: true, similarity: 1, requestProfile: profile}, nil, nil
}

func (e *Engine) completeResearchIntent(lease researchIntentLease, results []model.ResearchResult, err error) {
	if !lease.owner || lease.entry == nil {
		return
	}
	e.researchDedupeMu.Lock()
	entry, ok := e.researchDedupe[lease.entry.ID]
	if !ok {
		e.researchDedupeMu.Unlock()
		return
	}
	if err != nil {
		delete(e.researchDedupe, lease.entry.ID)
		if entry.InFlight {
			entry.InFlight = false
			close(entry.Done)
		}
		e.researchDedupeMu.Unlock()
		return
	}
	entry.Results = cloneResearchResults(uniqueResearchEvidence(results))
	entry.InFlight = false
	entry.Finished = time.Now().UTC()
	close(entry.Done)
	e.researchDedupeMu.Unlock()
}

func normalizeResearchIntent(value string) string {
	terms := researchTerms(value)
	if len(terms) == 0 {
		return strings.ToLower(strings.TrimSpace(value))
	}
	ordered := make([]string, 0, len(terms))
	for term := range terms {
		ordered = append(ordered, term)
	}
	sort.Strings(ordered)
	return strings.Join(ordered, " ")
}

func researchIntentSimilarity(a string, av []float64, b string, bv []float64) float64 {
	if len(av) > 0 && len(av) == len(bv) {
		return cosineVector(av, bv)
	}
	at := researchTerms(a)
	bt := researchTerms(b)
	if len(at) == 0 || len(bt) == 0 {
		if strings.EqualFold(strings.TrimSpace(a), strings.TrimSpace(b)) {
			return 1
		}
		return 0
	}
	intersection := 0
	union := len(at)
	for term := range bt {
		if at[term] {
			intersection++
		} else {
			union++
		}
	}
	if union == 0 {
		return 0
	}
	return float64(intersection) / float64(union)
}

func classifyResearchIntent(value string) researchIntentProfile {
	text := strings.ToLower(strings.TrimSpace(value))
	profile := researchIntentProfile{}

	// Primary intent is deliberately ordered from specific/high-risk technical
	// actions to generic configuration/definition. A forensic preservation task
	// therefore stays "evidence-preservation" even if the surrounding product is
	// an authentication system.
	switch {
	case containsResearchMarker(text, "forens", "digital evidence", "evidence preservation", "beweiss", "beweismittel", "chain of custody", "beweiskette", "volatile", "flüchtig", "hash", "checksum", "prüfsumm"):
		profile.Primary = "evidence-preservation"
	case containsResearchMarker(text, "incident response", "security incident", "sicherheitsvorfall", "vorfall untersuch", "ransomware response"):
		profile.Primary = "incident-response"
	case containsResearchMarker(text, "troubleshoot", "diagnos", "fehlerbeheb", "beheb", "root cause", "ursachen"):
		profile.Primary = "troubleshooting"
	case containsResearchMarker(text, "wiederherstell", "restore", "recovery", "recover", "rollback"):
		profile.Primary = "recovery"
	case containsResearchMarker(text, "hardening", "härt", "secure design", "sicher entwerf"):
		profile.Primary = "hardening"
	case containsResearchMarker(text, "authorization", "autoris", "berechtig", "least privilege", "privileg"):
		profile.Primary = "authorization"
	case containsResearchMarker(text, "authentication", "authentifiz", "multi-factor", "multifaktor", " mfa", "login", "sign-in", "signin"):
		profile.Primary = "authentication"
	case containsResearchMarker(text, "configur", "konfigur", "implement", "einricht", "deploy", "setup", "aktivier", "enable"):
		profile.Primary = "configuration"
	case containsResearchMarker(text, "compare", "vergleich", "unterschied", "difference", "versus", " vs "):
		profile.Primary = "comparison"
	case containsResearchMarker(text, "what is", "was ist", "definition", "grundlagen", "concept", "konzept"):
		profile.Primary = "definition"
	default:
		profile.Primary = "general"
	}

	focus := map[string][]string{
		"hash-integrity":   {"hash", "checksum", "prüfsumm", "sha-256", "sha256", "sha512", "md5"},
		"volatile-memory":  {"volatile", "flüchtig", "arbeitsspeicher", " ram", "memory acquisition", "reboot", "neustart"},
		"chain-of-custody": {"chain of custody", "beweiskette", "custody"},
		"egress-control":   {"egress control", "egress-control", "ausgehend", "outbound traffic"},
		"segmentation":     {"segmentation", "segmentierung", "network segment", "netzsegment"},
		"least-privilege":  {"least privilege", "geringste privileg", "minimal privilege"},
	}
	for name, markers := range focus {
		if containsResearchMarker(text, markers...) {
			profile.Focus = append(profile.Focus, name)
		}
	}

	entities := map[string][]string{
		"azure-entra":           {"azure active directory", "azure ad", "microsoft entra", "entra id"},
		"aws":                   {"amazon web services", " aws", "aws ", "aws-"},
		"kubernetes":            {"kubernetes", " k8s", "k8s ", "kubectl"},
		"windows":               {"microsoft windows", "windows server", "windows-netzwerk", "windows network"},
		"dns":                   {"authoritative dns", " dns", "dns ", "domain name system"},
		"mobile-authentication": {"mobile authentication", "mobile authentification", "mobile-authentifizierung", "mobile authentifizierung"},
		"multi-factor-auth":     {"multi-factor", "multifaktor", " mfa", "mfa ", "2fa"},
		"ransomware":            {"ransomware"},
	}
	for name, markers := range entities {
		if containsResearchMarker(text, markers...) {
			profile.Entities = append(profile.Entities, name)
		}
	}
	sort.Strings(profile.Focus)
	sort.Strings(profile.Entities)
	return profile
}

func researchIntentProfilesCompatible(a, b researchIntentProfile) (bool, string) {
	if a.Primary != "" && b.Primary != "" && a.Primary != "general" && b.Primary != "general" && a.Primary != b.Primary {
		return false, "primary_intent_mismatch"
	}
	if len(a.Focus) > 0 || len(b.Focus) > 0 {
		if !stringSliceIntersects(a.Focus, b.Focus) {
			return false, "technical_focus_mismatch"
		}
	}
	if len(a.Entities) > 0 && len(b.Entities) > 0 && !stringSliceIntersects(a.Entities, b.Entities) {
		return false, "core_entity_mismatch"
	}
	return true, "compatible"
}

func containsResearchMarker(value string, markers ...string) bool {
	for _, marker := range markers {
		if strings.Contains(value, marker) {
			return true
		}
	}
	return false
}

func stringSliceIntersects(a, b []string) bool {
	if len(a) == 0 || len(b) == 0 {
		return false
	}
	seen := make(map[string]struct{}, len(a))
	for _, value := range a {
		seen[value] = struct{}{}
	}
	for _, value := range b {
		if _, ok := seen[value]; ok {
			return true
		}
	}
	return false
}

func researchIntentProfileMetadata(prefix string, profile researchIntentProfile) map[string]any {
	if prefix != "" && !strings.HasSuffix(prefix, "_") {
		prefix += "_"
	}
	return map[string]any{
		prefix + "intent_class":    profile.Primary,
		prefix + "intent_focus":    append([]string(nil), profile.Focus...),
		prefix + "intent_entities": append([]string(nil), profile.Entities...),
	}
}

func researchDedupeLeaseMetadata(lease researchIntentLease) map[string]any {
	metadata := researchIntentProfileMetadata("request", lease.requestProfile)
	if lease.entry != nil {
		for key, value := range researchIntentProfileMetadata("cached", lease.entry.Profile) {
			metadata[key] = value
		}
		metadata["cached_research_intent"] = lease.entry.RawIntent
	}
	return metadata
}

func cloneResearchResults(values []model.ResearchResult) []model.ResearchResult {
	out := make([]model.ResearchResult, len(values))
	copy(out, values)
	for i := range out {
		out[i].CoveredGapIDs = append([]string(nil), values[i].CoveredGapIDs...)
	}
	return out
}

func remapResearchEvidenceToQuestion(values []model.ResearchResult, question model.ResearchQuestion) []model.ResearchResult {
	out := cloneResearchResults(values)
	for i := range out {
		out[i].CoveredGapIDs = unique(append(out[i].CoveredGapIDs, question.GapID))
		if strings.TrimSpace(out[i].AssessmentReason) != "" {
			out[i].AssessmentReason = fmt.Sprintf("Wiederverwendete semantisch äquivalente Recherche: %s", out[i].AssessmentReason)
		}
	}
	return out
}

func (e *Engine) researchDedupeStatus() map[string]any {
	e.researchDedupeMu.Lock()
	defer e.researchDedupeMu.Unlock()
	inflight, completed := 0, 0
	for _, entry := range e.researchDedupe {
		if entry.InFlight {
			inflight++
		} else {
			completed++
		}
	}
	return map[string]any{"threshold": e.Cfg.ResearchDedupeThreshold, "ttl": e.Cfg.ResearchDedupeTTL.String(), "inflight": inflight, "cached": completed, "intent_guard_filtered_comparisons": e.researchDedupeGuardFiltered, "intent_guard_primary_mismatch": e.researchDedupeGuardPrimaryMismatch, "intent_guard_focus_mismatch": e.researchDedupeGuardFocusMismatch, "intent_guard_entity_mismatch": e.researchDedupeGuardEntityMismatch}
}