package engine import ( "context" "fmt" "sort" "strings" "time" "github.com/local/glpi-neural-brain/internal/model" "github.com/local/glpi-neural-brain/internal/ollama" ) type researchDedupeEntry struct { ID string Kind string Intent string Vector []float64 Created time.Time Finished time.Time InFlight bool Done chan struct{} Results []model.ResearchResult } type researchIntentLease struct { entry *researchDedupeEntry owner bool similarity float64 } func (e *Engine) withSharedResearchWork(ctx context.Context, kind string, fn func() error) error { if e.sharedWork == nil { return fn() } release, err := e.sharedWork.Acquire(ctx) if err != nil { if e.Broker != nil { e.Broker.Publish(model.Activity{Type: "work.queue.rejected", Source: "brain", Phase: "queue", Message: "Gemeinsame Research/Ollama-Queue ist ausgelastet", Strength: .3, Metadata: map[string]any{"kind": kind, "error": err.Error(), "queue": e.sharedWork.Status()}}) } return err } defer release() return fn() } func (e *Engine) beginResearchIntent(ctx context.Context, kind, intent string) (researchIntentLease, []model.ResearchResult, error) { kind = strings.ToLower(strings.TrimSpace(kind)) if kind == "" { kind = "evidence" } intent = normalizeResearchIntent(intent) if intent == "" { return researchIntentLease{owner: true}, nil, nil } var vector []float64 if e.Ollama != nil { vectors, err := e.Ollama.Embed(ollama.WithLowPriority(ctx), []string{intent}) if err == nil && len(vectors) == 1 { vector = vectors[0] } } now := time.Now().UTC() ttl := e.Cfg.ResearchDedupeTTL if ttl <= 0 { ttl = 45 * time.Minute } threshold := e.Cfg.ResearchDedupeThreshold if threshold <= 0 { threshold = .92 } e.researchDedupeMu.Lock() if e.researchDedupe == nil { e.researchDedupe = map[string]*researchDedupeEntry{} } for id, entry := range e.researchDedupe { if !entry.InFlight && !entry.Finished.IsZero() && now.Sub(entry.Finished) > ttl { delete(e.researchDedupe, id) } } var best *researchDedupeEntry bestSimilarity := 0.0 for _, entry := range e.researchDedupe { if entry.Kind != kind { continue } similarity := researchIntentSimilarity(intent, vector, entry.Intent, entry.Vector) if similarity > bestSimilarity { bestSimilarity = similarity best = entry } } if best != nil && bestSimilarity >= threshold { done := best.Done inFlight := best.InFlight e.researchDedupeMu.Unlock() if inFlight { select { case <-done: case <-ctx.Done(): return researchIntentLease{}, nil, ctx.Err() } } e.researchDedupeMu.Lock() current, stillCached := e.researchDedupe[best.ID] if !stillCached { e.researchDedupeMu.Unlock() // The owner failed and removed its cache entry. Retry as a new // contender instead of treating a failed duplicate as an empty // successful research result. return e.beginResearchIntent(ctx, kind, intent) } results := cloneResearchResults(current.Results) e.researchDedupeMu.Unlock() return researchIntentLease{entry: current, owner: false, similarity: bestSimilarity}, results, nil } id := newResearchRunID("research-intent", intent) entry := &researchDedupeEntry{ID: id, Kind: kind, Intent: intent, Vector: append([]float64(nil), vector...), Created: now, InFlight: true, Done: make(chan struct{})} e.researchDedupe[id] = entry e.researchDedupeMu.Unlock() return researchIntentLease{entry: entry, owner: true, similarity: 1}, nil, nil } func (e *Engine) completeResearchIntent(lease researchIntentLease, results []model.ResearchResult, err error) { if !lease.owner || lease.entry == nil { return } e.researchDedupeMu.Lock() entry, ok := e.researchDedupe[lease.entry.ID] if !ok { e.researchDedupeMu.Unlock() return } if err != nil { delete(e.researchDedupe, lease.entry.ID) if entry.InFlight { entry.InFlight = false close(entry.Done) } e.researchDedupeMu.Unlock() return } entry.Results = cloneResearchResults(uniqueResearchEvidence(results)) entry.InFlight = false entry.Finished = time.Now().UTC() close(entry.Done) e.researchDedupeMu.Unlock() } func normalizeResearchIntent(value string) string { terms := researchTerms(value) if len(terms) == 0 { return strings.ToLower(strings.TrimSpace(value)) } ordered := make([]string, 0, len(terms)) for term := range terms { ordered = append(ordered, term) } sort.Strings(ordered) return strings.Join(ordered, " ") } func researchIntentSimilarity(a string, av []float64, b string, bv []float64) float64 { if len(av) > 0 && len(av) == len(bv) { return cosineVector(av, bv) } at := researchTerms(a) bt := researchTerms(b) if len(at) == 0 || len(bt) == 0 { if strings.EqualFold(strings.TrimSpace(a), strings.TrimSpace(b)) { return 1 } return 0 } intersection := 0 union := len(at) for term := range bt { if at[term] { intersection++ } else { union++ } } if union == 0 { return 0 } return float64(intersection) / float64(union) } func cloneResearchResults(values []model.ResearchResult) []model.ResearchResult { out := make([]model.ResearchResult, len(values)) copy(out, values) for i := range out { out[i].CoveredGapIDs = append([]string(nil), values[i].CoveredGapIDs...) } return out } func remapResearchEvidenceToQuestion(values []model.ResearchResult, question model.ResearchQuestion) []model.ResearchResult { out := cloneResearchResults(values) for i := range out { out[i].CoveredGapIDs = unique(append(out[i].CoveredGapIDs, question.GapID)) if strings.TrimSpace(out[i].AssessmentReason) != "" { out[i].AssessmentReason = fmt.Sprintf("Wiederverwendete semantisch äquivalente Recherche: %s", out[i].AssessmentReason) } } return out } func (e *Engine) researchDedupeStatus() map[string]any { e.researchDedupeMu.Lock() defer e.researchDedupeMu.Unlock() inflight, completed := 0, 0 for _, entry := range e.researchDedupe { if entry.InFlight { inflight++ } else { completed++ } } return map[string]any{"threshold": e.Cfg.ResearchDedupeThreshold, "ttl": e.Cfg.ResearchDedupeTTL.String(), "inflight": inflight, "cached": completed} }