Ch1
This commit is contained in:
@@ -0,0 +1,227 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
)
|
||||
|
||||
type researchDedupeEntry struct {
|
||||
ID string
|
||||
Kind string
|
||||
Intent string
|
||||
Vector []float64
|
||||
Created time.Time
|
||||
Finished time.Time
|
||||
InFlight bool
|
||||
Done chan struct{}
|
||||
Results []model.ResearchResult
|
||||
}
|
||||
|
||||
type researchIntentLease struct {
|
||||
entry *researchDedupeEntry
|
||||
owner bool
|
||||
similarity float64
|
||||
}
|
||||
|
||||
func (e *Engine) withSharedResearchWork(ctx context.Context, kind string, fn func() error) error {
|
||||
if e.sharedWork == nil {
|
||||
return fn()
|
||||
}
|
||||
release, err := e.sharedWork.Acquire(ctx)
|
||||
if err != nil {
|
||||
if e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "work.queue.rejected", Source: "brain", Phase: "queue", Message: "Gemeinsame Research/Ollama-Queue ist ausgelastet", Strength: .3, Metadata: map[string]any{"kind": kind, "error": err.Error(), "queue": e.sharedWork.Status()}})
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
return fn()
|
||||
}
|
||||
|
||||
func (e *Engine) beginResearchIntent(ctx context.Context, kind, intent string) (researchIntentLease, []model.ResearchResult, error) {
|
||||
kind = strings.ToLower(strings.TrimSpace(kind))
|
||||
if kind == "" {
|
||||
kind = "evidence"
|
||||
}
|
||||
intent = normalizeResearchIntent(intent)
|
||||
if intent == "" {
|
||||
return researchIntentLease{owner: true}, nil, nil
|
||||
}
|
||||
|
||||
var vector []float64
|
||||
if e.Ollama != nil {
|
||||
vectors, err := e.Ollama.Embed(ollama.WithLowPriority(ctx), []string{intent})
|
||||
if err == nil && len(vectors) == 1 {
|
||||
vector = vectors[0]
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
ttl := e.Cfg.ResearchDedupeTTL
|
||||
if ttl <= 0 {
|
||||
ttl = 45 * time.Minute
|
||||
}
|
||||
threshold := e.Cfg.ResearchDedupeThreshold
|
||||
if threshold <= 0 {
|
||||
threshold = .92
|
||||
}
|
||||
|
||||
e.researchDedupeMu.Lock()
|
||||
if e.researchDedupe == nil {
|
||||
e.researchDedupe = map[string]*researchDedupeEntry{}
|
||||
}
|
||||
for id, entry := range e.researchDedupe {
|
||||
if !entry.InFlight && !entry.Finished.IsZero() && now.Sub(entry.Finished) > ttl {
|
||||
delete(e.researchDedupe, id)
|
||||
}
|
||||
}
|
||||
|
||||
var best *researchDedupeEntry
|
||||
bestSimilarity := 0.0
|
||||
for _, entry := range e.researchDedupe {
|
||||
if entry.Kind != kind {
|
||||
continue
|
||||
}
|
||||
similarity := researchIntentSimilarity(intent, vector, entry.Intent, entry.Vector)
|
||||
if similarity > bestSimilarity {
|
||||
bestSimilarity = similarity
|
||||
best = entry
|
||||
}
|
||||
}
|
||||
if best != nil && bestSimilarity >= threshold {
|
||||
done := best.Done
|
||||
inFlight := best.InFlight
|
||||
e.researchDedupeMu.Unlock()
|
||||
if inFlight {
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
return researchIntentLease{}, nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
e.researchDedupeMu.Lock()
|
||||
current, stillCached := e.researchDedupe[best.ID]
|
||||
if !stillCached {
|
||||
e.researchDedupeMu.Unlock()
|
||||
// The owner failed and removed its cache entry. Retry as a new
|
||||
// contender instead of treating a failed duplicate as an empty
|
||||
// successful research result.
|
||||
return e.beginResearchIntent(ctx, kind, intent)
|
||||
}
|
||||
results := cloneResearchResults(current.Results)
|
||||
e.researchDedupeMu.Unlock()
|
||||
return researchIntentLease{entry: current, owner: false, similarity: bestSimilarity}, results, nil
|
||||
}
|
||||
|
||||
id := newResearchRunID("research-intent", intent)
|
||||
entry := &researchDedupeEntry{ID: id, Kind: kind, Intent: intent, Vector: append([]float64(nil), vector...), Created: now, InFlight: true, Done: make(chan struct{})}
|
||||
e.researchDedupe[id] = entry
|
||||
e.researchDedupeMu.Unlock()
|
||||
return researchIntentLease{entry: entry, owner: true, similarity: 1}, nil, nil
|
||||
}
|
||||
|
||||
func (e *Engine) completeResearchIntent(lease researchIntentLease, results []model.ResearchResult, err error) {
|
||||
if !lease.owner || lease.entry == nil {
|
||||
return
|
||||
}
|
||||
e.researchDedupeMu.Lock()
|
||||
entry, ok := e.researchDedupe[lease.entry.ID]
|
||||
if !ok {
|
||||
e.researchDedupeMu.Unlock()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
delete(e.researchDedupe, lease.entry.ID)
|
||||
if entry.InFlight {
|
||||
entry.InFlight = false
|
||||
close(entry.Done)
|
||||
}
|
||||
e.researchDedupeMu.Unlock()
|
||||
return
|
||||
}
|
||||
entry.Results = cloneResearchResults(uniqueResearchEvidence(results))
|
||||
entry.InFlight = false
|
||||
entry.Finished = time.Now().UTC()
|
||||
close(entry.Done)
|
||||
e.researchDedupeMu.Unlock()
|
||||
}
|
||||
|
||||
func normalizeResearchIntent(value string) string {
|
||||
terms := researchTerms(value)
|
||||
if len(terms) == 0 {
|
||||
return strings.ToLower(strings.TrimSpace(value))
|
||||
}
|
||||
ordered := make([]string, 0, len(terms))
|
||||
for term := range terms {
|
||||
ordered = append(ordered, term)
|
||||
}
|
||||
sort.Strings(ordered)
|
||||
return strings.Join(ordered, " ")
|
||||
}
|
||||
|
||||
func researchIntentSimilarity(a string, av []float64, b string, bv []float64) float64 {
|
||||
if len(av) > 0 && len(av) == len(bv) {
|
||||
return cosineVector(av, bv)
|
||||
}
|
||||
at := researchTerms(a)
|
||||
bt := researchTerms(b)
|
||||
if len(at) == 0 || len(bt) == 0 {
|
||||
if strings.EqualFold(strings.TrimSpace(a), strings.TrimSpace(b)) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
intersection := 0
|
||||
union := len(at)
|
||||
for term := range bt {
|
||||
if at[term] {
|
||||
intersection++
|
||||
} else {
|
||||
union++
|
||||
}
|
||||
}
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(intersection) / float64(union)
|
||||
}
|
||||
|
||||
func cloneResearchResults(values []model.ResearchResult) []model.ResearchResult {
|
||||
out := make([]model.ResearchResult, len(values))
|
||||
copy(out, values)
|
||||
for i := range out {
|
||||
out[i].CoveredGapIDs = append([]string(nil), values[i].CoveredGapIDs...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func remapResearchEvidenceToQuestion(values []model.ResearchResult, question model.ResearchQuestion) []model.ResearchResult {
|
||||
out := cloneResearchResults(values)
|
||||
for i := range out {
|
||||
out[i].CoveredGapIDs = unique(append(out[i].CoveredGapIDs, question.GapID))
|
||||
if strings.TrimSpace(out[i].AssessmentReason) != "" {
|
||||
out[i].AssessmentReason = fmt.Sprintf("Wiederverwendete semantisch äquivalente Recherche: %s", out[i].AssessmentReason)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (e *Engine) researchDedupeStatus() map[string]any {
|
||||
e.researchDedupeMu.Lock()
|
||||
defer e.researchDedupeMu.Unlock()
|
||||
inflight, completed := 0, 0
|
||||
for _, entry := range e.researchDedupe {
|
||||
if entry.InFlight {
|
||||
inflight++
|
||||
} else {
|
||||
completed++
|
||||
}
|
||||
}
|
||||
return map[string]any{"threshold": e.Cfg.ResearchDedupeThreshold, "ttl": e.Cfg.ResearchDedupeTTL.String(), "inflight": inflight, "cached": completed}
|
||||
}
|
||||
Reference in New Issue
Block a user