glpi-neural-brain/internal/engine/research_work.go

package engine

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/local/glpi-neural-brain/internal/model"
	"github.com/local/glpi-neural-brain/internal/ollama"
)

type researchDedupeEntry struct {
	ID       string
	Kind     string
	Intent   string
	Vector   []float64
	Created  time.Time
	Finished time.Time
	InFlight bool
	Done     chan struct{}
	Results  []model.ResearchResult
}

type researchIntentLease struct {
	entry      *researchDedupeEntry
	owner      bool
	similarity float64
}

func (e *Engine) withSharedResearchWork(ctx context.Context, kind string, fn func() error) error {
	if e.sharedWork == nil {
		return fn()
	}
	release, err := e.sharedWork.Acquire(ctx)
	if err != nil {
		if e.Broker != nil {
			e.Broker.Publish(model.Activity{Type: "work.queue.rejected", Source: "brain", Phase: "queue", Message: "Gemeinsame Research/Ollama-Queue ist ausgelastet", Strength: .3, Metadata: map[string]any{"kind": kind, "error": err.Error(), "queue": e.sharedWork.Status()}})
		}
		return err
	}
	defer release()
	return fn()
}

func (e *Engine) beginResearchIntent(ctx context.Context, kind, intent string) (researchIntentLease, []model.ResearchResult, error) {
	kind = strings.ToLower(strings.TrimSpace(kind))
	if kind == "" {
		kind = "evidence"
	}
	intent = normalizeResearchIntent(intent)
	if intent == "" {
		return researchIntentLease{owner: true}, nil, nil
	}

	var vector []float64
	if e.Ollama != nil {
		vectors, err := e.Ollama.Embed(ollama.WithLowPriority(ctx), []string{intent})
		if err == nil && len(vectors) == 1 {
			vector = vectors[0]
		}
	}

	now := time.Now().UTC()
	ttl := e.Cfg.ResearchDedupeTTL
	if ttl <= 0 {
		ttl = 45 * time.Minute
	}
	threshold := e.Cfg.ResearchDedupeThreshold
	if threshold <= 0 {
		threshold = .92
	}

	e.researchDedupeMu.Lock()
	if e.researchDedupe == nil {
		e.researchDedupe = map[string]*researchDedupeEntry{}
	}
	for id, entry := range e.researchDedupe {
		if !entry.InFlight && !entry.Finished.IsZero() && now.Sub(entry.Finished) > ttl {
			delete(e.researchDedupe, id)
		}
	}

	var best *researchDedupeEntry
	bestSimilarity := 0.0
	for _, entry := range e.researchDedupe {
		if entry.Kind != kind {
			continue
		}
		similarity := researchIntentSimilarity(intent, vector, entry.Intent, entry.Vector)
		if similarity > bestSimilarity {
			bestSimilarity = similarity
			best = entry
		}
	}
	if best != nil && bestSimilarity >= threshold {
		done := best.Done
		inFlight := best.InFlight
		e.researchDedupeMu.Unlock()
		if inFlight {
			select {
			case <-done:
			case <-ctx.Done():
				return researchIntentLease{}, nil, ctx.Err()
			}
		}
		e.researchDedupeMu.Lock()
		current, stillCached := e.researchDedupe[best.ID]
		if !stillCached {
			e.researchDedupeMu.Unlock()
			// The owner failed and removed its cache entry. Retry as a new
			// contender instead of treating a failed duplicate as an empty
			// successful research result.
			return e.beginResearchIntent(ctx, kind, intent)
		}
		results := cloneResearchResults(current.Results)
		e.researchDedupeMu.Unlock()
		return researchIntentLease{entry: current, owner: false, similarity: bestSimilarity}, results, nil
	}

	id := newResearchRunID("research-intent", intent)
	entry := &researchDedupeEntry{ID: id, Kind: kind, Intent: intent, Vector: append([]float64(nil), vector...), Created: now, InFlight: true, Done: make(chan struct{})}
	e.researchDedupe[id] = entry
	e.researchDedupeMu.Unlock()
	return researchIntentLease{entry: entry, owner: true, similarity: 1}, nil, nil
}

func (e *Engine) completeResearchIntent(lease researchIntentLease, results []model.ResearchResult, err error) {
	if !lease.owner || lease.entry == nil {
		return
	}
	e.researchDedupeMu.Lock()
	entry, ok := e.researchDedupe[lease.entry.ID]
	if !ok {
		e.researchDedupeMu.Unlock()
		return
	}
	if err != nil {
		delete(e.researchDedupe, lease.entry.ID)
		if entry.InFlight {
			entry.InFlight = false
			close(entry.Done)
		}
		e.researchDedupeMu.Unlock()
		return
	}
	entry.Results = cloneResearchResults(uniqueResearchEvidence(results))
	entry.InFlight = false
	entry.Finished = time.Now().UTC()
	close(entry.Done)
	e.researchDedupeMu.Unlock()
}

func normalizeResearchIntent(value string) string {
	terms := researchTerms(value)
	if len(terms) == 0 {
		return strings.ToLower(strings.TrimSpace(value))
	}
	ordered := make([]string, 0, len(terms))
	for term := range terms {
		ordered = append(ordered, term)
	}
	sort.Strings(ordered)
	return strings.Join(ordered, " ")
}

func researchIntentSimilarity(a string, av []float64, b string, bv []float64) float64 {
	if len(av) > 0 && len(av) == len(bv) {
		return cosineVector(av, bv)
	}
	at := researchTerms(a)
	bt := researchTerms(b)
	if len(at) == 0 || len(bt) == 0 {
		if strings.EqualFold(strings.TrimSpace(a), strings.TrimSpace(b)) {
			return 1
		}
		return 0
	}
	intersection := 0
	union := len(at)
	for term := range bt {
		if at[term] {
			intersection++
		} else {
			union++
		}
	}
	if union == 0 {
		return 0
	}
	return float64(intersection) / float64(union)
}

func cloneResearchResults(values []model.ResearchResult) []model.ResearchResult {
	out := make([]model.ResearchResult, len(values))
	copy(out, values)
	for i := range out {
		out[i].CoveredGapIDs = append([]string(nil), values[i].CoveredGapIDs...)
	}
	return out
}

func remapResearchEvidenceToQuestion(values []model.ResearchResult, question model.ResearchQuestion) []model.ResearchResult {
	out := cloneResearchResults(values)
	for i := range out {
		out[i].CoveredGapIDs = unique(append(out[i].CoveredGapIDs, question.GapID))
		if strings.TrimSpace(out[i].AssessmentReason) != "" {
			out[i].AssessmentReason = fmt.Sprintf("Wiederverwendete semantisch äquivalente Recherche: %s", out[i].AssessmentReason)
		}
	}
	return out
}

func (e *Engine) researchDedupeStatus() map[string]any {
	e.researchDedupeMu.Lock()
	defer e.researchDedupeMu.Unlock()
	inflight, completed := 0, 0
	for _, entry := range e.researchDedupe {
		if entry.InFlight {
			inflight++
		} else {
			completed++
		}
	}
	return map[string]any{"threshold": e.Cfg.ResearchDedupeThreshold, "ttl": e.Cfg.ResearchDedupeTTL.String(), "inflight": inflight, "cached": completed}
}