Agent-Mode mit Aufgabenteilung und automatischer Recherche

2026-08-07 21:57:23 +02:00
parent ffee925163
commit adbe69d24d
30 changed files with 2580 additions and 51 deletions
@@ -0,0 +1,698 @@
+package sourceagent
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"math/bits"
+	"net/url"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+	"unicode"
+
+	_ "modernc.org/sqlite"
+)
+
+type Store struct {
+	db *sql.DB
+	mu sync.Mutex
+}
+
+func OpenStore(dataDir string) (*Store, error) {
+	path := filepath.Join(dataDir, "source-agents.db")
+	db, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)")
+	if err != nil {
+		return nil, err
+	}
+	db.SetMaxOpenConns(4)
+	store := &Store{db: db}
+	if err := store.init(context.Background()); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return store, nil
+}
+
+func (s *Store) Close() error {
+	if s == nil || s.db == nil {
+		return nil
+	}
+	return s.db.Close()
+}
+
+func (s *Store) init(ctx context.Context) error {
+	statements := []string{
+		`CREATE TABLE IF NOT EXISTS source_agents (
+			id TEXT PRIMARY KEY, name TEXT NOT NULL, token_hash TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1,
+			created_at_ns INTEGER NOT NULL, updated_at_ns INTEGER NOT NULL, last_seen_ns INTEGER NOT NULL DEFAULT 0,
+			last_error TEXT NOT NULL DEFAULT '', version TEXT NOT NULL DEFAULT ''
+		) WITHOUT ROWID`,
+		`CREATE TABLE IF NOT EXISTS source_tasks (
+			id TEXT PRIMARY KEY, agent_id TEXT NOT NULL REFERENCES source_agents(id) ON DELETE CASCADE,
+			name TEXT NOT NULL, type TEXT NOT NULL, url TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1,
+			poll_interval TEXT NOT NULL DEFAULT '4h', categories_json TEXT NOT NULL DEFAULT '[]', max_items INTEGER NOT NULL DEFAULT 20,
+			config_json TEXT NOT NULL DEFAULT '{}', created_at_ns INTEGER NOT NULL, updated_at_ns INTEGER NOT NULL
+		) WITHOUT ROWID`,
+		`CREATE INDEX IF NOT EXISTS idx_source_tasks_agent ON source_tasks(agent_id, enabled, updated_at_ns)`,
+		`CREATE TABLE IF NOT EXISTS source_inbox (
+			id TEXT PRIMARY KEY, agent_id TEXT NOT NULL, task_id TEXT NOT NULL, external_id TEXT NOT NULL DEFAULT '',
+			url TEXT NOT NULL, canonical_url TEXT NOT NULL, title TEXT NOT NULL, published_at_ns INTEGER NOT NULL DEFAULT 0,
+			discovered_at_ns INTEGER NOT NULL DEFAULT 0, received_at_ns INTEGER NOT NULL, updated_at_ns INTEGER NOT NULL,
+			language TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', text_content TEXT NOT NULL,
+			content_sha256 TEXT NOT NULL, source_name TEXT NOT NULL DEFAULT '', source_base_url TEXT NOT NULL DEFAULT '',
+			categories_json TEXT NOT NULL DEFAULT '[]', metadata_json TEXT NOT NULL DEFAULT '{}', signature INTEGER NOT NULL DEFAULT 0,
+			status TEXT NOT NULL DEFAULT 'received', relevance REAL NOT NULL DEFAULT 0, matched_node_id TEXT NOT NULL DEFAULT '',
+			UNIQUE(agent_id, task_id, canonical_url, content_sha256)
+		) WITHOUT ROWID`,
+		`CREATE INDEX IF NOT EXISTS idx_source_inbox_status_received ON source_inbox(status, received_at_ns)`,
+		`CREATE INDEX IF NOT EXISTS idx_source_inbox_candidate_updated ON source_inbox(status, updated_at_ns DESC)`,
+	}
+	for _, statement := range statements {
+		if _, err := s.db.ExecContext(ctx, statement); err != nil {
+			return err
+		}
+	}
+	_, _ = s.db.ExecContext(ctx, `UPDATE source_inbox SET status='received' WHERE status='processing' AND updated_at_ns<?`, time.Now().UTC().Add(-10*time.Minute).UnixNano())
+	return nil
+}
+
+func GenerateToken() (string, string, error) {
+	buf := make([]byte, 32)
+	if _, err := rand.Read(buf); err != nil {
+		return "", "", err
+	}
+	token := "brain_agent_" + hex.EncodeToString(buf)
+	return token, hashToken(token), nil
+}
+
+func hashToken(token string) string {
+	sum := sha256.Sum256([]byte(strings.TrimSpace(token)))
+	return hex.EncodeToString(sum[:])
+}
+
+func randomID(prefix string) string {
+	buf := make([]byte, 12)
+	_, _ = rand.Read(buf)
+	return prefix + "-" + hex.EncodeToString(buf)
+}
+
+func normalizeID(value, prefix string) string {
+	value = strings.ToLower(strings.TrimSpace(value))
+	var b strings.Builder
+	for _, r := range value {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' {
+			b.WriteRune(r)
+		}
+	}
+	value = strings.Trim(b.String(), "-_")
+	if value == "" {
+		return randomID(prefix)
+	}
+	return value
+}
+
+func (s *Store) CreateAgent(ctx context.Context, id, name string) (Agent, string, error) {
+	id = normalizeID(id, "agent")
+	name = strings.TrimSpace(name)
+	if name == "" {
+		name = id
+	}
+	token, tokenHash, err := GenerateToken()
+	if err != nil {
+		return Agent{}, "", err
+	}
+	now := time.Now().UTC()
+	_, err = s.db.ExecContext(ctx, `INSERT INTO source_agents(id,name,token_hash,enabled,created_at_ns,updated_at_ns) VALUES(?,?,?,?,?,?)`, id, name, tokenHash, 1, now.UnixNano(), now.UnixNano())
+	if err != nil {
+		return Agent{}, "", err
+	}
+	return Agent{ID: id, Name: name, Enabled: true, CreatedAt: now, UpdatedAt: now}, token, nil
+}
+
+func (s *Store) RotateToken(ctx context.Context, id string) (string, error) {
+	token, tokenHash, err := GenerateToken()
+	if err != nil {
+		return "", err
+	}
+	result, err := s.db.ExecContext(ctx, `UPDATE source_agents SET token_hash=?, updated_at_ns=? WHERE id=?`, tokenHash, time.Now().UTC().UnixNano(), id)
+	if err != nil {
+		return "", err
+	}
+	n, _ := result.RowsAffected()
+	if n == 0 {
+		return "", sql.ErrNoRows
+	}
+	return token, nil
+}
+
+func (s *Store) SetAgentEnabled(ctx context.Context, id string, enabled bool) error {
+	v := 0
+	if enabled {
+		v = 1
+	}
+	result, err := s.db.ExecContext(ctx, `UPDATE source_agents SET enabled=?,updated_at_ns=? WHERE id=?`, v, time.Now().UTC().UnixNano(), id)
+	if err != nil {
+		return err
+	}
+	n, _ := result.RowsAffected()
+	if n == 0 {
+		return sql.ErrNoRows
+	}
+	return nil
+}
+
+func (s *Store) DeleteAgent(ctx context.Context, id string) error {
+	_, err := s.db.ExecContext(ctx, `DELETE FROM source_agents WHERE id=?`, id)
+	return err
+}
+
+func (s *Store) ListAgents(ctx context.Context) ([]Agent, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT a.id,a.name,a.enabled,a.created_at_ns,a.updated_at_ns,a.last_seen_ns,a.last_error,a.version,(SELECT COUNT(*) FROM source_tasks t WHERE t.agent_id=a.id) FROM source_agents a ORDER BY a.name,a.id`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Agent
+	for rows.Next() {
+		var a Agent
+		var en int
+		var c, u, seen int64
+		if err := rows.Scan(&a.ID, &a.Name, &en, &c, &u, &seen, &a.LastError, &a.Version, &a.TaskCount); err != nil {
+			return nil, err
+		}
+		a.Enabled = en != 0
+		a.CreatedAt = nsTime(c)
+		a.UpdatedAt = nsTime(u)
+		a.LastSeen = nsTime(seen)
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+
+func (s *Store) Authenticate(ctx context.Context, token string) (Agent, error) {
+	if strings.TrimSpace(token) == "" {
+		return Agent{}, errors.New("empty agent token")
+	}
+	var a Agent
+	var en int
+	var c, u, seen int64
+	err := s.db.QueryRowContext(ctx, `SELECT id,name,enabled,created_at_ns,updated_at_ns,last_seen_ns,last_error,version FROM source_agents WHERE token_hash=?`, hashToken(token)).Scan(&a.ID, &a.Name, &en, &c, &u, &seen, &a.LastError, &a.Version)
+	if err != nil {
+		return Agent{}, err
+	}
+	a.Enabled = en != 0
+	if !a.Enabled {
+		return Agent{}, errors.New("agent disabled")
+	}
+	a.CreatedAt = nsTime(c)
+	a.UpdatedAt = nsTime(u)
+	a.LastSeen = nsTime(seen)
+	return a, nil
+}
+
+func validateTask(t Task) (Task, error) {
+	t.ID = normalizeID(t.ID, "task")
+	t.Name = strings.TrimSpace(t.Name)
+	if t.Name == "" {
+		t.Name = t.ID
+	}
+	t.Type = strings.ToLower(strings.TrimSpace(t.Type))
+	switch t.Type {
+	case "rss", "atom", "sitemap", "web":
+	default:
+		return t, fmt.Errorf("unsupported source task type %q", t.Type)
+	}
+	u, err := url.Parse(strings.TrimSpace(t.URL))
+	if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
+		return t, fmt.Errorf("task URL must be absolute http(s)")
+	}
+	if u.User != nil {
+		return t, fmt.Errorf("task URL must not contain userinfo")
+	}
+	t.URL = u.String()
+	if strings.TrimSpace(t.PollInterval) == "" {
+		t.PollInterval = "4h"
+	}
+	d, err := time.ParseDuration(t.PollInterval)
+	if err != nil || d < 5*time.Minute || d > 30*24*time.Hour {
+		return t, fmt.Errorf("poll_interval must be between 5m and 720h")
+	}
+	if t.MaxItems <= 0 {
+		t.MaxItems = 20
+	}
+	if t.MaxItems > 500 {
+		return t, fmt.Errorf("max_items must be <=500")
+	}
+	if t.Config == nil {
+		t.Config = map[string]string{}
+	}
+	return t, nil
+}
+
+func (s *Store) UpsertTask(ctx context.Context, t Task) (Task, error) {
+	var err error
+	t, err = validateTask(t)
+	if err != nil {
+		return Task{}, err
+	}
+	if strings.TrimSpace(t.AgentID) == "" {
+		return Task{}, errors.New("agent_id required")
+	}
+	cats, _ := json.Marshal(uniqueStrings(t.Categories))
+	cfg, _ := json.Marshal(t.Config)
+	now := time.Now().UTC()
+	_, err = s.db.ExecContext(ctx, `INSERT INTO source_tasks(id,agent_id,name,type,url,enabled,poll_interval,categories_json,max_items,config_json,created_at_ns,updated_at_ns) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET agent_id=excluded.agent_id,name=excluded.name,type=excluded.type,url=excluded.url,enabled=excluded.enabled,poll_interval=excluded.poll_interval,categories_json=excluded.categories_json,max_items=excluded.max_items,config_json=excluded.config_json,updated_at_ns=excluded.updated_at_ns`, t.ID, t.AgentID, t.Name, t.Type, t.URL, boolInt(t.Enabled), t.PollInterval, string(cats), t.MaxItems, string(cfg), now.UnixNano(), now.UnixNano())
+	if err != nil {
+		return Task{}, err
+	}
+	t.UpdatedAt = now
+	if t.CreatedAt.IsZero() {
+		t.CreatedAt = now
+	}
+	return t, nil
+}
+
+func (s *Store) DeleteTask(ctx context.Context, id string) error {
+	_, err := s.db.ExecContext(ctx, `DELETE FROM source_tasks WHERE id=?`, id)
+	return err
+}
+func (s *Store) ListTasks(ctx context.Context, agentID string) ([]Task, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT id,agent_id,name,type,url,enabled,poll_interval,categories_json,max_items,config_json,created_at_ns,updated_at_ns FROM source_tasks WHERE (?='' OR agent_id=?) ORDER BY name,id`, agentID, agentID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Task
+	for rows.Next() {
+		var t Task
+		var en int
+		var cats, cfg string
+		var c, u int64
+		if err := rows.Scan(&t.ID, &t.AgentID, &t.Name, &t.Type, &t.URL, &en, &t.PollInterval, &cats, &t.MaxItems, &cfg, &c, &u); err != nil {
+			return nil, err
+		}
+		t.Enabled = en != 0
+		_ = json.Unmarshal([]byte(cats), &t.Categories)
+		_ = json.Unmarshal([]byte(cfg), &t.Config)
+		t.CreatedAt = nsTime(c)
+		t.UpdatedAt = nsTime(u)
+		out = append(out, t)
+	}
+	return out, rows.Err()
+}
+
+func (s *Store) RemoteConfig(ctx context.Context, agent Agent) (RemoteConfig, error) {
+	tasks, err := s.ListTasks(ctx, agent.ID)
+	if err != nil {
+		return RemoteConfig{}, err
+	}
+	return RemoteConfig{SchemaVersion: SchemaVersion, Agent: agent, Tasks: tasks, IssuedAt: time.Now().UTC()}, nil
+}
+
+func (s *Store) Heartbeat(ctx context.Context, agentID string, h Heartbeat) error {
+	_, err := s.db.ExecContext(ctx, `UPDATE source_agents SET last_seen_ns=?,last_error=?,version=?,updated_at_ns=? WHERE id=?`, time.Now().UTC().UnixNano(), strings.TrimSpace(h.LastError), strings.TrimSpace(h.Version), time.Now().UTC().UnixNano(), agentID)
+	return err
+}
+
+func normalizeDocument(d Document) (Document, error) {
+	d.URL = strings.TrimSpace(d.URL)
+	if d.URL == "" {
+		return d, errors.New("document url required")
+	}
+	u, err := url.Parse(d.URL)
+	if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
+		return d, errors.New("invalid document url")
+	}
+	d.CanonicalURL = strings.TrimSpace(d.CanonicalURL)
+	if d.CanonicalURL == "" {
+		d.CanonicalURL = d.URL
+	}
+	d.Title = strings.TrimSpace(d.Title)
+	d.Text = strings.TrimSpace(d.Text)
+	if len([]rune(d.Text)) < 80 {
+		return d, errors.New("document text too short")
+	}
+	if d.DiscoveredAt.IsZero() {
+		d.DiscoveredAt = time.Now().UTC()
+	}
+	if d.ExternalID == "" {
+		d.ExternalID = d.CanonicalURL
+	}
+	d.Categories = uniqueStrings(d.Categories)
+	if d.Metadata == nil {
+		d.Metadata = map[string]any{}
+	}
+	if d.ContentSHA256 == "" {
+		sum := sha256.Sum256([]byte(d.Text))
+		d.ContentSHA256 = hex.EncodeToString(sum[:])
+	}
+	return d, nil
+}
+
+func (s *Store) Ingest(ctx context.Context, agentID, taskID string, docs []Document) (IngestResult, error) {
+	var taskOwner string
+	if err := s.db.QueryRowContext(ctx, `SELECT agent_id FROM source_tasks WHERE id=? AND enabled=1`, taskID).Scan(&taskOwner); err != nil {
+		return IngestResult{}, fmt.Errorf("unknown or disabled source task: %w", err)
+	}
+	if taskOwner != agentID {
+		return IngestResult{}, errors.New("source task does not belong to authenticated agent")
+	}
+	result := IngestResult{BatchID: randomID("batch")}
+	now := time.Now().UTC()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return result, err
+	}
+	defer tx.Rollback()
+	for _, raw := range docs {
+		d, e := normalizeDocument(raw)
+		if e != nil {
+			result.Rejected++
+			continue
+		}
+		cats, _ := json.Marshal(d.Categories)
+		meta, _ := json.Marshal(d.Metadata)
+		idSum := sha256.Sum256([]byte(agentID + "\x00" + taskID + "\x00" + d.CanonicalURL + "\x00" + d.ContentSHA256))
+		id := hex.EncodeToString(idSum[:12])
+		sig := int64(simhash64(d.Title + "\n" + d.Text))
+		res, e := tx.ExecContext(ctx, `INSERT OR IGNORE INTO source_inbox(id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,signature,status) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, id, agentID, taskID, d.ExternalID, d.URL, d.CanonicalURL, d.Title, timeNS(d.PublishedAt), timeNS(d.DiscoveredAt), now.UnixNano(), now.UnixNano(), d.Language, d.ContentType, d.Text, d.ContentSHA256, d.SourceName, d.SourceBaseURL, string(cats), string(meta), sig, "received")
+		if e != nil {
+			result.Rejected++
+			continue
+		}
+		n, _ := res.RowsAffected()
+		if n == 0 {
+			result.Duplicates++
+		} else {
+			result.Accepted++
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return result, err
+	}
+	return result, nil
+}
+
+func (s *Store) ClaimInbox(ctx context.Context, limit int) ([]InboxDocument, error) {
+	if limit < 1 {
+		limit = 1
+	}
+	if limit > 100 {
+		limit = 100
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+	rows, err := tx.QueryContext(ctx, `SELECT id FROM source_inbox WHERE status='received' ORDER BY received_at_ns LIMIT ?`, limit)
+	if err != nil {
+		return nil, err
+	}
+	var ids []string
+	for rows.Next() {
+		var id string
+		if err := rows.Scan(&id); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		ids = append(ids, id)
+	}
+	rows.Close()
+	if len(ids) == 0 {
+		return nil, tx.Commit()
+	}
+	now := time.Now().UTC().UnixNano()
+	for _, id := range ids {
+		if _, err := tx.ExecContext(ctx, `UPDATE source_inbox SET status='processing',updated_at_ns=? WHERE id=? AND status='received'`, now, id); err != nil {
+			return nil, err
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return nil, err
+	}
+	return s.GetInboxByIDs(ctx, ids)
+}
+
+func (s *Store) CompleteClassification(ctx context.Context, id, status string, relevance float64, matchedNodeID string, meta map[string]any) error {
+	if status != "candidate" && status != "archived" {
+		return errors.New("invalid inbox classification status")
+	}
+	data, _ := json.Marshal(meta)
+	_, err := s.db.ExecContext(ctx, `UPDATE source_inbox SET status=?,relevance=?,matched_node_id=?,metadata_json=?,updated_at_ns=? WHERE id=?`, status, relevance, matchedNodeID, string(data), time.Now().UTC().UnixNano(), id)
+	return err
+}
+
+func (s *Store) MarkUsed(ctx context.Context, canonicalURLs []string) error {
+	now := time.Now().UTC().UnixNano()
+	for _, u := range uniqueStrings(canonicalURLs) {
+		if _, err := s.db.ExecContext(ctx, `UPDATE source_inbox SET status='used',updated_at_ns=? WHERE canonical_url=?`, now, u); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Store) GetInboxByIDs(ctx context.Context, ids []string) ([]InboxDocument, error) {
+	if len(ids) == 0 {
+		return nil, nil
+	}
+	out := make([]InboxDocument, 0, len(ids))
+	for _, id := range ids {
+		doc, err := s.getInbox(ctx, id)
+		if err == nil {
+			out = append(out, doc)
+		}
+	}
+	return out, nil
+}
+func (s *Store) getInbox(ctx context.Context, id string) (InboxDocument, error) {
+	row := s.db.QueryRowContext(ctx, `SELECT id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,status,relevance,matched_node_id FROM source_inbox WHERE id=?`, id)
+	return scanInbox(row)
+}
+
+type rowScanner interface{ Scan(...any) error }
+
+func scanInbox(row rowScanner) (InboxDocument, error) {
+	var d InboxDocument
+	var ext, urlv, canon, title, lang, ctype, text, sha, source, base, cats, meta, status, matched string
+	var pub, disc, recv, upd int64
+	var rel float64
+	err := row.Scan(&d.ID, &d.AgentID, &d.TaskID, &ext, &urlv, &canon, &title, &pub, &disc, &recv, &upd, &lang, &ctype, &text, &sha, &source, &base, &cats, &meta, &status, &rel, &matched)
+	if err != nil {
+		return d, err
+	}
+	d.Document = Document{ExternalID: ext, URL: urlv, CanonicalURL: canon, Title: title, PublishedAt: nsTime(pub), DiscoveredAt: nsTime(disc), Language: lang, ContentType: ctype, Text: text, ContentSHA256: sha, SourceName: source, SourceBaseURL: base}
+	_ = json.Unmarshal([]byte(cats), &d.Document.Categories)
+	_ = json.Unmarshal([]byte(meta), &d.Metadata)
+	d.Status = status
+	d.Relevance = rel
+	d.MatchedNodeID = matched
+	d.ReceivedAt = nsTime(recv)
+	d.UpdatedAt = nsTime(upd)
+	return d, nil
+}
+
+func (s *Store) ListInbox(ctx context.Context, status string, limit int) ([]InboxDocument, error) {
+	if limit < 1 {
+		limit = 100
+	}
+	if limit > 1000 {
+		limit = 1000
+	}
+	rows, err := s.db.QueryContext(ctx, `SELECT id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,status,relevance,matched_node_id FROM source_inbox WHERE (?='' OR status=?) ORDER BY received_at_ns DESC LIMIT ?`, status, status, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []InboxDocument
+	for rows.Next() {
+		d, err := scanInbox(rows)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, d)
+	}
+	return out, rows.Err()
+}
+
+func (s *Store) Stats(ctx context.Context) (InboxStats, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT status,COUNT(*) FROM source_inbox GROUP BY status`)
+	if err != nil {
+		return InboxStats{}, err
+	}
+	defer rows.Close()
+	var st InboxStats
+	for rows.Next() {
+		var status string
+		var n int
+		if err := rows.Scan(&status, &n); err != nil {
+			return st, err
+		}
+		st.Total += n
+		switch status {
+		case "received":
+			st.Received = n
+		case "processing":
+			st.Processing = n
+		case "candidate":
+			st.Candidate = n
+		case "archived":
+			st.Archived = n
+		case "used":
+			st.Used = n
+		}
+	}
+	return st, rows.Err()
+}
+
+func (s *Store) SearchCandidates(ctx context.Context, query string, limit int, maxAge time.Duration) ([]ScoredDocument, error) {
+	if limit < 1 {
+		limit = 3
+	}
+	if limit > 50 {
+		limit = 50
+	}
+	since := int64(0)
+	if maxAge > 0 {
+		since = time.Now().UTC().Add(-maxAge).UnixNano()
+	}
+	rows, err := s.db.QueryContext(ctx, `SELECT id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,status,relevance,matched_node_id,signature FROM source_inbox WHERE status IN ('candidate','used') AND (?=0 OR COALESCE(NULLIF(published_at_ns,0),received_at_ns)>=?) ORDER BY updated_at_ns DESC LIMIT 5000`, since, since)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	qsig := simhash64(query)
+	qterms := termSet(query)
+	var out []ScoredDocument
+	for rows.Next() {
+		var d InboxDocument
+		var ext, urlv, canon, title, lang, ctype, text, sha, source, base, cats, meta, status, matched string
+		var pub, disc, recv, upd, sig int64
+		var rel float64
+		if err := rows.Scan(&d.ID, &d.AgentID, &d.TaskID, &ext, &urlv, &canon, &title, &pub, &disc, &recv, &upd, &lang, &ctype, &text, &sha, &source, &base, &cats, &meta, &status, &rel, &matched, &sig); err != nil {
+			return nil, err
+		}
+		d.Document = Document{ExternalID: ext, URL: urlv, CanonicalURL: canon, Title: title, PublishedAt: nsTime(pub), DiscoveredAt: nsTime(disc), Language: lang, ContentType: ctype, Text: text, ContentSHA256: sha, SourceName: source, SourceBaseURL: base}
+		_ = json.Unmarshal([]byte(cats), &d.Document.Categories)
+		_ = json.Unmarshal([]byte(meta), &d.Metadata)
+		d.Status = status
+		d.Relevance = rel
+		d.MatchedNodeID = matched
+		d.ReceivedAt = nsTime(recv)
+		d.UpdatedAt = nsTime(upd)
+		hamming := bits.OnesCount64(qsig ^ uint64(sig))
+		hashScore := 1 - float64(hamming)/64.0
+		lex := jaccard(qterms, termSet(title+" "+text))
+		score := 0.45*hashScore + 0.45*lex + 0.10*rel
+		if lex < 0.03 && hashScore < 0.58 {
+			continue
+		}
+		out = append(out, ScoredDocument{InboxDocument: d, QueryScore: score})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].QueryScore > out[j].QueryScore })
+	if len(out) > limit {
+		out = out[:limit]
+	}
+	return out, nil
+}
+
+func boolInt(v bool) int {
+	if v {
+		return 1
+	}
+	return 0
+}
+func timeNS(t time.Time) int64 {
+	if t.IsZero() {
+		return 0
+	}
+	return t.UTC().UnixNano()
+}
+func nsTime(v int64) time.Time {
+	if v <= 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, v).UTC()
+}
+func uniqueStrings(values []string) []string {
+	seen := map[string]bool{}
+	out := make([]string, 0, len(values))
+	for _, v := range values {
+		v = strings.TrimSpace(v)
+		if v == "" || seen[v] {
+			continue
+		}
+		seen[v] = true
+		out = append(out, v)
+	}
+	return out
+}
+func termSet(value string) map[string]bool {
+	out := map[string]bool{}
+	for _, r := range strings.FieldsFunc(strings.ToLower(value), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) {
+		if len([]rune(r)) >= 3 {
+			out[r] = true
+		}
+	}
+	return out
+}
+func jaccard(a, b map[string]bool) float64 {
+	if len(a) == 0 || len(b) == 0 {
+		return 0
+	}
+	inter := 0
+	union := len(a)
+	for k := range b {
+		if a[k] {
+			inter++
+		} else {
+			union++
+		}
+	}
+	if union == 0 {
+		return 0
+	}
+	return float64(inter) / float64(union)
+}
+func simhash64(value string) uint64 {
+	weights := [64]int{}
+	for token := range termSet(value) {
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(token))
+		x := h.Sum64()
+		for i := 0; i < 64; i++ {
+			if x&(1<<i) != 0 {
+				weights[i]++
+			} else {
+				weights[i]--
+			}
+		}
+	}
+	var sig uint64
+	for i, w := range weights {
+		if w >= 0 {
+			sig |= 1 << i
+		}
+	}
+	return sig
+}
+
+func (s *Store) ReleaseInbox(ctx context.Context, id, message string) error {
+	meta, _ := json.Marshal(map[string]any{"classification_error": strings.TrimSpace(message)})
+	_, err := s.db.ExecContext(ctx, `UPDATE source_inbox SET status='received',metadata_json=?,updated_at_ns=? WHERE id=?`, string(meta), time.Now().UTC().UnixNano(), id)
+	return err
+}