Agent-Mode mit Aufgabenteilung und automatischer Recherche
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Mode string
|
||||
ListenAddr string
|
||||
DataDir string
|
||||
KnowledgeDirs []string
|
||||
@@ -108,6 +109,22 @@ type Config struct {
|
||||
LowPowerMode bool
|
||||
RuntimeDefaultsConfigured bool
|
||||
|
||||
SourceInboxEnabled bool
|
||||
SourceInboxInterval time.Duration
|
||||
SourceInboxBatchSize int
|
||||
SourceInboxMinSimilarity float64
|
||||
SourceInboxMinResults int
|
||||
SourceInboxFreshMaxAge time.Duration
|
||||
AgentBrainURL string
|
||||
AgentID string
|
||||
AgentToken string
|
||||
AgentConfigFile string
|
||||
AgentConfigRefresh time.Duration
|
||||
AgentHTTPTimeout time.Duration
|
||||
AgentConcurrency int
|
||||
AgentBatchSize int
|
||||
AgentAllowPrivate bool
|
||||
|
||||
GLPIKBEnabled bool
|
||||
GLPIURL string
|
||||
GLPIAPIVersion string
|
||||
@@ -139,6 +156,7 @@ func Load() (Config, error) {
|
||||
ollamaURLs[i] = strings.TrimRight(ollamaURLs[i], "/")
|
||||
}
|
||||
cfg := Config{
|
||||
Mode: strings.ToLower(env("BRAIN_MODE", "brain")),
|
||||
ListenAddr: env("BRAIN_LISTEN_ADDR", ":8090"),
|
||||
DataDir: abs,
|
||||
KnowledgeDirs: paths("BRAIN_KNOWLEDGE_DIRS"),
|
||||
@@ -235,6 +253,21 @@ func Load() (Config, error) {
|
||||
MaxDisplayNodes: integer("BRAIN_MAX_DISPLAY_NODES", 0),
|
||||
LowPowerMode: boolean("BRAIN_LOW_POWER_MODE", false),
|
||||
RuntimeDefaultsConfigured: true,
|
||||
SourceInboxEnabled: boolean("BRAIN_SOURCE_INBOX_ENABLED", true),
|
||||
SourceInboxInterval: duration("BRAIN_SOURCE_INBOX_INTERVAL", 30*time.Second),
|
||||
SourceInboxBatchSize: integer("BRAIN_SOURCE_INBOX_BATCH_SIZE", 12),
|
||||
SourceInboxMinSimilarity: number("BRAIN_SOURCE_INBOX_MIN_SIMILARITY", 0.55),
|
||||
SourceInboxMinResults: integer("BRAIN_SOURCE_INBOX_MIN_RESULTS", 2),
|
||||
SourceInboxFreshMaxAge: duration("BRAIN_SOURCE_INBOX_FRESH_MAX_AGE", 168*time.Hour),
|
||||
AgentBrainURL: strings.TrimRight(strings.TrimSpace(os.Getenv("BRAIN_AGENT_BRAIN_URL")), "/"),
|
||||
AgentID: strings.TrimSpace(os.Getenv("BRAIN_AGENT_ID")),
|
||||
AgentToken: strings.TrimSpace(os.Getenv("BRAIN_AGENT_TOKEN")),
|
||||
AgentConfigFile: strings.TrimSpace(os.Getenv("BRAIN_AGENT_CONFIG_FILE")),
|
||||
AgentConfigRefresh: duration("BRAIN_AGENT_CONFIG_REFRESH", 5*time.Minute),
|
||||
AgentHTTPTimeout: duration("BRAIN_AGENT_HTTP_TIMEOUT", 30*time.Second),
|
||||
AgentConcurrency: integer("BRAIN_AGENT_CONCURRENCY", 3),
|
||||
AgentBatchSize: integer("BRAIN_AGENT_BATCH_SIZE", 50),
|
||||
AgentAllowPrivate: boolean("BRAIN_AGENT_ALLOW_PRIVATE", false),
|
||||
GLPIKBEnabled: boolean("GLPI_KB_ENABLED", false),
|
||||
GLPIURL: strings.TrimRight(strings.TrimSpace(os.Getenv("GLPI_URL")), "/"),
|
||||
GLPIAPIVersion: env("GLPI_API_VERSION", "v2.3"),
|
||||
@@ -250,6 +283,45 @@ func Load() (Config, error) {
|
||||
GLPIKBSyncInterval: duration("GLPI_KB_SYNC_INTERVAL", 10*time.Minute),
|
||||
GLPIKBSource: env("GLPI_KB_SOURCE", "GLPI Knowledge Base"),
|
||||
}
|
||||
if cfg.Mode != "brain" && cfg.Mode != "agent" {
|
||||
return Config{}, fmt.Errorf("BRAIN_MODE must be brain or agent")
|
||||
}
|
||||
if cfg.Mode == "agent" {
|
||||
if cfg.AgentConfigRefresh < time.Minute || cfg.AgentConfigRefresh > 24*time.Hour {
|
||||
return Config{}, fmt.Errorf("BRAIN_AGENT_CONFIG_REFRESH must be between 1m and 24h")
|
||||
}
|
||||
if cfg.AgentHTTPTimeout < 5*time.Second || cfg.AgentHTTPTimeout > 5*time.Minute {
|
||||
return Config{}, fmt.Errorf("BRAIN_AGENT_HTTP_TIMEOUT must be between 5s and 5m")
|
||||
}
|
||||
if cfg.AgentConcurrency < 1 || cfg.AgentConcurrency > 16 {
|
||||
return Config{}, fmt.Errorf("BRAIN_AGENT_CONCURRENCY must be between 1 and 16")
|
||||
}
|
||||
if cfg.AgentBatchSize < 1 || cfg.AgentBatchSize > 500 {
|
||||
return Config{}, fmt.Errorf("BRAIN_AGENT_BATCH_SIZE must be between 1 and 500")
|
||||
}
|
||||
if cfg.AgentConfigFile == "" && (cfg.AgentBrainURL == "" || cfg.AgentID == "" || cfg.AgentToken == "") {
|
||||
return Config{}, fmt.Errorf("agent mode requires BRAIN_AGENT_BRAIN_URL, BRAIN_AGENT_ID and BRAIN_AGENT_TOKEN or BRAIN_AGENT_CONFIG_FILE")
|
||||
}
|
||||
if err := os.MkdirAll(cfg.DataDir, 0o750); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
if cfg.SourceInboxInterval < 5*time.Second || cfg.SourceInboxInterval > 24*time.Hour {
|
||||
return Config{}, fmt.Errorf("BRAIN_SOURCE_INBOX_INTERVAL must be between 5s and 24h")
|
||||
}
|
||||
if cfg.SourceInboxBatchSize < 1 || cfg.SourceInboxBatchSize > 100 {
|
||||
return Config{}, fmt.Errorf("BRAIN_SOURCE_INBOX_BATCH_SIZE must be between 1 and 100")
|
||||
}
|
||||
if cfg.SourceInboxMinSimilarity < 0 || cfg.SourceInboxMinSimilarity > 1 {
|
||||
return Config{}, fmt.Errorf("BRAIN_SOURCE_INBOX_MIN_SIMILARITY must be between 0 and 1")
|
||||
}
|
||||
if cfg.SourceInboxMinResults < 1 || cfg.SourceInboxMinResults > 20 {
|
||||
return Config{}, fmt.Errorf("BRAIN_SOURCE_INBOX_MIN_RESULTS must be between 1 and 20")
|
||||
}
|
||||
if cfg.SourceInboxFreshMaxAge < time.Hour || cfg.SourceInboxFreshMaxAge > 365*24*time.Hour {
|
||||
return Config{}, fmt.Errorf("BRAIN_SOURCE_INBOX_FRESH_MAX_AGE must be between 1h and 8760h")
|
||||
}
|
||||
if cfg.ScanInterval < 2*time.Second {
|
||||
return Config{}, fmt.Errorf("BRAIN_SCAN_INTERVAL must be at least 2s")
|
||||
}
|
||||
|
||||
@@ -187,3 +187,32 @@ func TestLoadRejectsInvalidArticleResearchStrategy(t *testing.T) {
|
||||
t.Fatal("expected invalid article research strategy error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAgentModeDoesNotRequireBrainDependencies(t *testing.T) {
|
||||
t.Setenv("BRAIN_DATA_DIR", t.TempDir())
|
||||
t.Setenv("BRAIN_MODE", "agent")
|
||||
t.Setenv("BRAIN_AGENT_BRAIN_URL", "http://brain:8090")
|
||||
t.Setenv("BRAIN_AGENT_ID", "poller-1")
|
||||
t.Setenv("BRAIN_AGENT_TOKEN", "brain_agent_test")
|
||||
t.Setenv("OLLAMA_URLS", "not-a-url")
|
||||
t.Setenv("BRAIN_AUTONOMOUS_RESEARCH_ENABLED", "true")
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Mode != "agent" || cfg.AgentID != "poller-1" || cfg.AgentBrainURL != "http://brain:8090" {
|
||||
t.Fatalf("unexpected agent config: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAgentModeRequiresBootstrapConnection(t *testing.T) {
|
||||
t.Setenv("BRAIN_DATA_DIR", t.TempDir())
|
||||
t.Setenv("BRAIN_MODE", "agent")
|
||||
t.Setenv("BRAIN_AGENT_BRAIN_URL", "")
|
||||
t.Setenv("BRAIN_AGENT_ID", "")
|
||||
t.Setenv("BRAIN_AGENT_TOKEN", "")
|
||||
t.Setenv("BRAIN_AGENT_CONFIG_FILE", "")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("expected missing agent bootstrap configuration error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +139,7 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
researchStrategy := e.effectiveArticleResearchStrategy()
|
||||
freshness := detectArticleFreshnessNeed(plan, relation, selected)
|
||||
initialWebResearch := false
|
||||
if e.ResearchEnabledForRuntime() {
|
||||
if e.evidenceAcquisitionEnabled() {
|
||||
switch researchStrategy {
|
||||
case "always":
|
||||
collected, report, researchErr := e.collectResearchMaterialForArticle(ctx, trigger, plan.SourceNodeIDs, selected, plan, brief, researchResults)
|
||||
@@ -160,7 +160,7 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
}
|
||||
}
|
||||
}
|
||||
e.Broker.Publish(model.Activity{Type: "article.research.strategy", Source: "brain", Phase: "knowledge-research-routing", NodeIDs: plan.SourceNodeIDs, Message: fmt.Sprintf("Artikelrecherche: %s · initiales Webmaterial: %t", researchStrategy, initialWebResearch), Strength: .44, Metadata: map[string]any{"trigger": trigger, "research_strategy": researchStrategy, "freshness_required": freshness.Required, "freshness_reason": freshness.Reason, "initial_web_research": initialWebResearch, "initial_research_queries": researchReport.Queries, "initial_research_fetched": researchReport.Fetched}})
|
||||
e.Broker.Publish(model.Activity{Type: "article.research.strategy", Source: "brain", Phase: "knowledge-research-routing", NodeIDs: plan.SourceNodeIDs, Message: fmt.Sprintf("Artikelrecherche: %s · initiales Webmaterial: %t", researchStrategy, initialWebResearch), Strength: .44, Metadata: map[string]any{"trigger": trigger, "research_strategy": researchStrategy, "freshness_required": freshness.Required, "freshness_reason": freshness.Reason, "initial_web_research": initialWebResearch, "initial_research_queries": researchReport.Queries, "source_inbox_results": researchReport.InboxResults, "initial_research_fetched": researchReport.Fetched}})
|
||||
|
||||
e.Broker.Publish(model.Activity{Type: "article.draft.started", Source: "brain", Phase: "knowledge-synthesis", NodeIDs: plan.SourceNodeIDs, Message: fmt.Sprintf("%s erstellt zuerst aus dem verfügbaren Evidenzsatz einen KB-Artikel; Webrecherche erfolgt nur bei Bedarf", e.Cfg.ArticleSynthesisModel), Strength: .95, Metadata: map[string]any{"trigger": trigger, "action": plan.Action, "article_type": plan.ArticleType, "target_article_id": plan.TargetArticleID, "source_count": len(selected), "research_material_count": len(researchResults), "research_rounds": researchReport.Rounds, "research_fetched": researchReport.Fetched, "research_strategy": researchStrategy, "freshness_required": freshness.Required, "synthesis_model": e.Cfg.ArticleSynthesisModel, "review_model": e.Cfg.ArticleReviewModel, "generation_depth": generationDepth}})
|
||||
|
||||
@@ -187,7 +187,7 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
// In adaptive mode Gemma is allowed to say that the internal KB is not
|
||||
// sufficient. Only then do we pay for a focused Web round. This happens
|
||||
// before the reviewer, so an obvious evidence gap does not waste a Qwen call.
|
||||
if researchStrategy == "adaptive" && !authorResearchAttempted && !freshness.Required && e.ResearchEnabledForRuntime() && (content.ResearchNeeded || content.FreshnessSensitive) {
|
||||
if researchStrategy == "adaptive" && !authorResearchAttempted && !freshness.Required && e.evidenceAcquisitionEnabled() && (content.ResearchNeeded || content.FreshnessSensitive) {
|
||||
queries := append([]string{}, content.ResearchQueries...)
|
||||
if content.ResearchNeeded && len(queries) == 0 {
|
||||
queries = append(queries, plan.ResearchQuery)
|
||||
@@ -235,11 +235,11 @@ func (e *Engine) synthesizeKnowledgeArticle(ctx context.Context, trigger string,
|
||||
break
|
||||
}
|
||||
repairAttempts++
|
||||
if len(quality.MissingEvidenceQueries) > 0 && e.ResearchEnabledForRuntime() {
|
||||
if len(quality.MissingEvidenceQueries) > 0 && e.evidenceAcquisitionEnabled() {
|
||||
e.Broker.Publish(model.Activity{Type: "article.review.research.started", Source: "brain", Phase: "quality-repair-research", NodeIDs: draft.SourceNodeIDs, Message: "Der Reviewer hat konkrete unbelegte Aussagen gefunden · nur diese Punkte werden nachrecherchiert", Strength: .84, Metadata: map[string]any{"trigger": trigger, "repair_round": repairAttempts, "queries": quality.MissingEvidenceQueries, "review_model": e.Cfg.ArticleReviewModel}})
|
||||
additional, repairReport := e.collectReviewerRepairResearch(ctx, trigger, plan.SourceNodeIDs, quality.MissingEvidenceQueries, repairAttempts, attemptedRepairURLs)
|
||||
researchResults = uniqueResearchEvidence(append(researchResults, additional...))
|
||||
e.Broker.Publish(model.Activity{Type: "article.review.research.completed", Source: "brain", Phase: "quality-repair-research", NodeIDs: draft.SourceNodeIDs, Message: fmt.Sprintf("Gezielte Nachrecherche beendet · %d zusätzliche Volltextquellen", len(additional)), Strength: .82, Metadata: map[string]any{"trigger": trigger, "repair_round": repairAttempts, "new_material": len(additional), "queries": repairReport.Queries, "search_results": repairReport.SearchResults, "fetched": repairReport.Fetched}})
|
||||
e.Broker.Publish(model.Activity{Type: "article.review.research.completed", Source: "brain", Phase: "quality-repair-research", NodeIDs: draft.SourceNodeIDs, Message: fmt.Sprintf("Gezielte Nachrecherche beendet · %d zusätzliche Volltextquellen", len(additional)), Strength: .82, Metadata: map[string]any{"trigger": trigger, "repair_round": repairAttempts, "new_material": len(additional), "queries": repairReport.Queries, "source_inbox_results": repairReport.InboxResults, "search_results": repairReport.SearchResults, "fetched": repairReport.Fetched}})
|
||||
}
|
||||
reviewCopy := quality
|
||||
reviewFeedback = &reviewCopy
|
||||
|
||||
@@ -145,13 +145,27 @@ func (e *Engine) collectAdaptiveInitialResearch(ctx context.Context, trigger str
|
||||
out := make([]model.ResearchResult, 0)
|
||||
report := articleResearchReport{Rounds: 1}
|
||||
for i, query := range queries {
|
||||
report.Queries++
|
||||
freshnessSensitive := containsFreshnessLanguage(query)
|
||||
inbox := e.sourceInboxResearch(ctx, query, fetchCap, freshnessSensitive)
|
||||
if len(inbox) > 0 {
|
||||
report.InboxResults += len(inbox)
|
||||
report.Accepted += len(inbox)
|
||||
out = uniqueResearchEvidence(append(out, inbox...))
|
||||
}
|
||||
if len(inbox) >= e.Cfg.SourceInboxMinResults || !e.ResearchEnabledForRuntime() {
|
||||
continue
|
||||
}
|
||||
remaining := fetchCap - len(inbox)
|
||||
if remaining < 1 {
|
||||
remaining = 1
|
||||
}
|
||||
language := "en-US"
|
||||
if looksGermanResearchQuery(query) {
|
||||
language = "de-DE"
|
||||
}
|
||||
question := model.ResearchQuestion{GapID: fmt.Sprintf("ADAPTIVE-%d", i+1), Question: query, Critical: true, ExpectActionable: containsActionableLanguage(query)}
|
||||
report.Queries++
|
||||
items, stats := e.executeArticleResearchQueryForSynthesis(ctx, trigger, nodeIDs, question, query, language, 1, attemptedURLs, fetchCap)
|
||||
items, stats := e.executeArticleResearchQueryForSynthesis(ctx, trigger, nodeIDs, question, query, language, 1, attemptedURLs, remaining)
|
||||
report.SearchResults += stats.SearchResults
|
||||
report.Fetched += stats.Fetched
|
||||
report.Accepted += stats.Accepted
|
||||
@@ -307,7 +321,9 @@ func (e *Engine) materializeGroundedResearchEvidence(articleID string, sources [
|
||||
}
|
||||
categories := categoriesFromArticleSources(sources)
|
||||
ids := make([]string, 0, len(results))
|
||||
usedURLs := make([]string, 0, len(results))
|
||||
for _, result := range results {
|
||||
usedURLs = append(usedURLs, result.URL)
|
||||
id := graph.ID("external", result.URL)
|
||||
path, contentHash, err := e.queueResearchEvidence(result)
|
||||
if err != nil {
|
||||
@@ -322,5 +338,8 @@ func (e *Engine) materializeGroundedResearchEvidence(articleID string, sources [
|
||||
e.Graph.UpsertNode(node)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if e.SourceInbox != nil && len(usedURLs) > 0 {
|
||||
_ = e.SourceInbox.MarkUsed(context.Background(), usedURLs)
|
||||
}
|
||||
return unique(ids)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ func (e *Engine) collectResearchMaterialForArticle(ctx context.Context, trigger
|
||||
}
|
||||
}
|
||||
|
||||
if !e.ResearchEnabledForRuntime() {
|
||||
if !e.evidenceAcquisitionEnabled() {
|
||||
return material, report, nil
|
||||
}
|
||||
maxRounds := e.Cfg.ArticleResearchRounds
|
||||
@@ -56,6 +56,23 @@ func (e *Engine) collectResearchMaterialForArticle(ctx context.Context, trigger
|
||||
if report.Queries >= maxQueries {
|
||||
break
|
||||
}
|
||||
inboxMaterial := e.sourceInboxResearch(ctx, question.Question, e.Cfg.ArticleResearchFetchResults, containsFreshnessLanguage(question.Question))
|
||||
if len(inboxMaterial) > 0 {
|
||||
report.InboxResults += len(inboxMaterial)
|
||||
report.Accepted += len(inboxMaterial)
|
||||
for _, item := range inboxMaterial {
|
||||
key := canonicalResearchURL(item.URL)
|
||||
if key == "" || seenURLs[key] {
|
||||
continue
|
||||
}
|
||||
seenURLs[key] = true
|
||||
material = append(material, item)
|
||||
collectedThisRound++
|
||||
}
|
||||
}
|
||||
if len(inboxMaterial) >= e.Cfg.SourceInboxMinResults || !e.ResearchEnabledForRuntime() {
|
||||
continue
|
||||
}
|
||||
lease, reused, err := e.beginResearchIntent(ctx, "synthesis-material", question.Question)
|
||||
if err != nil {
|
||||
return material, report, err
|
||||
@@ -201,13 +218,26 @@ func (e *Engine) collectReviewerRepairResearch(ctx context.Context, trigger stri
|
||||
if query == "" {
|
||||
continue
|
||||
}
|
||||
report.Queries++
|
||||
inbox := e.sourceInboxResearch(ctx, query, fetchCap, containsFreshnessLanguage(query))
|
||||
if len(inbox) > 0 {
|
||||
report.InboxResults += len(inbox)
|
||||
report.Accepted += len(inbox)
|
||||
out = uniqueResearchEvidence(append(out, inbox...))
|
||||
}
|
||||
if len(inbox) >= e.Cfg.SourceInboxMinResults || !e.ResearchEnabledForRuntime() {
|
||||
continue
|
||||
}
|
||||
remaining := fetchCap - len(inbox)
|
||||
if remaining < 1 {
|
||||
remaining = 1
|
||||
}
|
||||
language := "en-US"
|
||||
if looksGermanResearchQuery(query) {
|
||||
language = "de-DE"
|
||||
}
|
||||
question := model.ResearchQuestion{GapID: fmt.Sprintf("REVIEW-%d", i+1), Question: query, Critical: true, ExpectActionable: containsActionableLanguage(query)}
|
||||
report.Queries++
|
||||
items, stats := e.executeArticleResearchQueryForSynthesis(ctx, trigger, nodeIDs, question, query, language, round, attemptedURLs, fetchCap)
|
||||
items, stats := e.executeArticleResearchQueryForSynthesis(ctx, trigger, nodeIDs, question, query, language, round, attemptedURLs, remaining)
|
||||
report.SearchResults += stats.SearchResults
|
||||
report.Fetched += stats.Fetched
|
||||
report.Accepted += stats.Accepted
|
||||
|
||||
@@ -26,6 +26,7 @@ type articleResearchReport struct {
|
||||
Rejected int
|
||||
FetchFailed int
|
||||
SearchFailed int
|
||||
InboxResults int
|
||||
}
|
||||
|
||||
type rankedResearchCandidate struct {
|
||||
@@ -492,6 +493,7 @@ type queryExecutionStats struct {
|
||||
Rejected int
|
||||
FetchFailed int
|
||||
SearchFailed int
|
||||
InboxResults int
|
||||
}
|
||||
|
||||
func (e *Engine) executeArticleResearchQuery(ctx context.Context, trigger string, nodeIDs []string, question model.ResearchQuestion, query, language string, round int, attemptedURLs map[string]bool, fetchCaps ...int) ([]model.ResearchResult, queryExecutionStats) {
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
"github.com/local/glpi-neural-brain/internal/persist"
|
||||
"github.com/local/glpi-neural-brain/internal/research"
|
||||
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
||||
"github.com/local/glpi-neural-brain/internal/workqueue"
|
||||
)
|
||||
|
||||
@@ -63,6 +64,7 @@ type Engine struct {
|
||||
Scanner *ingest.KnowledgeScanner
|
||||
GLPIKB *ingest.GLPIKBSyncer
|
||||
Persistence *persist.Coordinator
|
||||
SourceInbox *sourceagent.Store
|
||||
|
||||
mu sync.Mutex
|
||||
stateMu sync.RWMutex
|
||||
@@ -329,6 +331,9 @@ func (e *Engine) Start(ctx context.Context) {
|
||||
}
|
||||
go e.idle(ctx)
|
||||
e.startAutonomousResearch(ctx)
|
||||
if e.SourceInbox != nil && e.Cfg.SourceInboxEnabled {
|
||||
go e.sourceInboxLoop(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) enrichmentScheduler(ctx context.Context) {
|
||||
@@ -1041,6 +1046,11 @@ func (e *Engine) Status() map[string]any {
|
||||
}
|
||||
e.stateMu.RUnlock()
|
||||
status["autonomous_research"] = e.AutonomousResearchStatus(context.Background())
|
||||
if e.SourceInbox != nil {
|
||||
if inbox, err := e.SourceInbox.Stats(context.Background()); err == nil {
|
||||
status["source_inbox"] = inbox
|
||||
}
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/ollama"
|
||||
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
||||
)
|
||||
|
||||
func (e *Engine) SetSourceInbox(store *sourceagent.Store) { e.SourceInbox = store }
|
||||
|
||||
func (e *Engine) evidenceAcquisitionEnabled() bool {
|
||||
return (e.SourceInbox != nil && e.Cfg.SourceInboxEnabled) || e.ResearchEnabledForRuntime()
|
||||
}
|
||||
|
||||
func (e *Engine) sourceInboxLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(e.Cfg.SourceInboxInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
e.processSourceInbox(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) processSourceInbox(ctx context.Context) {
|
||||
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled {
|
||||
return
|
||||
}
|
||||
items, err := e.SourceInbox.ClaimInbox(ctx, e.Cfg.SourceInboxBatchSize)
|
||||
if err != nil {
|
||||
slog.Warn("source inbox claim failed", "error", err)
|
||||
return
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return
|
||||
}
|
||||
texts := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
text := item.Document.Title + "\n" + strings.Join(item.Document.Categories, " · ") + "\n" + item.Document.Text
|
||||
if len([]rune(text)) > 12000 {
|
||||
text = string([]rune(text)[:12000])
|
||||
}
|
||||
texts = append(texts, text)
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ollama.WithLowPriority(ctx), 4*time.Minute)
|
||||
vectors, embedErr := e.Ollama.Embed(cctx, texts)
|
||||
cancel()
|
||||
if embedErr != nil || len(vectors) != len(items) {
|
||||
for _, item := range items {
|
||||
_ = e.SourceInbox.ReleaseInbox(ctx, item.ID, fmt.Sprint(embedErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
candidateCount := 0
|
||||
for i, item := range items {
|
||||
hits, stats := e.similarKnowledge(vectors[i], 1, e.effectiveLearningFilter(), 0)
|
||||
if len(hits) == 0 {
|
||||
_ = e.SourceInbox.ReleaseInbox(ctx, item.ID, "knowledge vectors are not ready")
|
||||
continue
|
||||
}
|
||||
relevance := 0.0
|
||||
matched := ""
|
||||
if len(hits) > 0 {
|
||||
relevance = hits[0].Score
|
||||
matched = hits[0].NodeID
|
||||
}
|
||||
status := "archived"
|
||||
if relevance >= e.Cfg.SourceInboxMinSimilarity {
|
||||
status = "candidate"
|
||||
candidateCount++
|
||||
}
|
||||
meta := make(map[string]any, len(item.Metadata)+4)
|
||||
for key, value := range item.Metadata {
|
||||
meta[key] = value
|
||||
}
|
||||
meta["processing_mode"] = e.RuntimeSettings().ProcessingMode
|
||||
meta["exact_comparisons"] = stats.ExactComparisons
|
||||
meta["coarse_comparisons"] = stats.CoarseComparisons
|
||||
meta["candidate_pool"] = stats.CandidatePool
|
||||
_ = e.SourceInbox.CompleteClassification(ctx, item.ID, status, relevance, matched, meta)
|
||||
}
|
||||
if e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "source.inbox.classified", Source: "brain", Phase: "source-inbox", Message: fmt.Sprintf("Source-Inbox: %d Dokumente geprüft · %d als Wissenskandidaten vorgemerkt", len(items), candidateCount), Strength: .36, Metadata: map[string]any{"documents": len(items), "candidates": candidateCount, "minimum_similarity": e.Cfg.SourceInboxMinSimilarity}})
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) sourceInboxResearch(ctx context.Context, query string, limit int, freshnessSensitive bool) []model.ResearchResult {
|
||||
if e.SourceInbox == nil || !e.Cfg.SourceInboxEnabled || limit < 1 {
|
||||
return nil
|
||||
}
|
||||
maxAge := time.Duration(0)
|
||||
if freshnessSensitive {
|
||||
maxAge = e.Cfg.SourceInboxFreshMaxAge
|
||||
}
|
||||
items, err := e.SourceInbox.SearchCandidates(ctx, query, limit, maxAge)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]model.ResearchResult, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item.QueryScore < .18 {
|
||||
continue
|
||||
}
|
||||
d := item.Document
|
||||
out = append(out, model.ResearchResult{Title: d.Title, URL: d.CanonicalURL, Snippet: clamp(d.Text, 1000), Content: d.Text, ContentType: d.ContentType, Query: query, Language: d.Language, Fetched: true, Relevant: true, Relevance: item.QueryScore, SourceQuality: "source_inbox", SourceQualityScore: .68, AssessmentReason: "Vorab durch Source-Agent gesammelt und vom Brain als thematisch passend zur Knowledgebase klassifiziert."})
|
||||
}
|
||||
if len(out) > 0 && e.Broker != nil {
|
||||
e.Broker.Publish(model.Activity{Type: "article.research.inbox", Source: "brain", Phase: "knowledge-research-routing", Message: fmt.Sprintf("Source-Inbox liefert %d bereits gecrawlte Kandidaten vor SearXNG", len(out)), Strength: .56, Metadata: map[string]any{"query": query, "results": len(out), "freshness_sensitive": freshnessSensitive}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
+28
-17
@@ -46,6 +46,33 @@ type FetchedPage struct {
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// NewSafeHTTPClient returns an HTTP client that applies the same SSRF protections
|
||||
// as FetchPage. It is intended for source/feed fetchers that need the raw response
|
||||
// body instead of extracted page text. Private, loopback and link-local targets are
|
||||
// rejected unless allowPrivate is explicitly enabled. Redirects are validated too.
|
||||
func NewSafeHTTPClient(allowPrivate bool, timeout time.Duration) *http.Client {
|
||||
if timeout <= 0 {
|
||||
timeout = 20 * time.Second
|
||||
}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = safeDialContext(allowPrivate)
|
||||
transport.MaxIdleConns = 4
|
||||
transport.MaxIdleConnsPerHost = 2
|
||||
transport.IdleConnTimeout = 20 * time.Second
|
||||
return &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("zu viele Weiterleitungen")
|
||||
}
|
||||
_, err := validateFetchURL(req.Context(), req.URL.String(), allowPrivate)
|
||||
return err
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) FetchPage(ctx context.Context, rawURL string, options FetchOptions) (FetchedPage, FetchDiagnostic, error) {
|
||||
started := time.Now()
|
||||
diagnostic := FetchDiagnostic{URL: strings.TrimSpace(rawURL)}
|
||||
@@ -79,23 +106,7 @@ func (c *Client) FetchPage(ctx context.Context, rawURL string, options FetchOpti
|
||||
return finish(FetchedPage{}, err)
|
||||
}
|
||||
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = safeDialContext(options.AllowPrivate)
|
||||
transport.MaxIdleConns = 4
|
||||
transport.MaxIdleConnsPerHost = 2
|
||||
transport.IdleConnTimeout = 20 * time.Second
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: options.Timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("zu viele Weiterleitungen")
|
||||
}
|
||||
_, err := validateFetchURL(req.Context(), req.URL.String(), options.AllowPrivate)
|
||||
return err
|
||||
},
|
||||
}
|
||||
client := NewSafeHTTPClient(options.AllowPrivate, options.Timeout)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,658 @@
|
||||
package sourceagent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/research"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type RunnerConfig struct {
|
||||
BrainURL string
|
||||
AgentID string
|
||||
Token string
|
||||
DataDir string
|
||||
ConfigFile string
|
||||
ConfigRefresh time.Duration
|
||||
HTTPTimeout time.Duration
|
||||
Concurrency int
|
||||
BatchSize int
|
||||
AllowPrivate bool
|
||||
Version string
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
cfg RunnerConfig
|
||||
http *http.Client
|
||||
sourceHTTP *http.Client
|
||||
state *localState
|
||||
mu sync.RWMutex
|
||||
remote RemoteConfig
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
type bootstrapConfig struct {
|
||||
BrainURL string `json:"brain_url"`
|
||||
AgentID string `json:"agent_id"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
func NewRunner(cfg RunnerConfig) (*Runner, error) {
|
||||
if strings.TrimSpace(cfg.ConfigFile) != "" {
|
||||
if data, err := os.ReadFile(cfg.ConfigFile); err == nil {
|
||||
var b bootstrapConfig
|
||||
if json.Unmarshal(data, &b) == nil {
|
||||
if cfg.BrainURL == "" {
|
||||
cfg.BrainURL = b.BrainURL
|
||||
}
|
||||
if cfg.AgentID == "" {
|
||||
cfg.AgentID = b.AgentID
|
||||
}
|
||||
if cfg.Token == "" {
|
||||
cfg.Token = b.Token
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cfg.BrainURL = strings.TrimRight(strings.TrimSpace(cfg.BrainURL), "/")
|
||||
cfg.AgentID = strings.TrimSpace(cfg.AgentID)
|
||||
cfg.Token = strings.TrimSpace(cfg.Token)
|
||||
if cfg.BrainURL == "" || cfg.AgentID == "" || cfg.Token == "" {
|
||||
return nil, errors.New("agent mode requires BRAIN_AGENT_BRAIN_URL, BRAIN_AGENT_ID and BRAIN_AGENT_TOKEN (or BRAIN_AGENT_CONFIG_FILE)")
|
||||
}
|
||||
u, err := url.Parse(cfg.BrainURL)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return nil, errors.New("BRAIN_AGENT_BRAIN_URL must be an absolute http(s) URL")
|
||||
}
|
||||
if u.User != nil {
|
||||
return nil, errors.New("BRAIN_AGENT_BRAIN_URL must not contain userinfo")
|
||||
}
|
||||
if cfg.ConfigRefresh < time.Minute {
|
||||
cfg.ConfigRefresh = 5 * time.Minute
|
||||
}
|
||||
if cfg.HTTPTimeout < 5*time.Second {
|
||||
cfg.HTTPTimeout = 30 * time.Second
|
||||
}
|
||||
if cfg.Concurrency < 1 {
|
||||
cfg.Concurrency = 3
|
||||
}
|
||||
if cfg.Concurrency > 16 {
|
||||
cfg.Concurrency = 16
|
||||
}
|
||||
if cfg.BatchSize < 1 {
|
||||
cfg.BatchSize = 50
|
||||
}
|
||||
if cfg.BatchSize > 500 {
|
||||
cfg.BatchSize = 500
|
||||
}
|
||||
state, err := openLocalState(cfg.DataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Runner{
|
||||
cfg: cfg,
|
||||
http: &http.Client{Timeout: cfg.HTTPTimeout},
|
||||
sourceHTTP: research.NewSafeHTTPClient(cfg.AllowPrivate, cfg.HTTPTimeout),
|
||||
state: state,
|
||||
wake: make(chan struct{}, 1),
|
||||
}, nil
|
||||
}
|
||||
func (r *Runner) Close() error {
|
||||
if r == nil || r.state == nil {
|
||||
return nil
|
||||
}
|
||||
return r.state.Close()
|
||||
}
|
||||
|
||||
func (r *Runner) Start(ctx context.Context) {
|
||||
r.loadCachedConfig()
|
||||
go r.loop(ctx)
|
||||
}
|
||||
|
||||
func (r *Runner) Status() map[string]any {
|
||||
r.mu.RLock()
|
||||
remote := r.remote
|
||||
r.mu.RUnlock()
|
||||
return map[string]any{"ok": true, "mode": "agent", "agent_id": r.cfg.AgentID, "brain_url": r.cfg.BrainURL, "configured_tasks": len(remote.Tasks), "config_issued_at": remote.IssuedAt, "version": r.cfg.Version}
|
||||
}
|
||||
|
||||
func (r *Runner) loop(ctx context.Context) {
|
||||
refresh := time.NewTicker(r.cfg.ConfigRefresh)
|
||||
defer refresh.Stop()
|
||||
run := time.NewTicker(30 * time.Second)
|
||||
defer run.Stop()
|
||||
_ = r.refreshConfig(ctx)
|
||||
r.runDue(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-refresh.C:
|
||||
_ = r.refreshConfig(ctx)
|
||||
case <-run.C:
|
||||
r.runDue(ctx)
|
||||
case <-r.wake:
|
||||
r.runDue(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) refreshConfig(ctx context.Context) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.cfg.BrainURL+"/api/v1/agent/config", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.auth(req)
|
||||
resp, err := r.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("brain config returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var cfg RemoteConfig
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Agent.ID != "" && cfg.Agent.ID != r.cfg.AgentID {
|
||||
return fmt.Errorf("brain returned config for unexpected agent %q", cfg.Agent.ID)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.remote = cfg
|
||||
r.mu.Unlock()
|
||||
r.saveCachedConfig(cfg)
|
||||
_ = r.sendHeartbeat(ctx, Heartbeat{AgentID: r.cfg.AgentID, Version: r.cfg.Version, Status: "online", Metadata: map[string]any{"configured_tasks": len(cfg.Tasks)}})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) runDue(ctx context.Context) {
|
||||
r.mu.RLock()
|
||||
tasks := append([]Task(nil), r.remote.Tasks...)
|
||||
r.mu.RUnlock()
|
||||
if len(tasks) == 0 {
|
||||
return
|
||||
}
|
||||
sem := make(chan struct{}, r.cfg.Concurrency)
|
||||
var wg sync.WaitGroup
|
||||
for _, task := range tasks {
|
||||
task := task
|
||||
if !task.Enabled || !r.state.Due(ctx, task) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
defer func() { <-sem }()
|
||||
r.runTask(ctx, task)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (r *Runner) runTask(ctx context.Context, task Task) {
|
||||
started := time.Now().UTC()
|
||||
docs, err := r.pollTask(ctx, task)
|
||||
if err == nil && len(docs) > 0 {
|
||||
for start := 0; start < len(docs); start += r.cfg.BatchSize {
|
||||
end := start + r.cfg.BatchSize
|
||||
if end > len(docs) {
|
||||
end = len(docs)
|
||||
}
|
||||
if sendErr := r.sendBatch(ctx, task.ID, docs[start:end]); sendErr != nil {
|
||||
err = sendErr
|
||||
break
|
||||
}
|
||||
for _, d := range docs[start:end] {
|
||||
_ = r.state.MarkSeen(ctx, task.ID, d.CanonicalURL, d.ContentSHA256)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = r.state.FinishTask(ctx, task.ID, started, err)
|
||||
h := Heartbeat{AgentID: r.cfg.AgentID, Version: r.cfg.Version, Status: "ok", LastRunAt: started, TasksChecked: 1, Documents: len(docs)}
|
||||
if err != nil {
|
||||
h.Status = "error"
|
||||
h.LastError = err.Error()
|
||||
slog.Warn("source agent task failed", "task", task.ID, "error", err)
|
||||
} else {
|
||||
slog.Info("source agent task completed", "task", task.ID, "documents", len(docs))
|
||||
}
|
||||
_ = r.sendHeartbeat(ctx, h)
|
||||
}
|
||||
|
||||
func (r *Runner) pollTask(ctx context.Context, task Task) ([]Document, error) {
|
||||
switch task.Type {
|
||||
case "rss", "atom":
|
||||
return r.pollFeed(ctx, task)
|
||||
case "sitemap":
|
||||
return r.pollSitemap(ctx, task)
|
||||
case "web":
|
||||
return r.pollWeb(ctx, task)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported task type %q", task.Type)
|
||||
}
|
||||
}
|
||||
|
||||
type feedEnvelope struct {
|
||||
Channel struct {
|
||||
Items []struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
GUID string `xml:"guid"`
|
||||
} `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
Entries []struct {
|
||||
Title string `xml:"title"`
|
||||
ID string `xml:"id"`
|
||||
Updated string `xml:"updated"`
|
||||
Published string `xml:"published"`
|
||||
Summary string `xml:"summary"`
|
||||
Content string `xml:"content"`
|
||||
Links []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
} `xml:"entry"`
|
||||
}
|
||||
|
||||
type feedItem struct {
|
||||
Title, Link, Summary string
|
||||
Published time.Time
|
||||
}
|
||||
|
||||
func (r *Runner) pollFeed(ctx context.Context, task Task) ([]Document, error) {
|
||||
body, finalURL, _, _, err := r.fetchRaw(ctx, task.URL, 5<<20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var env feedEnvelope
|
||||
if err := xml.Unmarshal(body, &env); err != nil {
|
||||
return nil, fmt.Errorf("feed XML: %w", err)
|
||||
}
|
||||
var items []feedItem
|
||||
for _, it := range env.Channel.Items {
|
||||
link := strings.TrimSpace(it.Link)
|
||||
if link == "" {
|
||||
link = strings.TrimSpace(it.GUID)
|
||||
}
|
||||
items = append(items, feedItem{Title: strings.TrimSpace(it.Title), Link: resolveLink(finalURL, link), Summary: stripMarkup(it.Description), Published: parsePublished(it.PubDate)})
|
||||
}
|
||||
for _, it := range env.Entries {
|
||||
link := ""
|
||||
for _, l := range it.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
link = l.Href
|
||||
break
|
||||
}
|
||||
}
|
||||
pub := parsePublished(it.Published)
|
||||
if pub.IsZero() {
|
||||
pub = parsePublished(it.Updated)
|
||||
}
|
||||
summary := it.Summary
|
||||
if summary == "" {
|
||||
summary = it.Content
|
||||
}
|
||||
items = append(items, feedItem{Title: strings.TrimSpace(it.Title), Link: resolveLink(finalURL, link), Summary: stripMarkup(summary), Published: pub})
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].Published.After(items[j].Published) })
|
||||
if len(items) > task.MaxItems {
|
||||
items = items[:task.MaxItems]
|
||||
}
|
||||
return r.materializeItems(ctx, task, items)
|
||||
}
|
||||
|
||||
type sitemapEnvelope struct {
|
||||
URLs []struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
} `xml:"url"`
|
||||
Sitemaps []struct {
|
||||
Loc string `xml:"loc"`
|
||||
} `xml:"sitemap"`
|
||||
}
|
||||
|
||||
func (r *Runner) pollSitemap(ctx context.Context, task Task) ([]Document, error) {
|
||||
items, err := r.collectSitemapItems(ctx, task.URL, task.MaxItems, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.materializeItems(ctx, task, items)
|
||||
}
|
||||
|
||||
func (r *Runner) collectSitemapItems(ctx context.Context, rawURL string, limit, depth int) ([]feedItem, error) {
|
||||
if limit <= 0 || depth > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
body, finalURL, _, _, err := r.fetchRaw(ctx, rawURL, 8<<20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sm sitemapEnvelope
|
||||
if err := xml.Unmarshal(body, &sm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]feedItem, 0, limit)
|
||||
for _, u := range sm.URLs {
|
||||
link := resolveLink(finalURL, u.Loc)
|
||||
if link == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, feedItem{Link: link, Published: parsePublished(u.LastMod)})
|
||||
if len(items) >= limit {
|
||||
return items, nil
|
||||
}
|
||||
}
|
||||
// Sitemap indexes are common on larger publishers. Follow a bounded number of
|
||||
// child maps once; source HTTP safety rules apply to every child URL.
|
||||
for i, child := range sm.Sitemaps {
|
||||
if len(items) >= limit || i >= 12 {
|
||||
break
|
||||
}
|
||||
childURL := resolveLink(finalURL, child.Loc)
|
||||
if childURL == "" {
|
||||
continue
|
||||
}
|
||||
more, childErr := r.collectSitemapItems(ctx, childURL, limit-len(items), depth+1)
|
||||
if childErr != nil {
|
||||
continue
|
||||
}
|
||||
items = append(items, more...)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
var hrefPattern = regexp.MustCompile(`(?is)<a\b[^>]*href\s*=\s*["']([^"'#]+)["'][^>]*>(.*?)</a>`)
|
||||
|
||||
func (r *Runner) pollWeb(ctx context.Context, task Task) ([]Document, error) {
|
||||
body, finalURL, _, _, err := r.fetchRaw(ctx, task.URL, 5<<20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matches := hrefPattern.FindAllStringSubmatch(string(body), -1)
|
||||
seen := map[string]bool{}
|
||||
items := make([]feedItem, 0, task.MaxItems)
|
||||
base, _ := url.Parse(finalURL)
|
||||
for _, m := range matches {
|
||||
link := resolveLink(finalURL, m[1])
|
||||
if link == "" || seen[link] {
|
||||
continue
|
||||
}
|
||||
u, err := url.Parse(link)
|
||||
if err != nil || u.Host != base.Host {
|
||||
continue
|
||||
}
|
||||
if !looksArticleLink(u.Path, stripMarkup(m[2])) {
|
||||
continue
|
||||
}
|
||||
seen[link] = true
|
||||
items = append(items, feedItem{Title: stripMarkup(m[2]), Link: link})
|
||||
if len(items) >= task.MaxItems {
|
||||
break
|
||||
}
|
||||
}
|
||||
return r.materializeItems(ctx, task, items)
|
||||
}
|
||||
|
||||
func (r *Runner) materializeItems(ctx context.Context, task Task, items []feedItem) ([]Document, error) {
|
||||
fetcher := research.New("")
|
||||
out := make([]Document, 0, len(items))
|
||||
for _, item := range items {
|
||||
if strings.TrimSpace(item.Link) == "" {
|
||||
continue
|
||||
}
|
||||
if r.state.SeenURL(ctx, task.ID, item.Link) && !strings.EqualFold(strings.TrimSpace(task.Config["refetch_seen"]), "true") {
|
||||
continue
|
||||
}
|
||||
page, diag, err := fetcher.FetchPage(ctx, item.Link, research.FetchOptions{MaxBytes: 2 << 20, MaxChars: 20000, Timeout: r.cfg.HTTPTimeout, AllowPrivate: r.cfg.AllowPrivate})
|
||||
text := strings.TrimSpace(item.Summary)
|
||||
title := strings.TrimSpace(item.Title)
|
||||
ctype := "text/html"
|
||||
final := item.Link
|
||||
if err == nil {
|
||||
if page.Content != "" {
|
||||
text = page.Content
|
||||
}
|
||||
if page.Title != "" {
|
||||
title = page.Title
|
||||
}
|
||||
if page.URL != "" {
|
||||
final = page.URL
|
||||
}
|
||||
ctype = page.ContentType
|
||||
} else if len([]rune(text)) < 80 {
|
||||
continue
|
||||
}
|
||||
if title == "" {
|
||||
title = final
|
||||
}
|
||||
sum := sha256.Sum256([]byte(text))
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
if r.state.SeenHash(ctx, task.ID, final, sha) {
|
||||
continue
|
||||
}
|
||||
baseURL := ""
|
||||
if u, e := url.Parse(task.URL); e == nil {
|
||||
baseURL = u.Scheme + "://" + u.Host
|
||||
}
|
||||
out = append(out, Document{ExternalID: final, URL: final, CanonicalURL: final, Title: title, PublishedAt: item.Published, DiscoveredAt: time.Now().UTC(), ContentType: ctype, Text: text, ContentSHA256: sha, SourceName: task.Name, SourceBaseURL: baseURL, Categories: task.Categories, Metadata: map[string]any{"agent_task_type": task.Type, "fetch_error_kind": diag.ErrorKind}})
|
||||
if len(out) >= task.MaxItems {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Runner) fetchRaw(ctx context.Context, raw string, max int64) ([]byte, string, string, string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
|
||||
if err != nil {
|
||||
return nil, "", "", "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "glpi-neural-brain-source-agent/1.0")
|
||||
req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.9, */*;q=0.5")
|
||||
resp, err := r.sourceHTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", "", "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, "", "", "", fmt.Errorf("source returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, max+1))
|
||||
if err != nil {
|
||||
return nil, "", "", "", err
|
||||
}
|
||||
if int64(len(body)) > max {
|
||||
return nil, "", "", "", errors.New("source response too large")
|
||||
}
|
||||
return body, resp.Request.URL.String(), resp.Header.Get("ETag"), resp.Header.Get("Last-Modified"), nil
|
||||
}
|
||||
|
||||
func (r *Runner) sendBatch(ctx context.Context, taskID string, docs []Document) error {
|
||||
payload := IngestBatch{SchemaVersion: SchemaVersion, AgentID: r.cfg.AgentID, TaskID: taskID, Documents: docs}
|
||||
data, _ := json.Marshal(payload)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.cfg.BrainURL+"/api/v1/agent/ingest", bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.auth(req)
|
||||
resp, err := r.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
return fmt.Errorf("brain ingest HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (r *Runner) sendHeartbeat(ctx context.Context, h Heartbeat) error {
|
||||
data, _ := json.Marshal(h)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.cfg.BrainURL+"/api/v1/agent/heartbeat", bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.auth(req)
|
||||
resp, err := r.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("heartbeat HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (r *Runner) auth(req *http.Request) {
|
||||
req.Header.Set("Authorization", "Bearer "+r.cfg.Token)
|
||||
req.Header.Set("X-Brain-Agent-ID", r.cfg.AgentID)
|
||||
}
|
||||
|
||||
func resolveLink(base, ref string) string {
|
||||
ref = strings.TrimSpace(html.UnescapeString(ref))
|
||||
if ref == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return b.ResolveReference(u).String()
|
||||
}
|
||||
func stripMarkup(v string) string {
|
||||
v = regexp.MustCompile(`(?is)<[^>]+>`).ReplaceAllString(v, " ")
|
||||
return strings.Join(strings.Fields(html.UnescapeString(v)), " ")
|
||||
}
|
||||
func looksArticleLink(path, label string) bool {
|
||||
p := strings.ToLower(path + " " + label)
|
||||
if strings.Contains(p, "/tag/") || strings.Contains(p, "/category/") || strings.Contains(p, "/author/") || strings.Contains(p, "login") || strings.Contains(p, "privacy") || strings.Contains(p, "impress") || strings.Contains(p, "kontakt") {
|
||||
return false
|
||||
}
|
||||
segments := strings.Split(strings.Trim(path, "/"), "/")
|
||||
return len(segments) >= 2 || strings.Contains(p, "news") || strings.Contains(p, "blog") || strings.Contains(p, "advis") || strings.Contains(p, "release") || strings.Contains(p, "security")
|
||||
}
|
||||
func parsePublished(v string) time.Time {
|
||||
v = strings.TrimSpace(v)
|
||||
for _, layout := range []string{time.RFC3339, time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, "2006-01-02"} {
|
||||
if t, err := time.Parse(layout, v); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func (r *Runner) configCachePath() string {
|
||||
return filepath.Join(r.cfg.DataDir, "source-agent-config-cache.json")
|
||||
}
|
||||
func (r *Runner) saveCachedConfig(cfg RemoteConfig) {
|
||||
data, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(r.configCachePath(), append(data, '\n'), 0o600)
|
||||
}
|
||||
func (r *Runner) loadCachedConfig() {
|
||||
data, err := os.ReadFile(r.configCachePath())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var cfg RemoteConfig
|
||||
if json.Unmarshal(data, &cfg) != nil || cfg.Agent.ID != r.cfg.AgentID {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.remote = cfg
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
type localState struct {
|
||||
db *sql.DB
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func openLocalState(dataDir string) (*localState, error) {
|
||||
if strings.TrimSpace(dataDir) == "" {
|
||||
dataDir = "./data"
|
||||
}
|
||||
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db, err := sql.Open("sqlite", "file:"+filepath.ToSlash(filepath.Join(dataDir, "source-agent-local.db"))+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st := &localState{db: db}
|
||||
for _, q := range []string{`CREATE TABLE IF NOT EXISTS task_state(task_id TEXT PRIMARY KEY,last_run_ns INTEGER NOT NULL DEFAULT 0,last_error TEXT NOT NULL DEFAULT '') WITHOUT ROWID`, `CREATE TABLE IF NOT EXISTS seen(task_id TEXT NOT NULL,url TEXT NOT NULL,content_sha256 TEXT NOT NULL,seen_at_ns INTEGER NOT NULL,PRIMARY KEY(task_id,url,content_sha256)) WITHOUT ROWID`, `CREATE INDEX IF NOT EXISTS idx_seen_task_url ON seen(task_id,url,seen_at_ns DESC)`} {
|
||||
if _, err := db.Exec(q); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
func (s *localState) Close() error { return s.db.Close() }
|
||||
func (s *localState) Due(ctx context.Context, t Task) bool {
|
||||
d, err := time.ParseDuration(t.PollInterval)
|
||||
if err != nil {
|
||||
d = 4 * time.Hour
|
||||
}
|
||||
var last int64
|
||||
err = s.db.QueryRowContext(ctx, `SELECT last_run_ns FROM task_state WHERE task_id=?`, t.ID).Scan(&last)
|
||||
return err == sql.ErrNoRows || err != nil || last == 0 || time.Since(time.Unix(0, last)) >= d
|
||||
}
|
||||
func (s *localState) FinishTask(ctx context.Context, id string, started time.Time, err error) error {
|
||||
msg := ""
|
||||
if err != nil {
|
||||
msg = err.Error()
|
||||
}
|
||||
_, e := s.db.ExecContext(ctx, `INSERT INTO task_state(task_id,last_run_ns,last_error) VALUES(?,?,?) ON CONFLICT(task_id) DO UPDATE SET last_run_ns=excluded.last_run_ns,last_error=excluded.last_error`, id, started.UnixNano(), msg)
|
||||
return e
|
||||
}
|
||||
func (s *localState) SeenURL(ctx context.Context, task, urlv string) bool {
|
||||
var n int
|
||||
_ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM seen WHERE task_id=? AND url=?`, task, urlv).Scan(&n)
|
||||
return n > 0
|
||||
}
|
||||
func (s *localState) SeenHash(ctx context.Context, task, urlv, sha string) bool {
|
||||
var n int
|
||||
_ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM seen WHERE task_id=? AND url=? AND content_sha256=?`, task, urlv, sha).Scan(&n)
|
||||
return n > 0
|
||||
}
|
||||
func (s *localState) MarkSeen(ctx context.Context, task, urlv, sha string) error {
|
||||
_, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO seen(task_id,url,content_sha256,seen_at_ns) VALUES(?,?,?,?)`, task, urlv, sha, time.Now().UTC().UnixNano())
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package sourceagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/bits"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/research"
|
||||
)
|
||||
|
||||
func TestValidateTaskSupportsPollerTypes(t *testing.T) {
|
||||
for _, typ := range []string{"rss", "atom", "sitemap", "web"} {
|
||||
task, err := validateTask(Task{ID: "security", AgentID: "a", Name: "Security", Type: typ, URL: "https://example.org/feed", Enabled: true, PollInterval: "2h", MaxItems: 25})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", typ, err)
|
||||
}
|
||||
if task.Type != typ || task.MaxItems != 25 {
|
||||
t.Fatalf("unexpected task: %+v", task)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTaskRejectsUnsafeShape(t *testing.T) {
|
||||
if _, err := validateTask(Task{Type: "ftp", URL: "https://example.org", PollInterval: "1h"}); err == nil {
|
||||
t.Fatal("expected unsupported type")
|
||||
}
|
||||
if _, err := validateTask(Task{Type: "rss", URL: "ftp://example.org/feed", PollInterval: "1h"}); err == nil {
|
||||
t.Fatal("expected URL validation error")
|
||||
}
|
||||
if _, err := validateTask(Task{Type: "rss", URL: "https://example.org/feed", PollInterval: "1m"}); err == nil {
|
||||
t.Fatal("expected interval validation error")
|
||||
}
|
||||
if _, err := validateTask(Task{Type: "rss", URL: "https://user:pass@example.org/feed", PollInterval: "1h"}); err == nil {
|
||||
t.Fatal("expected userinfo URL validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDocumentBuildsContentHash(t *testing.T) {
|
||||
d, err := normalizeDocument(Document{URL: "https://example.org/a", Title: "A", Text: "Dies ist ein ausreichend langer Dokumenttext mit sicherheitsrelevanten Informationen und mehreren Details für die Verarbeitung."})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.ContentSHA256 == "" || d.CanonicalURL != "https://example.org/a" || d.ExternalID != "https://example.org/a" {
|
||||
t.Fatalf("unexpected normalized document: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimhashKeepsRelatedTextCloser(t *testing.T) {
|
||||
base := simhash64("Windows Backup Repository Ransomware Schutz immutable storage MFA")
|
||||
related := simhash64("Ransomware Schutz für Windows Backup Repository mit MFA und immutable Storage")
|
||||
unrelated := simhash64("Kaffee Bohnen Espresso Maschine Mahlgrad Temperatur")
|
||||
if bits.OnesCount64(base^related) >= bits.OnesCount64(base^unrelated) {
|
||||
t.Fatalf("related text should hash closer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRawBlocksPrivateSourceByDefault(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("<rss/>")) }))
|
||||
defer srv.Close()
|
||||
r := &Runner{cfg: RunnerConfig{HTTPTimeout: time.Second}, sourceHTTP: research.NewSafeHTTPClient(false, time.Second)}
|
||||
if _, _, _, _, err := r.fetchRaw(context.Background(), srv.URL, 1024); err == nil {
|
||||
t.Fatal("expected private/loopback source URL to be blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSitemapIndexCollectsChildURLs(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
var base string
|
||||
mux.HandleFunc("/index.xml", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(`<sitemapindex><sitemap><loc>` + base + `/child.xml</loc></sitemap></sitemapindex>`))
|
||||
})
|
||||
mux.HandleFunc("/child.xml", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(`<urlset><url><loc>` + base + `/article-1</loc><lastmod>2026-08-07T12:00:00Z</lastmod></url></urlset>`))
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
base = srv.URL
|
||||
r := &Runner{cfg: RunnerConfig{HTTPTimeout: time.Second, AllowPrivate: true}, sourceHTTP: research.NewSafeHTTPClient(true, time.Second)}
|
||||
items, err := r.collectSitemapItems(context.Background(), srv.URL+"/index.xml", 10, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) != 1 || items[0].Link != srv.URL+"/article-1" {
|
||||
t.Fatalf("unexpected sitemap items: %+v", items)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,698 @@
|
||||
package sourceagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"math/bits"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func OpenStore(dataDir string) (*Store, error) {
|
||||
path := filepath.Join(dataDir, "source-agents.db")
|
||||
db, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(4)
|
||||
store := &Store{db: db}
|
||||
if err := store.init(context.Background()); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error {
|
||||
if s == nil || s.db == nil {
|
||||
return nil
|
||||
}
|
||||
return s.db.Close()
|
||||
}
|
||||
|
||||
func (s *Store) init(ctx context.Context) error {
|
||||
statements := []string{
|
||||
`CREATE TABLE IF NOT EXISTS source_agents (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, token_hash TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at_ns INTEGER NOT NULL, updated_at_ns INTEGER NOT NULL, last_seen_ns INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT NOT NULL DEFAULT '', version TEXT NOT NULL DEFAULT ''
|
||||
) WITHOUT ROWID`,
|
||||
`CREATE TABLE IF NOT EXISTS source_tasks (
|
||||
id TEXT PRIMARY KEY, agent_id TEXT NOT NULL REFERENCES source_agents(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL, type TEXT NOT NULL, url TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1,
|
||||
poll_interval TEXT NOT NULL DEFAULT '4h', categories_json TEXT NOT NULL DEFAULT '[]', max_items INTEGER NOT NULL DEFAULT 20,
|
||||
config_json TEXT NOT NULL DEFAULT '{}', created_at_ns INTEGER NOT NULL, updated_at_ns INTEGER NOT NULL
|
||||
) WITHOUT ROWID`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_source_tasks_agent ON source_tasks(agent_id, enabled, updated_at_ns)`,
|
||||
`CREATE TABLE IF NOT EXISTS source_inbox (
|
||||
id TEXT PRIMARY KEY, agent_id TEXT NOT NULL, task_id TEXT NOT NULL, external_id TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL, canonical_url TEXT NOT NULL, title TEXT NOT NULL, published_at_ns INTEGER NOT NULL DEFAULT 0,
|
||||
discovered_at_ns INTEGER NOT NULL DEFAULT 0, received_at_ns INTEGER NOT NULL, updated_at_ns INTEGER NOT NULL,
|
||||
language TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', text_content TEXT NOT NULL,
|
||||
content_sha256 TEXT NOT NULL, source_name TEXT NOT NULL DEFAULT '', source_base_url TEXT NOT NULL DEFAULT '',
|
||||
categories_json TEXT NOT NULL DEFAULT '[]', metadata_json TEXT NOT NULL DEFAULT '{}', signature INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'received', relevance REAL NOT NULL DEFAULT 0, matched_node_id TEXT NOT NULL DEFAULT '',
|
||||
UNIQUE(agent_id, task_id, canonical_url, content_sha256)
|
||||
) WITHOUT ROWID`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_source_inbox_status_received ON source_inbox(status, received_at_ns)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_source_inbox_candidate_updated ON source_inbox(status, updated_at_ns DESC)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := s.db.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, _ = s.db.ExecContext(ctx, `UPDATE source_inbox SET status='received' WHERE status='processing' AND updated_at_ns<?`, time.Now().UTC().Add(-10*time.Minute).UnixNano())
|
||||
return nil
|
||||
}
|
||||
|
||||
func GenerateToken() (string, string, error) {
|
||||
buf := make([]byte, 32)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
token := "brain_agent_" + hex.EncodeToString(buf)
|
||||
return token, hashToken(token), nil
|
||||
}
|
||||
|
||||
func hashToken(token string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(token)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func randomID(prefix string) string {
|
||||
buf := make([]byte, 12)
|
||||
_, _ = rand.Read(buf)
|
||||
return prefix + "-" + hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
func normalizeID(value, prefix string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
value = strings.Trim(b.String(), "-_")
|
||||
if value == "" {
|
||||
return randomID(prefix)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func (s *Store) CreateAgent(ctx context.Context, id, name string) (Agent, string, error) {
|
||||
id = normalizeID(id, "agent")
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = id
|
||||
}
|
||||
token, tokenHash, err := GenerateToken()
|
||||
if err != nil {
|
||||
return Agent{}, "", err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
_, err = s.db.ExecContext(ctx, `INSERT INTO source_agents(id,name,token_hash,enabled,created_at_ns,updated_at_ns) VALUES(?,?,?,?,?,?)`, id, name, tokenHash, 1, now.UnixNano(), now.UnixNano())
|
||||
if err != nil {
|
||||
return Agent{}, "", err
|
||||
}
|
||||
return Agent{ID: id, Name: name, Enabled: true, CreatedAt: now, UpdatedAt: now}, token, nil
|
||||
}
|
||||
|
||||
func (s *Store) RotateToken(ctx context.Context, id string) (string, error) {
|
||||
token, tokenHash, err := GenerateToken()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE source_agents SET token_hash=?, updated_at_ns=? WHERE id=?`, tokenHash, time.Now().UTC().UnixNano(), id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n == 0 {
|
||||
return "", sql.ErrNoRows
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (s *Store) SetAgentEnabled(ctx context.Context, id string, enabled bool) error {
|
||||
v := 0
|
||||
if enabled {
|
||||
v = 1
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE source_agents SET enabled=?,updated_at_ns=? WHERE id=?`, v, time.Now().UTC().UnixNano(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteAgent(ctx context.Context, id string) error {
|
||||
_, err := s.db.ExecContext(ctx, `DELETE FROM source_agents WHERE id=?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) ListAgents(ctx context.Context) ([]Agent, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT a.id,a.name,a.enabled,a.created_at_ns,a.updated_at_ns,a.last_seen_ns,a.last_error,a.version,(SELECT COUNT(*) FROM source_tasks t WHERE t.agent_id=a.id) FROM source_agents a ORDER BY a.name,a.id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Agent
|
||||
for rows.Next() {
|
||||
var a Agent
|
||||
var en int
|
||||
var c, u, seen int64
|
||||
if err := rows.Scan(&a.ID, &a.Name, &en, &c, &u, &seen, &a.LastError, &a.Version, &a.TaskCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.Enabled = en != 0
|
||||
a.CreatedAt = nsTime(c)
|
||||
a.UpdatedAt = nsTime(u)
|
||||
a.LastSeen = nsTime(seen)
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) Authenticate(ctx context.Context, token string) (Agent, error) {
|
||||
if strings.TrimSpace(token) == "" {
|
||||
return Agent{}, errors.New("empty agent token")
|
||||
}
|
||||
var a Agent
|
||||
var en int
|
||||
var c, u, seen int64
|
||||
err := s.db.QueryRowContext(ctx, `SELECT id,name,enabled,created_at_ns,updated_at_ns,last_seen_ns,last_error,version FROM source_agents WHERE token_hash=?`, hashToken(token)).Scan(&a.ID, &a.Name, &en, &c, &u, &seen, &a.LastError, &a.Version)
|
||||
if err != nil {
|
||||
return Agent{}, err
|
||||
}
|
||||
a.Enabled = en != 0
|
||||
if !a.Enabled {
|
||||
return Agent{}, errors.New("agent disabled")
|
||||
}
|
||||
a.CreatedAt = nsTime(c)
|
||||
a.UpdatedAt = nsTime(u)
|
||||
a.LastSeen = nsTime(seen)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func validateTask(t Task) (Task, error) {
|
||||
t.ID = normalizeID(t.ID, "task")
|
||||
t.Name = strings.TrimSpace(t.Name)
|
||||
if t.Name == "" {
|
||||
t.Name = t.ID
|
||||
}
|
||||
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
|
||||
switch t.Type {
|
||||
case "rss", "atom", "sitemap", "web":
|
||||
default:
|
||||
return t, fmt.Errorf("unsupported source task type %q", t.Type)
|
||||
}
|
||||
u, err := url.Parse(strings.TrimSpace(t.URL))
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return t, fmt.Errorf("task URL must be absolute http(s)")
|
||||
}
|
||||
if u.User != nil {
|
||||
return t, fmt.Errorf("task URL must not contain userinfo")
|
||||
}
|
||||
t.URL = u.String()
|
||||
if strings.TrimSpace(t.PollInterval) == "" {
|
||||
t.PollInterval = "4h"
|
||||
}
|
||||
d, err := time.ParseDuration(t.PollInterval)
|
||||
if err != nil || d < 5*time.Minute || d > 30*24*time.Hour {
|
||||
return t, fmt.Errorf("poll_interval must be between 5m and 720h")
|
||||
}
|
||||
if t.MaxItems <= 0 {
|
||||
t.MaxItems = 20
|
||||
}
|
||||
if t.MaxItems > 500 {
|
||||
return t, fmt.Errorf("max_items must be <=500")
|
||||
}
|
||||
if t.Config == nil {
|
||||
t.Config = map[string]string{}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpsertTask(ctx context.Context, t Task) (Task, error) {
|
||||
var err error
|
||||
t, err = validateTask(t)
|
||||
if err != nil {
|
||||
return Task{}, err
|
||||
}
|
||||
if strings.TrimSpace(t.AgentID) == "" {
|
||||
return Task{}, errors.New("agent_id required")
|
||||
}
|
||||
cats, _ := json.Marshal(uniqueStrings(t.Categories))
|
||||
cfg, _ := json.Marshal(t.Config)
|
||||
now := time.Now().UTC()
|
||||
_, err = s.db.ExecContext(ctx, `INSERT INTO source_tasks(id,agent_id,name,type,url,enabled,poll_interval,categories_json,max_items,config_json,created_at_ns,updated_at_ns) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET agent_id=excluded.agent_id,name=excluded.name,type=excluded.type,url=excluded.url,enabled=excluded.enabled,poll_interval=excluded.poll_interval,categories_json=excluded.categories_json,max_items=excluded.max_items,config_json=excluded.config_json,updated_at_ns=excluded.updated_at_ns`, t.ID, t.AgentID, t.Name, t.Type, t.URL, boolInt(t.Enabled), t.PollInterval, string(cats), t.MaxItems, string(cfg), now.UnixNano(), now.UnixNano())
|
||||
if err != nil {
|
||||
return Task{}, err
|
||||
}
|
||||
t.UpdatedAt = now
|
||||
if t.CreatedAt.IsZero() {
|
||||
t.CreatedAt = now
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteTask(ctx context.Context, id string) error {
|
||||
_, err := s.db.ExecContext(ctx, `DELETE FROM source_tasks WHERE id=?`, id)
|
||||
return err
|
||||
}
|
||||
func (s *Store) ListTasks(ctx context.Context, agentID string) ([]Task, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id,agent_id,name,type,url,enabled,poll_interval,categories_json,max_items,config_json,created_at_ns,updated_at_ns FROM source_tasks WHERE (?='' OR agent_id=?) ORDER BY name,id`, agentID, agentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Task
|
||||
for rows.Next() {
|
||||
var t Task
|
||||
var en int
|
||||
var cats, cfg string
|
||||
var c, u int64
|
||||
if err := rows.Scan(&t.ID, &t.AgentID, &t.Name, &t.Type, &t.URL, &en, &t.PollInterval, &cats, &t.MaxItems, &cfg, &c, &u); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.Enabled = en != 0
|
||||
_ = json.Unmarshal([]byte(cats), &t.Categories)
|
||||
_ = json.Unmarshal([]byte(cfg), &t.Config)
|
||||
t.CreatedAt = nsTime(c)
|
||||
t.UpdatedAt = nsTime(u)
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) RemoteConfig(ctx context.Context, agent Agent) (RemoteConfig, error) {
|
||||
tasks, err := s.ListTasks(ctx, agent.ID)
|
||||
if err != nil {
|
||||
return RemoteConfig{}, err
|
||||
}
|
||||
return RemoteConfig{SchemaVersion: SchemaVersion, Agent: agent, Tasks: tasks, IssuedAt: time.Now().UTC()}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Heartbeat(ctx context.Context, agentID string, h Heartbeat) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE source_agents SET last_seen_ns=?,last_error=?,version=?,updated_at_ns=? WHERE id=?`, time.Now().UTC().UnixNano(), strings.TrimSpace(h.LastError), strings.TrimSpace(h.Version), time.Now().UTC().UnixNano(), agentID)
|
||||
return err
|
||||
}
|
||||
|
||||
func normalizeDocument(d Document) (Document, error) {
|
||||
d.URL = strings.TrimSpace(d.URL)
|
||||
if d.URL == "" {
|
||||
return d, errors.New("document url required")
|
||||
}
|
||||
u, err := url.Parse(d.URL)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return d, errors.New("invalid document url")
|
||||
}
|
||||
d.CanonicalURL = strings.TrimSpace(d.CanonicalURL)
|
||||
if d.CanonicalURL == "" {
|
||||
d.CanonicalURL = d.URL
|
||||
}
|
||||
d.Title = strings.TrimSpace(d.Title)
|
||||
d.Text = strings.TrimSpace(d.Text)
|
||||
if len([]rune(d.Text)) < 80 {
|
||||
return d, errors.New("document text too short")
|
||||
}
|
||||
if d.DiscoveredAt.IsZero() {
|
||||
d.DiscoveredAt = time.Now().UTC()
|
||||
}
|
||||
if d.ExternalID == "" {
|
||||
d.ExternalID = d.CanonicalURL
|
||||
}
|
||||
d.Categories = uniqueStrings(d.Categories)
|
||||
if d.Metadata == nil {
|
||||
d.Metadata = map[string]any{}
|
||||
}
|
||||
if d.ContentSHA256 == "" {
|
||||
sum := sha256.Sum256([]byte(d.Text))
|
||||
d.ContentSHA256 = hex.EncodeToString(sum[:])
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (s *Store) Ingest(ctx context.Context, agentID, taskID string, docs []Document) (IngestResult, error) {
|
||||
var taskOwner string
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT agent_id FROM source_tasks WHERE id=? AND enabled=1`, taskID).Scan(&taskOwner); err != nil {
|
||||
return IngestResult{}, fmt.Errorf("unknown or disabled source task: %w", err)
|
||||
}
|
||||
if taskOwner != agentID {
|
||||
return IngestResult{}, errors.New("source task does not belong to authenticated agent")
|
||||
}
|
||||
result := IngestResult{BatchID: randomID("batch")}
|
||||
now := time.Now().UTC()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, raw := range docs {
|
||||
d, e := normalizeDocument(raw)
|
||||
if e != nil {
|
||||
result.Rejected++
|
||||
continue
|
||||
}
|
||||
cats, _ := json.Marshal(d.Categories)
|
||||
meta, _ := json.Marshal(d.Metadata)
|
||||
idSum := sha256.Sum256([]byte(agentID + "\x00" + taskID + "\x00" + d.CanonicalURL + "\x00" + d.ContentSHA256))
|
||||
id := hex.EncodeToString(idSum[:12])
|
||||
sig := int64(simhash64(d.Title + "\n" + d.Text))
|
||||
res, e := tx.ExecContext(ctx, `INSERT OR IGNORE INTO source_inbox(id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,signature,status) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, id, agentID, taskID, d.ExternalID, d.URL, d.CanonicalURL, d.Title, timeNS(d.PublishedAt), timeNS(d.DiscoveredAt), now.UnixNano(), now.UnixNano(), d.Language, d.ContentType, d.Text, d.ContentSHA256, d.SourceName, d.SourceBaseURL, string(cats), string(meta), sig, "received")
|
||||
if e != nil {
|
||||
result.Rejected++
|
||||
continue
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n == 0 {
|
||||
result.Duplicates++
|
||||
} else {
|
||||
result.Accepted++
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) ClaimInbox(ctx context.Context, limit int) ([]InboxDocument, error) {
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
rows, err := tx.QueryContext(ctx, `SELECT id FROM source_inbox WHERE status='received' ORDER BY received_at_ns LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
rows.Close()
|
||||
if len(ids) == 0 {
|
||||
return nil, tx.Commit()
|
||||
}
|
||||
now := time.Now().UTC().UnixNano()
|
||||
for _, id := range ids {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE source_inbox SET status='processing',updated_at_ns=? WHERE id=? AND status='received'`, now, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.GetInboxByIDs(ctx, ids)
|
||||
}
|
||||
|
||||
func (s *Store) CompleteClassification(ctx context.Context, id, status string, relevance float64, matchedNodeID string, meta map[string]any) error {
|
||||
if status != "candidate" && status != "archived" {
|
||||
return errors.New("invalid inbox classification status")
|
||||
}
|
||||
data, _ := json.Marshal(meta)
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE source_inbox SET status=?,relevance=?,matched_node_id=?,metadata_json=?,updated_at_ns=? WHERE id=?`, status, relevance, matchedNodeID, string(data), time.Now().UTC().UnixNano(), id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) MarkUsed(ctx context.Context, canonicalURLs []string) error {
|
||||
now := time.Now().UTC().UnixNano()
|
||||
for _, u := range uniqueStrings(canonicalURLs) {
|
||||
if _, err := s.db.ExecContext(ctx, `UPDATE source_inbox SET status='used',updated_at_ns=? WHERE canonical_url=?`, now, u); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetInboxByIDs(ctx context.Context, ids []string) ([]InboxDocument, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]InboxDocument, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
doc, err := s.getInbox(ctx, id)
|
||||
if err == nil {
|
||||
out = append(out, doc)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (s *Store) getInbox(ctx context.Context, id string) (InboxDocument, error) {
|
||||
row := s.db.QueryRowContext(ctx, `SELECT id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,status,relevance,matched_node_id FROM source_inbox WHERE id=?`, id)
|
||||
return scanInbox(row)
|
||||
}
|
||||
|
||||
type rowScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanInbox(row rowScanner) (InboxDocument, error) {
|
||||
var d InboxDocument
|
||||
var ext, urlv, canon, title, lang, ctype, text, sha, source, base, cats, meta, status, matched string
|
||||
var pub, disc, recv, upd int64
|
||||
var rel float64
|
||||
err := row.Scan(&d.ID, &d.AgentID, &d.TaskID, &ext, &urlv, &canon, &title, &pub, &disc, &recv, &upd, &lang, &ctype, &text, &sha, &source, &base, &cats, &meta, &status, &rel, &matched)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
d.Document = Document{ExternalID: ext, URL: urlv, CanonicalURL: canon, Title: title, PublishedAt: nsTime(pub), DiscoveredAt: nsTime(disc), Language: lang, ContentType: ctype, Text: text, ContentSHA256: sha, SourceName: source, SourceBaseURL: base}
|
||||
_ = json.Unmarshal([]byte(cats), &d.Document.Categories)
|
||||
_ = json.Unmarshal([]byte(meta), &d.Metadata)
|
||||
d.Status = status
|
||||
d.Relevance = rel
|
||||
d.MatchedNodeID = matched
|
||||
d.ReceivedAt = nsTime(recv)
|
||||
d.UpdatedAt = nsTime(upd)
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListInbox(ctx context.Context, status string, limit int) ([]InboxDocument, error) {
|
||||
if limit < 1 {
|
||||
limit = 100
|
||||
}
|
||||
if limit > 1000 {
|
||||
limit = 1000
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,status,relevance,matched_node_id FROM source_inbox WHERE (?='' OR status=?) ORDER BY received_at_ns DESC LIMIT ?`, status, status, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []InboxDocument
|
||||
for rows.Next() {
|
||||
d, err := scanInbox(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) Stats(ctx context.Context) (InboxStats, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT status,COUNT(*) FROM source_inbox GROUP BY status`)
|
||||
if err != nil {
|
||||
return InboxStats{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var st InboxStats
|
||||
for rows.Next() {
|
||||
var status string
|
||||
var n int
|
||||
if err := rows.Scan(&status, &n); err != nil {
|
||||
return st, err
|
||||
}
|
||||
st.Total += n
|
||||
switch status {
|
||||
case "received":
|
||||
st.Received = n
|
||||
case "processing":
|
||||
st.Processing = n
|
||||
case "candidate":
|
||||
st.Candidate = n
|
||||
case "archived":
|
||||
st.Archived = n
|
||||
case "used":
|
||||
st.Used = n
|
||||
}
|
||||
}
|
||||
return st, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) SearchCandidates(ctx context.Context, query string, limit int, maxAge time.Duration) ([]ScoredDocument, error) {
|
||||
if limit < 1 {
|
||||
limit = 3
|
||||
}
|
||||
if limit > 50 {
|
||||
limit = 50
|
||||
}
|
||||
since := int64(0)
|
||||
if maxAge > 0 {
|
||||
since = time.Now().UTC().Add(-maxAge).UnixNano()
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id,agent_id,task_id,external_id,url,canonical_url,title,published_at_ns,discovered_at_ns,received_at_ns,updated_at_ns,language,content_type,text_content,content_sha256,source_name,source_base_url,categories_json,metadata_json,status,relevance,matched_node_id,signature FROM source_inbox WHERE status IN ('candidate','used') AND (?=0 OR COALESCE(NULLIF(published_at_ns,0),received_at_ns)>=?) ORDER BY updated_at_ns DESC LIMIT 5000`, since, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
qsig := simhash64(query)
|
||||
qterms := termSet(query)
|
||||
var out []ScoredDocument
|
||||
for rows.Next() {
|
||||
var d InboxDocument
|
||||
var ext, urlv, canon, title, lang, ctype, text, sha, source, base, cats, meta, status, matched string
|
||||
var pub, disc, recv, upd, sig int64
|
||||
var rel float64
|
||||
if err := rows.Scan(&d.ID, &d.AgentID, &d.TaskID, &ext, &urlv, &canon, &title, &pub, &disc, &recv, &upd, &lang, &ctype, &text, &sha, &source, &base, &cats, &meta, &status, &rel, &matched, &sig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Document = Document{ExternalID: ext, URL: urlv, CanonicalURL: canon, Title: title, PublishedAt: nsTime(pub), DiscoveredAt: nsTime(disc), Language: lang, ContentType: ctype, Text: text, ContentSHA256: sha, SourceName: source, SourceBaseURL: base}
|
||||
_ = json.Unmarshal([]byte(cats), &d.Document.Categories)
|
||||
_ = json.Unmarshal([]byte(meta), &d.Metadata)
|
||||
d.Status = status
|
||||
d.Relevance = rel
|
||||
d.MatchedNodeID = matched
|
||||
d.ReceivedAt = nsTime(recv)
|
||||
d.UpdatedAt = nsTime(upd)
|
||||
hamming := bits.OnesCount64(qsig ^ uint64(sig))
|
||||
hashScore := 1 - float64(hamming)/64.0
|
||||
lex := jaccard(qterms, termSet(title+" "+text))
|
||||
score := 0.45*hashScore + 0.45*lex + 0.10*rel
|
||||
if lex < 0.03 && hashScore < 0.58 {
|
||||
continue
|
||||
}
|
||||
out = append(out, ScoredDocument{InboxDocument: d, QueryScore: score})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].QueryScore > out[j].QueryScore })
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func boolInt(v bool) int {
|
||||
if v {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func timeNS(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UTC().UnixNano()
|
||||
}
|
||||
func nsTime(v int64) time.Time {
|
||||
if v <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(0, v).UTC()
|
||||
}
|
||||
func uniqueStrings(values []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" || seen[v] {
|
||||
continue
|
||||
}
|
||||
seen[v] = true
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func termSet(value string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range strings.FieldsFunc(strings.ToLower(value), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) {
|
||||
if len([]rune(r)) >= 3 {
|
||||
out[r] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func jaccard(a, b map[string]bool) float64 {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
inter := 0
|
||||
union := len(a)
|
||||
for k := range b {
|
||||
if a[k] {
|
||||
inter++
|
||||
} else {
|
||||
union++
|
||||
}
|
||||
}
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
func simhash64(value string) uint64 {
|
||||
weights := [64]int{}
|
||||
for token := range termSet(value) {
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte(token))
|
||||
x := h.Sum64()
|
||||
for i := 0; i < 64; i++ {
|
||||
if x&(1<<i) != 0 {
|
||||
weights[i]++
|
||||
} else {
|
||||
weights[i]--
|
||||
}
|
||||
}
|
||||
}
|
||||
var sig uint64
|
||||
for i, w := range weights {
|
||||
if w >= 0 {
|
||||
sig |= 1 << i
|
||||
}
|
||||
}
|
||||
return sig
|
||||
}
|
||||
|
||||
func (s *Store) ReleaseInbox(ctx context.Context, id, message string) error {
|
||||
meta, _ := json.Marshal(map[string]any{"classification_error": strings.TrimSpace(message)})
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE source_inbox SET status='received',metadata_json=?,updated_at_ns=? WHERE id=?`, string(meta), time.Now().UTC().UnixNano(), id)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package sourceagent
|
||||
|
||||
import "time"
|
||||
|
||||
const SchemaVersion = 1
|
||||
|
||||
type Agent struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LastSeen time.Time `json:"last_seen,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
TaskCount int `json:"task_count,omitempty"`
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
AgentID string `json:"agent_id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
Enabled bool `json:"enabled"`
|
||||
PollInterval string `json:"poll_interval"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
MaxItems int `json:"max_items,omitempty"`
|
||||
Config map[string]string `json:"config,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type RemoteConfig struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Agent Agent `json:"agent"`
|
||||
Tasks []Task `json:"tasks"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
}
|
||||
|
||||
type Document struct {
|
||||
ExternalID string `json:"external_id,omitempty"`
|
||||
URL string `json:"url"`
|
||||
CanonicalURL string `json:"canonical_url,omitempty"`
|
||||
Title string `json:"title"`
|
||||
PublishedAt time.Time `json:"published_at,omitempty"`
|
||||
DiscoveredAt time.Time `json:"discovered_at,omitempty"`
|
||||
Language string `json:"language,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
Text string `json:"text"`
|
||||
ContentSHA256 string `json:"content_sha256,omitempty"`
|
||||
SourceName string `json:"source_name,omitempty"`
|
||||
SourceBaseURL string `json:"source_base_url,omitempty"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type IngestBatch struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
AgentID string `json:"agent_id,omitempty"`
|
||||
TaskID string `json:"task_id"`
|
||||
Documents []Document `json:"documents"`
|
||||
}
|
||||
|
||||
type IngestResult struct {
|
||||
Accepted int `json:"accepted"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Rejected int `json:"rejected"`
|
||||
BatchID string `json:"batch_id"`
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
AgentID string `json:"agent_id,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
LastRunAt time.Time `json:"last_run_at,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
TasksChecked int `json:"tasks_checked,omitempty"`
|
||||
Documents int `json:"documents,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type InboxDocument struct {
|
||||
ID string `json:"id"`
|
||||
AgentID string `json:"agent_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
Document Document `json:"document"`
|
||||
Status string `json:"status"`
|
||||
Relevance float64 `json:"relevance,omitempty"`
|
||||
MatchedNodeID string `json:"matched_node_id,omitempty"`
|
||||
ReceivedAt time.Time `json:"received_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type InboxStats struct {
|
||||
Total int `json:"total"`
|
||||
Received int `json:"received"`
|
||||
Processing int `json:"processing"`
|
||||
Candidate int `json:"candidate"`
|
||||
Archived int `json:"archived"`
|
||||
Used int `json:"used"`
|
||||
}
|
||||
|
||||
type ScoredDocument struct {
|
||||
InboxDocument
|
||||
QueryScore float64 `json:"query_score"`
|
||||
}
|
||||
+20
-4
@@ -18,16 +18,18 @@ import (
|
||||
"github.com/local/glpi-neural-brain/internal/engine"
|
||||
"github.com/local/glpi-neural-brain/internal/graph"
|
||||
"github.com/local/glpi-neural-brain/internal/model"
|
||||
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
||||
)
|
||||
|
||||
//go:embed static/*
|
||||
var assets embed.FS
|
||||
|
||||
type Server struct {
|
||||
Engine *engine.Engine
|
||||
Graph *graph.Store
|
||||
Broker *activity.Broker
|
||||
APIKey string
|
||||
Engine *engine.Engine
|
||||
Graph *graph.Store
|
||||
Broker *activity.Broker
|
||||
APIKey string
|
||||
SourceAgents *sourceagent.Store
|
||||
}
|
||||
|
||||
func (s *Server) Handler() http.Handler {
|
||||
@@ -55,6 +57,20 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("POST /api/glpi-kb/sync", s.handleGLPIKBSync)
|
||||
mux.HandleFunc("POST /api/flush", s.handleFlush)
|
||||
mux.HandleFunc("GET /api/state/export", s.handleStateExport)
|
||||
mux.HandleFunc("GET /api/source-agents", s.handleListSourceAgents)
|
||||
mux.HandleFunc("POST /api/source-agents", s.handleCreateSourceAgent)
|
||||
mux.HandleFunc("PATCH /api/source-agents/{id}", s.handlePatchSourceAgent)
|
||||
mux.HandleFunc("DELETE /api/source-agents/{id}", s.handleDeleteSourceAgent)
|
||||
mux.HandleFunc("POST /api/source-agents/{id}/rotate-token", s.handleRotateSourceAgentToken)
|
||||
mux.HandleFunc("GET /api/source-agents/{id}/tasks", s.handleListSourceTasks)
|
||||
mux.HandleFunc("POST /api/source-agents/{id}/tasks", s.handleCreateSourceTask)
|
||||
mux.HandleFunc("PUT /api/source-tasks/{id}", s.handleUpdateSourceTask)
|
||||
mux.HandleFunc("DELETE /api/source-tasks/{id}", s.handleDeleteSourceTask)
|
||||
mux.HandleFunc("GET /api/source-inbox", s.handleSourceInbox)
|
||||
mux.HandleFunc("GET /api/source-inbox/status", s.handleSourceInboxStatus)
|
||||
mux.HandleFunc("GET /api/v1/agent/config", s.handleAgentConfig)
|
||||
mux.HandleFunc("POST /api/v1/agent/heartbeat", s.handleAgentHeartbeat)
|
||||
mux.HandleFunc("POST /api/v1/agent/ingest", s.handleAgentIngest)
|
||||
sub, _ := fs.Sub(assets, "static")
|
||||
mux.HandleFunc("GET /analysis", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/analysis.html", http.StatusTemporaryRedirect)
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/local/glpi-neural-brain/internal/sourceagent"
|
||||
)
|
||||
|
||||
func (s *Server) sourceStoreAvailable(w http.ResponseWriter) bool {
|
||||
if s.SourceAgents != nil {
|
||||
return true
|
||||
}
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source agent service is not initialized"})
|
||||
return false
|
||||
}
|
||||
func (s *Server) adminAuthorized(w http.ResponseWriter, r *http.Request) bool {
|
||||
if s.authorized(r) {
|
||||
return true
|
||||
}
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Server) handleListSourceAgents(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
agents, err := s.SourceAgents.ListAgents(ctx)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
tasks, _ := s.SourceAgents.ListTasks(ctx, "")
|
||||
stats, _ := s.SourceAgents.Stats(ctx)
|
||||
writeJSON(w, 200, map[string]any{"agents": agents, "tasks": tasks, "inbox": stats})
|
||||
}
|
||||
func (s *Server) handleCreateSourceAgent(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := decode(r, &in); err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
a, token, err := s.SourceAgents.CreateAgent(ctx, in.ID, in.Name)
|
||||
if err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, map[string]any{"agent": a, "token": token, "token_notice": "Der Token wird nur in dieser Antwort im Klartext ausgegeben."})
|
||||
}
|
||||
func (s *Server) handlePatchSourceAgent(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
}
|
||||
if err := decode(r, &in); err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if in.Enabled == nil {
|
||||
writeJSON(w, 400, map[string]string{"error": "enabled is required"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.SourceAgents.SetAgentEnabled(ctx, r.PathValue("id"), *in.Enabled); err != nil {
|
||||
writeJSON(w, 404, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"ok": true})
|
||||
}
|
||||
func (s *Server) handleDeleteSourceAgent(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.SourceAgents.DeleteAgent(ctx, r.PathValue("id")); err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"ok": true})
|
||||
}
|
||||
func (s *Server) handleRotateSourceAgentToken(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
token, err := s.SourceAgents.RotateToken(ctx, r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeJSON(w, 404, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"token": token, "token_notice": "Der neue Token wird nur in dieser Antwort im Klartext ausgegeben."})
|
||||
}
|
||||
func (s *Server) handleListSourceTasks(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
tasks, err := s.SourceAgents.ListTasks(ctx, r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"tasks": tasks})
|
||||
}
|
||||
func (s *Server) handleCreateSourceTask(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
var t sourceagent.Task
|
||||
if err := decode(r, &t); err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
t.AgentID = r.PathValue("id")
|
||||
if strings.TrimSpace(t.ID) == "" {
|
||||
t.ID = "task-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.SourceAgents.UpsertTask(ctx, t)
|
||||
if err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, out)
|
||||
}
|
||||
func (s *Server) handleUpdateSourceTask(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
var t sourceagent.Task
|
||||
if err := decode(r, &t); err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
t.ID = r.PathValue("id")
|
||||
if t.AgentID == "" {
|
||||
writeJSON(w, 400, map[string]string{"error": "agent_id required"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.SourceAgents.UpsertTask(ctx, t)
|
||||
if err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
func (s *Server) handleDeleteSourceTask(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.SourceAgents.DeleteTask(ctx, r.PathValue("id")); err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"ok": true})
|
||||
}
|
||||
func (s *Server) handleSourceInbox(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
limit := 100
|
||||
if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n > 0 && n <= 1000 {
|
||||
limit = n
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
docs, err := s.SourceAgents.ListInbox(ctx, strings.TrimSpace(r.URL.Query().Get("status")), limit)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"documents": docs})
|
||||
}
|
||||
func (s *Server) handleSourceInboxStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.sourceStoreAvailable(w) || !s.adminAuthorized(w, r) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
st, err := s.SourceAgents.Stats(ctx)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, st)
|
||||
}
|
||||
|
||||
func bearerToken(r *http.Request) string {
|
||||
v := strings.TrimSpace(r.Header.Get("Authorization"))
|
||||
if strings.HasPrefix(v, "Bearer ") {
|
||||
return strings.TrimSpace(strings.TrimPrefix(v, "Bearer "))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func (s *Server) authenticateSourceAgent(r *http.Request) (sourceagent.Agent, error) {
|
||||
if s.SourceAgents == nil {
|
||||
return sourceagent.Agent{}, errors.New("source agent service unavailable")
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
a, err := s.SourceAgents.Authenticate(ctx, bearerToken(r))
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if want := strings.TrimSpace(r.Header.Get("X-Brain-Agent-ID")); want != "" && want != a.ID {
|
||||
return sourceagent.Agent{}, errors.New("agent id does not match token")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
func (s *Server) handleAgentConfig(w http.ResponseWriter, r *http.Request) {
|
||||
a, err := s.authenticateSourceAgent(r)
|
||||
if err != nil {
|
||||
writeJSON(w, 401, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
cfg, err := s.SourceAgents.RemoteConfig(ctx, a)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, cfg)
|
||||
}
|
||||
func (s *Server) handleAgentHeartbeat(w http.ResponseWriter, r *http.Request) {
|
||||
a, err := s.authenticateSourceAgent(r)
|
||||
if err != nil {
|
||||
writeJSON(w, 401, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
var h sourceagent.Heartbeat
|
||||
if err := decode(r, &h); err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if h.AgentID != "" && h.AgentID != a.ID {
|
||||
writeJSON(w, 403, map[string]string{"error": "agent_id mismatch"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.SourceAgents.Heartbeat(ctx, a.ID, h); err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"ok": true, "server_time": time.Now().UTC()})
|
||||
}
|
||||
func (s *Server) handleAgentIngest(w http.ResponseWriter, r *http.Request) {
|
||||
a, err := s.authenticateSourceAgent(r)
|
||||
if err != nil {
|
||||
writeJSON(w, 401, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
var batch sourceagent.IngestBatch
|
||||
dec := json.NewDecoder(io.LimitReader(r.Body, 32<<20))
|
||||
if err := dec.Decode(&batch); err != nil {
|
||||
writeJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if batch.AgentID != "" && batch.AgentID != a.ID {
|
||||
writeJSON(w, 403, map[string]string{"error": "agent_id mismatch"})
|
||||
return
|
||||
}
|
||||
if batch.SchemaVersion != 0 && batch.SchemaVersion != sourceagent.SchemaVersion {
|
||||
writeJSON(w, 400, map[string]string{"error": "unsupported schema_version"})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(batch.TaskID) == "" || len(batch.Documents) == 0 || len(batch.Documents) > 500 {
|
||||
writeJSON(w, 400, map[string]string{"error": "task_id and 1..500 documents are required"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := contextTimeout(r, 60*time.Second)
|
||||
defer cancel()
|
||||
result, err := s.SourceAgents.Ingest(ctx, a.ID, batch.TaskID, batch.Documents)
|
||||
if err != nil {
|
||||
writeJSON(w, 500, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
_ = s.SourceAgents.Heartbeat(ctx, a.ID, sourceagent.Heartbeat{AgentID: a.ID, Status: "ingest", Documents: result.Accepted})
|
||||
writeJSON(w, http.StatusAccepted, result)
|
||||
}
|
||||
@@ -61,6 +61,7 @@
|
||||
<button id="toggleEco" title="Optimierter Renderpfad für schwächere Systeme">ECO</button>
|
||||
<button id="resetView">Zentrieren</button>
|
||||
<button id="openSettings" title="Exakte KB-source-Filter und Laufzeitsteuerung">FILTER</button>
|
||||
<a class="analysis-dashboard-link" href="/source-agents.html" title="Verteilte Source-Agenten, Polling-Aufgaben und Source-Inbox">AGENTS</a>
|
||||
<a class="analysis-dashboard-link" href="/analysis.html" title="Technisches Analyse-Dashboard mit Lauf- und Graphhistorie">ANALYSE</a>
|
||||
<button id="enrichNow" class="think-action" title="Einen autonomen AI-THINK-Zyklus sofort starten"><span>AI-THINK</span><small>STARTEN</small></button>
|
||||
</nav>
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
:root{color-scheme:dark;font-family:Inter,ui-sans-serif,system-ui,sans-serif;background:#040810;color:#dcecff}*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 70% 0,#0b2637 0,#040810 42%);min-height:100vh}header{position:sticky;top:0;z-index:5;display:flex;justify-content:space-between;align-items:center;padding:18px 28px;background:#07111bd9;border-bottom:1px solid #173244;backdrop-filter:blur(18px)}header strong{display:block;letter-spacing:.16em;color:#66e3ff}header small{color:#7f9bad}nav{display:flex;gap:10px}a,button{color:#ccefff;background:#0a1a27;border:1px solid #285069;border-radius:9px;padding:9px 13px;text-decoration:none;cursor:pointer}button:hover,a:hover{border-color:#52dfff}main{width:min(1500px,96vw);margin:24px auto 60px;display:grid;gap:18px}.panel{background:#07121dcc;border:1px solid #17384b;border-radius:16px;padding:18px;box-shadow:0 20px 60px #0006}.panel h2{margin:0 0 8px;font-size:17px}.panel p{margin:0;color:#7896a9;font-size:13px}.auth{display:flex;gap:12px;align-items:center}.auth>div{margin-right:auto}.auth input{max-width:360px}.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}.stats article{padding:18px;border:1px solid #17384b;border-radius:14px;background:#07121dcc}.stats b{display:block;font-size:28px;color:#64e2ff}.stats span{color:#829dad}.grid{display:grid;grid-template-columns:1fr 1fr;gap:18px}label{display:grid;gap:6px;margin-top:12px;color:#8eaabb;font-size:12px}input,select{width:100%;padding:10px 11px;border:1px solid #25465b;border-radius:8px;background:#030a10;color:#e5f5ff}.two{display:grid;grid-template-columns:1fr 1fr;gap:10px}.panel>button{margin-top:14px}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(310px,1fr));gap:12px;margin-top:14px}.card{border:1px solid #19384b;border-radius:12px;padding:14px;background:#050d15}.card-head{display:flex;justify-content:space-between;gap:12px}.card small,.muted{color:#7692a3}.status-online{color:#68f6b1}.status-offline{color:#ffb06b}.actions{display:flex;flex-wrap:wrap;gap:7px;margin-top:12px}.actions button{padding:6px 9px;font-size:11px}.task{margin-top:10px;padding:9px;border-left:2px solid #31a7cd;background:#081722}.task b{font-size:12px}.task span{display:block;color:#7994a5;font-size:11px;overflow-wrap:anywhere}.token-output{white-space:pre-wrap;word-break:break-all;background:#02070c;border:1px solid #2a5871;padding:12px;border-radius:10px;color:#9ff0ff}.hidden{display:none}.title-row{display:flex;justify-content:space-between;align-items:center;gap:12px}.title-row select{width:auto}.table-wrap{overflow:auto;margin-top:14px}table{width:100%;border-collapse:collapse;font-size:12px}th,td{padding:10px;border-bottom:1px solid #142b39;text-align:left;vertical-align:top}th{color:#79a6bb}td a{padding:0;border:0;background:none;color:#79dcff}.pill{display:inline-block;padding:3px 7px;border-radius:99px;background:#123246;color:#aeefff}.pill.candidate{background:#143c30;color:#8dffc5}.pill.archived{background:#2b2f36;color:#aab3bd}.pill.received,.pill.processing{background:#3a3215;color:#ffe59a}#toast{position:fixed;right:20px;bottom:20px;max-width:440px;padding:12px 16px;border-radius:10px;background:#102c3c;border:1px solid #3a718c;opacity:0;pointer-events:none;transition:.2s}#toast.show{opacity:1}@media(max-width:850px){.grid,.stats{grid-template-columns:1fr}.auth{align-items:stretch;flex-direction:column}.auth>div{margin:0}header{padding:14px}.title-row{align-items:flex-start;flex-direction:column}}
|
||||
|
||||
label.check{display:flex;grid-template-columns:auto 1fr;align-items:center;gap:8px}.check input{width:auto}
|
||||
@@ -0,0 +1,24 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Source Agents · Neural Brain</title>
|
||||
<link rel="stylesheet" href="/source-agents.css">
|
||||
</head>
|
||||
<body>
|
||||
<header><div><strong>SOURCE AGENTS</strong><small>Verteilte Quellenbeobachtung · Inbox vor SearXNG</small></div><nav><a href="/">BRAIN</a><a href="/analysis.html">ANALYSE</a></nav></header>
|
||||
<main>
|
||||
<section class="panel auth"><div><h2>Administration</h2><p>Falls <code>BRAIN_API_KEY</code> gesetzt ist, wird er nur in dieser Browser-Session gehalten.</p></div><input id="apiKey" type="password" placeholder="BRAIN_API_KEY (optional)"><button id="saveKey">Übernehmen</button></section>
|
||||
<section class="stats" id="stats"><article><b id="agentCount">0</b><span>Agents</span></article><article><b id="taskCount">0</b><span>Tasks</span></article><article><b id="candidateCount">0</b><span>Inbox-Kandidaten</span></article><article><b id="receivedCount">0</b><span>Queue</span></article></section>
|
||||
<section class="grid">
|
||||
<article class="panel"><h2>Agent anlegen</h2><label>Name<input id="agentName" placeholder="Security News Frankfurt"></label><label>ID (optional)<input id="agentID" placeholder="security-frankfurt"></label><button id="createAgent">AGENT ERSTELLEN</button><pre id="tokenOutput" class="token-output hidden"></pre></article>
|
||||
<article class="panel"><h2>Polling-Aufgabe</h2><label>Agent<select id="taskAgent"></select></label><div class="two"><label>Typ<select id="taskType"><option>rss</option><option>atom</option><option>sitemap</option><option>web</option></select></label><label>Intervall<input id="taskInterval" value="4h"></label></div><label>Name<input id="taskName" placeholder="Microsoft Security Blog"></label><label>URL<input id="taskURL" placeholder="https://example.org/feed.xml"></label><label>Kategorien<input id="taskCategories" placeholder="Security, Microsoft"></label><label>Max. Elemente pro Lauf<input id="taskMaxItems" type="number" min="1" max="500" value="20"></label><label class="check"><input id="taskRefetchSeen" type="checkbox"> Bereits bekannte URLs erneut auf Inhaltsänderungen prüfen</label><button id="createTask">TASK SPEICHERN</button></article>
|
||||
</section>
|
||||
<section class="panel"><div class="title-row"><div><h2>Agents</h2><p>Tokenrechte: config · heartbeat · ingest. Kein Graph-/Adminzugriff.</p></div><button id="reload">AKTUALISIEREN</button></div><div id="agents" class="cards"></div></section>
|
||||
<section class="panel"><div class="title-row"><div><h2>Source Inbox</h2><p>Nur als <b>candidate</b> klassifizierte Dokumente werden vor SearXNG durchsucht. Graph-Nodes entstehen erst nach tatsächlichem Claim-Grounding.</p></div><select id="inboxFilter"><option value="">alle</option><option>received</option><option>processing</option><option>candidate</option><option>archived</option><option>used</option></select></div><div class="table-wrap"><table><thead><tr><th>Status</th><th>Titel / Quelle</th><th>Relevanz</th><th>Agent / Task</th><th>Eingang</th></tr></thead><tbody id="inbox"></tbody></table></div></section>
|
||||
</main>
|
||||
<div id="toast"></div>
|
||||
<script src="/source-agents.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,18 @@
|
||||
(() => {
|
||||
const $=id=>document.getElementById(id); let snapshot={agents:[],tasks:[],inbox:{}};
|
||||
$('apiKey').value=sessionStorage.getItem('brain_api_key')||'';
|
||||
$('saveKey').onclick=()=>{sessionStorage.setItem('brain_api_key',$('apiKey').value.trim());toast('API-Key für diese Session gespeichert.');loadAll()};
|
||||
function headers(){const h={'Content-Type':'application/json'},key=sessionStorage.getItem('brain_api_key')||'';if(key)h.Authorization=`Bearer ${key}`;return h}
|
||||
async function api(path,opt={}){const res=await fetch(path,{...opt,headers:{...headers(),...(opt.headers||{})}});const data=await res.json().catch(()=>({}));if(!res.ok)throw new Error(data.error||`HTTP ${res.status}`);return data}
|
||||
function toast(msg){const t=$('toast');t.textContent=msg;t.classList.add('show');setTimeout(()=>t.classList.remove('show'),3500)}
|
||||
function esc(v){return String(v??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]))}
|
||||
function when(v){if(!v)return 'nie';const d=new Date(v);if(Number.isNaN(d.getTime()))return 'nie';return d.toLocaleString()}
|
||||
async function loadAll(){try{snapshot=await api('/api/source-agents');render();await loadInbox()}catch(e){toast(e.message)}}
|
||||
function render(){const {agents=[],tasks=[],inbox={}}=snapshot;$('agentCount').textContent=agents.length;$('taskCount').textContent=tasks.length;$('candidateCount').textContent=inbox.candidate||0;$('receivedCount').textContent=(inbox.received||0)+(inbox.processing||0);$('taskAgent').innerHTML=agents.map(a=>`<option value="${esc(a.id)}">${esc(a.name)} · ${esc(a.id)}</option>`).join('');$('agents').innerHTML=agents.length?agents.map(a=>{const own=tasks.filter(t=>t.agent_id===a.id),online=a.last_seen&&Date.now()-new Date(a.last_seen).getTime()<15*60*1000;return `<div class="card"><div class="card-head"><div><b>${esc(a.name)}</b><small>${esc(a.id)}</small></div><span class="${online?'status-online':'status-offline'}">● ${online?'online':'offline'}</span></div><small>Version ${esc(a.version||'—')} · letzter Kontakt ${esc(when(a.last_seen))}</small>${a.last_error?`<p>${esc(a.last_error)}</p>`:''}<div class="actions"><button data-rotate="${esc(a.id)}">TOKEN ROTIEREN</button><button data-toggle="${esc(a.id)}" data-enabled="${a.enabled}">${a.enabled?'DEAKTIVIEREN':'AKTIVIEREN'}</button><button data-delete-agent="${esc(a.id)}">LÖSCHEN</button></div>${own.map(t=>`<div class="task"><b>${esc(t.name)} · ${esc(t.type)} · ${esc(t.poll_interval)}</b><span>${esc(t.url)}</span><div class="actions"><button data-delete-task="${esc(t.id)}">TASK LÖSCHEN</button></div></div>`).join('')}</div>`}).join(''):'<p class="muted">Noch keine Agenten konfiguriert.</p>';wireActions()}
|
||||
function wireActions(){document.querySelectorAll('[data-rotate]').forEach(b=>b.onclick=async()=>{try{const d=await api(`/api/source-agents/${b.dataset.rotate}/rotate-token`,{method:'POST'});showToken(b.dataset.rotate,d.token);await loadAll()}catch(e){toast(e.message)}});document.querySelectorAll('[data-toggle]').forEach(b=>b.onclick=async()=>{try{await api(`/api/source-agents/${b.dataset.toggle}`,{method:'PATCH',body:JSON.stringify({enabled:b.dataset.enabled!=='true'})});await loadAll()}catch(e){toast(e.message)}});document.querySelectorAll('[data-delete-agent]').forEach(b=>b.onclick=async()=>{if(!confirm('Agent inklusive Tasks löschen?'))return;try{await api(`/api/source-agents/${b.dataset.deleteAgent}`,{method:'DELETE'});await loadAll()}catch(e){toast(e.message)}});document.querySelectorAll('[data-delete-task]').forEach(b=>b.onclick=async()=>{try{await api(`/api/source-tasks/${b.dataset.deleteTask}`,{method:'DELETE'});await loadAll()}catch(e){toast(e.message)}})}
|
||||
function showToken(id,token){const brain=location.origin;$('tokenOutput').textContent=`Token nur jetzt kopieren:\n${token}\n\nAgent-ENV:\nBRAIN_MODE=agent\nBRAIN_AGENT_BRAIN_URL=${brain}\nBRAIN_AGENT_ID=${id}\nBRAIN_AGENT_TOKEN=${token}`;$('tokenOutput').classList.remove('hidden')}
|
||||
$('createAgent').onclick=async()=>{try{const d=await api('/api/source-agents',{method:'POST',body:JSON.stringify({id:$('agentID').value.trim(),name:$('agentName').value.trim()})});showToken(d.agent.id,d.token);$('agentName').value='';$('agentID').value='';await loadAll()}catch(e){toast(e.message)}};
|
||||
$('createTask').onclick=async()=>{const agent=$('taskAgent').value;if(!agent)return toast('Zuerst einen Agent anlegen.');const task={name:$('taskName').value.trim(),type:$('taskType').value,url:$('taskURL').value.trim(),enabled:true,poll_interval:$('taskInterval').value.trim(),categories:$('taskCategories').value.split(',').map(x=>x.trim()).filter(Boolean),max_items:Number($('taskMaxItems').value)||20,config:{refetch_seen:$('taskRefetchSeen').checked?'true':'false'}};try{await api(`/api/source-agents/${encodeURIComponent(agent)}/tasks`,{method:'POST',body:JSON.stringify(task)});$('taskName').value='';$('taskURL').value='';await loadAll();toast('Polling-Aufgabe gespeichert.')}catch(e){toast(e.message)}};
|
||||
async function loadInbox(){try{const status=$('inboxFilter').value;const d=await api(`/api/source-inbox?limit=100${status?`&status=${encodeURIComponent(status)}`:''}`);$('inbox').innerHTML=(d.documents||[]).map(x=>`<tr><td><span class="pill ${esc(x.status)}">${esc(x.status)}</span></td><td><a href="${esc(x.document.canonical_url||x.document.url)}" target="_blank" rel="noreferrer">${esc(x.document.title)}</a><br><span class="muted">${esc(x.document.source_name||x.document.source_base_url||'')}</span></td><td>${(Number(x.relevance||0)*100).toFixed(0)}%</td><td>${esc(x.agent_id)}<br><span class="muted">${esc(x.task_id)}</span></td><td>${esc(when(x.received_at))}</td></tr>`).join('')}catch(e){toast(e.message)}}
|
||||
$('inboxFilter').onchange=loadInbox;$('reload').onclick=loadAll;loadAll();setInterval(loadAll,30000);
|
||||
})();
|
||||
Reference in New Issue
Block a user