Cluster-Work
This commit is contained in:
@@ -38,6 +38,13 @@ type Config struct {
|
||||
EnrichStepDelay time.Duration
|
||||
EnrichBatchSize int
|
||||
EnrichAnchors int
|
||||
ProcessingMode string
|
||||
ClusterHashBits int
|
||||
ClusterHashTables int
|
||||
ClusterCandidatesPerAnchor int
|
||||
ClusterArticleCandidates int
|
||||
ClusterReviewEvidence int
|
||||
ClusterReviewContextChars int
|
||||
SimilarityThreshold float64
|
||||
RelationThreshold float64
|
||||
ArticleSynthesisEnabled bool
|
||||
@@ -155,6 +162,13 @@ func Load() (Config, error) {
|
||||
EnrichStepDelay: duration("BRAIN_ENRICH_STEP_DELAY", 3*time.Second),
|
||||
EnrichBatchSize: integer("BRAIN_ENRICH_BATCH_SIZE", 3),
|
||||
EnrichAnchors: integer("BRAIN_ENRICH_ANCHORS", 48),
|
||||
ProcessingMode: strings.ToLower(env("BRAIN_PROCESSING_MODE", "precise")),
|
||||
ClusterHashBits: integer("BRAIN_CLUSTER_HASH_BITS", 24),
|
||||
ClusterHashTables: integer("BRAIN_CLUSTER_HASH_TABLES", 2),
|
||||
ClusterCandidatesPerAnchor: integer("BRAIN_CLUSTER_CANDIDATES_PER_ANCHOR", 96),
|
||||
ClusterArticleCandidates: integer("BRAIN_CLUSTER_ARTICLE_CANDIDATES", 192),
|
||||
ClusterReviewEvidence: integer("BRAIN_CLUSTER_REVIEW_EVIDENCE", 8),
|
||||
ClusterReviewContextChars: integer("BRAIN_CLUSTER_REVIEW_CONTEXT_CHARS", 8000),
|
||||
SimilarityThreshold: number("BRAIN_SIMILARITY_THRESHOLD", 0.68),
|
||||
RelationThreshold: number("BRAIN_RELATION_THRESHOLD", 0.72),
|
||||
ArticleSynthesisEnabled: boolean("BRAIN_ARTICLE_SYNTHESIS_ENABLED", true),
|
||||
@@ -297,6 +311,29 @@ func Load() (Config, error) {
|
||||
if cfg.ArticleResearchMinQuality < 0 || cfg.ArticleResearchMinQuality > 1 {
|
||||
return Config{}, fmt.Errorf("BRAIN_ARTICLE_RESEARCH_MIN_QUALITY must be between 0 and 1")
|
||||
}
|
||||
switch cfg.ProcessingMode {
|
||||
case "precise", "clustered":
|
||||
default:
|
||||
return Config{}, fmt.Errorf("BRAIN_PROCESSING_MODE must be precise or clustered")
|
||||
}
|
||||
if cfg.ClusterHashBits < 8 || cfg.ClusterHashBits > 63 {
|
||||
return Config{}, fmt.Errorf("BRAIN_CLUSTER_HASH_BITS must be between 8 and 63")
|
||||
}
|
||||
if cfg.ClusterHashTables < 1 || cfg.ClusterHashTables > 4 {
|
||||
return Config{}, fmt.Errorf("BRAIN_CLUSTER_HASH_TABLES must be between 1 and 4")
|
||||
}
|
||||
if cfg.ClusterCandidatesPerAnchor < 16 || cfg.ClusterCandidatesPerAnchor > 2048 {
|
||||
return Config{}, fmt.Errorf("BRAIN_CLUSTER_CANDIDATES_PER_ANCHOR must be between 16 and 2048")
|
||||
}
|
||||
if cfg.ClusterArticleCandidates < 16 || cfg.ClusterArticleCandidates > 4096 {
|
||||
return Config{}, fmt.Errorf("BRAIN_CLUSTER_ARTICLE_CANDIDATES must be between 16 and 4096")
|
||||
}
|
||||
if cfg.ClusterReviewEvidence < 2 || cfg.ClusterReviewEvidence > 64 {
|
||||
return Config{}, fmt.Errorf("BRAIN_CLUSTER_REVIEW_EVIDENCE must be between 2 and 64")
|
||||
}
|
||||
if cfg.ClusterReviewContextChars < 4000 || cfg.ClusterReviewContextChars > 100000 {
|
||||
return Config{}, fmt.Errorf("BRAIN_CLUSTER_REVIEW_CONTEXT_CHARS must be between 4000 and 100000")
|
||||
}
|
||||
if cfg.ArticleResearchPageMaxBytes < 65536 || cfg.ArticleResearchPageMaxBytes > 16777216 {
|
||||
return Config{}, fmt.Errorf("BRAIN_ARTICLE_RESEARCH_PAGE_MAX_BYTES must be between 65536 and 16777216")
|
||||
}
|
||||
|
||||
@@ -149,3 +149,29 @@ func TestLoadRejectsAutonomousResearchWithoutSearXNG(t *testing.T) {
|
||||
t.Fatal("expected SEARXNG_URL validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadClusterProcessingMode(t *testing.T) {
|
||||
t.Setenv("BRAIN_DATA_DIR", t.TempDir())
|
||||
t.Setenv("BRAIN_PROCESSING_MODE", "clustered")
|
||||
t.Setenv("BRAIN_CLUSTER_HASH_BITS", "20")
|
||||
t.Setenv("BRAIN_CLUSTER_HASH_TABLES", "3")
|
||||
t.Setenv("BRAIN_CLUSTER_CANDIDATES_PER_ANCHOR", "80")
|
||||
t.Setenv("BRAIN_CLUSTER_ARTICLE_CANDIDATES", "160")
|
||||
t.Setenv("BRAIN_CLUSTER_REVIEW_EVIDENCE", "7")
|
||||
t.Setenv("BRAIN_CLUSTER_REVIEW_CONTEXT_CHARS", "7000")
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.ProcessingMode != "clustered" || cfg.ClusterHashBits != 20 || cfg.ClusterHashTables != 3 || cfg.ClusterCandidatesPerAnchor != 80 || cfg.ClusterArticleCandidates != 160 || cfg.ClusterReviewEvidence != 7 || cfg.ClusterReviewContextChars != 7000 {
|
||||
t.Fatalf("unexpected cluster config: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidProcessingMode(t *testing.T) {
|
||||
t.Setenv("BRAIN_DATA_DIR", t.TempDir())
|
||||
t.Setenv("BRAIN_PROCESSING_MODE", "turbo")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("expected invalid processing mode error")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user