Init
release-tag / Resolve release metadata (push) Successful in 30s
release-tag / Build knowledge (push) Failing after 4m51s
release-tag / Build control (push) Failing after 5m0s
release-tag / Build agent (push) Failing after 5m0s
release-tag / Build agent-data-init (push) Failing after 5m5s
release-tag / Build neuroforge-worker (push) Failing after 5m7s
release-tag / Build neuroforge (push) Failing after 5m9s
release-tag / Resolve release metadata (push) Successful in 30s
release-tag / Build knowledge (push) Failing after 4m51s
release-tag / Build control (push) Failing after 5m0s
release-tag / Build agent (push) Failing after 5m0s
release-tag / Build agent-data-init (push) Failing after 5m5s
release-tag / Build neuroforge-worker (push) Failing after 5m7s
release-tag / Build neuroforge (push) Failing after 5m9s
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAppEndpointsAcceptAdminTokenWithoutAppKeyReveal(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
sec := s.store.Secrets()
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/stats", nil)
|
||||
req.Header.Set("X-Admin-Token", sec.AdminToken)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("admin-auth app endpoint status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/stats", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+sec.AppAPIKey)
|
||||
rr = httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("app bearer status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/stats", nil)
|
||||
rr = httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauthenticated status=%d want=%d", rr.Code, http.StatusUnauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminDashboardDoesNotPutMaskedAppSecretInAuthorizationHeader(t *testing.T) {
|
||||
b, err := webFS.ReadFile("index.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
html := string(b)
|
||||
if strings.Contains(html, "getAppKey()") {
|
||||
t.Fatal("dashboard must not fetch App API key for its own application calls")
|
||||
}
|
||||
if strings.Contains(html, "Authorization':'Bearer '+(await getAppKey())") {
|
||||
t.Fatal("dashboard still builds Authorization from masked App API key")
|
||||
}
|
||||
if !strings.Contains(html, "validHeaderToken") {
|
||||
t.Fatal("dashboard should validate admin token before fetch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminDashboardChatInputDoesNotCollideWithWindowPrompt(t *testing.T) {
|
||||
b, err := webFS.ReadFile("index.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
html := string(b)
|
||||
if strings.Contains(html, `id="prompt"`) {
|
||||
t.Fatal("dashboard chat textarea must not use reserved browser global name prompt")
|
||||
}
|
||||
if strings.Contains(html, "input:prompt.value") || strings.Contains(html, "input: prompt.value") {
|
||||
t.Fatal("dashboard chat must not read window.prompt as the request input")
|
||||
}
|
||||
if !strings.Contains(html, `document.getElementById('chatPrompt')`) {
|
||||
t.Fatal("dashboard chat should resolve its textarea explicitly")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,767 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"neuroforge/internal/brain"
|
||||
"neuroforge/internal/core"
|
||||
"neuroforge/internal/cost"
|
||||
"neuroforge/internal/provider"
|
||||
"neuroforge/internal/store"
|
||||
)
|
||||
|
||||
//go:embed index.html
|
||||
var webFS embed.FS
|
||||
|
||||
type Server struct {
|
||||
store *store.Store
|
||||
brain *brain.Engine
|
||||
router *provider.Router
|
||||
cost *cost.Manager
|
||||
mux *http.ServeMux
|
||||
metrics *metricsRegistry
|
||||
inflight atomic.Int64
|
||||
}
|
||||
|
||||
func New(s *store.Store, b *brain.Engine, r *provider.Router, c *cost.Manager) *Server {
|
||||
x := &Server{store: s, brain: b, router: r, cost: c, mux: http.NewServeMux(), metrics: newMetricsRegistry()}
|
||||
x.routes()
|
||||
return x
|
||||
}
|
||||
func (s *Server) Handler() http.Handler {
|
||||
var h http.Handler = s.mux
|
||||
h = s.requestLimits(h)
|
||||
h = s.securityHeaders(h)
|
||||
h = s.logging(h)
|
||||
return h
|
||||
}
|
||||
|
||||
func (s *Server) routes() {
|
||||
s.mux.HandleFunc("GET /", s.index)
|
||||
s.mux.HandleFunc("GET /admin", s.index)
|
||||
s.mux.HandleFunc("GET /metrics", s.metricsEndpoint)
|
||||
s.mux.HandleFunc("GET /healthz", s.livez)
|
||||
s.mux.HandleFunc("GET /livez", s.livez)
|
||||
s.mux.HandleFunc("GET /readyz", s.readyz)
|
||||
s.mux.HandleFunc("GET /version", func(w http.ResponseWriter, r *http.Request) { s.json(w, 200, map[string]any{"version": "0.8.2"}) })
|
||||
s.mux.Handle("POST /api/v1/chat", s.appAuth(http.HandlerFunc(s.chat)))
|
||||
s.mux.Handle("POST /api/v1/learn", s.appAuth(http.HandlerFunc(s.learn)))
|
||||
s.mux.Handle("POST /api/v1/search", s.appAuth(http.HandlerFunc(s.search)))
|
||||
s.mux.Handle("POST /api/v1/search/vector", s.appAuth(http.HandlerFunc(s.searchVector)))
|
||||
s.mux.Handle("POST /api/v1/memory/import", s.appAuth(http.HandlerFunc(s.importMemory)))
|
||||
s.mux.Handle("POST /api/v1/feedback", s.appAuth(http.HandlerFunc(s.feedback)))
|
||||
s.mux.Handle("GET /api/v1/stats", s.appAuth(http.HandlerFunc(s.stats)))
|
||||
s.mux.Handle("GET /api/v1/goals", s.appAuth(http.HandlerFunc(s.goalsList)))
|
||||
s.mux.Handle("POST /api/v1/goals", s.appAuth(http.HandlerFunc(s.goalsCreate)))
|
||||
s.mux.Handle("GET /api/v1/goals/{id}", s.appAuth(http.HandlerFunc(s.goalsGet)))
|
||||
s.mux.Handle("PUT /api/v1/goals/{id}", s.appAuth(http.HandlerFunc(s.goalsPut)))
|
||||
s.mux.Handle("DELETE /api/v1/goals/{id}", s.appAuth(http.HandlerFunc(s.goalsDelete)))
|
||||
s.mux.Handle("POST /api/v1/goals/{id}/pause", s.appAuth(http.HandlerFunc(s.goalPause)))
|
||||
s.mux.Handle("POST /api/v1/goals/{id}/resume", s.appAuth(http.HandlerFunc(s.goalResume)))
|
||||
s.mux.Handle("POST /api/v1/goals/{id}/cycle", s.appAuth(http.HandlerFunc(s.goalCycle)))
|
||||
s.mux.Handle("GET /api/v1/goals/{id}/research/live", s.appAuth(http.HandlerFunc(s.goalResearchLive)))
|
||||
s.mux.Handle("GET /api/v1/goals/{id}/research/history", s.appAuth(http.HandlerFunc(s.goalResearchHistory)))
|
||||
s.mux.Handle("GET /api/v1/learning-cycles", s.appAuth(http.HandlerFunc(s.learningCycles)))
|
||||
s.mux.Handle("GET /api/v1/conflicts", s.appAuth(http.HandlerFunc(s.conflicts)))
|
||||
s.mux.Handle("POST /api/v1/ingest/text", s.appAuth(http.HandlerFunc(s.ingestText)))
|
||||
s.mux.Handle("POST /api/v1/ingest/document", s.appAuth(http.HandlerFunc(s.ingestDocument)))
|
||||
s.mux.Handle("GET /api/v1/sources", s.appAuth(http.HandlerFunc(s.sourcesList)))
|
||||
s.mux.Handle("GET /api/v1/sources/{id}", s.appAuth(http.HandlerFunc(s.sourceGet)))
|
||||
s.mux.Handle("POST /api/v1/research", s.appAuth(http.HandlerFunc(s.researchSearch)))
|
||||
s.mux.Handle("POST /api/v1/integrations/knowledge/upsert", s.appAuth(http.HandlerFunc(s.integrationKnowledgeUpsert)))
|
||||
s.mux.Handle("DELETE /api/v1/integrations/knowledge/{namespace}/{document_id}", s.appAuth(http.HandlerFunc(s.integrationKnowledgeDelete)))
|
||||
s.mux.Handle("POST /api/v1/integrations/knowledge/search", s.appAuth(http.HandlerFunc(s.integrationKnowledgeSearch)))
|
||||
s.mux.Handle("POST /api/v1/integrations/events", s.appAuth(http.HandlerFunc(s.integrationEvent)))
|
||||
s.mux.Handle("POST /api/v1/integrations/outcomes", s.appAuth(http.HandlerFunc(s.integrationValidatedOutcome)))
|
||||
s.mux.Handle("POST /api/v1/integrations/outcomes/search", s.appAuth(http.HandlerFunc(s.integrationValidatedOutcomeSearch)))
|
||||
s.mux.Handle("GET /api/v1/integrations/graph/research", s.appAuth(http.HandlerFunc(s.integrationResearchGraph)))
|
||||
s.mux.Handle("GET /api/v1/integrations/graph/brain", s.appAuth(http.HandlerFunc(s.integrationBrainGraph)))
|
||||
|
||||
s.mux.Handle("POST /internal/v1/cluster/request-vote", s.clusterAuth(http.HandlerFunc(s.clusterRequestVote)))
|
||||
s.mux.Handle("POST /internal/v1/cluster/heartbeat", s.clusterAuth(http.HandlerFunc(s.clusterHeartbeat)))
|
||||
s.mux.Handle("POST /internal/v1/cluster/prepare", s.clusterAuth(http.HandlerFunc(s.clusterPrepare)))
|
||||
s.mux.Handle("POST /internal/v1/cluster/commit", s.clusterAuth(http.HandlerFunc(s.clusterCommit)))
|
||||
s.mux.Handle("POST /internal/v1/cluster/abort", s.clusterAuth(http.HandlerFunc(s.clusterAbort)))
|
||||
s.mux.Handle("POST /internal/v1/cluster/propose/memory", s.clusterAuth(http.HandlerFunc(s.clusterProposeMemory)))
|
||||
s.mux.Handle("GET /internal/v1/cluster/decision/{id}", s.clusterAuth(http.HandlerFunc(s.clusterDecision)))
|
||||
s.mux.Handle("GET /internal/v1/cluster/status", s.clusterAuth(http.HandlerFunc(s.clusterStatus)))
|
||||
|
||||
s.mux.Handle("POST /api/v1/worker/claim", s.workerAuth(http.HandlerFunc(s.workerClaim)))
|
||||
s.mux.Handle("POST /api/v1/worker/complete", s.workerAuth(http.HandlerFunc(s.workerComplete)))
|
||||
|
||||
s.mux.Handle("GET /admin/api/status", s.adminAuth(http.HandlerFunc(s.adminStatus)))
|
||||
s.mux.Handle("GET /admin/api/config", s.adminAuth(http.HandlerFunc(s.adminGetConfig)))
|
||||
s.mux.Handle("PUT /admin/api/config", s.adminAuth(http.HandlerFunc(s.adminPutConfig)))
|
||||
s.mux.Handle("GET /admin/api/model-routing", s.adminAuth(http.HandlerFunc(s.adminGetModelRouting)))
|
||||
s.mux.Handle("PUT /admin/api/model-routing", s.adminAuth(http.HandlerFunc(s.adminPutModelRouting)))
|
||||
s.mux.Handle("GET /admin/api/secrets/status", s.adminAuth(http.HandlerFunc(s.adminSecretsStatus)))
|
||||
s.mux.Handle("GET /admin/api/secrets", s.adminAuth(http.HandlerFunc(s.adminGetSecrets)))
|
||||
s.mux.Handle("PUT /admin/api/secrets", s.adminAuth(http.HandlerFunc(s.adminPutSecrets)))
|
||||
s.mux.Handle("POST /admin/api/provider-health", s.adminAuth(http.HandlerFunc(s.adminProviderHealth)))
|
||||
s.mux.Handle("GET /admin/api/memories", s.adminAuth(http.HandlerFunc(s.adminMemories)))
|
||||
s.mux.Handle("DELETE /admin/api/memories/{id}", s.adminAuth(http.HandlerFunc(s.adminDeleteMemory)))
|
||||
s.mux.Handle("GET /admin/api/synapses", s.adminAuth(http.HandlerFunc(s.adminSynapses)))
|
||||
s.mux.Handle("GET /admin/api/usage", s.adminAuth(http.HandlerFunc(s.adminUsage)))
|
||||
s.mux.Handle("GET /admin/api/export", s.adminAuth(http.HandlerFunc(s.adminExport)))
|
||||
s.mux.Handle("POST /admin/api/consolidate", s.adminAuth(http.HandlerFunc(s.adminConsolidate)))
|
||||
s.mux.Handle("POST /admin/api/retention", s.adminAuth(http.HandlerFunc(s.adminRetention)))
|
||||
s.mux.Handle("POST /admin/api/autonomy", s.adminAuth(http.HandlerFunc(s.adminAutonomy)))
|
||||
s.mux.Handle("POST /admin/api/rebalance", s.adminAuth(http.HandlerFunc(s.adminRebalance)))
|
||||
s.mux.Handle("POST /admin/api/checkpoint", s.adminAuth(http.HandlerFunc(s.adminCheckpoint)))
|
||||
s.mux.Handle("GET /admin/api/wal", s.adminAuth(http.HandlerFunc(s.adminWAL)))
|
||||
s.mux.Handle("GET /admin/api/storage", s.adminAuth(http.HandlerFunc(s.adminStorageStatus)))
|
||||
s.mux.Handle("POST /admin/api/storage/compact", s.adminAuth(http.HandlerFunc(s.adminCompactSegments)))
|
||||
s.mux.Handle("POST /admin/api/storage/tier", s.adminAuth(http.HandlerFunc(s.adminTierStorage)))
|
||||
s.mux.Handle("POST /admin/api/index/merge", s.adminAuth(http.HandlerFunc(s.adminMergeIndex)))
|
||||
s.mux.Handle("GET /admin/api/index/disk", s.adminAuth(http.HandlerFunc(s.adminDiskANNStatus)))
|
||||
s.mux.Handle("POST /admin/api/index/disk/rebuild", s.adminAuth(http.HandlerFunc(s.adminDiskANNBuild)))
|
||||
s.mux.Handle("GET /admin/api/cluster", s.adminAuth(http.HandlerFunc(s.clusterStatus)))
|
||||
s.mux.Handle("POST /admin/api/cluster/repair", s.adminAuth(http.HandlerFunc(s.adminClusterRepair)))
|
||||
s.mux.Handle("POST /admin/api/conflicts/resolve", s.adminAuth(http.HandlerFunc(s.adminResolveConflict)))
|
||||
s.mux.Handle("GET /admin/api/knowledge/summary", s.adminAuth(http.HandlerFunc(s.adminKnowledgeSummary)))
|
||||
s.mux.Handle("GET /admin/api/knowledge/memories", s.adminAuth(http.HandlerFunc(s.adminKnowledgeMemories)))
|
||||
s.mux.Handle("GET /admin/api/knowledge/memory/{id}", s.adminAuth(http.HandlerFunc(s.adminKnowledgeMemory)))
|
||||
s.mux.Handle("GET /admin/api/knowledge/graph", s.adminAuth(http.HandlerFunc(s.adminKnowledgeGraph)))
|
||||
s.mux.Handle("GET /admin/api/knowledge/events", s.adminAuth(http.HandlerFunc(s.adminKnowledgeEvents)))
|
||||
s.mux.Handle("POST /admin/api/knowledge/search", s.adminAuth(http.HandlerFunc(s.adminKnowledgeSearch)))
|
||||
s.mux.Handle("GET /admin/api/learning-policy", s.adminAuth(http.HandlerFunc(s.adminGetLearningPolicy)))
|
||||
s.mux.Handle("PUT /admin/api/learning-policy", s.adminAuth(http.HandlerFunc(s.adminPutLearningPolicy)))
|
||||
s.mux.Handle("GET /admin/api/research", s.adminAuth(http.HandlerFunc(s.adminResearchGet)))
|
||||
s.mux.Handle("PUT /admin/api/research", s.adminAuth(http.HandlerFunc(s.adminResearchPut)))
|
||||
s.mux.Handle("POST /admin/api/research/test", s.adminAuth(http.HandlerFunc(s.adminResearchTest)))
|
||||
}
|
||||
|
||||
func (s *Server) index(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" && r.URL.Path != "/admin" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
b, err := webFS.ReadFile("index.html")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
w.Write(b)
|
||||
}
|
||||
|
||||
type statusWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func (w *statusWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
|
||||
|
||||
func (w *statusWriter) WriteHeader(code int) {
|
||||
if w.status != 0 {
|
||||
return
|
||||
}
|
||||
w.status = code
|
||||
w.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (w *statusWriter) Write(p []byte) (int, error) {
|
||||
if w.status == 0 {
|
||||
w.status = http.StatusOK
|
||||
}
|
||||
n, err := w.ResponseWriter.Write(p)
|
||||
w.bytes += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (s *Server) logging(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
sw := &statusWriter{ResponseWriter: w}
|
||||
next.ServeHTTP(sw, r)
|
||||
d := time.Since(start)
|
||||
s.metrics.observeHTTP(r.Method, normalizeMetricRoute(r), sw.status, sw.bytes, d)
|
||||
log.Printf("%s %s %d %s", r.Method, r.URL.Path, func() int {
|
||||
if sw.status == 0 {
|
||||
return http.StatusOK
|
||||
}
|
||||
return sw.status
|
||||
}(), d.Round(time.Millisecond))
|
||||
})
|
||||
}
|
||||
|
||||
func secureEqual(a, b string) bool {
|
||||
if len(a) == 0 || len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
|
||||
}
|
||||
|
||||
func bearer(r *http.Request) string {
|
||||
h := r.Header.Get("Authorization")
|
||||
if strings.HasPrefix(strings.ToLower(h), "bearer ") {
|
||||
return strings.TrimSpace(h[7:])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func (s *Server) appAuth(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.store.Config()
|
||||
if cfg.API.RequireKey {
|
||||
sec := s.store.Secrets()
|
||||
// The admin dashboard is already authenticated with the stronger admin
|
||||
// credential. Allow it to call application endpoints directly so the
|
||||
// browser never needs the App API key (which is masked by default in
|
||||
// production). External applications still authenticate with Bearer.
|
||||
adminOK := secureEqual(r.Header.Get("X-Admin-Token"), sec.AdminToken)
|
||||
appOK := secureEqual(bearer(r), sec.AppAPIKey)
|
||||
if !adminOK && !appOK {
|
||||
s.err(w, 401, errors.New("invalid app API key or admin token"))
|
||||
return
|
||||
}
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
func (s *Server) workerAuth(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !secureEqual(bearer(r), s.store.Secrets().WorkerToken) {
|
||||
s.err(w, 401, errors.New("invalid worker token"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
func (s *Server) adminAuth(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !secureEqual(r.Header.Get("X-Admin-Token"), s.store.Secrets().AdminToken) {
|
||||
s.err(w, 401, errors.New("invalid admin token"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) clusterAuth(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sec := s.store.Secrets()
|
||||
if sec.ClusterToken == "" || !secureEqual(r.Header.Get("X-Cluster-Token"), sec.ClusterToken) {
|
||||
s.err(w, 401, errors.New("invalid cluster token"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func decode(r *http.Request, v any) error {
|
||||
defer r.Body.Close()
|
||||
d := json.NewDecoder(io.LimitReader(r.Body, 128<<20))
|
||||
d.DisallowUnknownFields()
|
||||
return d.Decode(v)
|
||||
}
|
||||
func (s *Server) json(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
func (s *Server) err(w http.ResponseWriter, status int, err error) {
|
||||
s.json(w, status, map[string]any{"error": err.Error()})
|
||||
}
|
||||
|
||||
func (s *Server) chat(w http.ResponseWriter, r *http.Request) {
|
||||
var q brain.ChatRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
out, err := s.brain.Chat(r.Context(), q)
|
||||
if err != nil {
|
||||
s.err(w, 502, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
func (s *Server) learn(w http.ResponseWriter, r *http.Request) {
|
||||
var q brain.LearnRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
m, err := s.brain.Learn(r.Context(), q)
|
||||
if err != nil {
|
||||
s.err(w, 502, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 201, m)
|
||||
}
|
||||
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
Text string `json:"text"`
|
||||
K int `json:"k"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
hits, err := s.brain.Search(r.Context(), q.Text, q.K)
|
||||
if err != nil {
|
||||
s.err(w, 502, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, hits)
|
||||
}
|
||||
func (s *Server) searchVector(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
Vector []float32 `json:"vector"`
|
||||
K int `json:"k"`
|
||||
MinSimilarity *float64 `json:"min_similarity,omitempty"`
|
||||
GraphBonus *float64 `json:"graph_bonus,omitempty"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if len(q.Vector) == 0 {
|
||||
s.err(w, 400, errors.New("vector required"))
|
||||
return
|
||||
}
|
||||
cfg := s.store.Config()
|
||||
if q.K <= 0 {
|
||||
q.K = cfg.Brain.RecallK
|
||||
}
|
||||
min := cfg.Brain.MinSimilarity
|
||||
if q.MinSimilarity != nil {
|
||||
min = *q.MinSimilarity
|
||||
}
|
||||
bonus := cfg.Brain.GraphBonus
|
||||
if q.GraphBonus != nil {
|
||||
bonus = *q.GraphBonus
|
||||
}
|
||||
// Intentionally local-only: shard federation is one hop and must not recurse.
|
||||
hits := s.store.SearchVector(q.Vector, q.K, min, bonus)
|
||||
for i := range hits {
|
||||
if hits[i].Memory.ShardID == "" {
|
||||
hits[i].Memory.ShardID = cfg.Sharding.LocalShardID
|
||||
}
|
||||
}
|
||||
s.json(w, 200, hits)
|
||||
}
|
||||
|
||||
func (s *Server) importMemory(w http.ResponseWriter, r *http.Request) {
|
||||
var m core.Memory
|
||||
if err := decode(r, &m); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
out, err := s.brain.ImportMemory(r.Context(), m)
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 201, out)
|
||||
}
|
||||
|
||||
func (s *Server) feedback(w http.ResponseWriter, r *http.Request) {
|
||||
var q brain.FeedbackRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.brain.Feedback(q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
func (s *Server) stats(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, map[string]any{"stats": s.store.Stats(), "cost": s.cost.Totals()})
|
||||
}
|
||||
|
||||
func (s *Server) workerClaim(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if q.WorkerID == "" {
|
||||
s.err(w, 400, errors.New("worker_id required"))
|
||||
return
|
||||
}
|
||||
lease := s.store.Config().Worker.LeaseSeconds
|
||||
if lease < 10 {
|
||||
lease = 120
|
||||
}
|
||||
j, err := s.store.ClaimJob(q.WorkerID, time.Duration(lease)*time.Second)
|
||||
if err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
if j == nil {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, j)
|
||||
}
|
||||
func (s *Server) workerComplete(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
JobID string `json:"job_id"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
j, err := s.store.CompleteJob(q.JobID, q.WorkerID, q.Result, q.Error)
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.brain.ApplyJobResult(j); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
|
||||
func (s *Server) adminStatus(w http.ResponseWriter, r *http.Request) {
|
||||
obs := s.store.ObservabilitySnapshot()
|
||||
maint := s.store.MaintenanceStatus()
|
||||
stats := map[string]any{
|
||||
"revision": obs.Revision, "memories": obs.Memories, "synapses": obs.Synapses, "goals": obs.Goals, "learning_cycles": obs.LearningCycles,
|
||||
"pending_jobs": obs.JobsQueued + obs.JobsClaimed, "hnsw_nodes": obs.HNSWNodes, "hnsw_dimensions": obs.HNSWDimensions,
|
||||
"disk_pq_items": obs.DiskPQItems, "disk_pq_bytes": obs.DiskPQBytes, "index_mode": obs.IndexMode, "remote_shards": obs.RemoteShards, "maintenance": maint,
|
||||
}
|
||||
cfg := s.store.Config()
|
||||
sec := s.store.Secrets()
|
||||
providers := make([]map[string]any, 0, len(cfg.Ollama)+1)
|
||||
for _, o := range cfg.Ollama {
|
||||
providers = append(providers, map[string]any{"provider": "ollama", "id": o.ID, "name": o.Name, "model": o.ChatModel, "enabled": o.Enabled})
|
||||
}
|
||||
providers = append(providers, map[string]any{"provider": "openai", "model": cfg.OpenAI.ChatModel, "enabled": cfg.OpenAI.Enabled, "configured": sec.OpenAIAPIKey != ""})
|
||||
tiering := map[string]any{
|
||||
"hot_memories": obs.HotMemories, "cold_memories": obs.ColdMemories, "hot_bytes": obs.HotBytes, "tier_evictions_total": obs.TierEvictions,
|
||||
"page_cache": map[string]any{"enabled": obs.PageCacheEnabled, "max_bytes": obs.PageCacheMaxBytes, "bytes": obs.PageCacheBytes, "entries": obs.PageCacheEntries, "hits": obs.PageCacheHits, "misses": obs.PageCacheMisses, "evictions": obs.PageCacheEvicts},
|
||||
}
|
||||
cluster := map[string]any{
|
||||
"enabled": obs.ClusterEnabled, "node_id": obs.ClusterNodeID, "leader_id": obs.ClusterLeaderID, "role": obs.ClusterRole, "term": obs.ClusterTerm,
|
||||
"last_index": obs.ClusterLastIndex, "commit_index": obs.ClusterCommitIndex, "peers": obs.ClusterPeers, "voters": obs.ClusterVoters, "quorum": obs.ClusterQuorum,
|
||||
"replicated_log": obs.ClusterLog,
|
||||
}
|
||||
s.json(w, 200, map[string]any{
|
||||
"stats": stats, "wal": s.store.WALStatus(),
|
||||
"storage": map[string]any{"memory_segments": obs.Segments, "index_snapshot": map[string]any{"revision": obs.IndexSnapshotRevision, "deltas": obs.IndexDeltaCount, "segmented": cfg.Storage.IndexSegments.Enabled}, "tiering": tiering, "disk_ann": s.store.DiskANNStatus()},
|
||||
"cluster": cluster, "cost": s.cost.Totals(), "providers": providers, "usage": s.store.RecentUsage(25),
|
||||
"observability": obs, "runtime": currentRuntimeSnapshot(), "http": s.metrics.dashboardSnapshot(),
|
||||
})
|
||||
}
|
||||
func (s *Server) adminGetConfig(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.Config())
|
||||
}
|
||||
func (s *Server) adminPutConfig(w http.ResponseWriter, r *http.Request) {
|
||||
var c core.Config
|
||||
if err := decode(r, &c); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.ValidateConfig(c); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateConfig(c); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "admin.config_changed", Summary: "Runtime configuration updated", Reason: "PUT /admin/api/config", Actor: "admin"})
|
||||
s.json(w, 200, c)
|
||||
}
|
||||
|
||||
type modelRoutingLearning struct {
|
||||
AutoRewardEnabled bool `json:"auto_reward_enabled"`
|
||||
AutoRewardMode string `json:"auto_reward_mode"`
|
||||
ConsolidationEnabled bool `json:"consolidation_enabled"`
|
||||
ConsolidationUseLLM bool `json:"consolidation_use_llm"`
|
||||
AutonomyEnabled bool `json:"autonomy_enabled"`
|
||||
AutonomyUseLLM bool `json:"autonomy_use_llm"`
|
||||
}
|
||||
|
||||
type modelRoutingSettings struct {
|
||||
Routing core.RoutingConfig `json:"routing"`
|
||||
Ollama []core.OllamaServer `json:"ollama"`
|
||||
Learning modelRoutingLearning `json:"learning"`
|
||||
}
|
||||
|
||||
type modelRoutingUpdate struct {
|
||||
Routing *core.RoutingConfig `json:"routing,omitempty"`
|
||||
Ollama *[]core.OllamaServer `json:"ollama,omitempty"`
|
||||
Learning *modelRoutingLearning `json:"learning,omitempty"`
|
||||
}
|
||||
|
||||
func modelRoutingFromConfig(c core.Config) modelRoutingSettings {
|
||||
var out modelRoutingSettings
|
||||
out.Routing = c.Routing
|
||||
out.Ollama = append([]core.OllamaServer(nil), c.Ollama...)
|
||||
out.Learning.AutoRewardEnabled = c.Brain.AutoReward.Enabled
|
||||
out.Learning.AutoRewardMode = c.Brain.AutoReward.Mode
|
||||
out.Learning.ConsolidationEnabled = c.Brain.Consolidation.Enabled
|
||||
out.Learning.ConsolidationUseLLM = c.Brain.Consolidation.UseLLM
|
||||
out.Learning.AutonomyEnabled = c.Autonomy.Enabled
|
||||
out.Learning.AutonomyUseLLM = c.Autonomy.UseLLM
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Server) adminGetModelRouting(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, modelRoutingFromConfig(s.store.Config()))
|
||||
}
|
||||
|
||||
func (s *Server) adminPutModelRouting(w http.ResponseWriter, r *http.Request) {
|
||||
var q modelRoutingUpdate
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if q.Routing == nil && q.Ollama == nil && q.Learning == nil {
|
||||
s.err(w, 400, errors.New("routing, ollama or learning is required"))
|
||||
return
|
||||
}
|
||||
c := s.store.Config()
|
||||
if q.Routing != nil {
|
||||
c.Routing = *q.Routing
|
||||
}
|
||||
if q.Ollama != nil {
|
||||
c.Ollama = append([]core.OllamaServer(nil), (*q.Ollama)...)
|
||||
}
|
||||
if q.Learning != nil {
|
||||
c.Brain.AutoReward.Enabled = q.Learning.AutoRewardEnabled
|
||||
c.Brain.AutoReward.Mode = q.Learning.AutoRewardMode
|
||||
c.Brain.Consolidation.Enabled = q.Learning.ConsolidationEnabled
|
||||
c.Brain.Consolidation.UseLLM = q.Learning.ConsolidationUseLLM
|
||||
c.Autonomy.Enabled = q.Learning.AutonomyEnabled
|
||||
c.Autonomy.UseLLM = q.Learning.AutonomyUseLLM
|
||||
}
|
||||
if err := s.store.ValidateConfig(c); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateConfig(c); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "admin.model_routing_changed", Summary: "Model routing / Ollama configuration updated", Reason: "PUT /admin/api/model-routing", Actor: "admin", Metadata: map[string]string{"chat_provider": c.Routing.ChatProvider, "embedding_provider": c.Routing.EmbeddingProvider}})
|
||||
s.json(w, 200, modelRoutingFromConfig(c))
|
||||
}
|
||||
|
||||
func (s *Server) adminSecretsStatus(w http.ResponseWriter, r *http.Request) {
|
||||
sec := s.store.Secrets()
|
||||
s.json(w, 200, map[string]any{"openai_configured": sec.OpenAIAPIKey != "", "app_key_configured": sec.AppAPIKey != "", "worker_token_configured": sec.WorkerToken != "", "metrics_token_configured": sec.MetricsToken != "", "shard_tokens": len(sec.ShardAPIToken), "cluster_token_configured": sec.ClusterToken != ""})
|
||||
}
|
||||
func maskedSecret(v string) string {
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
if len(v) <= 4 {
|
||||
return "••••"
|
||||
}
|
||||
return "••••••••" + v[len(v)-4:]
|
||||
}
|
||||
func (s *Server) adminGetSecrets(w http.ResponseWriter, r *http.Request) {
|
||||
sec := s.store.Secrets()
|
||||
reveal := r.URL.Query().Get("reveal") == "1" && s.store.Config().Security.AllowSecretReveal
|
||||
if reveal {
|
||||
s.json(w, 200, map[string]any{"revealed": true, "app_api_key": sec.AppAPIKey, "worker_token": sec.WorkerToken, "metrics_token": sec.MetricsToken, "shard_api_tokens": sec.ShardAPIToken, "cluster_token": sec.ClusterToken})
|
||||
return
|
||||
}
|
||||
maskedShards := map[string]string{}
|
||||
for k, v := range sec.ShardAPIToken {
|
||||
maskedShards[k] = maskedSecret(v)
|
||||
}
|
||||
s.json(w, 200, map[string]any{"revealed": false, "reveal_allowed": s.store.Config().Security.AllowSecretReveal, "app_api_key": maskedSecret(sec.AppAPIKey), "worker_token": maskedSecret(sec.WorkerToken), "metrics_token": maskedSecret(sec.MetricsToken), "shard_api_tokens": maskedShards, "cluster_token": maskedSecret(sec.ClusterToken)})
|
||||
}
|
||||
func (s *Server) adminPutSecrets(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
OpenAIAPIKey string `json:"openai_api_key,omitempty"`
|
||||
AppAPIKey string `json:"app_api_key,omitempty"`
|
||||
WorkerToken string `json:"worker_token,omitempty"`
|
||||
MetricsToken string `json:"metrics_token,omitempty"`
|
||||
ShardAPIToken map[string]string `json:"shard_api_tokens,omitempty"`
|
||||
ClusterToken string `json:"cluster_token,omitempty"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
sec := s.store.Secrets()
|
||||
if q.OpenAIAPIKey != "" {
|
||||
sec.OpenAIAPIKey = q.OpenAIAPIKey
|
||||
}
|
||||
if q.AppAPIKey != "" {
|
||||
sec.AppAPIKey = q.AppAPIKey
|
||||
}
|
||||
if q.WorkerToken != "" {
|
||||
sec.WorkerToken = q.WorkerToken
|
||||
}
|
||||
if q.MetricsToken != "" {
|
||||
sec.MetricsToken = q.MetricsToken
|
||||
}
|
||||
if q.ShardAPIToken != nil {
|
||||
sec.ShardAPIToken = q.ShardAPIToken
|
||||
}
|
||||
if q.ClusterToken != "" {
|
||||
sec.ClusterToken = q.ClusterToken
|
||||
}
|
||||
if err := s.store.UpdateSecrets(sec); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "admin.secrets_changed", Summary: "One or more service credentials were updated", Reason: "PUT /admin/api/secrets", Actor: "admin"})
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
func (s *Server) adminProviderHealth(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 8*time.Second)
|
||||
defer cancel()
|
||||
s.json(w, 200, s.router.Health(ctx))
|
||||
}
|
||||
func (s *Server) adminMemories(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
all := s.store.MemoriesSnapshot()
|
||||
if len(all) > limit {
|
||||
all = all[len(all)-limit:]
|
||||
}
|
||||
s.json(w, 200, all)
|
||||
}
|
||||
func (s *Server) adminDeleteMemory(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
m, _ := s.store.GetMemory(id)
|
||||
if err := s.store.DeleteMemory(id); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
if m != nil {
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "memory.deleted", MemoryID: id, Summary: "Memory deleted by administrator", Reason: "DELETE /admin/api/memories/{id}", Actor: "admin", Metadata: map[string]string{"kind": m.Kind, "memory_type": m.MemoryType, "truth_key": m.TruthKey}})
|
||||
}
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
func (s *Server) adminSynapses(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.SynapsesSnapshot())
|
||||
}
|
||||
func (s *Server) adminUsage(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
s.json(w, 200, s.store.RecentUsage(limit))
|
||||
}
|
||||
func (s *Server) adminExport(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.ExportSafe())
|
||||
}
|
||||
|
||||
func (s *Server) adminConsolidate(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := s.brain.Consolidate(r.Context())
|
||||
if err != nil {
|
||||
// A cycle can partially succeed and still report a synthesis/provider error.
|
||||
s.json(w, 207, map[string]any{"result": out, "warning": err.Error()})
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) securityHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store.Config().Security.SecureHeaders {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Cross-Origin-Opener-Policy", "same-origin")
|
||||
w.Header().Set("X-Permitted-Cross-Domain-Policies", "none")
|
||||
w.Header().Set("X-Frame-Options", "DENY")
|
||||
w.Header().Set("Referrer-Policy", "no-referrer")
|
||||
w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=()")
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'self'; img-src 'self' data:; connect-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'")
|
||||
}
|
||||
if r.URL.Path == "/admin" || strings.HasPrefix(r.URL.Path, "/admin/api/") {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
}
|
||||
if r.Header.Get("X-Request-ID") == "" {
|
||||
r.Header.Set("X-Request-ID", store.NewID("req"))
|
||||
}
|
||||
w.Header().Set("X-Request-ID", r.Header.Get("X-Request-ID"))
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) requestLimits(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.store.Config().HTTP
|
||||
if cfg.MaxBodyBytes > 0 && r.Body != nil {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, cfg.MaxBodyBytes)
|
||||
}
|
||||
if r.URL.Path == "/healthz" || r.URL.Path == "/livez" || r.URL.Path == "/readyz" || r.URL.Path == "/metrics" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
n := s.inflight.Add(1)
|
||||
defer s.inflight.Add(-1)
|
||||
max := int64(cfg.MaxConcurrentRequests)
|
||||
if max <= 0 {
|
||||
max = 128
|
||||
}
|
||||
if n > max {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
s.err(w, http.StatusServiceUnavailable, errors.New("server is at the configured concurrent request limit"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) livez(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, http.StatusOK, map[string]any{"ok": true, "status": "alive", "time": time.Now().UTC(), "version": "0.8.2"})
|
||||
}
|
||||
|
||||
func (s *Server) readyz(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.store.Config()
|
||||
sec := s.store.Secrets()
|
||||
obs := s.store.ObservabilitySnapshot()
|
||||
components := map[string]any{}
|
||||
configOK := s.store.ValidateConfig(cfg) == nil
|
||||
components["config"] = configOK
|
||||
components["app_auth"] = !cfg.API.RequireKey || sec.AppAPIKey != ""
|
||||
hasOllamaChat, hasOllamaEmbed := false, false
|
||||
for _, o := range cfg.Ollama {
|
||||
if !o.Enabled {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(o.ChatModel) != "" {
|
||||
hasOllamaChat = true
|
||||
}
|
||||
if strings.TrimSpace(o.EmbeddingModel) != "" {
|
||||
hasOllamaEmbed = true
|
||||
}
|
||||
}
|
||||
openAIReady := cfg.OpenAI.Enabled && sec.OpenAIAPIKey != ""
|
||||
chatReady := (cfg.Routing.ChatProvider == "openai" && openAIReady) || (cfg.Routing.ChatProvider == "ollama" && hasOllamaChat) || ((cfg.Routing.ChatProvider == "auto" || cfg.Routing.ChatProvider == "") && (hasOllamaChat || openAIReady))
|
||||
embedReady := (cfg.Routing.EmbeddingProvider == "openai" && openAIReady) || (cfg.Routing.EmbeddingProvider == "ollama" && hasOllamaEmbed) || ((cfg.Routing.EmbeddingProvider == "auto" || cfg.Routing.EmbeddingProvider == "") && (hasOllamaEmbed || openAIReady))
|
||||
components["chat_route_configured"] = chatReady
|
||||
components["embedding_route_configured"] = embedReady
|
||||
clusterReady := !cfg.Cluster.Enabled || obs.ClusterLeaderID != ""
|
||||
components["cluster"] = clusterReady
|
||||
ready := configOK && chatReady && embedReady && clusterReady && (!cfg.API.RequireKey || sec.AppAPIKey != "")
|
||||
status := http.StatusOK
|
||||
if !ready {
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
s.json(w, status, map[string]any{"ok": ready, "status": map[bool]string{true: "ready", false: "not_ready"}[ready], "components": components, "revision": obs.Revision, "time": time.Now().UTC()})
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1,viewport-fit=cover">
|
||||
<title>NeuroForge · Knowledge OS</title>
|
||||
<style>
|
||||
:root{
|
||||
color-scheme:dark;--bg:#080b12;--panel:#101622;--panel2:#151d2c;--line:#26334a;--muted:#91a0b7;--text:#eef3fb;
|
||||
--accent:#7aa7ff;--accent2:#65d6b5;--warn:#f7c36d;--bad:#ff7c8d;--shadow:0 18px 48px rgba(0,0,0,.28);
|
||||
font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif
|
||||
}
|
||||
*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 15% 0,#10192b 0,#080b12 36rem);color:var(--text)}
|
||||
button,input,textarea,select{font:inherit}button{cursor:pointer}.app{min-height:100vh;display:grid;grid-template-columns:238px minmax(0,1fr)}
|
||||
.sidebar{position:sticky;top:0;height:100vh;padding:22px 16px;border-right:1px solid var(--line);background:rgba(8,11,18,.9);backdrop-filter:blur(16px);z-index:20}
|
||||
.brand{display:flex;align-items:center;gap:12px;padding:4px 8px 24px}.mark{width:36px;height:36px;border-radius:12px;background:linear-gradient(145deg,var(--accent),#8d7cff);box-shadow:0 8px 24px #637cff45}.brand b{font-size:20px}.brand small{display:block;color:var(--muted);margin-top:2px}
|
||||
.nav{display:grid;gap:5px}.nav button{display:flex;align-items:center;gap:10px;width:100%;border:0;background:transparent;color:var(--muted);padding:10px 12px;border-radius:11px;text-align:left}.nav button:hover{background:#151e2e;color:var(--text)}.nav button.active{background:#1b2840;color:#fff;box-shadow:inset 3px 0 var(--accent)}.nav .ico{width:20px;text-align:center}
|
||||
.sidebarFoot{position:absolute;left:16px;right:16px;bottom:18px}.statusChip{display:flex;align-items:center;gap:8px;padding:10px 12px;border:1px solid var(--line);border-radius:12px;background:#0e1420;color:var(--muted);font-size:13px}.dot{width:8px;height:8px;border-radius:50%;background:var(--muted)}.dot.ok{background:var(--accent2);box-shadow:0 0 12px #65d6b566}.dot.bad{background:var(--bad)}
|
||||
.main{min-width:0;padding:26px 30px 60px}.topbar{display:flex;justify-content:space-between;gap:16px;align-items:center;margin-bottom:22px}.topbar h1{font-size:25px;margin:0}.topbar p{margin:5px 0 0;color:var(--muted);font-size:14px}.actions{display:flex;gap:8px;align-items:center;flex-wrap:wrap}
|
||||
.btn{border:1px solid transparent;border-radius:10px;padding:9px 13px;font-weight:650;background:var(--accent);color:#07101e}.btn:hover{filter:brightness(1.06)}.btn.secondary{background:#172133;border-color:#2d3c55;color:var(--text)}.btn.ghost{background:transparent;border-color:var(--line);color:var(--muted)}.btn.danger{background:#3a1821;border-color:#65303e;color:#ffb8c2}.btn:disabled{opacity:.45;cursor:not-allowed}
|
||||
.view{display:none}.view.active{display:block}.grid{display:grid;gap:14px}.metrics{grid-template-columns:repeat(4,minmax(0,1fr));margin-bottom:14px}.metric{border:1px solid var(--line);background:linear-gradient(160deg,#131b2a,#0e141f);border-radius:16px;padding:17px;box-shadow:var(--shadow)}.metric .label{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.06em}.metric .value{font-size:28px;font-weight:750;margin-top:5px}.metric .sub{font-size:12px;color:var(--muted);margin-top:4px}
|
||||
.card{border:1px solid var(--line);background:rgba(16,22,34,.94);border-radius:16px;padding:18px;box-shadow:var(--shadow)}.card h2{font-size:17px;margin:0 0 14px}.card h3{font-size:14px;margin:18px 0 9px}.two{grid-template-columns:minmax(0,1.35fr) minmax(300px,.65fr)}.three{grid-template-columns:repeat(3,minmax(0,1fr))}.split{display:grid;grid-template-columns:minmax(0,1.7fr) minmax(320px,.72fr);gap:14px}
|
||||
label{display:block;color:var(--muted);font-size:12px;margin-bottom:5px}.field{margin-bottom:12px}.control{width:100%;border:1px solid #30405b;background:#0b111c;color:var(--text);border-radius:10px;padding:10px 11px;outline:none}.control:focus{border-color:var(--accent);box-shadow:0 0 0 3px #7aa7ff18}textarea.control{min-height:120px;resize:vertical}.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}.row{display:flex;gap:10px;align-items:center}.row>*{min-width:0}.grow{flex:1}.checks{display:flex;gap:14px;flex-wrap:wrap}.checks label{display:flex;align-items:center;gap:7px;color:var(--text);font-size:13px;margin:0}.checks input{accent-color:var(--accent)}
|
||||
.pill{display:inline-flex;align-items:center;gap:5px;padding:4px 8px;border-radius:99px;background:#1a2639;color:#b9c8dd;font-size:11px;border:1px solid #2c3b55}.pill.good{background:#102b25;border-color:#225445;color:#8de0c5}.pill.warn{background:#2b2414;border-color:#5b4923;color:#f5cf85}.pill.bad{background:#321821;border-color:#66303e;color:#ffacb8}.muted{color:var(--muted)}.small{font-size:12px}.error{color:#ffadb8}.good{color:#83dec1}.warn{color:#f7ce84}.pre{white-space:pre-wrap;word-break:break-word;background:#090e17;border:1px solid #202c40;border-radius:10px;padding:11px;max-height:360px;overflow:auto}.scroll{max-height:590px;overflow:auto}.list{display:grid;gap:8px}.item{border:1px solid #26344a;background:#0d1420;border-radius:12px;padding:12px}.item.click{cursor:pointer}.item.click:hover{border-color:#3c5377;background:#111b2b}.itemTitle{display:flex;justify-content:space-between;gap:10px;align-items:flex-start}.itemTitle b{font-size:13px}.item p{margin:6px 0;color:#ccd7e7;font-size:13px;line-height:1.42}.meta{display:flex;gap:6px;flex-wrap:wrap;color:var(--muted);font-size:11px}
|
||||
.progress{height:7px;border-radius:99px;background:#1a2536;overflow:hidden}.progress span{display:block;height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:inherit}
|
||||
.pipeline{display:grid;grid-template-columns:repeat(7,minmax(120px,1fr));gap:8px}.stage{position:relative;padding:12px;border:1px solid var(--line);border-radius:12px;background:#0d1420}.stage b{display:block;font-size:12px}.stage small{display:block;color:var(--muted);margin-top:4px}.stage:not(:last-child):after{content:"→";position:absolute;right:-8px;top:28px;color:var(--accent);z-index:2}
|
||||
.graphShell{position:relative;border:1px solid var(--line);border-radius:16px;background:radial-gradient(circle at center,#121d30,#090d15 72%);overflow:hidden;min-height:620px}.graphShell canvas{width:100%;height:620px;display:block;touch-action:none}.graphHud{position:absolute;left:12px;top:12px;display:flex;gap:7px;flex-wrap:wrap;pointer-events:none}.graphHud .pill{background:#0b111ddd;backdrop-filter:blur(8px)}.graphTools{position:absolute;right:12px;top:12px;display:flex;gap:6px}.graphTools button{width:34px;height:34px;border-radius:10px;padding:0}.graphHint{position:absolute;left:12px;bottom:10px;color:#8697b1;font-size:11px;background:#080d16c7;padding:5px 8px;border-radius:8px}.legend{display:flex;gap:10px;flex-wrap:wrap;margin:9px 0 0}.legend span{font-size:11px;color:var(--muted)}.sw{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:5px}
|
||||
.drop{border:1px dashed #405575;border-radius:14px;padding:25px;text-align:center;background:#0b121e;transition:.15s}.drop.drag{border-color:var(--accent);background:#111d31}.drop input{max-width:100%}.sourceGrid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:8px}.sourceIcon{width:34px;height:34px;display:grid;place-items:center;border-radius:10px;background:#1a2639;color:var(--accent)}
|
||||
.goalGrid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:12px}.goal{border:1px solid var(--line);border-radius:14px;padding:15px;background:#0d1420}.goalHead{display:flex;justify-content:space-between;gap:12px}.goal h3{margin:0 0 4px;font-size:15px}.goal .progress{margin:12px 0 7px}.goalActions{display:flex;gap:6px;flex-wrap:wrap;margin-top:10px}
|
||||
.researchLive{display:none;margin-top:14px;overflow:hidden}.researchLive.show{display:block}.liveHead{display:flex;justify-content:space-between;gap:14px;align-items:flex-start}.liveTitle{display:flex;gap:10px;align-items:center}.livePulse{width:10px;height:10px;border-radius:50%;background:var(--muted);flex:0 0 auto;margin-top:5px}.livePulse.running{background:var(--accent2);box-shadow:0 0 0 0 rgba(101,214,181,.5);animation:researchPulse 1.6s infinite}.livePulse.warn{background:var(--warn)}.livePulse.error{background:var(--bad)}@keyframes researchPulse{0%{box-shadow:0 0 0 0 rgba(101,214,181,.45)}70%{box-shadow:0 0 0 9px rgba(101,214,181,0)}100%{box-shadow:0 0 0 0 rgba(101,214,181,0)}}
|
||||
.liveStats{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));gap:8px;margin:14px 0}.liveStat{border:1px solid #24334a;border-radius:11px;background:#0b121e;padding:10px}.liveStat b{font-size:19px;display:block}.liveStat span{font-size:10px;text-transform:uppercase;letter-spacing:.05em;color:var(--muted)}.queryChips{display:flex;gap:6px;flex-wrap:wrap;margin:8px 0 14px}.queryChip{font-size:11px;padding:6px 8px;border:1px solid #30435f;border-radius:9px;background:#0d1726;color:#c8d7ec;max-width:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
.eventBoard{display:grid;grid-template-columns:repeat(auto-fit,minmax(245px,1fr));gap:9px}.eventLane{border:1px solid #223149;border-radius:13px;background:#0a101a;min-height:180px;overflow:hidden}.eventLaneHead{display:flex;align-items:center;justify-content:space-between;padding:10px 11px;border-bottom:1px solid #223149;background:#0f1724;position:sticky;top:0;z-index:1}.eventLaneHead b{font-size:12px}.eventLaneBody{padding:7px;display:grid;gap:6px;max-height:360px;overflow:auto}.researchEvent{border:1px solid #25354c;border-radius:9px;background:#0d1521;padding:8px 9px;animation:eventIn .18s ease-out}.researchEvent.running{border-color:#2e6959}.researchEvent.error{border-color:#65303e}.researchEvent.warn,.researchEvent.skipped{border-color:#624e26}.researchEvent .eventTop{display:flex;justify-content:space-between;gap:8px;font-size:10px;color:var(--muted)}.researchEvent b{font-size:11px;display:block;margin:4px 0}.researchEvent p{font-size:11px;line-height:1.35;margin:3px 0;color:#c9d5e6}.researchEvent a{color:var(--accent);font-size:10px;display:block;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.researchEvent .preview{border-left:2px solid #304866;padding-left:6px;color:#aebdd1}.emptyLane{color:#687990;font-size:11px;padding:10px}@keyframes eventIn{from{transform:translateY(3px);opacity:.45}to{transform:none;opacity:1}}
|
||||
.researchHistory{margin-top:12px;display:flex;gap:7px;overflow:auto;padding-bottom:2px}.historyRun{min-width:205px;border:1px solid #26354d;border-radius:10px;background:#0c1420;padding:9px}.historyRun b{font-size:11px}.historyRun .meta{margin-top:5px}
|
||||
.tableWrap{overflow:auto}table{width:100%;border-collapse:collapse;font-size:12px}th,td{text-align:left;padding:9px;border-bottom:1px solid #223047;vertical-align:top}th{color:var(--muted);font-weight:600}.kv{display:grid;grid-template-columns:130px 1fr;gap:7px 12px;font-size:12px}.kv div:nth-child(odd){color:var(--muted)}
|
||||
.toast{position:fixed;right:20px;bottom:20px;max-width:440px;padding:12px 14px;border:1px solid #334766;border-radius:12px;background:#111b2b;color:#eef3fb;box-shadow:var(--shadow);z-index:80;display:none}.toast.show{display:block}.toast.bad{border-color:#673241;background:#28131a}
|
||||
.mobileNav{display:none}
|
||||
@media(max-width:1100px){.metrics{grid-template-columns:repeat(2,1fr)}.liveStats{grid-template-columns:repeat(3,1fr)}.pipeline{grid-template-columns:repeat(4,1fr)}.pipeline .stage:nth-child(4):after{display:none}.split,.two{grid-template-columns:1fr}.graphShell canvas{height:560px}.graphShell{min-height:560px}}
|
||||
@media(max-width:760px){.app{display:block}.liveStats{grid-template-columns:repeat(2,1fr)}.eventLaneBody{max-height:280px}.sidebar{display:none}.main{padding:18px 14px 90px}.topbar{align-items:flex-start}.topbar h1{font-size:21px}.topbar .actions .btn.ghost{display:none}.metrics{grid-template-columns:1fr 1fr}.metric .value{font-size:23px}.three,.goalGrid,.sourceGrid{grid-template-columns:1fr}.pipeline{grid-template-columns:1fr 1fr}.pipeline .stage:nth-child(even):after{display:none}.pipeline .stage:nth-child(4):after{display:none}.graphShell,.graphShell canvas{min-height:500px;height:500px}.mobileNav{position:fixed;display:flex;left:8px;right:8px;bottom:8px;z-index:60;background:#0c121ddd;border:1px solid #29384f;border-radius:15px;padding:6px;backdrop-filter:blur(18px);overflow-x:auto}.mobileNav button{flex:1;min-width:64px;border:0;background:transparent;color:var(--muted);padding:8px 6px;border-radius:9px;font-size:10px}.mobileNav button.active{background:#1b2840;color:#fff}.kv{grid-template-columns:100px 1fr}}
|
||||
@media(max-width:430px){.metrics{grid-template-columns:1fr}.pipeline{grid-template-columns:1fr}.stage:after{display:none!important}.row.stack{flex-direction:column;align-items:stretch}}
|
||||
.goalPaused{opacity:.78;border-style:dashed}.pill.warn{color:#ffd27d;border-color:rgba(255,210,125,.35);background:rgba(255,210,125,.08)}</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<aside class="sidebar">
|
||||
<div class="brand"><div class="mark"></div><div><b>NeuroForge</b><small>Knowledge OS · v0.8</small></div></div>
|
||||
<nav class="nav" id="nav">
|
||||
<button class="active" data-view="dashboard"><span class="ico">◈</span>Übersicht</button>
|
||||
<button data-view="knowledge"><span class="ico">◎</span>Wissensraum</button>
|
||||
<button data-view="sources"><span class="ico">↥</span>Quellen & Import</button>
|
||||
<button data-view="research"><span class="ico">⌕</span>Research</button>
|
||||
<button data-view="goals"><span class="ico">◉</span>Ziele & Autonomie</button>
|
||||
<button data-view="chat"><span class="ico">◇</span>Chat</button>
|
||||
<button data-view="models"><span class="ico">⬡</span>Modelle</button>
|
||||
<button data-view="ops"><span class="ico">⚙</span>Betrieb</button>
|
||||
</nav>
|
||||
<div class="sidebarFoot"><div class="statusChip"><span class="dot" id="sideDot"></span><span id="sideStatus">nicht verbunden</span></div></div>
|
||||
</aside>
|
||||
<main class="main">
|
||||
<header class="topbar"><div><h1 id="pageTitle">Übersicht</h1><p id="pageSub">Was das System weiß, lernt und gerade tut.</p></div><div class="actions"><button class="btn ghost" id="refreshBtn">Aktualisieren</button><button class="btn secondary" id="tokenBtn">Admin-Token</button></div></header>
|
||||
|
||||
<section class="view active" id="view-dashboard">
|
||||
<div class="grid metrics">
|
||||
<div class="metric"><div class="label">Memories</div><div class="value" id="mMem">–</div><div class="sub" id="mTypes">Wissensknoten</div></div>
|
||||
<div class="metric"><div class="label">Synapsen</div><div class="value" id="mSyn">–</div><div class="sub">gewichtete Beziehungen</div></div>
|
||||
<div class="metric"><div class="label">Quellen</div><div class="value" id="mSources">–</div><div class="sub">Dokumente · Text · Web</div></div>
|
||||
<div class="metric"><div class="label">Aktive Ziele</div><div class="value" id="mGoals">–</div><div class="sub" id="mNextGoal">autonomer Lernplan</div></div>
|
||||
</div>
|
||||
<div class="grid two">
|
||||
<div class="card"><h2>Lernpfad</h2><div class="pipeline">
|
||||
<div class="stage"><b>1 · Quelle</b><small>Chat, Text, Dokument, Web</small></div>
|
||||
<div class="stage"><b>2 · Evidence</b><small>Extrahieren, chunken, deduplizieren</small></div>
|
||||
<div class="stage"><b>3 · Embedding</b><small>stabiler semantischer Raum</small></div>
|
||||
<div class="stage"><b>4 · Recall</b><small>HNSW + PQ + Synapsen</small></div>
|
||||
<div class="stage"><b>5 · Reason</b><small>Actor/Critic mit Evidenz</small></div>
|
||||
<div class="stage"><b>6 · Learn</b><small>Reward + Provenance + Konflikte</small></div>
|
||||
<div class="stage"><b>7 · Consolidate</b><small>Episoden → Langzeitwissen</small></div>
|
||||
</div><p class="muted small">Web-Ergebnisse und Dokumente werden als quellengebundene Evidence-Memories gespeichert. Erst der normale Recall-/Konsolidierungspfad macht daraus stärkeres semantisches Wissen.</p></div>
|
||||
<div class="card"><h2>Systemzustand</h2><div id="systemState" class="list"></div></div>
|
||||
</div>
|
||||
<div class="grid two" style="margin-top:14px"><div class="card"><h2>Letzte Lernereignisse</h2><div class="list scroll" id="dashEvents"></div></div><div class="card"><h2>Aktive Ziele</h2><div class="list" id="dashGoals"></div></div></div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-knowledge">
|
||||
<div class="row stack" style="margin-bottom:12px"><div class="grow"><input id="knowledgeQuery" class="control" placeholder="Wissen semantisch durchsuchen …"></div><button class="btn" id="knowledgeSearchBtn">Suchen</button><button class="btn secondary" id="graphReloadBtn">Graph neu laden</button></div>
|
||||
<div class="split">
|
||||
<div><div class="graphShell"><canvas id="knowledgeCanvas"></canvas><div class="graphHud"><span class="pill" id="lodBadge">LOD · Übersicht</span><span class="pill" id="graphCount">0 Knoten</span><span class="pill" id="graphZoom">100%</span></div><div class="graphTools"><button class="btn secondary" id="zoomInBtn">+</button><button class="btn secondary" id="zoomOutBtn">−</button><button class="btn secondary" id="fitBtn">⌂</button></div><div class="graphHint">Mausrad/Pinch: Zoom · Ziehen: Pan · Klick: Memory öffnen · LOD blendet Details progressiv ein</div></div><div class="legend"><span><i class="sw" style="background:#7aa7ff"></i>semantic</span><span><i class="sw" style="background:#65d6b5"></i>episodic</span><span><i class="sw" style="background:#f7c36d"></i>procedural</span><span><i class="sw" style="background:#bd8cff"></i>working/evidence</span></div></div>
|
||||
<div class="card"><h2>Inspektor</h2><div id="knowledgeInspector" class="scroll"><p class="muted">Klicke einen Knoten oder suche nach einem Begriff. Hier siehst du Herkunft, Confidence, Reward, Verknüpfungen und Lernhistorie.</p></div></div>
|
||||
</div>
|
||||
<div class="card" style="margin-top:14px"><h2>Recall-Erklärung</h2><div id="knowledgeHits" class="list"></div></div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-sources">
|
||||
<div class="grid two">
|
||||
<div class="card"><h2>Dokument importieren</h2><div class="drop" id="dropZone"><p><b>Datei hier ablegen</b> oder auswählen</p><p class="muted small">TXT · Markdown · HTML · JSON · CSV/TSV · DOCX · PDF*</p><input id="docFile" type="file" class="control" accept=".txt,.md,.markdown,.html,.htm,.json,.csv,.tsv,.docx,.pdf,text/*,application/json,application/pdf"></div><div class="row stack" style="margin-top:12px"><div class="grow"><label>Titel</label><input id="docTitle" class="control" placeholder="optional"></div><div class="grow"><label>Tags (Komma)</label><input id="docTags" class="control" placeholder="nvidia,cuda"></div><div style="width:110px"><label>Trust 0–1</label><input id="docTrust" class="control" type="number" min="0" max="1" step="0.05" value="1"></div></div><button class="btn" id="uploadBtn">Dokument lernen</button><p class="muted small">* PDF wird serverseitig über <span class="mono">pdftotext</span> extrahiert; das Release-Containerimage enthält das Tool.</p></div>
|
||||
<div class="card"><h2>Text einspeisen</h2><div class="field"><label>Titel / Kontext</label><input id="textTitle" class="control" placeholder="z. B. Interne Notiz: NVIDIA Infrastruktur"></div><div class="field"><label>Text</label><textarea id="textInput" class="control" placeholder="Hier Wissen, Notizen, Protokolle oder kopierten Dokumenttext einfügen …"></textarea></div><div class="row stack"><div class="grow"><label>Quelle/URI (optional)</label><input id="textURI" class="control" placeholder="https://… oder interne://…"></div><div style="width:110px"><label>Trust 0–1</label><input id="textTrust" class="control" type="number" min="0" max="1" step="0.05" value="1"></div></div><button class="btn" id="textLearnBtn">Text lernen</button></div>
|
||||
</div>
|
||||
<div class="card" style="margin-top:14px"><div class="itemTitle"><h2>Quellenbibliothek</h2><button class="btn ghost" id="sourcesReloadBtn">Neu laden</button></div><div id="sourcesList" class="sourceGrid"></div></div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-research">
|
||||
<div class="grid two">
|
||||
<div class="card"><h2>SearXNG Research</h2><div class="row stack"><div class="grow"><label>Suchanfrage</label><input id="researchQuery" class="control" placeholder="z. B. NVIDIA Blackwell Architektur offizielle Dokumentation"></div><div style="width:130px"><label>Max Ergebnisse</label><input id="researchMax" class="control" type="number" min="1" max="50" value="8"></div></div><div class="checks"><label><input type="checkbox" id="researchLearn"> Ergebnisse lernen</label><label><input type="checkbox" id="researchFetch" checked> Seiten abrufen</label></div><div class="row" style="margin-top:12px"><button class="btn" id="researchBtn">Suchen</button><button class="btn secondary" id="researchTestBtn">Verbindung testen</button></div><div id="researchResults" class="list" style="margin-top:14px"></div></div>
|
||||
<div class="card"><h2>Research-Konfiguration</h2><div class="checks"><label><input type="checkbox" id="rcEnabled"> Research aktiv</label><label><input type="checkbox" id="rcSearx"> SearXNG aktiv</label><label><input type="checkbox" id="rcGoal"> Goal-Research</label></div><div class="field" style="margin-top:12px"><label>SearXNG Base URL</label><input id="rcBase" class="control mono" placeholder="http://searxng:8080"></div><div class="row stack"><div class="grow"><label>Sprache</label><input id="rcLang" class="control" value="de-DE"></div><div class="grow"><label>Kategorien</label><input id="rcCats" class="control" value="general,science,it"></div></div><div class="row stack"><div class="grow"><label>Queries / Goal-Cycle</label><input id="rcQueries" class="control" type="number" min="1" max="20"></div><div class="grow"><label>Seiten / Cycle</label><input id="rcPages" class="control" type="number" min="0" max="50"></div><div class="grow"><label>Goal-Intervall (min)</label><input id="rcInterval" class="control" type="number" min="1"></div></div><div class="field"><label>Authorization-Header (optional, wird nicht zurückgelesen)</label><input id="rcAuth" class="control mono" type="password" placeholder="Bearer …"></div><div class="checks"><label><input type="checkbox" id="rcFetch"> Webseiten nachladen</label><label><input type="checkbox" id="rcAuto"> Autonomie aktiv</label><label><input type="checkbox" id="rcImmediate"> neue Goals sofort einplanen</label></div><button class="btn" id="researchSaveBtn" style="margin-top:12px">Research speichern</button><p class="muted small">Private Ziel-URLs sind für den Web-Fetch standardmäßig blockiert (SSRF-Schutz). Die lokale SearXNG-Adresse ist davon nicht betroffen, weil sie nur als Such-API angesprochen wird. PDF/DOCX/TXT/MD/CSV/JSON-Dateitreffer werden automatisch über die Dokument-Pipeline extrahiert, gechunkt und als quellengebundene Evidence verarbeitet.</p></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-goals">
|
||||
<div class="grid two"><div class="card"><h2>Neues Ziel</h2><div class="field"><label>Titel</label><input id="goalTitle" class="control" placeholder="NVIDIA"></div><div class="field"><label>Beschreibung</label><textarea id="goalDesc" class="control" placeholder="Welche Wissenslücke soll das System autonom schließen?"></textarea></div><div class="field"><label>Messbares Ziel</label><input id="goalTarget" class="control" placeholder="z. B. 100 hochwertige, quellengebundene Wissenseinträge"></div><div class="row stack"><div class="grow"><label>Priorität 1–100</label><input id="goalPriority" class="control" type="number" min="1" max="100" value="50"></div><div class="grow"><label>Intervall (Minuten)</label><input id="goalInterval" class="control" type="number" min="1" value="10"></div></div><button class="btn" id="goalCreateBtn">Ziel anlegen</button></div><div class="card"><h2>So arbeitet ein Goal</h2><div class="pre">FÄLLIG\n ↓\nSEARCH PLAN (Actor)\n ↓\nSEARXNG → Web Evidence\n ↓\nChunk + Embed + Dedup\n ↓\nRecall vorhandenes + neues Wissen\n ↓\nPredict → Evaluate → Learn\n ↓\nNextCycleAt / Backoff</div><p class="muted small">Ein neues Ziel wird bei aktivierter Autonomie sofort fällig. Danach besitzt jedes Goal sein eigenes Intervall und Fehler-Backoff.</p></div></div>
|
||||
<div class="card" style="margin-top:14px"><div class="itemTitle"><h2>Ziele</h2><button class="btn secondary" id="runAutonomyBtn">Alle fälligen jetzt</button></div><div class="goalGrid" id="goalsList"></div></div>
|
||||
<div class="card researchLive" id="goalResearchLive">
|
||||
<div class="liveHead"><div class="liveTitle"><span class="livePulse" id="livePulse"></span><div><h2 style="margin:0" id="liveGoalTitle">Live Research</h2><div class="muted small" id="liveRunMeta">Noch kein Research-Lauf ausgewählt.</div></div></div><div class="row"><button class="btn ghost" id="liveRefreshBtn">Aktualisieren</button><button class="btn ghost" id="liveCloseBtn">Schließen</button></div></div>
|
||||
<div class="liveStats"><div class="liveStat"><b id="lsResults">0</b><span>Treffer</span></div><div class="liveStat"><b id="lsDownloads">0</b><span>Downloads</span></div><div class="liveStat"><b id="lsClaims">0</b><span>Claims</span></div><div class="liveStat"><b id="lsNew">0</b><span>Neu gelernt</span></div><div class="liveStat"><b id="lsDup">0</b><span>Duplikate</span></div><div class="liveStat"><b id="lsConfirm">0</b><span>Bestätigt</span></div></div>
|
||||
<div><label>Aktuelle / geplante Suchqueries</label><div class="queryChips" id="liveQueries"><span class="muted small">–</span></div></div>
|
||||
<div class="eventBoard">
|
||||
<div class="eventLane"><div class="eventLaneHead"><b>⌕ Queries</b><span class="pill" id="lcQueries">0</span></div><div class="eventLaneBody" id="laneQueries"></div></div>
|
||||
<div class="eventLane"><div class="eventLaneHead"><b>↗ Gefundene URLs</b><span class="pill" id="lcResults">0</span></div><div class="eventLaneBody" id="laneResults"></div></div>
|
||||
<div class="eventLane"><div class="eventLaneHead"><b>⇣ Downloads / Quellen</b><span class="pill" id="lcDownloads">0</span></div><div class="eventLaneBody" id="laneDownloads"></div></div>
|
||||
<div class="eventLane"><div class="eventLaneHead"><b>◈ Claims / Evidenz</b><span class="pill" id="lcClaims">0</span></div><div class="eventLaneBody" id="laneClaims"></div></div>
|
||||
<div class="eventLane"><div class="eventLaneHead"><b>✓ Duplikate / Bestätigung</b><span class="pill" id="lcDedupe">0</span></div><div class="eventLaneBody" id="laneDedupe"></div></div>
|
||||
<div class="eventLane"><div class="eventLaneHead"><b>! Verworfen / Fehler</b><span class="pill" id="lcRejected">0</span></div><div class="eventLaneBody" id="laneRejected"></div></div>
|
||||
</div>
|
||||
<h3>Letzte Research-Läufe</h3><div class="researchHistory" id="researchHistory"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-chat">
|
||||
<div class="card"><h2>Chat gegen das lernende System</h2><div class="row stack"><div class="grow"><label>Session</label><input id="chatSession" class="control" value="web-demo"></div><div class="grow"><label>Provider</label><select id="chatProvider" class="control"><option value="">Auto</option><option value="ollama">Ollama</option><option value="openai">OpenAI</option></select></div></div><div class="field"><label>Nachricht</label><textarea id="chatPrompt" class="control" style="min-height:180px" placeholder="Frage stellen …"></textarea></div><div class="row"><button class="btn" id="chatBtn">Senden & lernen</button><button class="btn danger" id="chatAbortBtn" disabled>Abbrechen</button><span class="muted small" id="chatElapsed"></span></div><div id="chatOutput" class="pre" style="margin-top:14px;min-height:80px"></div></div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-models">
|
||||
<div class="grid two"><div class="card"><h2>Modelle & Routing</h2><p class="muted small">Hier siehst du den aktiven Routing-Block. Für feine Änderungen kannst du den JSON-Editor verwenden.</p><div id="modelCards" class="list"></div><button class="btn secondary" id="modelHealthBtn" style="margin-top:12px">Ollama-Verbindungen prüfen</button><div id="modelHealth" class="pre" style="margin-top:10px"></div></div><div class="card"><h2>Routing JSON</h2><textarea id="modelJSON" class="control mono" style="min-height:410px"></textarea><button class="btn" id="modelSaveBtn">Routing speichern</button></div></div>
|
||||
</section>
|
||||
|
||||
<section class="view" id="view-ops">
|
||||
<div class="grid three"><div class="card"><h2>Index</h2><div id="opsIndex" class="list"></div><div class="row"><button class="btn secondary" id="pqBuildBtn">PQ rebuild</button><button class="btn ghost" id="indexMergeBtn">HNSW merge</button></div></div><div class="card"><h2>Storage</h2><div id="opsStorage" class="list"></div><div class="row"><button class="btn secondary" id="checkpointBtn">Checkpoint</button><button class="btn ghost" id="compactBtn">Kompaktieren</button></div></div><div class="card"><h2>Observability</h2><div id="opsObs" class="list"></div><div class="field"><label>Prometheus</label><div class="pre mono">GET /metrics\nAuthorization: Bearer <METRICS_TOKEN></div></div></div></div>
|
||||
<div class="card" style="margin-top:14px"><h2>Gesamtkonfiguration</h2><textarea id="configJSON" class="control mono" style="min-height:420px"></textarea><button class="btn" id="configSaveBtn">Konfiguration speichern</button></div>
|
||||
</section>
|
||||
</main>
|
||||
</div>
|
||||
<nav class="mobileNav" id="mobileNav"><button class="active" data-view="dashboard">Home</button><button data-view="knowledge">Graph</button><button data-view="sources">Import</button><button data-view="research">Web</button><button data-view="goals">Goals</button><button data-view="chat">Chat</button><button data-view="ops">Ops</button></nav>
|
||||
<div class="toast" id="toast"></div>
|
||||
<script>
|
||||
'use strict';
|
||||
const $=id=>document.getElementById(id), $$=q=>[...document.querySelectorAll(q)];
|
||||
let admin=sessionStorage.getItem('neuroforge_admin_token')||'', currentView='dashboard', chatAbort=null, graphData={nodes:[],edges:[]}, graphPos=new Map(), selectedMemory='', graphLayoutReady=false, researchWatch={goalId:'',goalTitle:'',runId:'',after:0,events:[],run:null,timer:null};
|
||||
const viewMeta={dashboard:['Übersicht','Was das System weiß, lernt und gerade tut.'],knowledge:['Wissensraum','Interaktiver Graph mit Level-of-Detail und echter Recall-Erklärung.'],sources:['Quellen & Import','Dokumente und Text werden zu nachvollziehbarer Evidence.'],research:['Research','SearXNG suchen, Webseiten abrufen und Quellen kontrolliert lernen.'],goals:['Ziele & Autonomie','Per-Goal Scheduler, Research und Fehler-Backoff.'],chat:['Chat','Interaktiv mit Recall, Lernen und Abbruch.'],models:['Modelle & Routing','Actor, Embeddings und Ollama-Nodes.'],ops:['Betrieb','Index, Storage, Observability und Rohkonfiguration.']};
|
||||
function toast(msg,bad=false){let t=$('toast');t.textContent=msg;t.className='toast show'+(bad?' bad':'');clearTimeout(t._timer);t._timer=setTimeout(()=>t.className='toast',4500)}
|
||||
function esc(s){return String(s??'').replace(/[&<>"']/g,m=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[m]))}
|
||||
function safeHref(raw){try{let u=new URL(String(raw||''),location.origin);return (u.protocol==='http:'||u.protocol==='https:')?u.href:'#'}catch{return '#'}}
|
||||
function fmtN(n){return Number(n||0).toLocaleString('de-DE')}function fmtBytes(n){n=Number(n||0);if(n<1024)return n+' B';if(n<1048576)return(n/1024).toFixed(1)+' KiB';if(n<1073741824)return(n/1048576).toFixed(1)+' MiB';return(n/1073741824).toFixed(2)+' GiB'}
|
||||
function when(d){if(!d)return'–';let x=new Date(d),ms=x-Date.now(),a=Math.abs(ms),m=Math.round(a/60000);if(a<60000)return ms>0?'gleich':'gerade';if(m<60)return(ms>0?'in ':'vor ')+m+' min';let h=Math.round(m/60);return(ms>0?'in ':'vor ')+h+' h'}
|
||||
function validHeaderToken(value){return typeof value==='string'&&/^[\x21-\x7E]+$/.test(value)}
|
||||
function tokenHeaders(json=true){if(!admin)throw new Error('Admin-Token fehlt.');if(!validHeaderToken(admin))throw new Error('Admin-Token enthält Leerzeichen/Unicode. Bitte echten ASCII-Token neu eingeben.');let h={'X-Admin-Token':admin};if(json)h['Content-Type']='application/json';return h}
|
||||
async function req(path,opt={}){let o={...opt,headers:{...(opt.headers||{}),...tokenHeaders(opt.json!==false)}};let r=await fetch(path,o),txt=await r.text();let data=null;try{data=txt?JSON.parse(txt):null}catch{data=txt}if(!r.ok)throw new Error((data&&data.error)||data||`HTTP ${r.status}`);return data}
|
||||
async function setToken(){let v=prompt('Admin-Token',admin||'');if(v===null)return;v=v.trim();if(!v)return;admin=v;sessionStorage.setItem('neuroforge_admin_token',v);await refreshAll()}
|
||||
function switchView(v){currentView=v;$$('.view').forEach(x=>x.classList.toggle('active',x.id==='view-'+v));$$('[data-view]').forEach(x=>x.classList.toggle('active',x.dataset.view===v));$('pageTitle').textContent=viewMeta[v][0];$('pageSub').textContent=viewMeta[v][1];if(v!=='goals')stopResearchPolling();if(v==='knowledge')loadKnowledgeGraph();if(v==='sources')loadSources();if(v==='research')loadResearchConfig();if(v==='goals'){loadGoals();if(researchWatch.goalId)startResearchPolling()}if(v==='models')loadModels();if(v==='ops')loadOps()}
|
||||
$$('[data-view]').forEach(b=>b.addEventListener('click',()=>switchView(b.dataset.view)));$('tokenBtn').addEventListener('click',setToken);$('refreshBtn').addEventListener('click',refreshAll);
|
||||
|
||||
async function refreshAll(){if(!admin){$('sideStatus').textContent='Admin-Token setzen';return}try{let [s,k,g,src]=await Promise.all([req('/admin/api/status'),req('/admin/api/knowledge/summary'),req('/api/v1/goals'),req('/api/v1/sources?limit=8')]);$('sideDot').className='dot ok';$('sideStatus').textContent='bereit · rev '+s.revision;$('mMem').textContent=fmtN(s.memories);$('mSyn').textContent=fmtN(s.synapses);$('mSources').textContent=fmtN(k.sources||src.length);let active=(g||[]).filter(x=>x.status==='active');$('mGoals').textContent=active.length;$('mNextGoal').textContent=active.length?('nächster '+when(active.map(x=>x.next_cycle_at).filter(Boolean).sort()[0])):'keine aktiven Ziele';$('mTypes').textContent=Object.entries(k.by_type||{}).map(([a,b])=>a+' '+b).join(' · ');renderSystemState(s);renderEvents(k.recent_events||[],'dashEvents');renderDashboardGoals(active.slice(0,5));if(currentView==='ops')loadOps()}catch(e){$('sideDot').className='dot bad';$('sideStatus').textContent='Fehler';toast(e.message,true)}}
|
||||
function renderSystemState(s){let o=s.observability||{},m=s.maintenance||{};$('systemState').innerHTML=[['Index',`${s.index_mode||o.index_mode||'–'} · HNSW ${fmtN(s.hnsw_nodes)} · PQ ${fmtN(s.disk_pq_items)}`],['Jobs',fmtN(s.pending_jobs)+' ausstehend'],['Cluster',s.cluster?.enabled?(s.cluster.role+' · term '+s.cluster.term):'deaktiviert'],['Letzter Autonomie-Lauf',m.last_autonomy_run?new Date(m.last_autonomy_run).toLocaleString():'–'],['Letzter Fehler',m.last_error||'keiner']].map(([a,b])=>`<div class="item"><div class="itemTitle"><b>${esc(a)}</b><span class="muted small">${esc(b)}</span></div></div>`).join('')}
|
||||
function renderEvents(xs,id){$(id).innerHTML=xs.length?xs.map(e=>`<div class="item"><div class="itemTitle"><b>${esc(e.type)}</b><span class="pill">${esc(e.actor||'system')}</span></div><p>${esc(e.summary||'')}</p><div class="meta"><span>${new Date(e.created_at).toLocaleString()}</span>${e.reason?`<span>${esc(e.reason)}</span>`:''}</div></div>`).join(''):'<p class="muted">Noch keine Lernereignisse.</p>'}
|
||||
function renderDashboardGoals(xs){$('dashGoals').innerHTML=xs.length?xs.map(g=>`<div class="item"><div class="itemTitle"><b>${esc(g.title)}</b><span class="pill ${g.auto_run?'good':''}">${g.auto_run?'AUTO':'MANUELL'}</span></div><div class="progress" style="margin:9px 0"><span style="width:${Math.round((g.progress||0)*100)}%"></span></div><div class="meta"><span>${Math.round((g.progress||0)*100)}%</span><span>${g.research_enabled?'Research an':'Research aus'}</span><span>${g.next_cycle_at?when(g.next_cycle_at):'nicht geplant'}</span></div></div>`).join(''):'<p class="muted">Keine aktiven Ziele.</p>'}
|
||||
|
||||
// ---- Knowledge graph with client-side LOD ----
|
||||
const canvas=$('knowledgeCanvas'),ctx=canvas.getContext('2d'),cam={x:0,y:0,z:1},pointer={down:false,x:0,y:0,moved:false},typeColors={semantic:'#7aa7ff',episodic:'#65d6b5',procedural:'#f7c36d',working:'#bd8cff'};
|
||||
function hash(s){let h=2166136261;for(let i=0;i<s.length;i++){h^=s.charCodeAt(i);h=Math.imul(h,16777619)}return h>>>0}function rand01(s){return(hash(s)%100000)/100000}
|
||||
function layoutGraph(){graphPos.clear();let types=[...new Set(graphData.nodes.map(n=>n.memory_type||'other'))],centers={};types.forEach((t,i)=>{let a=i/types.length*Math.PI*2;centers[t]={x:Math.cos(a)*260,y:Math.sin(a)*210}});for(let n of graphData.nodes){let c=centers[n.memory_type||'other']||{x:0,y:0},a=rand01(n.id)*Math.PI*2,r=40+rand01(n.id+'r')*170;graphPos.set(n.id,{x:c.x+Math.cos(a)*r,y:c.y+Math.sin(a)*r,vx:0,vy:0,n})}for(let it=0;it<110;it++){for(let e of graphData.edges){let a=graphPos.get(e.a),b=graphPos.get(e.b);if(!a||!b)continue;let dx=b.x-a.x,dy=b.y-a.y,d=Math.hypot(dx,dy)||1,target=65+70/(1+Math.abs(Number(e.weight||0))),f=(d-target)*.0035,fx=dx/d*f,fy=dy/d*f;a.vx+=fx;a.vy+=fy;b.vx-=fx;b.vy-=fy}for(let p of graphPos.values()){let c=centers[p.n.memory_type||'other']||{x:0,y:0};p.vx+=(c.x-p.x)*.0007;p.vy+=(c.y-p.y)*.0007;p.vx*=.86;p.vy*=.86;p.x+=p.vx;p.y+=p.vy}}graphLayoutReady=true;fitGraph();drawGraph()}
|
||||
function resizeCanvas(){let r=canvas.getBoundingClientRect(),dpr=devicePixelRatio||1;canvas.width=Math.max(1,Math.floor(r.width*dpr));canvas.height=Math.max(1,Math.floor(r.height*dpr));ctx.setTransform(dpr,0,0,dpr,0,0);drawGraph()}new ResizeObserver(resizeCanvas).observe(canvas);
|
||||
function worldToScreen(p){let r=canvas.getBoundingClientRect();return{x:r.width/2+cam.x+p.x*cam.z,y:r.height/2+cam.y+p.y*cam.z}}function screenToWorld(x,y){let r=canvas.getBoundingClientRect();return{x:(x-r.width/2-cam.x)/cam.z,y:(y-r.height/2-cam.y)/cam.z}}
|
||||
function lod(){return cam.z<.62?0:cam.z<1.45?1:2}function drawGraph(){let r=canvas.getBoundingClientRect();ctx.clearRect(0,0,r.width,r.height);let L=lod();$('lodBadge').textContent='LOD · '+(['Übersicht','Netz','Detail'][L]);$('graphZoom').textContent=Math.round(cam.z*100)+'%';if(!graphLayoutReady)return;if(L===0){drawClusters();return}let edges=graphData.edges.slice(0,L===1?Math.min(graphData.edges.length,graphData.nodes.length*2):graphData.edges.length);for(let e of edges){let a=graphPos.get(e.a),b=graphPos.get(e.b);if(!a||!b)continue;let A=worldToScreen(a),B=worldToScreen(b),w=Math.abs(Number(e.weight||0));ctx.strokeStyle=Number(e.weight||0)<0?'rgba(255,124,141,.28)':'rgba(117,153,207,'+(L===2?.32:.18)+')';ctx.lineWidth=Math.min(4,.45+w*.8);ctx.beginPath();ctx.moveTo(A.x,A.y);ctx.lineTo(B.x,B.y);ctx.stroke()}for(let p of graphPos.values()){let s=worldToScreen(p),sel=p.n.id===selectedMemory,rr=(sel?8:5)*(L===2?1.15:1);ctx.fillStyle=typeColors[p.n.memory_type]||'#bd8cff';ctx.beginPath();ctx.arc(s.x,s.y,rr,0,Math.PI*2);ctx.fill();if(sel){ctx.strokeStyle='#fff';ctx.lineWidth=2;ctx.stroke()}let important=(p.n.salience||0)>1.15||(p.n.access_count||0)>4;if(sel||(L===2&&important)){ctx.fillStyle='#dce7f7';ctx.font='11px system-ui';let txt=(p.n.truth_key||p.n.text||p.n.kind||p.n.id).slice(0,34);ctx.fillText(txt,s.x+10,s.y+4)}}}
|
||||
function drawClusters(){let groups={};for(let p of graphPos.values()){let t=p.n.memory_type||'other';(groups[t]??=[]).push(p)}for(let[t,xs]of Object.entries(groups)){let x=xs.reduce((a,p)=>a+p.x,0)/xs.length,y=xs.reduce((a,p)=>a+p.y,0)/xs.length,s=worldToScreen({x,y}),rad=Math.min(58,20+Math.sqrt(xs.length)*6);ctx.fillStyle=(typeColors[t]||'#bd8cff')+'44';ctx.strokeStyle=typeColors[t]||'#bd8cff';ctx.lineWidth=2;ctx.beginPath();ctx.arc(s.x,s.y,rad,0,Math.PI*2);ctx.fill();ctx.stroke();ctx.fillStyle='#eef3fb';ctx.font='600 13px system-ui';ctx.textAlign='center';ctx.fillText(t,s.x,s.y-2);ctx.fillStyle='#a9b7cb';ctx.font='11px system-ui';ctx.fillText(xs.length+' nodes',s.x,s.y+15)}ctx.textAlign='left'}
|
||||
function fitGraph(){if(!graphPos.size)return;let xs=[...graphPos.values()].map(p=>p.x),ys=[...graphPos.values()].map(p=>p.y),w=Math.max(...xs)-Math.min(...xs)+140,h=Math.max(...ys)-Math.min(...ys)+140,r=canvas.getBoundingClientRect();cam.z=Math.max(.25,Math.min(1.15,Math.min(r.width/w,r.height/h)));cam.x=cam.y=0;drawGraph()}
|
||||
canvas.addEventListener('wheel',e=>{e.preventDefault();let before=screenToWorld(e.offsetX,e.offsetY),factor=Math.exp(-e.deltaY*.0012);cam.z=Math.max(.2,Math.min(5,cam.z*factor));let after=worldToScreen(before);cam.x+=e.offsetX-after.x;cam.y+=e.offsetY-after.y;drawGraph()},{passive:false});canvas.addEventListener('pointerdown',e=>{pointer.down=true;pointer.x=e.clientX;pointer.y=e.clientY;pointer.moved=false;canvas.setPointerCapture(e.pointerId)});canvas.addEventListener('pointermove',e=>{if(!pointer.down)return;let dx=e.clientX-pointer.x,dy=e.clientY-pointer.y;if(Math.hypot(dx,dy)>2)pointer.moved=true;cam.x+=dx;cam.y+=dy;pointer.x=e.clientX;pointer.y=e.clientY;drawGraph()});canvas.addEventListener('pointerup',e=>{pointer.down=false;if(pointer.moved)return;let rect=canvas.getBoundingClientRect(),x=e.clientX-rect.left,y=e.clientY-rect.top;if(lod()===0){cam.z=.8;drawGraph();return}let best=null,bd=15;for(let p of graphPos.values()){let s=worldToScreen(p),d=Math.hypot(s.x-x,s.y-y);if(d<bd){best=p;bd=d}}if(best)openMemory(best.n.id)});$('zoomInBtn').addEventListener('click',()=>{cam.z=Math.min(5,cam.z*1.3);drawGraph()});$('zoomOutBtn').addEventListener('click',()=>{cam.z=Math.max(.2,cam.z/1.3);drawGraph()});$('fitBtn').addEventListener('click',fitGraph);
|
||||
async function loadKnowledgeGraph(center=''){try{let d=await req('/admin/api/knowledge/graph?depth=3&max_nodes=600'+(center?'¢er='+encodeURIComponent(center):''));graphData=d;$('graphCount').textContent=fmtN(d.nodes.length)+' Knoten · '+fmtN(d.edges.length)+' Kanten';layoutGraph()}catch(e){toast(e.message,true)}}$('graphReloadBtn').addEventListener('click',()=>loadKnowledgeGraph(selectedMemory));
|
||||
$('knowledgeSearchBtn').addEventListener('click',searchKnowledge);$('knowledgeQuery').addEventListener('keydown',e=>{if(e.key==='Enter')searchKnowledge()});async function searchKnowledge(){let q=$('knowledgeQuery').value.trim();if(!q)return;try{let d=await req('/admin/api/knowledge/search',{method:'POST',body:JSON.stringify({text:q,k:16})});$('knowledgeHits').innerHTML=(d.hits||[]).map(h=>`<div class="item click" data-mid="${esc(h.memory.id)}"><div class="itemTitle"><b>${esc(h.memory.text.slice(0,120))}</b><span class="pill">${Number(h.score||0).toFixed(3)}</span></div><div class="meta"><span>${esc(h.candidate_source||'')}</span><span>sim ${Number(h.similarity||0).toFixed(3)}</span><span>base ${Number(h.base_score||0).toFixed(3)}</span><span>graph +${Number(h.graph_boost||0).toFixed(3)}</span></div></div>`).join('')||'<p class="muted">Keine Treffer.</p>';$$('#knowledgeHits [data-mid]').forEach(x=>x.addEventListener('click',()=>openMemory(x.dataset.mid)));if(d.hits?.[0]){selectedMemory=d.hits[0].memory.id;await loadKnowledgeGraph(selectedMemory);await openMemory(selectedMemory,false)}}catch(e){toast(e.message,true)}}
|
||||
async function openMemory(id,recenter=true){selectedMemory=id;try{let d=await req('/admin/api/knowledge/memory/'+encodeURIComponent(id)),m=d.memory||{},p=m.provenance||{};$('knowledgeInspector').innerHTML=`<div class="meta"><span class="pill">${esc(m.memory_type)}</span><span class="pill">${esc(m.kind)}</span><span class="pill ${m.status==='active'?'good':m.status==='conflicted'?'warn':''}">${esc(m.status||'active')}</span></div><h3>${esc(m.truth_key||p.source_title||m.id)}</h3><div style="white-space:pre-wrap;line-height:1.5;font-size:13px">${esc(m.text)}</div><h3>Qualität</h3><div class="kv"><div>Confidence</div><div>${Number(m.confidence||0).toFixed(3)}</div><div>Salience</div><div>${Number(m.salience||0).toFixed(3)}</div><div>Reward</div><div>${Number(m.reward||0).toFixed(3)}</div><div>Evidenzquellen</div><div>${fmtN(m.evidence_count||m.evidence_source_ids?.length||0)}</div><div>Aufrufe</div><div>${fmtN(m.access_count)}</div></div><h3>Provenance</h3><div class="kv"><div>Quelle</div><div>${esc(p.source||'legacy/unknown')}</div><div>Source-ID</div><div class="mono">${esc(p.source_id||'–')}</div><div>Titel</div><div>${esc(p.source_title||'–')}</div><div>URI</div><div>${p.source_uri?`<a href="${esc(safeHref(p.source_uri))}" target="_blank" rel="noopener" style="color:var(--accent)">${esc(p.source_uri)}</a>`:'–'}</div><div>Chunk</div><div>${p.chunk_index||'–'} / ${p.chunk_count||'–'}</div><div>Embedding</div><div>${esc([p.embedding_model,p.embedding_node_id].filter(Boolean).join(' · ')||'–')}</div><div>Generierung</div><div>${esc([p.generation_model,p.generation_node_id].filter(Boolean).join(' · ')||'–')}</div></div><h3>Stärkste Synapsen</h3>${(d.edges||[]).slice(0,12).map(e=>`<div class="item click" data-mid="${esc(e.a===id?e.b:e.a)}"><div class="itemTitle"><span class="mono small">${esc(e.a===id?e.b:e.a)}</span><span class="pill">w ${Number(e.weight).toFixed(2)}</span></div></div>`).join('')||'<p class="muted">Keine Synapsen.</p>'}<h3>Timeline</h3>${(d.events||[]).map(e=>`<div class="item"><b>${esc(e.type)}</b><p>${esc(e.summary)}</p><div class="meta">${new Date(e.created_at).toLocaleString()}</div></div>`).join('')||'<p class="muted">Keine Events.</p>'}`;$$('#knowledgeInspector [data-mid]').forEach(x=>x.addEventListener('click',()=>openMemory(x.dataset.mid)));drawGraph();if(recenter)loadKnowledgeGraph(id)}catch(e){toast(e.message,true)}}
|
||||
|
||||
// ---- Sources & ingestion ----
|
||||
async function loadSources(){try{let xs=await req('/api/v1/sources?limit=100');$('sourcesList').innerHTML=xs.length?xs.map(s=>`<div class="item"><div class="itemTitle"><div class="row"><span class="sourceIcon">${s.type==='document'?'▤':s.type==='web'?'⌁':'T'}</span><div><b>${esc(s.title)}</b><div class="meta"><span>${esc(s.type)}</span><span>${s.chunk_count||0} chunks</span><span>${fmtBytes(s.bytes)}</span></div></div></div><span class="pill ${s.status==='ready'?'good':s.status==='error'?'bad':'warn'}">${esc(s.status)}</span></div>${s.uri?`<p class="mono small">${esc(s.uri)}</p>`:''}${s.error?`<p class="error small">${esc(s.error)}</p>`:''}</div>`).join(''):'<p class="muted">Noch keine externen Quellen.</p>';$('mSources').textContent=fmtN(xs.length)}catch(e){toast(e.message,true)}}$('sourcesReloadBtn').addEventListener('click',loadSources);
|
||||
const dz=$('dropZone'),fi=$('docFile');['dragenter','dragover'].forEach(n=>dz.addEventListener(n,e=>{e.preventDefault();dz.classList.add('drag')}));['dragleave','drop'].forEach(n=>dz.addEventListener(n,e=>{e.preventDefault();dz.classList.remove('drag')}));dz.addEventListener('drop',e=>{if(e.dataTransfer.files.length)fi.files=e.dataTransfer.files});$('uploadBtn').addEventListener('click',async()=>{let f=fi.files[0];if(!f)return toast('Bitte eine Datei auswählen.',true);let fd=new FormData();fd.append('file',f);fd.append('title',$('docTitle').value);fd.append('tags',$('docTags').value);fd.append('trust',$('docTrust').value);try{let r=await fetch('/api/v1/ingest/document',{method:'POST',headers:tokenHeaders(false),body:fd}),txt=await r.text(),d;try{d=JSON.parse(txt)}catch{d=txt}if(!r.ok)throw new Error(d.error||d);toast(`Dokument gelernt: ${d.chunks} Chunks, ${d.duplicates} Duplikate`);fi.value='';loadSources();refreshAll()}catch(e){toast(e.message,true)}});
|
||||
$('textLearnBtn').addEventListener('click',async()=>{try{let d=await req('/api/v1/ingest/text',{method:'POST',body:JSON.stringify({title:$('textTitle').value||'Manual text',text:$('textInput').value,source_uri:$('textURI').value,trust:Number($('textTrust').value||1),source_type:'text',memory_type:'semantic'})});toast(`Text gelernt: ${d.chunks} Chunks`);$('textInput').value='';loadSources();refreshAll()}catch(e){toast(e.message,true)}});
|
||||
|
||||
// ---- Research ----
|
||||
async function loadResearchConfig(){try{let d=await req('/admin/api/research'),r=d.research||{},s=r.searxng||{},w=r.web_fetch||{},g=r.goal||{},a=d.autonomy||{};$('rcEnabled').checked=!!r.enabled;$('rcSearx').checked=!!s.enabled;$('rcGoal').checked=!!g.enabled;$('rcBase').value=s.base_url||'';$('rcLang').value=s.language||'de-DE';$('rcCats').value=s.categories||'general';$('rcQueries').value=g.max_queries_per_cycle||2;$('rcPages').value=g.max_pages_per_cycle??4;$('rcFetch').checked=!!w.enabled;$('rcAuto').checked=!!a.enabled;$('rcImmediate').checked=!!a.run_on_goal_create;$('rcInterval').value=a.default_goal_interval_minutes||10;$('rcAuth').placeholder=d.searxng_auth_configured?'konfiguriert – leer lassen zum Behalten':'Bearer …'}catch(e){toast(e.message,true)}}
|
||||
$('researchSaveBtn').addEventListener('click',async()=>{try{let current=await req('/admin/api/research'),r=current.research||{};r.enabled=$('rcEnabled').checked;r.searxng={...(r.searxng||{}),enabled:$('rcSearx').checked,base_url:$('rcBase').value.trim(),language:$('rcLang').value.trim(),categories:$('rcCats').value.trim()};r.web_fetch={...(r.web_fetch||{}),enabled:$('rcFetch').checked};r.goal={...(r.goal||{}),enabled:$('rcGoal').checked,max_queries_per_cycle:Number($('rcQueries').value||2),max_pages_per_cycle:Number($('rcPages').value||0)};let body={research:r,ingestion:current.ingestion,autonomy:{enabled:$('rcAuto').checked,run_on_goal_create:$('rcImmediate').checked,default_goal_interval_minutes:Number($('rcInterval').value||10)}};if($('rcAuth').value)body.searxng_auth_header=$('rcAuth').value;await req('/admin/api/research',{method:'PUT',body:JSON.stringify(body)});$('rcAuth').value='';toast('Research-Konfiguration gespeichert.')}catch(e){toast(e.message,true)}});
|
||||
$('researchBtn').addEventListener('click',()=>runResearch(false));$('researchTestBtn').addEventListener('click',()=>runResearch(true));
|
||||
function researchKind(x){let file=!!(x.filename||x.mimetype||String(x.template||'').toLowerCase().includes('file')||/\.(pdf|docx|txt|md|markdown|csv|tsv|json|ya?ml)(?:$|[?#])/i.test(x.url||''));return file?'DOKUMENT':'WEB'}
|
||||
async function runResearch(test){let q=$('researchQuery').value.trim()||(test?'NVIDIA CUDA':'');if(!q)return toast('Suchanfrage fehlt.',true);$('researchResults').innerHTML='<p class="muted">SearXNG sucht …</p>';try{let d=test?await req('/admin/api/research/test',{method:'POST',body:JSON.stringify({query:q})}):await req('/api/v1/research',{method:'POST',body:JSON.stringify({query:q,learn:$('researchLearn').checked,fetch_pages:$('researchFetch').checked,max_results:Number($('researchMax').value||8)})});$('researchResults').innerHTML=(d.results||[]).map(x=>`<div class="item"><div class="itemTitle"><b>${esc(x.title||x.filename||x.url)}</b><span class="pill ${researchKind(x)==='DOKUMENT'?'good':''}">${researchKind(x)}</span></div><p>${esc(x.content||x.abstract||'')}</p><div class="meta"><span>${esc(x.mimetype||x.engine||((x.engines||[])[0]||'web'))}</span>${x.filename?`<span>${esc(x.filename)}</span>`:''}${x.size?`<span>${esc(x.size)}</span>`:''}<a href="${esc(safeHref(x.url))}" target="_blank" rel="noopener" style="color:var(--accent)">${esc(x.url)}</a></div></div>`).join('')||'<p class="muted">Keine Ergebnisse.</p>';if(d.ingested)toast(`${d.ingested} Evidence-Memories gelernt${d.documents_ingested?` · ${d.documents_ingested} Dokument(e) verarbeitet`:''}.`);if(d.errors?.length)toast(d.errors[0],true);if(d.ingested)loadSources()}catch(e){$('researchResults').innerHTML='<p class="error">'+esc(e.message)+'</p>';toast(e.message,true)}}
|
||||
|
||||
// ---- Goals + live research ----
|
||||
function stopResearchPolling(){if(researchWatch.timer){clearInterval(researchWatch.timer);researchWatch.timer=null}}
|
||||
function startResearchPolling(){stopResearchPolling();if(!researchWatch.goalId||currentView!=='goals')return;pollGoalResearch();researchWatch.timer=setInterval(pollGoalResearch,1200)}
|
||||
function researchLane(ev){
|
||||
if(['plan.started','plan.completed','plan.fallback','query.planned','search.started','search.completed'].includes(ev.type))return'queries';
|
||||
if(ev.type==='search.result')return'results';
|
||||
if(['download.started','download.completed','source.processing','source.ingested','source.duplicate'].includes(ev.type))return'downloads';
|
||||
if(['claim.extracted','evidence.learned'].includes(ev.type))return'claims';
|
||||
if(['evidence.duplicate','evidence.corroborated'].includes(ev.type))return'dedupe';
|
||||
if(ev.status==='error'||ev.status==='skipped'||['source.rejected','evidence.skipped','evidence.error','search.error'].includes(ev.type))return'rejected';
|
||||
return'queries';
|
||||
}
|
||||
function eventLabel(ev){return ({'query.planned':'Query','search.started':'SearXNG','search.completed':'SearXNG','search.result':'Treffer','download.started':'Download','download.completed':'Geladen','source.processing':'Extraktion','source.ingested':'Quelle','source.duplicate':'Quelle bekannt','claim.extracted':'Claim-Kandidat','evidence.learned':'Neu gelernt','evidence.duplicate':'Duplikat','evidence.corroborated':'Bestätigt','source.rejected':'Verworfen','evidence.skipped':'Verworfen','evidence.error':'Fehler','plan.fallback':'Fallback','run.started':'Start','run.finished':'Ende'}[ev.type]||ev.type)}
|
||||
function liveEventCard(ev,terminals){let status=ev.status||'ok';if(ev.type==='download.started'&&terminals.has(ev.url))status='ok';let link=ev.url?`<a href="${esc(safeHref(ev.url))}" target="_blank" rel="noopener">${esc(ev.url)}</a>`:'';let preview=ev.preview?`<p class="preview">${esc(ev.preview)}</p>`:'';let score=ev.similarity?` · sim ${Number(ev.similarity).toFixed(3)}`:ev.score?` · score ${Number(ev.score).toFixed(3)}`:'';return `<div class="researchEvent ${esc(status)}"><div class="eventTop"><span>${esc(eventLabel(ev))}${score}</span><span>${new Date(ev.created_at).toLocaleTimeString()}</span></div><b>${esc(ev.title||ev.query||ev.message||'Event')}</b>${ev.message&&ev.message!==(ev.title||ev.query)?`<p>${esc(ev.message)}</p>`:''}${preview}${link}</div>`}
|
||||
function renderResearchLive(){let run=researchWatch.run,events=researchWatch.events;if(!run){$('liveRunMeta').textContent='Für dieses Ziel existiert noch kein Research-Lauf.';$('livePulse').className='livePulse';return}
|
||||
let st=run.stats||{},running=run.status==='running';$('livePulse').className='livePulse '+(running?'running':run.status==='completed_with_errors'?'warn':run.status==='failed'||run.status==='cancelled'?'error':'');let elapsed=run.started_at?Math.max(0,Math.round(((run.completed_at?new Date(run.completed_at):new Date())-new Date(run.started_at))/1000)):0;$('liveRunMeta').textContent=`${run.status} · Run ${run.id} · ${elapsed}s · letztes Event #${run.last_seq||0}${run.last_error?' · '+run.last_error:''}`;
|
||||
$('lsResults').textContent=fmtN(st.results);$('lsDownloads').textContent=fmtN(st.downloads_completed);$('lsClaims').textContent=fmtN(st.claims);$('lsNew').textContent=fmtN(st.new_evidence);$('lsDup').textContent=fmtN(st.duplicates);$('lsConfirm').textContent=fmtN(st.corroborations);$('liveQueries').innerHTML=(run.queries||[]).map(q=>`<span class="queryChip" title="${esc(q)}">${esc(q)}</span>`).join('')||'<span class="muted small">Noch keine Query geplant.</span>';
|
||||
let terminals=new Set(events.filter(e=>['download.completed','source.rejected'].includes(e.type)&&e.url).map(e=>e.url));let groups={queries:[],results:[],downloads:[],claims:[],dedupe:[],rejected:[]};for(let ev of events){let lane=researchLane(ev);(groups[lane]||groups.queries).push(ev)}
|
||||
let ids={queries:'laneQueries',results:'laneResults',downloads:'laneDownloads',claims:'laneClaims',dedupe:'laneDedupe',rejected:'laneRejected'},counts={queries:'lcQueries',results:'lcResults',downloads:'lcDownloads',claims:'lcClaims',dedupe:'lcDedupe',rejected:'lcRejected'};for(let k of Object.keys(ids)){let xs=groups[k].slice(-40).reverse();$(counts[k]).textContent=fmtN(groups[k].length);$(ids[k]).innerHTML=xs.length?xs.map(e=>liveEventCard(e,terminals)).join(''):'<div class="emptyLane">Noch keine Events.</div>'}
|
||||
}
|
||||
async function loadResearchHistory(){if(!researchWatch.goalId)return;try{let xs=await req('/api/v1/goals/'+encodeURIComponent(researchWatch.goalId)+'/research/history?limit=8');$('researchHistory').innerHTML=xs.length?xs.map(r=>`<div class="historyRun"><div class="itemTitle"><b>${new Date(r.started_at).toLocaleString()}</b><span class="pill ${r.status==='completed'?'good':r.status==='running'?'warn':r.status==='failed'?'bad':''}">${esc(r.status)}</span></div><div class="meta"><span>${r.stats?.results||0} Treffer</span><span>${r.stats?.new_evidence||0} neu</span><span>${r.stats?.corroborations||0} bestätigt</span></div></div>`).join(''):'<span class="muted small">Noch keine Research-Läufe.</span>'}catch(e){console.warn(e)}}
|
||||
async function pollGoalResearch(){if(!researchWatch.goalId||currentView!=='goals')return;try{let q=`?after=${researchWatch.after}`+(researchWatch.runId?`&run_id=${encodeURIComponent(researchWatch.runId)}`:'');let d=await req('/api/v1/goals/'+encodeURIComponent(researchWatch.goalId)+'/research/live'+q);if(!d.run){researchWatch.run=null;researchWatch.events=[];researchWatch.after=0;renderResearchLive();return}if(d.reset||researchWatch.runId!==d.run.id){researchWatch.runId=d.run.id;researchWatch.after=0;researchWatch.events=[]}researchWatch.run=d.run;if(d.events?.length){researchWatch.events.push(...d.events);if(researchWatch.events.length>500)researchWatch.events=researchWatch.events.slice(-500);researchWatch.after=Math.max(researchWatch.after,...d.events.map(e=>Number(e.seq||0)))}renderResearchLive();if(d.run.status!=='running')loadResearchHistory()}catch(e){$('liveRunMeta').textContent='Live-Research konnte nicht geladen werden: '+e.message;$('livePulse').className='livePulse error'}}
|
||||
function openGoalResearch(id,title){researchWatch.goalId=id;researchWatch.goalTitle=title||id;researchWatch.runId='';researchWatch.after=0;researchWatch.events=[];researchWatch.run=null;$('goalResearchLive').classList.add('show');$('liveGoalTitle').textContent='Live Research · '+researchWatch.goalTitle;renderResearchLive();loadResearchHistory();startResearchPolling();setTimeout(()=>$('goalResearchLive').scrollIntoView({behavior:'smooth',block:'start'}),20)}
|
||||
$('liveCloseBtn').addEventListener('click',()=>{$('goalResearchLive').classList.remove('show');stopResearchPolling()});$('liveRefreshBtn').addEventListener('click',()=>{researchWatch.after=0;researchWatch.events=[];researchWatch.runId='';pollGoalResearch();loadResearchHistory()});
|
||||
async function loadGoals(){try{let xs=await req('/api/v1/goals');$('goalsList').innerHTML=xs.length?xs.map(g=>{let active=g.status==='active',paused=g.status==='paused';return `<div class="goal ${paused?'goalPaused':''}"><div class="goalHead"><div><h3>${esc(g.title)}</h3><div class="meta"><span>Priorität ${g.priority}</span><span>${g.research_enabled?'⌕ Research':'ohne Research'}</span><span>${g.auto_run?'AUTO':'MANUELL'}</span></div></div><span class="pill ${active?'good':paused?'warn':''}">${paused?'PAUSIERT':esc(g.status)}</span></div><p class="muted small">${esc(g.description||'')}</p><div class="progress"><span style="width:${Math.round((g.progress||0)*100)}%"></span></div><div class="meta"><span>${Math.round((g.progress||0)*100)}%</span><span>Intervall ${g.interval_minutes||'–'} min</span><span>${paused?'Scheduler angehalten':('Nächster: '+(g.next_cycle_at?when(g.next_cycle_at):'–'))}</span></div>${g.next_action?`<p><b>Nächste Aktion:</b> ${esc(g.next_action)}</p>`:''}${g.last_error?`<p class="error small">${esc(g.last_error)}</p>`:''}<div class="goalActions">${active?`<button class="btn secondary" data-goal-action="cycle" data-goal-id="${esc(g.id)}" data-goal-title="${esc(g.title)}">Jetzt lernen</button><button class="btn ghost" data-goal-action="pause" data-goal-id="${esc(g.id)}">Pausieren</button>`:''}${paused?`<button class="btn secondary" data-goal-action="resume" data-goal-id="${esc(g.id)}">Fortsetzen</button>`:''}${g.research_enabled?`<button class="btn ghost" data-research-live="${esc(g.id)}" data-goal-title="${esc(g.title)}">Live Research</button>`:''}<button class="btn ghost" data-focus="${esc(g.id)}">Graph</button><button class="btn danger" data-goal-action="delete" data-goal-id="${esc(g.id)}" data-goal-title="${esc(g.title)}">Löschen</button></div></div>`}).join(''):'<p class="muted">Noch keine Ziele.</p>';$$('[data-goal-action]').forEach(b=>b.addEventListener('click',()=>goalAction(b)));$$('[data-research-live]').forEach(b=>b.addEventListener('click',()=>openGoalResearch(b.dataset.researchLive,b.dataset.goalTitle)));$$('[data-focus]').forEach(b=>b.addEventListener('click',()=>{switchView('knowledge');$('knowledgeQuery').value='goal:'+b.dataset.focus;searchKnowledge()}))}catch(e){toast(e.message,true)}}
|
||||
async function goalAction(b){let id=b.dataset.goalId,action=b.dataset.goalAction;if(action==='delete'&&!confirm(`Ziel „${b.dataset.goalTitle||id}“ wirklich löschen? Bereits gelerntes Wissen bleibt erhalten.`))return;if(action==='cycle')openGoalResearch(id,b.dataset.goalTitle||id);let old=b.textContent;try{b.disabled=true;b.textContent=action==='cycle'?'läuft …':action==='delete'?'lösche …':'speichere …';let path='/api/v1/goals/'+encodeURIComponent(id)+(action==='cycle'?'/cycle':action==='pause'?'/pause':action==='resume'?'/resume':'');let d=await req(path,{method:action==='delete'?'DELETE':'POST',body:action==='delete'?undefined:'{}'});if(action==='cycle')toast(`Zyklus fertig · Evaluation ${Number(d.evaluation).toFixed(2)} · ${d.sources_ingested||0} Quellen`);else if(action==='pause')toast('Ziel pausiert. Der Scheduler verarbeitet es nicht weiter.');else if(action==='resume')toast('Ziel wieder aktiv. Nächster Lauf wurde eingeplant.');else{toast('Ziel gelöscht. Bereits gelerntes Wissen bleibt erhalten.');if(researchWatch.goalId===id){$('goalResearchLive').classList.remove('show');stopResearchPolling();researchWatch.goalId=''}}await loadGoals();refreshAll()}catch(e){toast(e.message,true)}finally{b.disabled=false;b.textContent=old}}
|
||||
$('goalCreateBtn').addEventListener('click',async()=>{let title=$('goalTitle').value.trim();if(!title)return toast('Titel fehlt.',true);try{let g=await req('/api/v1/goals',{method:'POST',body:JSON.stringify({title,description:$('goalDesc').value,target:$('goalTarget').value,priority:Number($('goalPriority').value||50),interval_minutes:Number($('goalInterval').value||10),status:'active'})});toast(`Ziel ${g.title} angelegt – erster Lauf ist eingeplant.`);$('goalTitle').value='';$('goalDesc').value='';loadGoals();refreshAll()}catch(e){toast(e.message,true)}});$('runAutonomyBtn').addEventListener('click',async()=>{try{let d=await req('/admin/api/autonomy',{method:'POST',body:'{}'});toast(`${d.cycles?.length||0} fällige Zyklen ausgeführt.`);loadGoals()}catch(e){toast(e.message,true)}});
|
||||
|
||||
// ---- Chat ----
|
||||
$('chatBtn').addEventListener('click',async()=>{let input=document.getElementById('chatPrompt').value.trim();if(!input)return toast('Nachricht fehlt.',true);chatAbort=new AbortController();$('chatAbortBtn').disabled=false;$('chatBtn').disabled=true;let start=Date.now(),timer=setInterval(()=>$('chatElapsed').textContent=Math.round((Date.now()-start)/1000)+' s',1000);$('chatOutput').textContent='Ollama / NeuroForge arbeitet …';try{let r=await fetch('/api/v1/chat',{method:'POST',headers:tokenHeaders(),signal:chatAbort.signal,body:JSON.stringify({session_id:$('chatSession').value,input,provider:$('chatProvider').value})}),txt=await r.text(),d;try{d=JSON.parse(txt)}catch{d=txt}if(!r.ok)throw new Error(d.error||d);$('chatOutput').textContent=(d.answer||'')+'\n\n— '+(d.provider||'')+' · '+(d.model||'')+' · Recall '+(d.recalled?.length||0)+' · Reward '+Number(d.auto_reward||0).toFixed(3);refreshAll()}catch(e){$('chatOutput').textContent=e.name==='AbortError'?'Abgebrochen.':e.message}finally{clearInterval(timer);$('chatElapsed').textContent='';$('chatAbortBtn').disabled=true;$('chatBtn').disabled=false;chatAbort=null}});$('chatAbortBtn').addEventListener('click',()=>chatAbort?.abort());
|
||||
|
||||
// ---- Models ----
|
||||
async function loadModels(){try{let d=await req('/admin/api/model-routing');$('modelJSON').value=JSON.stringify(d,null,2);let nodes=d.ollama||[];$('modelCards').innerHTML=nodes.map(n=>`<div class="item"><div class="itemTitle"><b>${esc(n.name||n.id)}</b><span class="pill ${n.enabled?'good':''}">${n.enabled?'aktiv':'aus'}</span></div><p class="mono small">${esc(n.base_url)}</p><div class="meta"><span>Chat: ${esc(n.chat_model)}</span><span>Embed: ${esc(n.embedding_model)}</span><span>ctx ${n.num_ctx||'default'}</span><span>think ${esc(n.think||'off')}</span></div></div>`).join('')}catch(e){toast(e.message,true)}}$('modelSaveBtn').addEventListener('click',async()=>{try{let d=JSON.parse($('modelJSON').value);await req('/admin/api/model-routing',{method:'PUT',body:JSON.stringify(d)});toast('Modelle & Routing gespeichert.');loadModels()}catch(e){toast(e.message,true)}});$('modelHealthBtn').addEventListener('click',async()=>{try{$('modelHealth').textContent='prüfe …';let d=await req('/admin/api/provider-health',{method:'POST',body:'{}'});$('modelHealth').textContent=JSON.stringify(d,null,2)}catch(e){$('modelHealth').textContent=e.message}});
|
||||
|
||||
// ---- Operations ----
|
||||
async function loadOps(){try{let [s,st,cfg]=await Promise.all([req('/admin/api/status'),req('/admin/api/storage'),req('/admin/api/config')]);$('opsIndex').innerHTML=`<div class="item">HNSW <b>${fmtN(s.hnsw_nodes)}</b></div><div class="item">Disk-PQ <b>${fmtN(s.disk_pq_items)}</b></div><div class="item">Mode <b>${esc(s.index_mode||'–')}</b></div>`;$('opsStorage').innerHTML=`<div class="item">Revision <b>${fmtN(s.revision)}</b></div><div class="item">Segmente <b>${fmtN(st.segments?.files||st.memory_segments?.files||0)}</b></div><div class="item">Disk <b>${fmtBytes(st.segments?.bytes||st.memory_segments?.bytes||0)}</b></div>`;$('opsObs').innerHTML=`<div class="item">Pending jobs <b>${fmtN(s.pending_jobs)}</b></div><div class="item">Remote shards <b>${fmtN(s.remote_shards)}</b></div><div class="item">Letzter Fehler <span class="small">${esc(s.maintenance?.last_error||'keiner')}</span></div>`;$('configJSON').value=JSON.stringify(cfg,null,2)}catch(e){toast(e.message,true)}}$('configSaveBtn').addEventListener('click',async()=>{try{let c=JSON.parse($('configJSON').value);await req('/admin/api/config',{method:'PUT',body:JSON.stringify(c)});toast('Konfiguration gespeichert.')}catch(e){toast(e.message,true)}});$('checkpointBtn').addEventListener('click',()=>opPost('/admin/api/checkpoint','Checkpoint erstellt'));$('compactBtn').addEventListener('click',()=>opPost('/admin/api/storage/compact','Segmente kompaktiert'));$('pqBuildBtn').addEventListener('click',()=>opPost('/admin/api/index/disk/rebuild','PQ-Index neu gebaut'));$('indexMergeBtn').addEventListener('click',()=>opPost('/admin/api/index/merge','Index gemerged'));async function opPost(path,msg){try{await req(path,{method:'POST',body:'{}'});toast(msg);loadOps()}catch(e){toast(e.message,true)}}
|
||||
|
||||
async function boot(){if(!admin){await setToken();if(!admin)return}await refreshAll();setInterval(()=>{if(admin&&['dashboard','goals'].includes(currentView))refreshAll()},15000)}boot();
|
||||
</script>
|
||||
</body></html>
|
||||
@@ -0,0 +1,255 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
type integrationKnowledgeChunk struct {
|
||||
Index int `json:"index"`
|
||||
Text string `json:"text"`
|
||||
Vector []float32 `json:"vector"`
|
||||
ContentHash string `json:"content_hash,omitempty"`
|
||||
}
|
||||
|
||||
type integrationKnowledgeUpsert struct {
|
||||
Namespace string `json:"namespace"`
|
||||
DocumentID string `json:"document_id"`
|
||||
Title string `json:"title,omitempty"`
|
||||
SourceURI string `json:"source_uri,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Confidence float64 `json:"confidence,omitempty"`
|
||||
Chunks []integrationKnowledgeChunk `json:"chunks"`
|
||||
}
|
||||
|
||||
type integrationKnowledgeSearch struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Vector []float32 `json:"vector"`
|
||||
K int `json:"k"`
|
||||
MinSimilarity *float64 `json:"min_similarity,omitempty"`
|
||||
}
|
||||
|
||||
type integrationEventRequest struct {
|
||||
Type string `json:"type"`
|
||||
Source string `json:"source"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Query string `json:"query,omitempty"`
|
||||
Hits []struct {
|
||||
ID string `json:"id"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
} `json:"hits,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
func integrationSource(namespace string) string {
|
||||
return "integration:" + strings.ToLower(strings.TrimSpace(namespace))
|
||||
}
|
||||
|
||||
func integrationMemoryID(namespace, documentID string, chunk int) string {
|
||||
sum := sha256.Sum256([]byte(strings.ToLower(strings.TrimSpace(namespace)) + "\x00" + strings.TrimSpace(documentID) + fmt.Sprintf("\x00chunk\x00%d", chunk)))
|
||||
return "ik_" + hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
func validIntegrationName(v string) bool {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" || len(v) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, r := range v {
|
||||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' || r == ':') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) integrationKnowledgeUpsert(w http.ResponseWriter, r *http.Request) {
|
||||
var q integrationKnowledgeUpsert
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
q.Namespace = strings.TrimSpace(q.Namespace)
|
||||
q.DocumentID = strings.TrimSpace(q.DocumentID)
|
||||
if !validIntegrationName(q.Namespace) || !validIntegrationName(q.DocumentID) {
|
||||
s.err(w, http.StatusBadRequest, errors.New("namespace/document_id contains unsupported characters"))
|
||||
return
|
||||
}
|
||||
if len(q.Chunks) == 0 || len(q.Chunks) > 512 {
|
||||
s.err(w, http.StatusBadRequest, errors.New("chunks must contain 1..512 entries"))
|
||||
return
|
||||
}
|
||||
source := integrationSource(q.Namespace)
|
||||
confidence := q.Confidence
|
||||
if confidence <= 0 {
|
||||
confidence = 1
|
||||
}
|
||||
if confidence > 1 {
|
||||
confidence = 1
|
||||
}
|
||||
|
||||
// Snapshot once so document replacement is O(total memories + chunks), not
|
||||
// O(chunks * total memories), and batch deletes rebuild ANN indexes once.
|
||||
existing := make(map[string]core.Memory)
|
||||
for _, m := range s.store.MemoriesSnapshot() {
|
||||
if m.Provenance.Source == source && m.Provenance.SourceMemoryID == q.DocumentID && m.Kind == "knowledge.chunk" {
|
||||
existing[m.ID] = m
|
||||
}
|
||||
}
|
||||
|
||||
desired := make(map[string]bool, len(q.Chunks))
|
||||
createItems := make([]core.Memory, 0, len(q.Chunks))
|
||||
deleteIDs := make([]string, 0, len(existing))
|
||||
created, updated, unchanged := 0, 0, 0
|
||||
seenIndexes := make(map[int]struct{}, len(q.Chunks))
|
||||
sort.Slice(q.Chunks, func(i, j int) bool { return q.Chunks[i].Index < q.Chunks[j].Index })
|
||||
for _, chunk := range q.Chunks {
|
||||
if chunk.Index < 0 || strings.TrimSpace(chunk.Text) == "" || len(chunk.Vector) == 0 {
|
||||
s.err(w, http.StatusBadRequest, errors.New("each chunk requires non-negative index, text and vector"))
|
||||
return
|
||||
}
|
||||
if _, duplicate := seenIndexes[chunk.Index]; duplicate {
|
||||
s.err(w, http.StatusBadRequest, fmt.Errorf("duplicate chunk index %d", chunk.Index))
|
||||
return
|
||||
}
|
||||
seenIndexes[chunk.Index] = struct{}{}
|
||||
id := integrationMemoryID(q.Namespace, q.DocumentID, chunk.Index)
|
||||
desired[id] = true
|
||||
current, exists := existing[id]
|
||||
if exists && current.Provenance.ContentHash == chunk.ContentHash && current.Provenance.SourceMemoryID == q.DocumentID && len(current.Vector) == len(chunk.Vector) {
|
||||
unchanged++
|
||||
continue
|
||||
}
|
||||
if exists {
|
||||
deleteIDs = append(deleteIDs, id)
|
||||
updated++
|
||||
} else {
|
||||
created++
|
||||
}
|
||||
tags := append([]string(nil), q.Tags...)
|
||||
tags = append(tags, "integration", "namespace:"+q.Namespace, "document:"+q.DocumentID, "record:chunk")
|
||||
createItems = append(createItems, core.Memory{
|
||||
ID: id, Kind: "knowledge.chunk", MemoryType: core.MemorySemantic,
|
||||
Text: chunk.Text, Vector: append([]float32(nil), chunk.Vector...), Tags: tags,
|
||||
Salience: 1, Confidence: confidence,
|
||||
Provenance: core.MemoryProvenance{
|
||||
Source: source, Actor: "knowledge-sync", SourceMemoryID: q.DocumentID,
|
||||
SourceURI: strings.TrimSpace(q.SourceURI), SourceTitle: strings.TrimSpace(q.Title),
|
||||
ChunkIndex: chunk.Index, ChunkCount: len(q.Chunks), ContentHash: chunk.ContentHash,
|
||||
},
|
||||
})
|
||||
}
|
||||
deleted := 0
|
||||
for id := range existing {
|
||||
if !desired[id] {
|
||||
deleteIDs = append(deleteIDs, id)
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
if err := s.store.DeleteMemoriesBatch(deleteIDs); err != nil {
|
||||
s.err(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.AddMemoriesBatch(createItems); err != nil {
|
||||
s.err(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{
|
||||
Type: "integration.knowledge.synced", Summary: "External knowledge document synchronized", Actor: q.Namespace,
|
||||
Metadata: map[string]string{"namespace": q.Namespace, "document_id": q.DocumentID, "created": fmt.Sprint(created), "updated": fmt.Sprint(updated), "deleted": fmt.Sprint(deleted), "unchanged": fmt.Sprint(unchanged)},
|
||||
})
|
||||
s.json(w, http.StatusOK, map[string]any{"ok": true, "document_id": q.DocumentID, "created": created, "updated": updated, "deleted": deleted, "unchanged": unchanged})
|
||||
}
|
||||
|
||||
func (s *Server) integrationKnowledgeDelete(w http.ResponseWriter, r *http.Request) {
|
||||
namespace := strings.TrimSpace(r.PathValue("namespace"))
|
||||
documentID := strings.TrimSpace(r.PathValue("document_id"))
|
||||
if !validIntegrationName(namespace) || !validIntegrationName(documentID) {
|
||||
s.err(w, http.StatusBadRequest, errors.New("invalid namespace or document id"))
|
||||
return
|
||||
}
|
||||
source := integrationSource(namespace)
|
||||
ids := make([]string, 0)
|
||||
for _, m := range s.store.MemoriesSnapshot() {
|
||||
if m.Provenance.Source == source && m.Provenance.SourceMemoryID == documentID && m.Kind == "knowledge.chunk" {
|
||||
ids = append(ids, m.ID)
|
||||
}
|
||||
}
|
||||
if err := s.store.DeleteMemoriesBatch(ids); err != nil {
|
||||
s.err(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
deleted := len(ids)
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "integration.knowledge.deleted", Summary: "External knowledge document removed", Actor: namespace, Metadata: map[string]string{"namespace": namespace, "document_id": documentID, "deleted": fmt.Sprint(deleted)}})
|
||||
s.json(w, http.StatusOK, map[string]any{"ok": true, "deleted": deleted})
|
||||
}
|
||||
|
||||
func (s *Server) integrationKnowledgeSearch(w http.ResponseWriter, r *http.Request) {
|
||||
var q integrationKnowledgeSearch
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
q.Namespace = strings.TrimSpace(q.Namespace)
|
||||
if !validIntegrationName(q.Namespace) || len(q.Vector) == 0 {
|
||||
s.err(w, http.StatusBadRequest, errors.New("namespace and vector are required"))
|
||||
return
|
||||
}
|
||||
if q.K <= 0 {
|
||||
q.K = 128
|
||||
}
|
||||
if q.K > 500 {
|
||||
q.K = 500
|
||||
}
|
||||
min := -1.0
|
||||
if q.MinSimilarity != nil {
|
||||
min = *q.MinSimilarity
|
||||
}
|
||||
hits := s.store.SearchVectorByProvenanceSource(q.Vector, q.K, min, 0, integrationSource(q.Namespace))
|
||||
s.json(w, http.StatusOK, hits)
|
||||
}
|
||||
|
||||
func (s *Server) integrationEvent(w http.ResponseWriter, r *http.Request) {
|
||||
var q integrationEventRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
q.Type = strings.TrimSpace(q.Type)
|
||||
q.Source = strings.TrimSpace(q.Source)
|
||||
if q.Type == "" || q.Source == "" {
|
||||
s.err(w, http.StatusBadRequest, errors.New("type and source are required"))
|
||||
return
|
||||
}
|
||||
meta := map[string]string{}
|
||||
for k, v := range q.Metadata {
|
||||
if strings.TrimSpace(k) != "" {
|
||||
meta[k] = fmt.Sprint(v)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(q.Query) != "" {
|
||||
meta["query"] = q.Query
|
||||
}
|
||||
if len(q.Hits) > 0 {
|
||||
meta["hit_count"] = fmt.Sprint(len(q.Hits))
|
||||
limit := len(q.Hits)
|
||||
if limit > 8 {
|
||||
limit = 8
|
||||
}
|
||||
for i := 0; i < limit; i++ {
|
||||
meta[fmt.Sprintf("hit_%d", i+1)] = fmt.Sprintf("%s:%.4f", q.Hits[i].ID, q.Hits[i].Score)
|
||||
}
|
||||
}
|
||||
if err := s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: q.Type, Summary: strings.TrimSpace(q.Message), Actor: q.Source, Reason: "integration event", Metadata: meta}); err != nil {
|
||||
s.err(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusAccepted, map[string]bool{"ok": true})
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func integrationRequest(t *testing.T, s *Server, method, path, token, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
if body != "" {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
func TestIntegrationKnowledgeLifecycleAndNamespaceIsolation(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
key := s.store.Secrets().AppAPIKey
|
||||
|
||||
unauth := integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/knowledge/upsert", "", `{"namespace":"agent","document_id":"KB-1","chunks":[{"index":0,"text":"vpn","vector":[1,0],"content_hash":"a"}]}`)
|
||||
if unauth.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauth status=%d body=%s", unauth.Code, unauth.Body.String())
|
||||
}
|
||||
|
||||
body := `{"namespace":"agent","document_id":"KB-1","title":"VPN","chunks":[{"index":0,"text":"vpn gateway","vector":[1,0],"content_hash":"a"},{"index":1,"text":"reset token","vector":[0,1],"content_hash":"b"}]}`
|
||||
rr := integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/knowledge/upsert", key, body)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("upsert status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var first map[string]any
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first["created"] != float64(2) || first["updated"] != float64(0) {
|
||||
t.Fatalf("unexpected first upsert: %#v", first)
|
||||
}
|
||||
|
||||
// A second namespace with the same vector must never leak into agent search.
|
||||
other := `{"namespace":"other","document_id":"KB-X","chunks":[{"index":0,"text":"other vpn","vector":[1,0],"content_hash":"x"}]}`
|
||||
rr = integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/knowledge/upsert", key, other)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("other upsert status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
search := integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/knowledge/search", key, `{"namespace":"agent","vector":[1,0],"k":10}`)
|
||||
if search.Code != http.StatusOK {
|
||||
t.Fatalf("search status=%d body=%s", search.Code, search.Body.String())
|
||||
}
|
||||
var hits []struct {
|
||||
Memory struct {
|
||||
Text string `json:"text"`
|
||||
Provenance struct {
|
||||
Source string `json:"source"`
|
||||
SourceMemoryID string `json:"source_memory_id"`
|
||||
} `json:"provenance"`
|
||||
} `json:"memory"`
|
||||
}
|
||||
if err := json.Unmarshal(search.Body.Bytes(), &hits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hits) == 0 || hits[0].Memory.Provenance.SourceMemoryID != "KB-1" {
|
||||
t.Fatalf("unexpected scoped hits: %+v", hits)
|
||||
}
|
||||
for _, h := range hits {
|
||||
if h.Memory.Provenance.Source != "integration:agent" || h.Memory.Provenance.SourceMemoryID == "KB-X" {
|
||||
t.Fatalf("namespace leak: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// Replace both existing chunks in one request and remove chunk 1.
|
||||
update := `{"namespace":"agent","document_id":"KB-1","title":"VPN updated","chunks":[{"index":0,"text":"vpn gateway updated","vector":[1,0],"content_hash":"a2"}]}`
|
||||
rr = integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/knowledge/upsert", key, update)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("update status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var changed map[string]any
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &changed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed["updated"] != float64(1) || changed["deleted"] != float64(1) {
|
||||
t.Fatalf("unexpected replacement counts: %#v", changed)
|
||||
}
|
||||
|
||||
event := integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/events", key, `{"type":"knowledge.search","source":"agent","query":"vpn","message":"search completed","hits":[{"id":"KB-1","score":0.98}]}`)
|
||||
if event.Code != http.StatusAccepted {
|
||||
t.Fatalf("event status=%d body=%s", event.Code, event.Body.String())
|
||||
}
|
||||
|
||||
deleted := integrationRequest(t, s, http.MethodDelete, "/api/v1/integrations/knowledge/agent/KB-1", key, "")
|
||||
if deleted.Code != http.StatusOK || !strings.Contains(deleted.Body.String(), `"deleted":1`) {
|
||||
t.Fatalf("delete status=%d body=%s", deleted.Code, deleted.Body.String())
|
||||
}
|
||||
search = integrationRequest(t, s, http.MethodPost, "/api/v1/integrations/knowledge/search", key, `{"namespace":"agent","vector":[1,0],"k":10}`)
|
||||
if search.Code != http.StatusOK {
|
||||
t.Fatalf("post-delete search status=%d body=%s", search.Code, search.Body.String())
|
||||
}
|
||||
if strings.Contains(search.Body.String(), "KB-1") {
|
||||
t.Fatalf("deleted document still searchable: %s", search.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
type integrationGraphNode struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Label string `json:"label"`
|
||||
Group string `json:"group,omitempty"`
|
||||
Community string `json:"community,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
Meta map[string]any `json:"meta,omitempty"`
|
||||
}
|
||||
|
||||
type integrationGraphEdge struct {
|
||||
ID string `json:"id"`
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
Kind string `json:"kind"`
|
||||
Label string `json:"label,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Weight float64 `json:"weight,omitempty"`
|
||||
Meta map[string]any `json:"meta,omitempty"`
|
||||
}
|
||||
|
||||
type integrationGraphPayload struct {
|
||||
Scope string `json:"scope"`
|
||||
Title string `json:"title"`
|
||||
Nodes []integrationGraphNode `json:"nodes"`
|
||||
Edges []integrationGraphEdge `json:"edges"`
|
||||
Meta map[string]any `json:"meta,omitempty"`
|
||||
}
|
||||
|
||||
// integrationResearchGraph exposes only bounded research provenance metadata.
|
||||
// Full source bodies and prompts stay behind their existing dedicated APIs.
|
||||
func (s *Server) integrationResearchGraph(w http.ResponseWriter, r *http.Request) {
|
||||
limit := graphBoundedInt(r.URL.Query().Get("runs"), 6, 1, 20)
|
||||
maxEvents := graphBoundedInt(r.URL.Query().Get("max_events"), 320, 20, 800)
|
||||
runs := s.store.ResearchRunsSnapshot("", limit)
|
||||
g := integrationGraphPayload{Scope: "research", Title: "Research Provenance", Meta: map[string]any{"runs": len(runs), "max_events": maxEvents}}
|
||||
seen := map[string]bool{}
|
||||
addNode := func(n integrationGraphNode) {
|
||||
if n.ID == "" || seen[n.ID] {
|
||||
return
|
||||
}
|
||||
seen[n.ID] = true
|
||||
g.Nodes = append(g.Nodes, n)
|
||||
}
|
||||
addEdge := func(e integrationGraphEdge) {
|
||||
if e.ID == "" {
|
||||
e.ID = e.From + "->" + e.To + ":" + e.Kind
|
||||
}
|
||||
g.Edges = append(g.Edges, e)
|
||||
}
|
||||
eventsLeft := maxEvents
|
||||
for _, run := range runs {
|
||||
goalID := "goal:" + run.GoalID
|
||||
runID := "research-run:" + run.ID
|
||||
addNode(integrationGraphNode{ID: goalID, Kind: "research_goal", Label: graphCompact(firstGraphNonEmpty(run.GoalTitle, run.GoalID), 90), Group: "research", Community: "goal", Status: "goal"})
|
||||
addNode(integrationGraphNode{ID: runID, Kind: "research_run", Label: graphCompact(firstGraphNonEmpty(run.GoalTitle, run.ID), 90), Group: "research", Community: "run", Status: run.Status, Meta: map[string]any{"started_at": run.StartedAt, "completed_at": run.CompletedAt, "stats": run.Stats, "last_error": graphCompact(run.LastError, 180)}})
|
||||
addEdge(integrationGraphEdge{From: goalID, To: runID, Kind: "research_cycle", Status: run.Status})
|
||||
for _, q := range run.Queries {
|
||||
qid := "query:" + run.ID + ":" + shortGraphHash(q)
|
||||
addNode(integrationGraphNode{ID: qid, Kind: "query", Label: graphCompact(q, 100), Group: "research", Community: "search", Status: "planned"})
|
||||
addEdge(integrationGraphEdge{From: runID, To: qid, Kind: "planned_query"})
|
||||
}
|
||||
for _, ev := range run.Events {
|
||||
if eventsLeft <= 0 {
|
||||
break
|
||||
}
|
||||
eventsLeft--
|
||||
qid := ""
|
||||
if strings.TrimSpace(ev.Query) != "" {
|
||||
qid = "query:" + run.ID + ":" + shortGraphHash(ev.Query)
|
||||
addNode(integrationGraphNode{ID: qid, Kind: "query", Label: graphCompact(ev.Query, 100), Group: "research", Community: "search"})
|
||||
}
|
||||
sourceID := ""
|
||||
if ev.SourceID != "" {
|
||||
sourceID = "source:" + ev.SourceID
|
||||
} else if ev.URL != "" {
|
||||
sourceID = "url:" + shortGraphHash(ev.URL)
|
||||
}
|
||||
if sourceID != "" {
|
||||
status := ev.Status
|
||||
if status == "" {
|
||||
status = "seen"
|
||||
}
|
||||
addNode(integrationGraphNode{ID: sourceID, Kind: "source", Label: graphCompact(firstGraphNonEmpty(ev.Title, ev.URL, ev.SourceID), 100), Group: "source", Community: "research-source", Status: status, Score: ev.Score, Meta: map[string]any{"url": ev.URL, "source_id": ev.SourceID, "phase": ev.Phase, "engine": ev.Metadata["engine"], "mimetype": ev.Metadata["mimetype"]}})
|
||||
from := runID
|
||||
if qid != "" {
|
||||
from = qid
|
||||
}
|
||||
addEdge(integrationGraphEdge{From: from, To: sourceID, Kind: graphResearchEdgeKind(ev.Type), Label: ev.Type, Status: ev.Status, Weight: ev.Score})
|
||||
}
|
||||
if ev.Type == "claim.extracted" {
|
||||
cid := "claim:" + run.ID + ":" + strconv.FormatUint(ev.Seq, 10)
|
||||
addNode(integrationGraphNode{ID: cid, Kind: "claim", Label: graphCompact(ev.Preview, 120), Group: "evidence", Community: "claim", Status: ev.Status, Score: ev.Confidence, Meta: map[string]any{"phase": ev.Phase, "message": graphCompact(ev.Message, 160)}})
|
||||
from := runID
|
||||
if sourceID != "" {
|
||||
from = sourceID
|
||||
}
|
||||
addEdge(integrationGraphEdge{From: from, To: cid, Kind: "claim_extracted", Status: ev.Status})
|
||||
}
|
||||
if ev.MemoryID != "" {
|
||||
mid := "memory:" + ev.MemoryID
|
||||
status := ev.Status
|
||||
if strings.Contains(ev.Type, "corroborated") {
|
||||
status = "corroborated"
|
||||
}
|
||||
if strings.Contains(ev.Type, "duplicate") {
|
||||
status = "duplicate"
|
||||
}
|
||||
addNode(integrationGraphNode{ID: mid, Kind: "memory", Label: graphCompact(firstGraphNonEmpty(ev.Preview, ev.Message, ev.MemoryID), 120), Group: "brain", Community: "evidence", Status: status, Score: firstGraphScore(ev.Confidence, ev.Similarity), Meta: map[string]any{"memory_id": ev.MemoryID, "event": ev.Type, "similarity": ev.Similarity, "confidence": ev.Confidence}})
|
||||
from := runID
|
||||
if sourceID != "" {
|
||||
from = sourceID
|
||||
}
|
||||
kind := "learned_as"
|
||||
if strings.Contains(ev.Type, "corroborated") {
|
||||
kind = "corroborates"
|
||||
} else if strings.Contains(ev.Type, "duplicate") {
|
||||
kind = "matches_existing"
|
||||
}
|
||||
addEdge(integrationGraphEdge{From: from, To: mid, Kind: kind, Label: ev.Type, Status: ev.Status, Weight: firstGraphScore(ev.Confidence, ev.Similarity)})
|
||||
}
|
||||
}
|
||||
}
|
||||
s.json(w, http.StatusOK, g)
|
||||
}
|
||||
|
||||
// integrationBrainGraph is a bounded, redacted operational graph. It is not a
|
||||
// memory export: vectors and full text are omitted, and the caller controls only
|
||||
// the visualization window size.
|
||||
func (s *Server) integrationBrainGraph(w http.ResponseWriter, r *http.Request) {
|
||||
maxNodes := graphBoundedInt(r.URL.Query().Get("max_nodes"), 320, 50, 700)
|
||||
memories := s.store.MemoriesSnapshot()
|
||||
sort.SliceStable(memories, func(i, j int) bool {
|
||||
a, b := memoryGraphPriority(memories[i]), memoryGraphPriority(memories[j])
|
||||
if a == b {
|
||||
return memories[i].CreatedAt.After(memories[j].CreatedAt)
|
||||
}
|
||||
return a > b
|
||||
})
|
||||
if len(memories) > maxNodes {
|
||||
memories = memories[:maxNodes]
|
||||
}
|
||||
g := integrationGraphPayload{Scope: "brain", Title: "NeuroForge Brain", Meta: map[string]any{"nodes_budget": maxNodes, "total_memories": len(s.store.MemoriesSnapshot())}}
|
||||
seen := map[string]core.Memory{}
|
||||
for _, m := range memories {
|
||||
seen[m.ID] = m
|
||||
label := graphCompact(firstGraphNonEmpty(m.Provenance.SourceTitle, m.TruthKey, m.Text, m.ID), 110)
|
||||
community := m.Provenance.Source
|
||||
if community == "" {
|
||||
community = m.MemoryType
|
||||
}
|
||||
g.Nodes = append(g.Nodes, integrationGraphNode{ID: "memory:" + m.ID, Kind: "memory_" + m.MemoryType, Label: label, Group: "brain", Community: graphCompact(community, 48), Status: firstGraphNonEmpty(m.Status, core.MemoryActive), Score: m.Salience, Meta: map[string]any{"memory_id": m.ID, "kind": m.Kind, "source": m.Provenance.Source, "confidence": m.Confidence, "reward": m.Reward, "salience": m.Salience, "access_count": m.AccessCount, "created_at": m.CreatedAt, "source_id": m.Provenance.SourceMemoryID}})
|
||||
}
|
||||
for _, syn := range s.store.SynapsesSnapshot() {
|
||||
_, aok := seen[syn.A]
|
||||
_, bok := seen[syn.B]
|
||||
if !aok || !bok {
|
||||
continue
|
||||
}
|
||||
g.Edges = append(g.Edges, integrationGraphEdge{ID: "syn:" + syn.A + ":" + syn.B, From: "memory:" + syn.A, To: "memory:" + syn.B, Kind: "synapse", Weight: syn.Weight, Meta: map[string]any{"similarity": syn.Similarity, "activations": syn.Activations}})
|
||||
}
|
||||
for _, m := range memories {
|
||||
for _, old := range m.Supersedes {
|
||||
if _, ok := seen[old]; ok {
|
||||
g.Edges = append(g.Edges, integrationGraphEdge{From: "memory:" + m.ID, To: "memory:" + old, Kind: "supersedes", Status: "active", Weight: 1})
|
||||
}
|
||||
}
|
||||
for _, old := range m.ConsolidatedFrom {
|
||||
if _, ok := seen[old]; ok {
|
||||
g.Edges = append(g.Edges, integrationGraphEdge{From: "memory:" + old, To: "memory:" + m.ID, Kind: "consolidated_into", Weight: 1})
|
||||
}
|
||||
}
|
||||
}
|
||||
s.json(w, http.StatusOK, g)
|
||||
}
|
||||
|
||||
func graphBoundedInt(raw string, def, min, max int) int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(raw))
|
||||
if err != nil || n < min {
|
||||
return def
|
||||
}
|
||||
if n > max {
|
||||
return max
|
||||
}
|
||||
return n
|
||||
}
|
||||
func graphCompact(v string, n int) string {
|
||||
v = strings.Join(strings.Fields(strings.TrimSpace(v)), " ")
|
||||
rr := []rune(v)
|
||||
if n > 0 && len(rr) > n {
|
||||
return string(rr[:n]) + "…"
|
||||
}
|
||||
return v
|
||||
}
|
||||
func firstGraphNonEmpty(xs ...string) string {
|
||||
for _, x := range xs {
|
||||
if strings.TrimSpace(x) != "" {
|
||||
return strings.TrimSpace(x)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func firstGraphScore(xs ...float64) float64 {
|
||||
for _, x := range xs {
|
||||
if x != 0 {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func shortGraphHash(s string) string {
|
||||
var h uint64 = 1469598103934665603
|
||||
for _, b := range []byte(s) {
|
||||
h ^= uint64(b)
|
||||
h *= 1099511628211
|
||||
}
|
||||
return fmt.Sprintf("%x", h)
|
||||
}
|
||||
func graphResearchEdgeKind(t string) string {
|
||||
if strings.HasPrefix(t, "search.") {
|
||||
return "search_result"
|
||||
}
|
||||
if strings.HasPrefix(t, "download.") {
|
||||
return "fetched"
|
||||
}
|
||||
if strings.HasPrefix(t, "source.") {
|
||||
return "source_event"
|
||||
}
|
||||
return "research_event"
|
||||
}
|
||||
func memoryGraphPriority(m core.Memory) float64 {
|
||||
p := m.Salience + m.Confidence*.5 + float64(m.AccessCount)*.01
|
||||
if m.Status == core.MemoryActive {
|
||||
p += .5
|
||||
}
|
||||
if strings.HasPrefix(m.Provenance.Source, "glpi.outcome.") {
|
||||
p += 1
|
||||
}
|
||||
if strings.HasPrefix(m.Provenance.Source, "integration:") {
|
||||
p += .4
|
||||
}
|
||||
return p
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func appGraphRequest(t *testing.T, s *Server, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+s.store.Secrets().AppAPIKey)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
func TestIntegrationGraphEndpointsRequireAppKey(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
for _, path := range []string{"/api/v1/integrations/graph/brain", "/api/v1/integrations/graph/research"} {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("%s status=%d body=%s", path, rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationBrainGraphIsBoundedAndRedacted(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
m := &core.Memory{ID: "m-graph", Kind: "fact", MemoryType: core.MemorySemantic, Text: "secretly long operational text", Vector: []float32{1, 2, 3}, VectorDim: 3, Salience: .9, Confidence: .8, Status: core.MemoryActive, CreatedAt: time.Now().UTC(), Provenance: core.MemoryProvenance{Source: "glpi.outcome.accepted", SourceTitle: "VPN fix"}}
|
||||
if err := s.store.AddMemory(m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr := appGraphRequest(t, s, "/api/v1/integrations/graph/brain?max_nodes=50")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if strings.Contains(rr.Body.String(), `"vector"`) || strings.Contains(rr.Body.String(), "secretly long operational text") {
|
||||
t.Fatalf("graph leaked full memory data: %s", rr.Body.String())
|
||||
}
|
||||
var g integrationGraphPayload
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &g); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(g.Nodes) == 0 || g.Nodes[0].Kind == "" {
|
||||
t.Fatalf("missing graph nodes: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegrationResearchGraphShowsProvenanceChain(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
run, err := s.store.StartResearchRun("goal-1", "VPN research")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, ev := range []core.ResearchEvent{
|
||||
{Type: "query.planned", Query: "vpn client issue"},
|
||||
{Type: "search.result", Query: "vpn client issue", URL: "https://example.invalid/vpn", Title: "VPN source", SourceID: "src-1", Score: .7},
|
||||
{Type: "evidence.learned", SourceID: "src-1", MemoryID: "m-research", Preview: "verified workaround", Confidence: .65},
|
||||
} {
|
||||
if _, err := s.store.AddResearchEvent(run.ID, ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := s.store.FinishResearchRun(run.ID, "completed", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr := appGraphRequest(t, s, "/api/v1/integrations/graph/research?runs=2&max_events=50")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
body := rr.Body.String()
|
||||
for _, want := range []string{"research_goal", "query", "source", "memory", "learned_as"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("missing %q in %s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func (s *Server) adminKnowledgeSummary(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, http.StatusOK, s.store.KnowledgeSummary())
|
||||
}
|
||||
|
||||
func (s *Server) adminKnowledgeMemories(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
var before time.Time
|
||||
if raw := strings.TrimSpace(r.URL.Query().Get("before")); raw != "" {
|
||||
var err error
|
||||
before, err = time.Parse(time.RFC3339Nano, raw)
|
||||
if err != nil {
|
||||
s.err(w, 400, errors.New("before must be RFC3339"))
|
||||
return
|
||||
}
|
||||
}
|
||||
out := s.store.KnowledgeMemories(limit, before, r.URL.Query().Get("memory_type"), r.URL.Query().Get("status"), r.URL.Query().Get("kind"), r.URL.Query().Get("source"))
|
||||
s.json(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s *Server) adminKnowledgeMemory(w http.ResponseWriter, r *http.Request) {
|
||||
out, ok := s.store.KnowledgeMemoryDetail(r.PathValue("id"))
|
||||
if !ok {
|
||||
s.err(w, 404, errors.New("memory not found"))
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s *Server) adminKnowledgeGraph(w http.ResponseWriter, r *http.Request) {
|
||||
depth, _ := strconv.Atoi(r.URL.Query().Get("depth"))
|
||||
maxNodes, _ := strconv.Atoi(r.URL.Query().Get("max_nodes"))
|
||||
s.json(w, http.StatusOK, s.store.KnowledgeGraph(r.URL.Query().Get("center"), depth, maxNodes))
|
||||
}
|
||||
|
||||
func (s *Server) adminKnowledgeEvents(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
s.json(w, http.StatusOK, s.store.RecentKnowledgeEvents(limit))
|
||||
}
|
||||
|
||||
func (s *Server) adminKnowledgeSearch(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
Text string `json:"text"`
|
||||
K int `json:"k"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
q.Text = strings.TrimSpace(q.Text)
|
||||
if q.Text == "" {
|
||||
s.err(w, 400, errors.New("text is required"))
|
||||
return
|
||||
}
|
||||
if q.K <= 0 {
|
||||
q.K = 12
|
||||
}
|
||||
if q.K > 50 {
|
||||
q.K = 50
|
||||
}
|
||||
hits, err := s.brain.Search(r.Context(), q.Text, q.K)
|
||||
if err != nil {
|
||||
s.err(w, 502, err)
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusOK, map[string]any{
|
||||
"query": q.Text,
|
||||
"hits": hits,
|
||||
"formula": "score = similarity × salience_factor × type_weight × confidence_factor + graph_boost",
|
||||
"note": "candidate_source shows whether a candidate came from HNSW, Disk-PQ, full scan, or a synapse expansion; final similarity is always computed against the original vector when available.",
|
||||
})
|
||||
}
|
||||
|
||||
type learningPolicySettings struct {
|
||||
AutoLearn bool `json:"auto_learn"`
|
||||
Policy core.LearningPolicyConfig `json:"policy"`
|
||||
}
|
||||
|
||||
func (s *Server) adminGetLearningPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
c := s.store.Config()
|
||||
s.json(w, http.StatusOK, learningPolicySettings{AutoLearn: c.Brain.AutoLearn, Policy: c.Brain.LearningPolicy})
|
||||
}
|
||||
|
||||
func (s *Server) adminPutLearningPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
var q learningPolicySettings
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
c := s.store.Config()
|
||||
c.Brain.AutoLearn = q.AutoLearn
|
||||
c.Brain.LearningPolicy = q.Policy
|
||||
if err := s.store.ValidateConfig(c); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateConfig(c); err != nil {
|
||||
s.err(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{
|
||||
Type: "admin.learning_policy_changed", Summary: "Learning policy updated", Reason: "PUT /admin/api/learning-policy", Actor: "admin",
|
||||
Metadata: map[string]string{"auto_learn": strconv.FormatBool(q.AutoLearn), "enabled": strconv.FormatBool(q.Policy.Enabled), "duplicate_similarity": strconv.FormatFloat(q.Policy.DuplicateSimilarity, 'f', 4, 64)},
|
||||
})
|
||||
s.json(w, http.StatusOK, q)
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKnowledgeExplorerEndToEnd(t *testing.T) {
|
||||
fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/embed":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{{1, 0, 0}}, "prompt_eval_count": 2})
|
||||
case "/api/chat":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"message": map[string]any{"content": "PostgreSQL with pgvector."}, "prompt_eval_count": 4, "eval_count": 3})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer fake.Close()
|
||||
|
||||
s, _ := newMetricsTestServer(t)
|
||||
cfg := s.store.Config()
|
||||
cfg.Ollama[0].BaseURL = fake.URL
|
||||
cfg.Brain.ExternalRelinkWorker = false
|
||||
cfg.Brain.LearningPolicy.DuplicateSimilarity = .9999
|
||||
if err := s.store.UpdateConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sec := s.store.Secrets()
|
||||
|
||||
learn := httptest.NewRequest(http.MethodPost, "/api/v1/learn", strings.NewReader(`{"text":"Project Aurora uses PostgreSQL with pgvector.","kind":"knowledge","memory_type":"semantic","confidence":0.95}`))
|
||||
learn.Header.Set("Authorization", "Bearer "+sec.AppAPIKey)
|
||||
learn.Header.Set("Content-Type", "application/json")
|
||||
lr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(lr, learn)
|
||||
if lr.Code != http.StatusCreated {
|
||||
t.Fatalf("learn status=%d body=%s", lr.Code, lr.Body.String())
|
||||
}
|
||||
|
||||
sum := httptest.NewRequest(http.MethodGet, "/admin/api/knowledge/summary", nil)
|
||||
sum.Header.Set("X-Admin-Token", sec.AdminToken)
|
||||
sr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(sr, sum)
|
||||
if sr.Code != 200 || !strings.Contains(sr.Body.String(), "api.learn") {
|
||||
t.Fatalf("summary status=%d body=%s", sr.Code, sr.Body.String())
|
||||
}
|
||||
|
||||
search := httptest.NewRequest(http.MethodPost, "/admin/api/knowledge/search", strings.NewReader(`{"text":"Aurora database","k":5}`))
|
||||
search.Header.Set("X-Admin-Token", sec.AdminToken)
|
||||
search.Header.Set("Content-Type", "application/json")
|
||||
xr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(xr, search)
|
||||
if xr.Code != 200 {
|
||||
t.Fatalf("search status=%d body=%s", xr.Code, xr.Body.String())
|
||||
}
|
||||
body := xr.Body.String()
|
||||
for _, want := range []string{"base_score", "confidence_factor", "candidate_source", "Project Aurora"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("search missing %q body=%s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLearningPolicyAdminRoundTrip(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
admin := s.store.Secrets().AdminToken
|
||||
body := `{"auto_learn":true,"policy":{"enabled":true,"learn_chat_inputs":true,"learn_chat_responses":false,"allow_explicit_learn":true,"allow_imports":false,"learn_goal_cycles":false,"min_confidence":0.4,"duplicate_similarity":0.97,"semantic_min_confirmations":4,"semantic_min_confidence":0.7,"archive_negative_responses":true,"negative_archive_threshold":-0.8,"max_memory_text_chars":12000,"source_trust":{"chat.input":1,"chat.response":0.8,"api.learn":1,"api.import":0.5,"goal-cycle":0.8,"consolidation":1}}}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/admin/api/learning-policy", strings.NewReader(body))
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
cfg := s.store.Config()
|
||||
if cfg.Brain.LearningPolicy.LearnChatResponses || cfg.Brain.LearningPolicy.AllowImports || cfg.Brain.LearningPolicy.SemanticMinConfirmations != 4 {
|
||||
t.Fatalf("policy not applied: %+v", cfg.Brain.LearningPolicy)
|
||||
}
|
||||
get := httptest.NewRequest(http.MethodGet, "/admin/api/learning-policy", nil)
|
||||
get.Header.Set("X-Admin-Token", admin)
|
||||
getRR := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(getRR, get)
|
||||
if getRR.Code != 200 {
|
||||
t.Fatalf("get status=%d", getRR.Code)
|
||||
}
|
||||
var x learningPolicySettings
|
||||
if err := json.Unmarshal(getRR.Body.Bytes(), &x); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !x.AutoLearn || x.Policy.AllowImports {
|
||||
t.Fatalf("unexpected response %+v", x)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecretsAreMaskedByDefault(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
sec := s.store.Secrets()
|
||||
sec.ShardAPIToken = map[string]string{"remote": "super-secret-value"}
|
||||
if err := s.store.UpdateSecrets(sec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/api/secrets", nil)
|
||||
req.Header.Set("X-Admin-Token", sec.AdminToken)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("status=%d", rr.Code)
|
||||
}
|
||||
if strings.Contains(rr.Body.String(), "super-secret-value") {
|
||||
t.Fatal("secret leaked in masked response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessEndpoint(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
req := httptest.NewRequest(http.MethodGet, "/readyz", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,450 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var requestDurationBuckets = [...]float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}
|
||||
|
||||
type httpMetricKey struct {
|
||||
Method string
|
||||
Route string
|
||||
Code int
|
||||
}
|
||||
|
||||
type httpMetric struct {
|
||||
Count uint64
|
||||
Bytes uint64
|
||||
Sum float64
|
||||
Buckets [len(requestDurationBuckets)]uint64
|
||||
}
|
||||
|
||||
type metricsRegistry struct {
|
||||
mu sync.RWMutex
|
||||
started time.Time
|
||||
http map[httpMetricKey]*httpMetric
|
||||
}
|
||||
|
||||
func newMetricsRegistry() *metricsRegistry {
|
||||
return &metricsRegistry{started: time.Now(), http: map[httpMetricKey]*httpMetric{}}
|
||||
}
|
||||
|
||||
func normalizeMetricRoute(r *http.Request) string {
|
||||
route := strings.TrimSpace(r.Pattern)
|
||||
if route == "" {
|
||||
return "unmatched"
|
||||
}
|
||||
if strings.HasPrefix(route, r.Method+" ") {
|
||||
route = strings.TrimSpace(strings.TrimPrefix(route, r.Method+" "))
|
||||
}
|
||||
return route
|
||||
}
|
||||
|
||||
func (m *metricsRegistry) observeHTTP(method, route string, code int, bytes int64, d time.Duration) {
|
||||
if code == 0 {
|
||||
code = http.StatusOK
|
||||
}
|
||||
key := httpMetricKey{Method: method, Route: route, Code: code}
|
||||
sec := d.Seconds()
|
||||
m.mu.Lock()
|
||||
x := m.http[key]
|
||||
if x == nil {
|
||||
x = &httpMetric{}
|
||||
m.http[key] = x
|
||||
}
|
||||
x.Count++
|
||||
if bytes > 0 {
|
||||
x.Bytes += uint64(bytes)
|
||||
}
|
||||
x.Sum += sec
|
||||
for i, upper := range requestDurationBuckets {
|
||||
if sec <= upper {
|
||||
x.Buckets[i]++
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
type httpDashboardRoute struct {
|
||||
Method string `json:"method"`
|
||||
Route string `json:"route"`
|
||||
Requests uint64 `json:"requests"`
|
||||
Errors uint64 `json:"errors"`
|
||||
AverageMS float64 `json:"average_ms"`
|
||||
ApproxP95MS float64 `json:"approx_p95_ms"`
|
||||
ResponseBytes uint64 `json:"response_bytes"`
|
||||
}
|
||||
|
||||
type httpDashboardSnapshot struct {
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
UptimeSeconds float64 `json:"uptime_seconds"`
|
||||
Requests uint64 `json:"requests"`
|
||||
Errors4xx uint64 `json:"errors_4xx"`
|
||||
Errors5xx uint64 `json:"errors_5xx"`
|
||||
ResponseBytes uint64 `json:"response_bytes"`
|
||||
AverageMS float64 `json:"average_ms"`
|
||||
ApproxP95MS float64 `json:"approx_p95_ms"`
|
||||
Routes []httpDashboardRoute `json:"routes"`
|
||||
}
|
||||
|
||||
func approxP95(count uint64, buckets []uint64) float64 {
|
||||
if count == 0 {
|
||||
return 0
|
||||
}
|
||||
target := uint64(math.Ceil(float64(count) * 0.95))
|
||||
for i, n := range buckets {
|
||||
if n >= target {
|
||||
return requestDurationBuckets[i] * 1000
|
||||
}
|
||||
}
|
||||
return requestDurationBuckets[len(requestDurationBuckets)-1] * 1000
|
||||
}
|
||||
|
||||
func (m *metricsRegistry) dashboardSnapshot() httpDashboardSnapshot {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
out := httpDashboardSnapshot{StartedAt: m.started, UptimeSeconds: time.Since(m.started).Seconds()}
|
||||
type agg struct {
|
||||
count, errors, bytes uint64
|
||||
sum float64
|
||||
buckets [len(requestDurationBuckets)]uint64
|
||||
}
|
||||
byRoute := map[[2]string]*agg{}
|
||||
var allBuckets [len(requestDurationBuckets)]uint64
|
||||
for key, x := range m.http {
|
||||
out.Requests += x.Count
|
||||
out.ResponseBytes += x.Bytes
|
||||
out.AverageMS += x.Sum * 1000
|
||||
if key.Code >= 400 && key.Code < 500 {
|
||||
out.Errors4xx += x.Count
|
||||
}
|
||||
if key.Code >= 500 {
|
||||
out.Errors5xx += x.Count
|
||||
}
|
||||
for i := range allBuckets {
|
||||
allBuckets[i] += x.Buckets[i]
|
||||
}
|
||||
k := [2]string{key.Method, key.Route}
|
||||
a := byRoute[k]
|
||||
if a == nil {
|
||||
a = &agg{}
|
||||
byRoute[k] = a
|
||||
}
|
||||
a.count += x.Count
|
||||
a.bytes += x.Bytes
|
||||
a.sum += x.Sum
|
||||
if key.Code >= 400 {
|
||||
a.errors += x.Count
|
||||
}
|
||||
for i := range a.buckets {
|
||||
a.buckets[i] += x.Buckets[i]
|
||||
}
|
||||
}
|
||||
if out.Requests > 0 {
|
||||
out.AverageMS /= float64(out.Requests)
|
||||
}
|
||||
out.ApproxP95MS = approxP95(out.Requests, allBuckets[:])
|
||||
for k, a := range byRoute {
|
||||
r := httpDashboardRoute{Method: k[0], Route: k[1], Requests: a.count, Errors: a.errors, ResponseBytes: a.bytes}
|
||||
if a.count > 0 {
|
||||
r.AverageMS = a.sum * 1000 / float64(a.count)
|
||||
}
|
||||
r.ApproxP95MS = approxP95(a.count, a.buckets[:])
|
||||
out.Routes = append(out.Routes, r)
|
||||
}
|
||||
sort.Slice(out.Routes, func(i, j int) bool {
|
||||
if out.Routes[i].Requests == out.Routes[j].Requests {
|
||||
return out.Routes[i].Route < out.Routes[j].Route
|
||||
}
|
||||
return out.Routes[i].Requests > out.Routes[j].Requests
|
||||
})
|
||||
if len(out.Routes) > 20 {
|
||||
out.Routes = out.Routes[:20]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type runtimeSnapshot struct {
|
||||
Goroutines int `json:"goroutines"`
|
||||
HeapAlloc uint64 `json:"heap_alloc_bytes"`
|
||||
HeapInuse uint64 `json:"heap_inuse_bytes"`
|
||||
HeapObjects uint64 `json:"heap_objects"`
|
||||
SysBytes uint64 `json:"sys_bytes"`
|
||||
NumGC uint32 `json:"gc_cycles_total"`
|
||||
PauseTotalNS uint64 `json:"gc_pause_total_ns"`
|
||||
}
|
||||
|
||||
func currentRuntimeSnapshot() runtimeSnapshot {
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
return runtimeSnapshot{
|
||||
Goroutines: runtime.NumGoroutine(), HeapAlloc: ms.HeapAlloc, HeapInuse: ms.HeapInuse,
|
||||
HeapObjects: ms.HeapObjects, SysBytes: ms.Sys, NumGC: ms.NumGC, PauseTotalNS: ms.PauseTotalNs,
|
||||
}
|
||||
}
|
||||
|
||||
func metricEscape(v string) string {
|
||||
v = strings.ReplaceAll(v, `\`, `\\`)
|
||||
v = strings.ReplaceAll(v, "\n", `\n`)
|
||||
v = strings.ReplaceAll(v, `"`, `\"`)
|
||||
return v
|
||||
}
|
||||
|
||||
func metricLabels(labels ...string) string {
|
||||
if len(labels) == 0 {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteByte('{')
|
||||
for i := 0; i+1 < len(labels); i += 2 {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(labels[i])
|
||||
b.WriteString(`="`)
|
||||
b.WriteString(metricEscape(labels[i+1]))
|
||||
b.WriteByte('"')
|
||||
}
|
||||
b.WriteByte('}')
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func promHeader(b *strings.Builder, name, help, typ string) {
|
||||
fmt.Fprintf(b, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ)
|
||||
}
|
||||
|
||||
func promSample(b *strings.Builder, name string, value any, labels ...string) {
|
||||
fmt.Fprintf(b, "%s%s %v\n", name, metricLabels(labels...), value)
|
||||
}
|
||||
|
||||
func boolFloat(v bool) int {
|
||||
if v {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *Server) metricsEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
sec := s.store.Secrets()
|
||||
token := bearer(r)
|
||||
if sec.MetricsToken == "" || (!secureEqual(token, sec.MetricsToken) && !secureEqual(token, sec.AdminToken)) {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="neuroforge-metrics"`)
|
||||
s.err(w, http.StatusUnauthorized, fmt.Errorf("invalid metrics token"))
|
||||
return
|
||||
}
|
||||
|
||||
st := s.store.ObservabilitySnapshot()
|
||||
costs := s.cost.Totals()
|
||||
cfg := s.store.Config()
|
||||
rt := currentRuntimeSnapshot()
|
||||
httpSnap := s.metrics.dashboardSnapshot()
|
||||
var b strings.Builder
|
||||
b.Grow(24 << 10)
|
||||
|
||||
promHeader(&b, "neuroforge_up", "Whether the NeuroForge process is serving metrics.", "gauge")
|
||||
promSample(&b, "neuroforge_up", 1)
|
||||
promHeader(&b, "neuroforge_uptime_seconds", "Process uptime in seconds.", "gauge")
|
||||
promSample(&b, "neuroforge_uptime_seconds", strconv.FormatFloat(httpSnap.UptimeSeconds, 'f', 3, 64))
|
||||
promHeader(&b, "neuroforge_revision", "Current persisted NeuroForge state revision.", "gauge")
|
||||
promSample(&b, "neuroforge_revision", st.Revision)
|
||||
|
||||
promHeader(&b, "neuroforge_memories", "Current number of memory records.", "gauge")
|
||||
promSample(&b, "neuroforge_memories", st.Memories)
|
||||
promHeader(&b, "neuroforge_sources", "Current number of registered knowledge sources.", "gauge")
|
||||
promSample(&b, "neuroforge_sources", st.Sources)
|
||||
promHeader(&b, "neuroforge_synapses", "Current number of synapse edges.", "gauge")
|
||||
promSample(&b, "neuroforge_synapses", st.Synapses)
|
||||
promHeader(&b, "neuroforge_goals", "Current number of goal records.", "gauge")
|
||||
promSample(&b, "neuroforge_goals", st.Goals)
|
||||
promHeader(&b, "neuroforge_learning_cycles", "Current number of retained learning-cycle records.", "gauge")
|
||||
promSample(&b, "neuroforge_learning_cycles", st.LearningCycles)
|
||||
promHeader(&b, "neuroforge_knowledge_events", "Current number of retained explainability/knowledge events.", "gauge")
|
||||
promSample(&b, "neuroforge_knowledge_events", st.KnowledgeEvents)
|
||||
|
||||
promHeader(&b, "neuroforge_jobs", "Current worker jobs by bounded status class.", "gauge")
|
||||
promSample(&b, "neuroforge_jobs", st.JobsQueued, "status", "queued")
|
||||
promSample(&b, "neuroforge_jobs", st.JobsClaimed, "status", "claimed")
|
||||
promSample(&b, "neuroforge_jobs", st.JobsDone, "status", "done")
|
||||
promSample(&b, "neuroforge_jobs", st.JobsFailed, "status", "failed")
|
||||
|
||||
promHeader(&b, "neuroforge_hnsw_nodes", "Current number of vectors in hot HNSW indexes.", "gauge")
|
||||
promSample(&b, "neuroforge_hnsw_nodes", st.HNSWNodes)
|
||||
promHeader(&b, "neuroforge_hnsw_dimensions", "Number of active HNSW dimensionality indexes.", "gauge")
|
||||
promSample(&b, "neuroforge_hnsw_dimensions", st.HNSWDimensions)
|
||||
promHeader(&b, "neuroforge_disk_pq_items", "Current number of items indexed in disk PQ.", "gauge")
|
||||
promSample(&b, "neuroforge_disk_pq_items", st.DiskPQItems)
|
||||
promHeader(&b, "neuroforge_disk_pq_bytes", "Disk bytes used by disk PQ indexes.", "gauge")
|
||||
promSample(&b, "neuroforge_disk_pq_bytes", st.DiskPQBytes)
|
||||
promHeader(&b, "neuroforge_index_mode", "Active vector index mode as a one-hot info gauge.", "gauge")
|
||||
promSample(&b, "neuroforge_index_mode", 1, "mode", st.IndexMode)
|
||||
promHeader(&b, "neuroforge_index_delta_segments", "Current HNSW delta segment count.", "gauge")
|
||||
promSample(&b, "neuroforge_index_delta_segments", st.IndexDeltaCount)
|
||||
promHeader(&b, "neuroforge_disk_pq_building", "Whether a disk PQ rebuild is currently running.", "gauge")
|
||||
promSample(&b, "neuroforge_disk_pq_building", boolFloat(st.DiskANNBuilding))
|
||||
vj := s.store.VectorJournalStats()
|
||||
promHeader(&b, "neuroforge_vector_journal_raw_bytes", "Raw vector bytes represented by the rebuildable vector journal.", "gauge")
|
||||
promSample(&b, "neuroforge_vector_journal_raw_bytes", vj.VectorRawBytes)
|
||||
promHeader(&b, "neuroforge_vector_journal_stored_bytes", "Stored vector payload bytes after raw/DEFLATE/SQAR selection.", "gauge")
|
||||
promSample(&b, "neuroforge_vector_journal_stored_bytes", vj.VectorStoredBytes)
|
||||
promHeader(&b, "neuroforge_vector_journal_compression_savings_percent", "Vector journal payload savings percent.", "gauge")
|
||||
promSample(&b, "neuroforge_vector_journal_compression_savings_percent", strconv.FormatFloat(vj.CompressionSavingsPct, 'f', 3, 64))
|
||||
promHeader(&b, "neuroforge_vector_journal_sqar_blocks", "Number of vector journal blocks encoded with SQAR.", "gauge")
|
||||
promSample(&b, "neuroforge_vector_journal_sqar_blocks", vj.SQARBlocks)
|
||||
promHeader(&b, "neuroforge_vector_journal_compressed_blocks", "Number of compressed vector journal blocks.", "gauge")
|
||||
promSample(&b, "neuroforge_vector_journal_compressed_blocks", vj.CompressedBlocks)
|
||||
|
||||
promHeader(&b, "neuroforge_memory_segment_bytes", "Bytes used by authoritative memory segments.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_segment_bytes", st.Segments.Bytes)
|
||||
promHeader(&b, "neuroforge_memory_segments", "Number of authoritative memory segment files.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_segments", st.Segments.Segments)
|
||||
promHeader(&b, "neuroforge_memory_segment_records", "Number of records in memory segments.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_segment_records", st.Segments.Records)
|
||||
promHeader(&b, "neuroforge_memory_segment_tombstones", "Current tombstone count in memory segments.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_segment_tombstones", st.Segments.Tombstones)
|
||||
promHeader(&b, "neuroforge_memory_mmap_segments", "Current number of mmap-backed sealed segments.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_mmap_segments", st.Segments.MmapSegments)
|
||||
|
||||
promHeader(&b, "neuroforge_memory_tier_memories", "Memory bodies by hot/cold tier.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_tier_memories", st.HotMemories, "tier", "hot")
|
||||
promSample(&b, "neuroforge_memory_tier_memories", st.ColdMemories, "tier", "cold")
|
||||
promHeader(&b, "neuroforge_memory_hot_bytes", "Approximate bytes held by hot memory bodies.", "gauge")
|
||||
promSample(&b, "neuroforge_memory_hot_bytes", st.HotBytes)
|
||||
promHeader(&b, "neuroforge_memory_tier_evictions_total", "Total memory-body evictions from the hot tier.", "counter")
|
||||
promSample(&b, "neuroforge_memory_tier_evictions_total", st.TierEvictions)
|
||||
|
||||
promHeader(&b, "neuroforge_page_cache_bytes", "Current page-cache bytes.", "gauge")
|
||||
promSample(&b, "neuroforge_page_cache_bytes", st.PageCacheBytes)
|
||||
promHeader(&b, "neuroforge_page_cache_max_bytes", "Configured page-cache byte limit.", "gauge")
|
||||
promSample(&b, "neuroforge_page_cache_max_bytes", st.PageCacheMaxBytes)
|
||||
promHeader(&b, "neuroforge_page_cache_entries", "Current page-cache entries.", "gauge")
|
||||
promSample(&b, "neuroforge_page_cache_entries", st.PageCacheEntries)
|
||||
promHeader(&b, "neuroforge_page_cache_hits_total", "Total page-cache hits.", "counter")
|
||||
promSample(&b, "neuroforge_page_cache_hits_total", st.PageCacheHits)
|
||||
promHeader(&b, "neuroforge_page_cache_misses_total", "Total page-cache misses.", "counter")
|
||||
promSample(&b, "neuroforge_page_cache_misses_total", st.PageCacheMisses)
|
||||
promHeader(&b, "neuroforge_page_cache_evictions_total", "Total page-cache evictions.", "counter")
|
||||
promSample(&b, "neuroforge_page_cache_evictions_total", st.PageCacheEvicts)
|
||||
|
||||
promHeader(&b, "neuroforge_wal_events_since_checkpoint", "WAL events written since the last checkpoint.", "gauge")
|
||||
promSample(&b, "neuroforge_wal_events_since_checkpoint", st.WALEventsSinceCheckpoint)
|
||||
|
||||
promHeader(&b, "neuroforge_cluster_enabled", "Whether cluster mode is enabled.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_enabled", boolFloat(st.ClusterEnabled))
|
||||
promHeader(&b, "neuroforge_cluster_term", "Current cluster election term.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_term", st.ClusterTerm)
|
||||
promHeader(&b, "neuroforge_cluster_log_index", "Current cluster last log index.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_log_index", st.ClusterLastIndex)
|
||||
promHeader(&b, "neuroforge_cluster_commit_index", "Current committed cluster log index.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_commit_index", st.ClusterCommitIndex)
|
||||
promHeader(&b, "neuroforge_cluster_quorum", "Configured or calculated voting quorum.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_quorum", st.ClusterQuorum)
|
||||
promHeader(&b, "neuroforge_cluster_voters", "Current configured voting nodes including local node.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_voters", st.ClusterVoters)
|
||||
promHeader(&b, "neuroforge_cluster_info", "Static cluster identity information for this target.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_info", 1, "node_id", st.ClusterNodeID, "leader_id", st.ClusterLeaderID, "role", st.ClusterRole)
|
||||
promHeader(&b, "neuroforge_cluster_replicated_log_bytes", "Bytes used by the replicated cluster log.", "gauge")
|
||||
promSample(&b, "neuroforge_cluster_replicated_log_bytes", st.ClusterLog.Bytes)
|
||||
|
||||
promHeader(&b, "neuroforge_openai_cost_usd", "Recorded OpenAI cost in USD for the current day or month.", "gauge")
|
||||
promSample(&b, "neuroforge_openai_cost_usd", strconv.FormatFloat(costs["daily_usd"], 'f', 9, 64), "period", "day")
|
||||
promSample(&b, "neuroforge_openai_cost_usd", strconv.FormatFloat(costs["monthly_usd"], 'f', 9, 64), "period", "month")
|
||||
promHeader(&b, "neuroforge_openai_budget_usd", "Configured OpenAI budget in USD.", "gauge")
|
||||
promSample(&b, "neuroforge_openai_budget_usd", strconv.FormatFloat(cfg.OpenAI.DailyBudgetUSD, 'f', 9, 64), "period", "day")
|
||||
promSample(&b, "neuroforge_openai_budget_usd", strconv.FormatFloat(cfg.OpenAI.MonthlyBudgetUSD, 'f', 9, 64), "period", "month")
|
||||
|
||||
promHeader(&b, "neuroforge_runtime_goroutines", "Current Go goroutine count.", "gauge")
|
||||
promSample(&b, "neuroforge_runtime_goroutines", rt.Goroutines)
|
||||
promHeader(&b, "neuroforge_runtime_heap_alloc_bytes", "Current Go heap allocation bytes.", "gauge")
|
||||
promSample(&b, "neuroforge_runtime_heap_alloc_bytes", rt.HeapAlloc)
|
||||
promHeader(&b, "neuroforge_runtime_heap_inuse_bytes", "Current Go heap in-use bytes.", "gauge")
|
||||
promSample(&b, "neuroforge_runtime_heap_inuse_bytes", rt.HeapInuse)
|
||||
promHeader(&b, "neuroforge_runtime_heap_objects", "Current number of allocated heap objects.", "gauge")
|
||||
promSample(&b, "neuroforge_runtime_heap_objects", rt.HeapObjects)
|
||||
promHeader(&b, "neuroforge_runtime_sys_bytes", "Total bytes obtained from the OS by the Go runtime.", "gauge")
|
||||
promSample(&b, "neuroforge_runtime_sys_bytes", rt.SysBytes)
|
||||
promHeader(&b, "neuroforge_runtime_gc_cycles_total", "Completed Go garbage-collection cycles.", "counter")
|
||||
promSample(&b, "neuroforge_runtime_gc_cycles_total", rt.NumGC)
|
||||
promHeader(&b, "neuroforge_runtime_gc_pause_seconds_total", "Cumulative Go stop-the-world GC pause time in seconds.", "counter")
|
||||
promSample(&b, "neuroforge_runtime_gc_pause_seconds_total", strconv.FormatFloat(float64(rt.PauseTotalNS)/1e9, 'f', 9, 64))
|
||||
|
||||
m := s.metrics
|
||||
m.mu.RLock()
|
||||
keys := make([]httpMetricKey, 0, len(m.http))
|
||||
for key := range m.http {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].Route != keys[j].Route {
|
||||
return keys[i].Route < keys[j].Route
|
||||
}
|
||||
if keys[i].Method != keys[j].Method {
|
||||
return keys[i].Method < keys[j].Method
|
||||
}
|
||||
return keys[i].Code < keys[j].Code
|
||||
})
|
||||
promHeader(&b, "neuroforge_http_requests_total", "Total HTTP requests by method, normalized route and response code.", "counter")
|
||||
for _, key := range keys {
|
||||
x := m.http[key]
|
||||
promSample(&b, "neuroforge_http_requests_total", x.Count, "method", key.Method, "route", key.Route, "code", strconv.Itoa(key.Code))
|
||||
}
|
||||
promHeader(&b, "neuroforge_http_response_bytes_total", "Total HTTP response bytes by method, normalized route and response code.", "counter")
|
||||
for _, key := range keys {
|
||||
x := m.http[key]
|
||||
promSample(&b, "neuroforge_http_response_bytes_total", x.Bytes, "method", key.Method, "route", key.Route, "code", strconv.Itoa(key.Code))
|
||||
}
|
||||
promHeader(&b, "neuroforge_http_request_duration_seconds", "HTTP request duration histogram by method and normalized route.", "histogram")
|
||||
type routeKey struct{ method, route string }
|
||||
type routeAgg struct {
|
||||
count uint64
|
||||
sum float64
|
||||
buckets [len(requestDurationBuckets)]uint64
|
||||
}
|
||||
aggs := map[routeKey]*routeAgg{}
|
||||
for _, key := range keys {
|
||||
x := m.http[key]
|
||||
rk := routeKey{key.Method, key.Route}
|
||||
a := aggs[rk]
|
||||
if a == nil {
|
||||
a = &routeAgg{}
|
||||
aggs[rk] = a
|
||||
}
|
||||
a.count += x.Count
|
||||
a.sum += x.Sum
|
||||
for i := range a.buckets {
|
||||
a.buckets[i] += x.Buckets[i]
|
||||
}
|
||||
}
|
||||
routes := make([]routeKey, 0, len(aggs))
|
||||
for k := range aggs {
|
||||
routes = append(routes, k)
|
||||
}
|
||||
sort.Slice(routes, func(i, j int) bool {
|
||||
if routes[i].route == routes[j].route {
|
||||
return routes[i].method < routes[j].method
|
||||
}
|
||||
return routes[i].route < routes[j].route
|
||||
})
|
||||
for _, rk := range routes {
|
||||
a := aggs[rk]
|
||||
for i, upper := range requestDurationBuckets {
|
||||
promSample(&b, "neuroforge_http_request_duration_seconds_bucket", a.buckets[i], "method", rk.method, "route", rk.route, "le", strconv.FormatFloat(upper, 'f', -1, 64))
|
||||
}
|
||||
promSample(&b, "neuroforge_http_request_duration_seconds_bucket", a.count, "method", rk.method, "route", rk.route, "le", "+Inf")
|
||||
promSample(&b, "neuroforge_http_request_duration_seconds_sum", strconv.FormatFloat(a.sum, 'f', 9, 64), "method", rk.method, "route", rk.route)
|
||||
promSample(&b, "neuroforge_http_request_duration_seconds_count", a.count, "method", rk.method, "route", rk.route)
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(b.String()))
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"neuroforge/internal/brain"
|
||||
"neuroforge/internal/cost"
|
||||
"neuroforge/internal/provider"
|
||||
"neuroforge/internal/store"
|
||||
)
|
||||
|
||||
func newMetricsTestServer(t *testing.T) (*Server, string) {
|
||||
t.Helper()
|
||||
s, err := store.New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
r := provider.NewRouter(s)
|
||||
c := cost.New(s)
|
||||
b := brain.New(s, r, c)
|
||||
return New(s, b, r, c), s.Secrets().MetricsToken
|
||||
}
|
||||
|
||||
func TestMetricsEndpointRequiresBearerToken(t *testing.T) {
|
||||
s, token := newMetricsTestServer(t)
|
||||
if token == "" {
|
||||
t.Fatal("metrics token was not generated")
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauthorized status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsEndpointExportsNormalizedHTTPMetrics(t *testing.T) {
|
||||
s, token := newMetricsTestServer(t)
|
||||
|
||||
// Generate one instrumented request before scraping.
|
||||
health := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
||||
healthRR := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(healthRR, health)
|
||||
if healthRR.Code != http.StatusOK {
|
||||
t.Fatalf("health status=%d", healthRR.Code)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("metrics status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if got := rr.Header().Get("Content-Type"); !strings.Contains(got, "text/plain") || !strings.Contains(got, "version=0.0.4") {
|
||||
t.Fatalf("unexpected content type %q", got)
|
||||
}
|
||||
body := rr.Body.String()
|
||||
for _, want := range []string{
|
||||
"# TYPE neuroforge_memories gauge",
|
||||
"neuroforge_up 1",
|
||||
"neuroforge_http_requests_total{method=\"GET\",route=\"/healthz\",code=\"200\"} 1",
|
||||
"neuroforge_http_request_duration_seconds_bucket{method=\"GET\",route=\"/healthz\",le=\"+Inf\"} 1",
|
||||
"neuroforge_page_cache_hits_total",
|
||||
"neuroforge_openai_budget_usd",
|
||||
"neuroforge_vector_journal_compression_savings_percent",
|
||||
"neuroforge_vector_journal_sqar_blocks",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("metrics output missing %q\n%s", want, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestModelRoutingAcceptsSimpleOllamaConfig(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
admin := s.store.Secrets().AdminToken
|
||||
before := s.store.Config()
|
||||
|
||||
body := `{
|
||||
"routing": {
|
||||
"chat_provider": "ollama",
|
||||
"embedding_provider": "ollama",
|
||||
"chat_node_id": "brain-01",
|
||||
"embedding_node_id": "brain-01",
|
||||
"critic": {"provider":"ollama","model":"critic-model","node_id":"brain-01"}
|
||||
},
|
||||
"ollama": [{
|
||||
"id": "brain-01",
|
||||
"name": "Primary Brain",
|
||||
"base_url": "http://127.0.0.1:11434",
|
||||
"chat_model": "qwen-chat",
|
||||
"embedding_model": "qwen-embed",
|
||||
"weight": 1,
|
||||
"enabled": true
|
||||
}]
|
||||
}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/admin/api/model-routing", strings.NewReader(body))
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
cfg := s.store.Config()
|
||||
if cfg.Routing.ChatProvider != "ollama" || cfg.Routing.EmbeddingProvider != "ollama" {
|
||||
t.Fatalf("routing not updated: %+v", cfg.Routing)
|
||||
}
|
||||
if cfg.Routing.Critic.Model != "critic-model" || cfg.Routing.Critic.NodeID != "brain-01" {
|
||||
t.Fatalf("critic route not updated: %+v", cfg.Routing.Critic)
|
||||
}
|
||||
if len(cfg.Ollama) != 1 || cfg.Ollama[0].ChatModel != "qwen-chat" || cfg.Ollama[0].EmbeddingModel != "qwen-embed" {
|
||||
t.Fatalf("ollama nodes not updated: %+v", cfg.Ollama)
|
||||
}
|
||||
// Omitting the optional learning section must preserve the current learning behavior.
|
||||
if cfg.Brain.AutoReward.Enabled != before.Brain.AutoReward.Enabled || cfg.Brain.AutoReward.Mode != before.Brain.AutoReward.Mode || cfg.Brain.Consolidation.UseLLM != before.Brain.Consolidation.UseLLM {
|
||||
t.Fatalf("omitted learning settings changed unexpectedly")
|
||||
}
|
||||
|
||||
var response modelRoutingSettings
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Routing.ChatNodeID != "brain-01" {
|
||||
t.Fatalf("response missing node pin: %+v", response.Routing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelRoutingRejectsUnknownPinnedNode(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
admin := s.store.Secrets().AdminToken
|
||||
body := `{"routing":{"chat_provider":"ollama","embedding_provider":"ollama","chat_node_id":"missing"}}`
|
||||
req := httptest.NewRequest(http.MethodPut, "/admin/api/model-routing", strings.NewReader(body))
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"neuroforge/internal/brain"
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
type validatedOutcomeRequest struct {
|
||||
OutcomeID string `json:"outcome_id"`
|
||||
RunID string `json:"run_id"`
|
||||
TicketID int64 `json:"ticket_id"`
|
||||
Decision string `json:"decision"` // accepted | corrected
|
||||
TicketInput string `json:"ticket_input"`
|
||||
ProposedReply string `json:"proposed_reply,omitempty"`
|
||||
ConfirmedReply string `json:"confirmed_reply"`
|
||||
CategoryID int64 `json:"category_id,omitempty"`
|
||||
CategoryName string `json:"category_name,omitempty"`
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||||
SupersedesID string `json:"supersedes_id,omitempty"`
|
||||
Actor string `json:"actor"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// integrationValidatedOutcome is deliberately narrower than /api/v1/learn.
|
||||
// Only a human-confirmed or human-corrected operational outcome can enter this
|
||||
// path, and trusted provenance is assigned server-side rather than accepted
|
||||
// from the caller.
|
||||
func (s *Server) integrationValidatedOutcome(w http.ResponseWriter, r *http.Request) {
|
||||
var q validatedOutcomeRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
q.OutcomeID = strings.TrimSpace(q.OutcomeID)
|
||||
q.RunID = strings.TrimSpace(q.RunID)
|
||||
q.Decision = strings.ToLower(strings.TrimSpace(q.Decision))
|
||||
q.TicketInput = strings.TrimSpace(q.TicketInput)
|
||||
q.ProposedReply = strings.TrimSpace(q.ProposedReply)
|
||||
q.ConfirmedReply = strings.TrimSpace(q.ConfirmedReply)
|
||||
q.CategoryName = strings.TrimSpace(q.CategoryName)
|
||||
q.KnowledgeID = strings.TrimSpace(q.KnowledgeID)
|
||||
q.SupersedesID = strings.TrimSpace(q.SupersedesID)
|
||||
q.Actor = strings.TrimSpace(q.Actor)
|
||||
q.Note = strings.TrimSpace(q.Note)
|
||||
|
||||
if q.OutcomeID == "" || q.RunID == "" || q.TicketID <= 0 || q.TicketInput == "" || q.ConfirmedReply == "" || q.Actor == "" {
|
||||
s.err(w, http.StatusBadRequest, errors.New("outcome_id, run_id, ticket_id, ticket_input, confirmed_reply and actor are required"))
|
||||
return
|
||||
}
|
||||
if q.Decision != "accepted" && q.Decision != "corrected" {
|
||||
s.err(w, http.StatusBadRequest, errors.New("decision must be accepted or corrected"))
|
||||
return
|
||||
}
|
||||
if q.Decision == "accepted" && q.ProposedReply == "" {
|
||||
s.err(w, http.StatusBadRequest, errors.New("accepted outcomes require proposed_reply"))
|
||||
return
|
||||
}
|
||||
if len([]rune(q.TicketInput)) > 12000 || len([]rune(q.ConfirmedReply)) > 12000 || len([]rune(q.Note)) > 4000 {
|
||||
s.err(w, http.StatusRequestEntityTooLarge, errors.New("validated outcome exceeds size limits"))
|
||||
return
|
||||
}
|
||||
|
||||
source := "glpi.outcome." + q.Decision
|
||||
confidence := 0.99
|
||||
if q.Decision == "corrected" {
|
||||
confidence = 1.0
|
||||
}
|
||||
var text strings.Builder
|
||||
text.WriteString("GLPI helpdesk outcome verified by a technician.\n\nProblem:\n")
|
||||
text.WriteString(q.TicketInput)
|
||||
text.WriteString("\n\nVerified solution:\n")
|
||||
text.WriteString(q.ConfirmedReply)
|
||||
if q.CategoryName != "" || q.CategoryID > 0 {
|
||||
text.WriteString("\n\nCategory: ")
|
||||
if q.CategoryName != "" {
|
||||
text.WriteString(q.CategoryName)
|
||||
}
|
||||
if q.CategoryID > 0 {
|
||||
text.WriteString(" (#")
|
||||
text.WriteString(strconv.FormatInt(q.CategoryID, 10))
|
||||
text.WriteString(")")
|
||||
}
|
||||
}
|
||||
|
||||
tags := []string{"integration:glpi", "validated:human", "outcome:" + q.Decision, "ticket:" + strconv.FormatInt(q.TicketID, 10), "run:" + q.RunID}
|
||||
if q.CategoryID > 0 {
|
||||
tags = append(tags, "category:"+strconv.FormatInt(q.CategoryID, 10))
|
||||
}
|
||||
if q.KnowledgeID != "" {
|
||||
tags = append(tags, "knowledge:"+q.KnowledgeID)
|
||||
}
|
||||
if q.SupersedesID != "" {
|
||||
tags = append(tags, "supersedes-outcome:"+q.SupersedesID)
|
||||
}
|
||||
|
||||
m, err := s.brain.Learn(r.Context(), brain.LearnRequest{
|
||||
Text: text.String(),
|
||||
Kind: "validated_outcome",
|
||||
MemoryType: core.MemorySemantic,
|
||||
Tags: tags,
|
||||
Salience: 1.2,
|
||||
Confidence: confidence,
|
||||
Source: source,
|
||||
Actor: q.Actor,
|
||||
SourceID: q.OutcomeID,
|
||||
SourceURI: fmt.Sprintf("glpi://Ticket/%d#run=%s", q.TicketID, q.RunID),
|
||||
Note: q.Note,
|
||||
})
|
||||
if err != nil {
|
||||
s.err(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
supersededMemoryID := ""
|
||||
if q.SupersedesID != "" {
|
||||
if prior, ok := s.store.MemoryByProvenanceSourceID(q.SupersedesID); ok && prior.Memory.ID != m.ID {
|
||||
if err := s.store.SupersedeMemory(prior.Memory.ID, m.ID); err != nil {
|
||||
s.err(w, http.StatusInternalServerError, fmt.Errorf("persist outcome supersession: %w", err))
|
||||
return
|
||||
}
|
||||
supersededMemoryID = prior.Memory.ID
|
||||
}
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{
|
||||
Type: "integration.outcome_validated", MemoryID: m.ID,
|
||||
Summary: "Human-confirmed GLPI ticket outcome learned",
|
||||
Reason: "technician explicitly accepted or corrected the AI proposal",
|
||||
Actor: q.Actor,
|
||||
Metadata: map[string]string{"source": source, "outcome_id": q.OutcomeID, "run_id": q.RunID, "ticket_id": strconv.FormatInt(q.TicketID, 10), "decision": q.Decision, "knowledge_id": q.KnowledgeID, "supersedes_outcome_id": q.SupersedesID},
|
||||
})
|
||||
s.json(w, http.StatusCreated, map[string]any{"memory": m, "outcome_id": q.OutcomeID, "decision": q.Decision, "source": source, "superseded_memory_id": supersededMemoryID})
|
||||
}
|
||||
|
||||
type validatedOutcomeSearchRequest struct {
|
||||
Text string `json:"text"`
|
||||
K int `json:"k"`
|
||||
MinSimilarity float64 `json:"min_similarity,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Server) integrationValidatedOutcomeSearch(w http.ResponseWriter, r *http.Request) {
|
||||
var q validatedOutcomeSearchRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
q.Text = strings.TrimSpace(q.Text)
|
||||
if q.Text == "" {
|
||||
s.err(w, http.StatusBadRequest, errors.New("text is required"))
|
||||
return
|
||||
}
|
||||
if len([]rune(q.Text)) > 12000 {
|
||||
s.err(w, http.StatusRequestEntityTooLarge, errors.New("search text exceeds size limit"))
|
||||
return
|
||||
}
|
||||
if q.K <= 0 {
|
||||
q.K = 8
|
||||
}
|
||||
if q.K > 50 {
|
||||
q.K = 50
|
||||
}
|
||||
if q.MinSimilarity == 0 {
|
||||
q.MinSimilarity = 0.50
|
||||
}
|
||||
hits, err := s.brain.SearchByProvenanceSources(r.Context(), q.Text, q.K, q.MinSimilarity, "glpi.outcome.accepted", "glpi.outcome.corrected")
|
||||
if err != nil {
|
||||
s.err(w, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusOK, hits)
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidatedOutcomeLearnsTrustedProvenance(t *testing.T) {
|
||||
fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/embed" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{{1, 0, 0}}})
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer fake.Close()
|
||||
|
||||
s, _ := newMetricsTestServer(t)
|
||||
cfg := s.store.Config()
|
||||
cfg.Ollama[0].BaseURL = fake.URL
|
||||
cfg.Brain.ExternalRelinkWorker = false
|
||||
cfg.Brain.LearningPolicy.SourceTrust["glpi.outcome.accepted"] = 1
|
||||
cfg.Brain.LearningPolicy.SourceTrust["glpi.outcome.corrected"] = 1
|
||||
cfg.Brain.LearningPolicy.LearnChatResponses = false
|
||||
if err := s.store.UpdateConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sec := s.store.Secrets()
|
||||
|
||||
body := `{"outcome_id":"out-1","run_id":"run-1","ticket_id":42,"decision":"accepted","ticket_input":"VPN verbindet nicht","proposed_reply":"VPN Client neu starten","confirmed_reply":"VPN Client neu starten","category_id":5,"category_name":"VPN","knowledge_id":"kb-vpn","actor":"tech-a"}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/integrations/outcomes", strings.NewReader(body))
|
||||
req.Header.Set("Authorization", "Bearer "+sec.AppAPIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusCreated {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var out struct {
|
||||
Memory struct {
|
||||
ID string `json:"id"`
|
||||
Provenance struct {
|
||||
Source string `json:"source"`
|
||||
Actor string `json:"actor"`
|
||||
SourceID string `json:"source_id"`
|
||||
} `json:"provenance"`
|
||||
} `json:"memory"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Memory.ID == "" || out.Memory.Provenance.Source != "glpi.outcome.accepted" || out.Memory.Provenance.Actor != "tech-a" || out.Memory.Provenance.SourceID != "out-1" {
|
||||
t.Fatalf("unexpected outcome memory: %#v body=%s", out, rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatedOutcomeRejectsUnconfirmedDecision(t *testing.T) {
|
||||
s, _ := newMetricsTestServer(t)
|
||||
sec := s.store.Secrets()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/integrations/outcomes", strings.NewReader(`{"outcome_id":"o","run_id":"r","ticket_id":1,"decision":"rejected","ticket_input":"x","confirmed_reply":"y","actor":"tech"}`))
|
||||
req.Header.Set("Authorization", "Bearer "+sec.AppAPIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatedOutcomeCorrectionSupersedesPriorMemoryAndSearchesOnlyActiveRevision(t *testing.T) {
|
||||
fake := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/embed" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
text := strings.ToLower(strings.TrimSpace(fmt.Sprint(body["input"])))
|
||||
vec := []float32{1, 0, 0}
|
||||
if strings.Contains(text, "korrigierte loesung") {
|
||||
vec = []float32{0, 1, 0}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{vec}})
|
||||
}))
|
||||
defer fake.Close()
|
||||
|
||||
s, _ := newMetricsTestServer(t)
|
||||
cfg := s.store.Config()
|
||||
cfg.Ollama[0].BaseURL = fake.URL
|
||||
cfg.Brain.ExternalRelinkWorker = false
|
||||
cfg.Brain.LearningPolicy.DuplicateSimilarity = 0.99999
|
||||
cfg.Brain.LearningPolicy.SourceTrust["glpi.outcome.accepted"] = 1
|
||||
cfg.Brain.LearningPolicy.SourceTrust["glpi.outcome.corrected"] = 1
|
||||
if err := s.store.UpdateConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sec := s.store.Secrets()
|
||||
post := func(body string) map[string]any {
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/integrations/outcomes", strings.NewReader(body))
|
||||
req.Header.Set("Authorization", "Bearer "+sec.AppAPIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusCreated {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
oldOut := post(`{"outcome_id":"old","run_id":"r1","ticket_id":7,"decision":"accepted","ticket_input":"Drucker druckt nicht","proposed_reply":"Treiber neu starten","confirmed_reply":"Treiber neu starten","actor":"tech"}`)
|
||||
oldID := oldOut["memory"].(map[string]any)["id"].(string)
|
||||
newOut := post(`{"outcome_id":"new","run_id":"r2","ticket_id":7,"decision":"corrected","ticket_input":"Drucker druckt nicht","proposed_reply":"Treiber neu starten","confirmed_reply":"Korrigierte Loesung: Printserver Queue bereinigen","supersedes_id":"old","actor":"tech"}`)
|
||||
newID := newOut["memory"].(map[string]any)["id"].(string)
|
||||
if oldID == newID {
|
||||
t.Fatal("correction must create a distinct memory")
|
||||
}
|
||||
var oldStatus string
|
||||
for _, m := range s.store.MemoriesSnapshot() {
|
||||
if m.ID == oldID {
|
||||
oldStatus = m.Status
|
||||
}
|
||||
}
|
||||
if oldStatus != "superseded" {
|
||||
t.Fatalf("old status=%q, want superseded", oldStatus)
|
||||
}
|
||||
newMem, ok := s.store.GetMemory(newID)
|
||||
if !ok {
|
||||
t.Fatal("corrected memory missing")
|
||||
}
|
||||
if strings.Contains(newMem.Text, "Treiber neu starten") {
|
||||
t.Fatalf("superseded AI proposal leaked into active corrected memory: %q", newMem.Text)
|
||||
}
|
||||
|
||||
search := httptest.NewRequest(http.MethodPost, "/api/v1/integrations/outcomes/search", strings.NewReader(`{"text":"Drucker korrigierte Loesung","k":10,"min_similarity":0}`))
|
||||
search.Header.Set("Authorization", "Bearer "+sec.AppAPIKey)
|
||||
search.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rr, search)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("search status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var hits []struct {
|
||||
Memory struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"memory"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &hits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range hits {
|
||||
if h.Memory.ID == oldID {
|
||||
t.Fatal("superseded outcome leaked into active retrieval")
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, h := range hits {
|
||||
found = found || h.Memory.ID == newID
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("corrected outcome not found; hits=%#v", hits)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func (s *Server) goalResearchLive(w http.ResponseWriter, r *http.Request) {
|
||||
goalID := r.PathValue("id")
|
||||
run, ok := s.store.LatestResearchRun(goalID)
|
||||
if !ok {
|
||||
s.json(w, http.StatusOK, map[string]any{"run": nil, "events": []any{}, "reset": false})
|
||||
return
|
||||
}
|
||||
after, _ := strconv.ParseUint(r.URL.Query().Get("after"), 10, 64)
|
||||
clientRunID := r.URL.Query().Get("run_id")
|
||||
reset := clientRunID != "" && clientRunID != run.ID
|
||||
if reset {
|
||||
after = 0
|
||||
}
|
||||
events := run.Events
|
||||
if after > 0 {
|
||||
filtered := events[:0:0]
|
||||
for _, ev := range events {
|
||||
if ev.Seq > after {
|
||||
filtered = append(filtered, ev)
|
||||
}
|
||||
}
|
||||
events = filtered
|
||||
}
|
||||
// Return metadata separately from the event delta so a 1s UI poll stays
|
||||
// bounded even when the persisted run keeps a larger audit tail.
|
||||
run.Events = nil
|
||||
s.json(w, http.StatusOK, map[string]any{"run": run, "events": events, "reset": reset})
|
||||
}
|
||||
|
||||
func (s *Server) goalResearchHistory(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 50 {
|
||||
limit = 10
|
||||
}
|
||||
runs := s.store.ResearchRunsSnapshot(r.PathValue("id"), limit)
|
||||
for i := range runs {
|
||||
// History cards need summary/stats, not hundreds of event rows. The live
|
||||
// endpoint exposes the latest run's detailed trace on demand.
|
||||
runs[i].Events = nil
|
||||
}
|
||||
s.json(w, http.StatusOK, runs)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func TestGoalResearchLiveReturnsEventDelta(t *testing.T) {
|
||||
srv, _ := newMetricsTestServer(t)
|
||||
g := core.Goal{Title: "Live NVIDIA", Status: core.GoalActive, Priority: 50}
|
||||
if err := srv.store.UpsertGoal(&g); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, err := srv.store.StartResearchRun(g.ID, g.Title)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ev1, err := srv.store.AddResearchEvent(run.ID, core.ResearchEvent{Type: "query.planned", Query: "NVIDIA CUDA", Status: "ok"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = srv.store.AddResearchEvent(run.ID, core.ResearchEvent{Type: "search.result", URL: "https://example.com", Title: "Example", Status: "ok"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/goals/"+g.ID+"/research/live?run_id="+run.ID+"&after="+jsonNumber(ev1.Seq), nil)
|
||||
req.Header.Set("X-Admin-Token", srv.store.Secrets().AdminToken)
|
||||
rr := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var out struct {
|
||||
Run core.ResearchRun `json:"run"`
|
||||
Events []core.ResearchEvent `json:"events"`
|
||||
Reset bool `json:"reset"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Reset || out.Run.ID != run.ID || len(out.Events) != 1 || out.Events[0].Type != "search.result" {
|
||||
t.Fatalf("unexpected live delta %#v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func jsonNumber(v uint64) string {
|
||||
b, _ := json.Marshal(v)
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func (s *Server) goalsList(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, http.StatusOK, s.store.GoalsSnapshot())
|
||||
}
|
||||
|
||||
func (s *Server) goalsCreate(w http.ResponseWriter, r *http.Request) {
|
||||
var g core.Goal
|
||||
if err := decode(r, &g); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.UpsertGoal(&g); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusCreated, g)
|
||||
}
|
||||
|
||||
func (s *Server) goalsGet(w http.ResponseWriter, r *http.Request) {
|
||||
g, ok := s.store.GetGoal(r.PathValue("id"))
|
||||
if !ok {
|
||||
s.err(w, 404, errors.New("goal not found"))
|
||||
return
|
||||
}
|
||||
s.json(w, 200, g)
|
||||
}
|
||||
|
||||
func (s *Server) goalsPut(w http.ResponseWriter, r *http.Request) {
|
||||
var g core.Goal
|
||||
if err := decode(r, &g); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
g.ID = r.PathValue("id")
|
||||
if old, ok := s.store.GetGoal(g.ID); ok && g.CreatedAt.IsZero() {
|
||||
g.CreatedAt = old.CreatedAt
|
||||
}
|
||||
if err := s.store.UpsertGoal(&g); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, g)
|
||||
}
|
||||
|
||||
func (s *Server) goalsDelete(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
goal, _ := s.store.GetGoal(id)
|
||||
if err := s.store.DeleteGoal(id); err != nil {
|
||||
s.err(w, 404, err)
|
||||
return
|
||||
}
|
||||
meta := map[string]string{"goal_id": id}
|
||||
if goal != nil {
|
||||
meta["title"] = goal.Title
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "goal.deleted", Summary: "Goal deleted", Reason: "administrator/user deleted goal", Actor: "goal-control", Metadata: meta})
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
|
||||
func (s *Server) goalPause(w http.ResponseWriter, r *http.Request) {
|
||||
g, err := s.store.PauseGoal(r.PathValue("id"))
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "goal.paused", Summary: "Goal paused: " + g.Title, Reason: "manual pause", Actor: "goal-control", Metadata: map[string]string{"goal_id": g.ID}})
|
||||
s.json(w, 200, g)
|
||||
}
|
||||
|
||||
func (s *Server) goalResume(w http.ResponseWriter, r *http.Request) {
|
||||
g, err := s.store.ResumeGoal(r.PathValue("id"))
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "goal.resumed", Summary: "Goal resumed: " + g.Title, Reason: "manual resume", Actor: "goal-control", Metadata: map[string]string{"goal_id": g.ID}})
|
||||
s.json(w, 200, g)
|
||||
}
|
||||
|
||||
func (s *Server) goalCycle(w http.ResponseWriter, r *http.Request) {
|
||||
cycle, err := s.brain.RunGoalCycle(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, cycle)
|
||||
}
|
||||
|
||||
func (s *Server) learningCycles(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
s.json(w, 200, s.store.RecentLearningCycles(limit))
|
||||
}
|
||||
|
||||
func (s *Server) conflicts(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.ConflictsSnapshot())
|
||||
}
|
||||
|
||||
func (s *Server) adminRetention(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := s.store.RunRetention(time.Now().UTC())
|
||||
if err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) adminAutonomy(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.brain.RunAutonomy(r.Context()))
|
||||
}
|
||||
|
||||
func (s *Server) adminRebalance(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
if r.ContentLength != 0 {
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
s.json(w, 200, s.brain.RebalanceShards(ctx, q.DryRun))
|
||||
}
|
||||
|
||||
func (s *Server) adminCheckpoint(w http.ResponseWriter, r *http.Request) {
|
||||
if err := s.store.ForceCheckpoint(); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, map[string]any{"ok": true, "wal": s.store.WALStatus()})
|
||||
}
|
||||
|
||||
func (s *Server) adminWAL(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.WALStatus())
|
||||
}
|
||||
|
||||
func (s *Server) adminResolveConflict(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
TruthKey string `json:"truth_key"`
|
||||
WinnerID string `json:"winner_id"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if q.TruthKey == "" || q.WinnerID == "" {
|
||||
s.err(w, 400, errors.New("truth_key and winner_id are required"))
|
||||
return
|
||||
}
|
||||
if err := s.store.ResolveConflict(q.TruthKey, q.WinnerID); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
_ = s.store.AddKnowledgeEvent(core.KnowledgeEvent{Type: "conflict.resolved", MemoryID: q.WinnerID, Summary: "Knowledge conflict resolved by administrator", Reason: "winner selected for truth key", Actor: "admin", Metadata: map[string]string{"truth_key": q.TruthKey}})
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func (s *Server) clusterPrepare(w http.ResponseWriter, r *http.Request) {
|
||||
var entry core.ClusterEntry
|
||||
if err := decode(r, &entry); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.brain.ClusterPrepare(entry); err != nil {
|
||||
s.err(w, 409, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, map[string]any{"ok": true, "entry_id": entry.ID, "index": entry.Index})
|
||||
}
|
||||
|
||||
func (s *Server) clusterCommit(w http.ResponseWriter, r *http.Request) {
|
||||
var entry core.ClusterEntry
|
||||
if err := decode(r, &entry); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.brain.ClusterCommit(entry); err != nil {
|
||||
s.err(w, 409, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, map[string]any{"ok": true, "entry_id": entry.ID, "index": entry.Index})
|
||||
}
|
||||
|
||||
func (s *Server) clusterAbort(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if q.ID == "" {
|
||||
s.err(w, 400, errors.New("id required"))
|
||||
return
|
||||
}
|
||||
if err := s.brain.ClusterAbort(q.ID); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, map[string]bool{"ok": true})
|
||||
}
|
||||
|
||||
func (s *Server) clusterProposeMemory(w http.ResponseWriter, r *http.Request) {
|
||||
var m core.Memory
|
||||
if err := decode(r, &m); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.brain.ClusterProposeMemory(ctx, &m); err != nil {
|
||||
s.err(w, 503, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 201, m)
|
||||
}
|
||||
|
||||
func (s *Server) clusterDecision(w http.ResponseWriter, r *http.Request) {
|
||||
d, ok := s.store.ClusterDecision(r.PathValue("id"))
|
||||
if !ok {
|
||||
s.err(w, 404, errors.New("cluster decision not found"))
|
||||
return
|
||||
}
|
||||
s.json(w, 200, d)
|
||||
}
|
||||
|
||||
func (s *Server) clusterStatus(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.ClusterStatus())
|
||||
}
|
||||
|
||||
func (s *Server) adminClusterRepair(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
s.json(w, 200, s.brain.RepairCluster(ctx))
|
||||
}
|
||||
|
||||
func (s *Server) adminCompactSegments(w http.ResponseWriter, r *http.Request) {
|
||||
stats, err := s.store.CompactMemorySegments()
|
||||
if err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.ForceCheckpoint(); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, stats)
|
||||
}
|
||||
|
||||
func (s *Server) adminStorageStatus(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, map[string]any{
|
||||
"wal": s.store.WALStatus(),
|
||||
"memory_segments": s.store.SegmentStats(),
|
||||
"index_snapshot": s.store.IndexSnapshotStatus(),
|
||||
"tiering": s.store.TieringStatus(),
|
||||
"cluster_log": s.store.ClusterLogStats(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func (s *Server) clusterRequestVote(w http.ResponseWriter, r *http.Request) {
|
||||
var req core.ClusterVoteRequest
|
||||
if err := decode(r, &req); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
resp, err := s.brain.ClusterVote(req)
|
||||
if err != nil {
|
||||
s.err(w, 409, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, resp)
|
||||
}
|
||||
|
||||
func (s *Server) clusterHeartbeat(w http.ResponseWriter, r *http.Request) {
|
||||
var req core.ClusterHeartbeat
|
||||
if err := decode(r, &req); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
resp, err := s.brain.ClusterHeartbeat(req)
|
||||
if err != nil {
|
||||
s.err(w, 409, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, resp)
|
||||
}
|
||||
|
||||
func (s *Server) adminTierStorage(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.TierMemoryBodies(time.Now().UTC()))
|
||||
}
|
||||
|
||||
func (s *Server) adminMergeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := s.store.CompactIndexSegments()
|
||||
if err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package httpapi
|
||||
|
||||
import "net/http"
|
||||
|
||||
func (s *Server) adminDiskANNStatus(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, 200, s.store.DiskANNStatus())
|
||||
}
|
||||
|
||||
func (s *Server) adminDiskANNBuild(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := s.store.RebuildDiskANN()
|
||||
if err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"neuroforge/internal/brain"
|
||||
)
|
||||
|
||||
func (s *Server) ingestText(w http.ResponseWriter, r *http.Request) {
|
||||
var q brain.IngestTextRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
out, err := s.brain.IngestText(r.Context(), q)
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusCreated, out)
|
||||
}
|
||||
|
||||
func (s *Server) ingestDocument(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.store.Config()
|
||||
max := cfg.Ingestion.MaxDocumentBytes
|
||||
if max <= 0 {
|
||||
max = 25 << 20
|
||||
}
|
||||
// requestLimits already caps the full body; this is a second, route-specific
|
||||
// bound on the actual uploaded file.
|
||||
if err := r.ParseMultipartForm(max + (1 << 20)); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
f, hdr, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
s.err(w, 400, errors.New("multipart field 'file' is required"))
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(f, max+1))
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if int64(len(data)) > max {
|
||||
s.err(w, http.StatusRequestEntityTooLarge, errors.New("document exceeds configured max_document_bytes"))
|
||||
return
|
||||
}
|
||||
trust, _ := strconv.ParseFloat(strings.TrimSpace(r.FormValue("trust")), 64)
|
||||
tags := splitCSV(r.FormValue("tags"))
|
||||
name := filepath.Base(hdr.Filename)
|
||||
out, err := s.brain.IngestDocument(r.Context(), name, hdr.Header.Get("Content-Type"), r.FormValue("title"), data, tags, trust)
|
||||
if err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
s.json(w, http.StatusCreated, out)
|
||||
}
|
||||
|
||||
func (s *Server) sourcesList(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if limit > 1000 {
|
||||
limit = 1000
|
||||
}
|
||||
s.json(w, 200, s.store.SourcesSnapshot(limit))
|
||||
}
|
||||
|
||||
func (s *Server) sourceGet(w http.ResponseWriter, r *http.Request) {
|
||||
x, ok := s.store.GetSource(r.PathValue("id"))
|
||||
if !ok {
|
||||
s.err(w, 404, errors.New("source not found"))
|
||||
return
|
||||
}
|
||||
s.json(w, 200, x)
|
||||
}
|
||||
|
||||
func (s *Server) researchSearch(w http.ResponseWriter, r *http.Request) {
|
||||
var q brain.ResearchRequest
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
out, err := s.brain.Research(r.Context(), q)
|
||||
if err != nil {
|
||||
s.err(w, 502, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
|
||||
func (s *Server) adminResearchGet(w http.ResponseWriter, r *http.Request) {
|
||||
c := s.store.Config()
|
||||
sec := s.store.Secrets()
|
||||
s.json(w, 200, map[string]any{"research": c.Research, "ingestion": c.Ingestion, "autonomy": map[string]any{"enabled": c.Autonomy.Enabled, "run_on_goal_create": c.Autonomy.RunOnGoalCreate, "default_goal_interval_minutes": c.Autonomy.DefaultGoalIntervalMinutes}, "searxng_auth_configured": strings.TrimSpace(sec.SearXNGAuthHeader) != ""})
|
||||
}
|
||||
|
||||
func (s *Server) adminResearchPut(w http.ResponseWriter, r *http.Request) {
|
||||
var patch map[string]json.RawMessage
|
||||
if err := decode(r, &patch); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
c := s.store.Config()
|
||||
base, _ := json.Marshal(c)
|
||||
var root map[string]json.RawMessage
|
||||
_ = json.Unmarshal(base, &root)
|
||||
for _, key := range []string{"research", "ingestion"} {
|
||||
if v, ok := patch[key]; ok {
|
||||
root[key] = v
|
||||
}
|
||||
}
|
||||
if raw, ok := patch["autonomy"]; ok {
|
||||
var a map[string]json.RawMessage
|
||||
_ = json.Unmarshal(root["autonomy"], &a)
|
||||
var ap map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &ap); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
for k, v := range ap {
|
||||
a[k] = v
|
||||
}
|
||||
root["autonomy"], _ = json.Marshal(a)
|
||||
}
|
||||
merged, _ := json.Marshal(root)
|
||||
if err := json.Unmarshal(merged, &c); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.ValidateConfig(c); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateConfig(c); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
if raw, ok := patch["searxng_auth_header"]; ok {
|
||||
var auth string
|
||||
if json.Unmarshal(raw, &auth) == nil && strings.TrimSpace(auth) != "" {
|
||||
for _, r := range auth {
|
||||
if r < 0x20 || r > 0x7e {
|
||||
s.err(w, 400, errors.New("searxng_auth_header must contain printable ASCII only"))
|
||||
return
|
||||
}
|
||||
}
|
||||
sec := s.store.Secrets()
|
||||
sec.SearXNGAuthHeader = auth
|
||||
if err := s.store.UpdateSecrets(sec); err != nil {
|
||||
s.err(w, 500, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
s.adminResearchGet(w, r)
|
||||
}
|
||||
|
||||
func (s *Server) adminResearchTest(w http.ResponseWriter, r *http.Request) {
|
||||
var q struct {
|
||||
Query string `json:"query"`
|
||||
}
|
||||
if r.ContentLength != 0 {
|
||||
if err := decode(r, &q); err != nil {
|
||||
s.err(w, 400, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(q.Query) == "" {
|
||||
q.Query = "NVIDIA GPU CUDA"
|
||||
}
|
||||
out, err := s.brain.Research(r.Context(), brain.ResearchRequest{Query: q.Query, Learn: false, FetchPages: false, MaxResults: 5})
|
||||
if err != nil {
|
||||
s.err(w, 502, err)
|
||||
return
|
||||
}
|
||||
s.json(w, 200, out)
|
||||
}
|
||||
|
||||
func splitCSV(s string) []string {
|
||||
var out []string
|
||||
for _, x := range strings.Split(s, ",") {
|
||||
x = strings.TrimSpace(x)
|
||||
if x != "" {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"neuroforge/internal/core"
|
||||
)
|
||||
|
||||
func TestGoalPauseResumeDeleteEndpoints(t *testing.T) {
|
||||
srv, _ := newMetricsTestServer(t)
|
||||
admin := srv.store.Secrets().AdminToken
|
||||
g := core.Goal{Title: "NVIDIA research", Description: "learn NVIDIA", Status: core.GoalActive, Priority: 70, AutoRun: true, IntervalMinutes: 10}
|
||||
if err := srv.store.UpsertGoal(&g); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
call := func(method, path string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(`{}`))
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rr, req)
|
||||
return rr
|
||||
}
|
||||
if rr := call(http.MethodPost, "/api/v1/goals/"+g.ID+"/pause"); rr.Code != http.StatusOK {
|
||||
t.Fatalf("pause status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
paused, _ := srv.store.GetGoal(g.ID)
|
||||
if paused.Status != core.GoalPaused || !paused.NextCycleAt.IsZero() {
|
||||
t.Fatalf("goal not paused correctly %#v", paused)
|
||||
}
|
||||
if rr := call(http.MethodPost, "/api/v1/goals/"+g.ID+"/cycle"); rr.Code == http.StatusOK {
|
||||
t.Fatal("paused goal must not run a manual cycle")
|
||||
}
|
||||
if rr := call(http.MethodPost, "/api/v1/goals/"+g.ID+"/resume"); rr.Code != http.StatusOK {
|
||||
t.Fatalf("resume status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
active, _ := srv.store.GetGoal(g.ID)
|
||||
if active.Status != core.GoalActive {
|
||||
t.Fatalf("goal not active after resume %#v", active)
|
||||
}
|
||||
if rr := call(http.MethodDelete, "/api/v1/goals/"+g.ID); rr.Code != http.StatusOK {
|
||||
t.Fatalf("delete status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if _, ok := srv.store.GetGoal(g.ID); ok {
|
||||
t.Fatal("goal still exists after delete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchIngestsSearXNGDocumentResult(t *testing.T) {
|
||||
var doc bytes.Buffer
|
||||
zw := zip.NewWriter(&doc)
|
||||
w, err := zw.Create("word/document.xml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`<?xml version="1.0"?><w:document xmlns:w="x"><w:body><w:p><w:r><w:t>NVIDIA CUDA document evidence from SearXNG.</w:t></w:r></w:p></w:body></w:document>`))
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
docSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="cuda.docx"`)
|
||||
_, _ = w.Write(doc.Bytes())
|
||||
}))
|
||||
defer docSrv.Close()
|
||||
searx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"results": []map[string]any{{
|
||||
"title": "CUDA paper", "url": docSrv.URL + "/cuda.docx", "content": "document result", "engine": "test", "template": "file.html", "filename": "cuda.docx", "mimetype": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
}}})
|
||||
}))
|
||||
defer searx.Close()
|
||||
ollama := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/embed" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{{1, 0, 0, 0}}, "prompt_eval_count": 1})
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ollama.Close()
|
||||
|
||||
srv, _ := newMetricsTestServer(t)
|
||||
cfg := srv.store.Config()
|
||||
cfg.Ollama[0].BaseURL = ollama.URL
|
||||
cfg.Routing.EmbeddingProvider = "ollama"
|
||||
cfg.Brain.ExternalRelinkWorker = false
|
||||
cfg.Research.Enabled = true
|
||||
cfg.Research.SearXNG.Enabled = true
|
||||
cfg.Research.SearXNG.BaseURL = searx.URL
|
||||
cfg.Research.WebFetch.Enabled = true
|
||||
cfg.Research.WebFetch.AllowPrivateTargets = true
|
||||
if err := srv.store.UpdateConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/research", strings.NewReader(`{"query":"CUDA docs","learn":true,"fetch_pages":true,"max_results":1,"max_pages":1}`))
|
||||
req.Header.Set("X-Admin-Token", srv.store.Secrets().AdminToken)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("research status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out["documents_ingested"].(float64) != 1 {
|
||||
t.Fatalf("expected one ingested document, got %s", rr.Body.String())
|
||||
}
|
||||
found := false
|
||||
for _, src := range srv.store.SourcesSnapshot(20) {
|
||||
if src.Type == "research-document" && strings.Contains(src.FileName, "cuda.docx") && src.ChunkCount > 0 {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("research document source not persisted: %#v", srv.store.SourcesSnapshot(20))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestV8TextDocumentAndResearchEndpoints(t *testing.T) {
|
||||
ollama := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/embed" {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float32{{1, 0, 0, 0}}, "prompt_eval_count": 1})
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer ollama.Close()
|
||||
searx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("format") != "json" {
|
||||
t.Errorf("format=%q", r.URL.Query().Get("format"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"results": []map[string]any{{"title": "GPU", "url": "https://example.com/gpu", "content": "GPU evidence from search", "engine": "test"}}})
|
||||
}))
|
||||
defer searx.Close()
|
||||
|
||||
srv, _ := newMetricsTestServer(t)
|
||||
cfg := srv.store.Config()
|
||||
cfg.Ollama[0].BaseURL = ollama.URL
|
||||
cfg.Routing.EmbeddingProvider = "ollama"
|
||||
cfg.Brain.ExternalRelinkWorker = false
|
||||
cfg.Research.Enabled = true
|
||||
cfg.Research.SearXNG.Enabled = true
|
||||
cfg.Research.SearXNG.BaseURL = searx.URL
|
||||
cfg.Research.WebFetch.Enabled = false
|
||||
if err := srv.store.UpdateConfig(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := srv.store.Secrets().AdminToken
|
||||
|
||||
// Plain text source.
|
||||
body := `{"title":"manual","text":"NVIDIA CUDA fact","trust":0.9}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/ingest/text", strings.NewReader(body))
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusCreated {
|
||||
t.Fatalf("text ingest status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
// Multipart text document.
|
||||
var mb bytes.Buffer
|
||||
mw := multipart.NewWriter(&mb)
|
||||
fw, _ := mw.CreateFormFile("file", "note.txt")
|
||||
_, _ = fw.Write([]byte("A second document fact about CUDA."))
|
||||
_ = mw.WriteField("title", "note")
|
||||
_ = mw.Close()
|
||||
req = httptest.NewRequest(http.MethodPost, "/api/v1/ingest/document", &mb)
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
rr = httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusCreated {
|
||||
t.Fatalf("document ingest status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
// SearXNG-backed research learns the snippet without web-page fetching.
|
||||
req = httptest.NewRequest(http.MethodPost, "/api/v1/research", strings.NewReader(`{"query":"NVIDIA CUDA","learn":true,"fetch_pages":false}`))
|
||||
req.Header.Set("X-Admin-Token", admin)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr = httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("research status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if len(srv.store.SourcesSnapshot(10)) < 3 {
|
||||
t.Fatalf("expected source records, got %#v", srv.store.SourcesSnapshot(10))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user