Update 1.5.0

2026-08-27 07:51:39 +02:00
parent 1decb831d6
commit 8c67c7a7fa
58 changed files with 10769 additions and 627 deletions
--- a/scripts/backup-data.sh
+++ b/scripts/backup-data.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+set -eu
+ENV_FILE=${ENV_FILE:-.env}
+DEST_ROOT=${1:-./release-backups}
+[ -f "$ENV_FILE" ] || { echo "backup: missing $ENV_FILE" >&2; exit 1; }
+getv() { sed -n "s/^$1=//p" "$ENV_FILE" | tail -n1 | tr -d '\r'; }
+tag=$(getv IMAGE_TAG)
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+dest="$DEST_ROOT/$stamp"
+mkdir -p "$dest"
+
+./scripts/preflight.sh
+restart() { docker compose --env-file "$ENV_FILE" up -d knowledge neuroforge neuroforge-worker agent >/dev/null 2>&1 || true; }
+trap restart EXIT INT TERM
+
+echo "backup: entering short maintenance stop"
+docker compose --env-file "$ENV_FILE" stop neuroforge-worker agent neuroforge knowledge
+docker compose --env-file "$ENV_FILE" create neuroforge agent >/dev/null
+mkdir -p "$dest/neuroforge-data" "$dest/agent-data" "$dest/knowledge" "$dest/staging" "$dest/knowledge-backups"
+docker compose --env-file "$ENV_FILE" cp neuroforge:/app/data/. "$dest/neuroforge-data/"
+docker compose --env-file "$ENV_FILE" cp agent:/app/data/. "$dest/agent-data/"
+
+copy_host() {
+  src=$1; out=$2
+  [ -d "$src" ] || return 0
+  cp -a "$src"/. "$out"/
+}
+copy_host "$(getv KB_DATA_PATH)" "$dest/knowledge"
+copy_host "$(getv KB_STAGING_PATH)" "$dest/staging"
+copy_host "$(getv KB_BACKUP_PATH)" "$dest/knowledge-backups"
+printf 'IMAGE_TAG=%s\nCREATED_AT=%s\n' "$tag" "$stamp" > "$dest/MANIFEST"
+( cd "$dest" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS )
+echo "backup: verified snapshot written to $dest"
--- a/scripts/check-compose-env.py
+++ b/scripts/check-compose-env.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Fail if production Compose broadens Agent secrets or misses runtime env knobs."""
+from __future__ import annotations
+import re
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except Exception as exc:  # PyYAML is intentionally only a release-dev dependency.
+    print(f"compose-env-check: PyYAML unavailable: {exc}", file=sys.stderr)
+    raise SystemExit(2)
+
+root = Path(__file__).resolve().parents[1]
+compose = yaml.safe_load((root / "docker-compose.yml").read_text())
+agent = compose["services"]["agent"]
+if "env_file" in agent:
+    raise SystemExit("compose-env-check: production Agent must not use env_file")
+actual = set((agent.get("environment") or {}).keys())
+
+# Canonical host-config variables are those documented in .env.example and
+# literally referenced by non-test Agent runtime source. Internal service values
+# are added separately below.
+documented: set[str] = set()
+for line in (root / ".env.example").read_text().splitlines():
+    match = re.match(r"([A-Z][A-Z0-9_]+)=", line)
+    if match:
+        documented.add(match.group(1))
+source = "\n".join(
+    p.read_text(errors="ignore")
+    for p in (root / "services/agent").rglob("*.go")
+    if not p.name.endswith("_test.go")
+)
+expected = {name for name in documented if re.search(rf'"{re.escape(name)}"', source)}
+expected.update({"NEUROFORGE_URL", "NEUROFORGE_API_KEY", "BRAIN_ACTIVITY_URL", "BRAIN_ACTIVITY_API_KEY"})
+missing = sorted(expected - actual)
+if missing:
+    print("compose-env-check: Agent runtime variables missing from explicit environment:", file=sys.stderr)
+    print("\n".join(f"  {name}" for name in missing), file=sys.stderr)
+    raise SystemExit(1)
+
+forbidden = {
+    "NEUROFORGE_ADMIN_TOKEN",
+    "NEUROFORGE_WORKER_TOKEN",
+    "NEUROFORGE_METRICS_TOKEN",
+    "NEUROFORGE_CLUSTER_TOKEN",
+    "NEUROFORGE_CONTROL_READ_TOKEN",
+    "KB_INTEGRATION_TOKEN",
+    "BASIC_AUTH_PASSWORD",
+    "CONTROL_BASIC_AUTH_PASSWORD",
+}
+leaked = sorted(forbidden & actual)
+if leaked:
+    print("compose-env-check: unrelated privileged secrets exposed to Agent: " + ", ".join(leaked), file=sys.stderr)
+    raise SystemExit(1)
+
+knowledge = compose["services"]["knowledge"]
+if "env_file" in knowledge:
+    raise SystemExit("compose-env-check: production Knowledge must not use env_file")
+print(f"compose-env-check: passed ({len(actual)} explicit Agent environment entries)")
--- a/scripts/generate-secrets.sh
+++ b/scripts/generate-secrets.sh
@@ -7,6 +7,8 @@ cat <<OUT
 # Mega service tokens
 NEUROFORGE_ADMIN_TOKEN=$(gen)
 NEUROFORGE_APP_API_KEY=$(gen)
+NEUROFORGE_INTEGRATION_TOKEN=$(gen)
+NEUROFORGE_CONTROL_READ_TOKEN=$(gen)
 NEUROFORGE_WORKER_TOKEN=$(gen)
 NEUROFORGE_METRICS_TOKEN=$(gen)
 KB_INTEGRATION_TOKEN=$(gen)
@@ -16,5 +18,6 @@ SEARXNG_SECRET=$(gen)
 # Web/UI and optional webhook secrets
 WEB_PASSWORD=$(gen)
 BASIC_AUTH_PASSWORD=$(gen)
+CONTROL_BASIC_AUTH_PASSWORD=$(gen)
 WEBHOOK_SECRET=$(gen)
 OUT
--- a/scripts/go-live.sh
+++ b/scripts/go-live.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+ENV_FILE=${ENV_FILE:-.env}
+./scripts/preflight.sh
+echo "go-live: pulling immutable registry images"
+docker compose --env-file "$ENV_FILE" pull
+echo "go-live: starting stack"
+docker compose --env-file "$ENV_FILE" up -d --remove-orphans
+echo "go-live: current container state"
+docker compose --env-file "$ENV_FILE" ps
+cat <<'MSG'
+go-live: container start completed. Verify the host-specific smoke gates from docs/GO-LIVE-v1.5.0.md before enabling write automation.
+MSG
--- a/scripts/preflight.sh
+++ b/scripts/preflight.sh
@@ -0,0 +1,90 @@
+#!/bin/sh
+set -eu
+
+ENV_FILE=${ENV_FILE:-.env}
+STATIC_ONLY=false
+[ "${1:-}" = "--static" ] && STATIC_ONLY=true
+
+fail() { echo "preflight: ERROR: $*" >&2; exit 1; }
+info() { echo "preflight: $*"; }
+
+[ -f docker-compose.yml ] || fail "run from the repository root"
+
+# Production compose must never silently fall back to source builds or broad .env injection.
+if grep -Eq '^[[:space:]]+build:' docker-compose.yml; then fail "production docker-compose.yml contains build:"; fi
+if grep -Eq '^[[:space:]]+env_file:' docker-compose.yml; then fail "production docker-compose.yml contains env_file:"; fi
+for image in neuroforge neuroforge-worker agent agent-data-init knowledge control; do
+  grep -Fq "git.send.nrw/sendnrw/glpi-neuroforge-mega-${image}:\${IMAGE_TAG:" docker-compose.yml || fail "registry image mapping missing for ${image}"
+done
+
+if [ "$STATIC_ONLY" = true ]; then
+  if grep -Eq '^IMAGE_TAG[[:space:]]*=[[:space:]]*latest([[:space:]]|$)' .env.example; then fail ".env.example sets IMAGE_TAG=latest"; fi
+  info "static compose/source checks passed"
+  exit 0
+fi
+
+[ -f "$ENV_FILE" ] || fail "$ENV_FILE not found (copy .env.example and replace every placeholder)"
+getv() { sed -n "s/^$1=//p" "$ENV_FILE" | tail -n 1 | tr -d '\r'; }
+check_not_placeholder() {
+  name=$1; value=$(getv "$name")
+  [ -n "$value" ] || fail "$name is empty"
+  upper=$(printf '%s' "$value" | tr '[:lower:]' '[:upper:]')
+  case "$upper" in *CHANGE_ME*|*CHANGEME*|*PLACEHOLDER*) fail "$name still contains a placeholder";; esac
+}
+check_secret() {
+  name=$1; min=$2
+  check_not_placeholder "$name"
+  value=$(getv "$name")
+  [ "${#value}" -ge "$min" ] || fail "$name must contain at least $min characters"
+}
+
+tag=$(getv IMAGE_TAG)
+[ -n "$tag" ] || fail "IMAGE_TAG is empty"
+[ "$tag" != latest ] || fail "IMAGE_TAG=latest is forbidden for production"
+case "$tag" in *[!A-Za-z0-9._-]*) fail "IMAGE_TAG contains invalid characters";; esac
+
+for spec in \
+  NEUROFORGE_ADMIN_TOKEN:24 \
+  NEUROFORGE_APP_API_KEY:24 \
+  NEUROFORGE_INTEGRATION_TOKEN:24 \
+  NEUROFORGE_CONTROL_READ_TOKEN:24 \
+  NEUROFORGE_WORKER_TOKEN:24 \
+  NEUROFORGE_METRICS_TOKEN:24 \
+  KB_INTEGRATION_TOKEN:24 \
+  CONTROL_READ_TOKEN:24 \
+  BASIC_AUTH_PASSWORD:12 \
+  CONTROL_BASIC_AUTH_PASSWORD:12 \
+  WEB_PASSWORD:12; do
+  check_secret "${spec%%:*}" "${spec##*:}"
+done
+
+for name in GLPI_URL GLPI_CLIENT_ID GLPI_CLIENT_SECRET GLPI_USERNAME GLPI_PASSWORD; do
+  check_not_placeholder "$name"
+done
+web_anon=$(printf '%s' "$(getv WEB_ALLOW_ANONYMOUS)" | tr '[:upper:]' '[:lower:]')
+if [ "$web_anon" != "true" ]; then
+  check_not_placeholder WEB_USERNAME
+fi
+research=$(printf '%s' "$(getv NEUROFORGE_SEARXNG_ENABLED)" | tr '[:upper:]' '[:lower:]')
+if [ "$research" = "true" ]; then
+  check_secret SEARXNG_SECRET 24
+fi
+
+# Trust-boundary tokens must not be reused across roles.
+seen=''
+for name in NEUROFORGE_ADMIN_TOKEN NEUROFORGE_APP_API_KEY NEUROFORGE_INTEGRATION_TOKEN NEUROFORGE_CONTROL_READ_TOKEN NEUROFORGE_WORKER_TOKEN NEUROFORGE_METRICS_TOKEN KB_INTEGRATION_TOKEN CONTROL_READ_TOKEN; do
+  value=$(getv "$name")
+  case "|$seen|" in *"|$value|"*) fail "$name reuses another service token";; esac
+  seen=${seen:+$seen|}$value
+done
+
+command -v docker >/dev/null 2>&1 || fail "docker is not installed"
+docker compose version >/dev/null 2>&1 || fail "Docker Compose v2 is unavailable"
+docker compose --env-file "$ENV_FILE" config -q || fail "docker compose config validation failed"
+images=$(docker compose --env-file "$ENV_FILE" config --images)
+for image in neuroforge neuroforge-worker agent agent-data-init knowledge control; do
+  expected="git.send.nrw/sendnrw/glpi-neuroforge-mega-${image}:${tag}"
+  printf '%s\n' "$images" | grep -Fxq "$expected" || fail "resolved image missing: $expected"
+done
+printf '%s\n' "$images" | grep -E 'git\.send\.nrw/sendnrw/glpi-neuroforge-mega-.*:latest$' >/dev/null && fail "a project image resolved to latest"
+info "production preflight passed for IMAGE_TAG=$tag"
--- a/scripts/release-gate.sh
+++ b/scripts/release-gate.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+set -eu
+ROOT=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+cd "$ROOT"
+./scripts/preflight.sh --static
+./scripts/check-compose-env.py
+./scripts/secret-scan.sh
+for module in platform/neuroforge services/agent services/knowledge services/control; do
+  echo "release-gate: $module: test"
+  (cd "$module" && go test ./...)
+  echo "release-gate: $module: vet"
+  (cd "$module" && go vet ./...)
+  echo "release-gate: $module: build"
+  (cd "$module" && go build ./...)
+done
+
+echo "release-gate: targeted race checks"
+(cd platform/neuroforge && go test -race ./internal/store ./internal/brain ./internal/httpapi)
+(cd services/agent && go test -race ./internal/state ./internal/knowledge ./internal/learning ./internal/agent)
+(cd services/knowledge && go test -race ./internal/store ./internal/staging ./cmd/server)
+(cd services/control && go test -race .)
+
+echo "release-gate: passed local source gates"
--- a/scripts/research-up.sh
+++ b/scripts/research-up.sh
@@ -15,5 +15,5 @@ export SEARXNG_SECRET=$secret
 cd "$ROOT"
 NEUROFORGE_RESEARCH_ENABLED=true \
 NEUROFORGE_SEARXNG_ENABLED=true \
-docker compose --profile research up -d searxng neuroforge neuroforge-worker
+docker compose --profile research up -d ollama knowledge searxng neuroforge neuroforge-worker
 printf '%s\n' 'SearXNG + NeuroForge research are running. Autonomy remains controlled by NEUROFORGE_AUTONOMY_ENABLED.'
--- a/scripts/restore-data.sh
+++ b/scripts/restore-data.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+set -eu
+ENV_FILE=${ENV_FILE:-.env}
+SNAPSHOT=${1:-}
+HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-busybox:1.36}
+[ -n "$SNAPSHOT" ] && [ -d "$SNAPSHOT" ] || { echo "usage: $0 SNAPSHOT_DIR" >&2; exit 2; }
+[ -f "$ENV_FILE" ] || { echo "restore: missing $ENV_FILE" >&2; exit 1; }
+( cd "$SNAPSHOT" && sha256sum -c SHA256SUMS )
+./scripts/preflight.sh
+getv() { sed -n "s/^$1=//p" "$ENV_FILE" | tail -n1 | tr -d '\r'; }
+
+restart() { docker compose --env-file "$ENV_FILE" up -d knowledge neuroforge neuroforge-worker agent >/dev/null 2>&1 || true; }
+trap restart EXIT INT TERM
+
+docker compose --env-file "$ENV_FILE" stop neuroforge-worker agent neuroforge knowledge
+docker compose --env-file "$ENV_FILE" create neuroforge agent >/dev/null
+volume_for() {
+  service=$1; destination=$2
+  cid=$(docker compose --env-file "$ENV_FILE" ps -aq "$service")
+  [ -n "$cid" ] || return 1
+  docker inspect -f "{{range .Mounts}}{{if eq .Destination \"$destination\"}}{{.Name}}{{end}}{{end}}" "$cid"
+}
+restore_volume() {
+  service=$1; destination=$2; source=$3
+  volume=$(volume_for "$service" "$destination")
+  [ -n "$volume" ] || { echo "restore: volume for $service:$destination not found" >&2; exit 1; }
+  src=$(CDPATH= cd -- "$source" && pwd)
+  docker run --rm -v "$volume:/target" -v "$src:/source:ro" "$HELPER_IMAGE" sh -eu -c 'rm -rf /target/* /target/.[!.]* /target/..?* 2>/dev/null || true; cp -a /source/. /target/'
+}
+restore_volume neuroforge /app/data "$SNAPSHOT/neuroforge-data"
+restore_volume agent /app/data "$SNAPSHOT/agent-data"
+restore_host() {
+  src=$1; dst=$2
+  [ -d "$src" ] || return 0
+  mkdir -p "$dst"
+  find "$dst" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
+  cp -a "$src"/. "$dst"/
+}
+restore_host "$SNAPSHOT/knowledge" "$(getv KB_DATA_PATH)"
+restore_host "$SNAPSHOT/staging" "$(getv KB_STAGING_PATH)"
+restore_host "$SNAPSHOT/knowledge-backups" "$(getv KB_BACKUP_PATH)"
+echo "restore: snapshot restored; services will be restarted"
--- a/scripts/secret-scan.sh
+++ b/scripts/secret-scan.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+set -eu
+fail=0
+bad() { echo "secret-scan: ERROR: $*" >&2; fail=1; }
+
+ROOT=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+cd "$ROOT"
+TMP_BASE=${TMPDIR:-/tmp}/neuroforge-secret-scan.$$
+FILES="$TMP_BASE.files"
+KEYS="$TMP_BASE.keys"
+TOKENS="$TMP_BASE.tokens"
+trap 'rm -f "$FILES" "$KEYS" "$TOKENS"' EXIT HUP INT TERM
+
+# Release archives intentionally do not contain .git. Use Git's tracked-file view
+# when available, otherwise scan every regular file in the extracted release.
+if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+  git ls-files > "$FILES"
+  HAVE_GIT=true
+else
+  find . -type f ! -path './.git/*' -print | sed 's#^\./##' | sort > "$FILES"
+  HAVE_GIT=false
+fi
+
+# Private keys and common live-token shapes must not be committed. Placeholders in
+# templates/docs are intentionally allowed.
+if grep -E '(^|/)\.env$|\.pem$|\.p12$|\.pfx$|(^|/)id_rsa$|(^|/)id_ed25519$' "$FILES" >/dev/null; then
+  bad "private environment/key material found"
+fi
+
+if [ "$HAVE_GIT" = true ]; then
+  if git grep -nE -- '-----BEGIN (RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----' -- ':!*.example' ':!*.md' >"$KEYS" 2>/dev/null; then
+    cat "$KEYS" >&2; bad "private key material found"
+  fi
+  if git grep -nE -- '(AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{24,})' -- ':!*.example' >"$TOKENS" 2>/dev/null; then
+    cat "$TOKENS" >&2; bad "token-like credential found"
+  fi
+else
+  # Fallback for release ZIPs: recurse through extracted source while preserving
+  # the same exclusions as the Git-backed scan.
+  if grep -RInE --exclude='*.example' --exclude='*.md' --exclude-dir='.git' -- \
+      '-----BEGIN (RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----' . >"$KEYS" 2>/dev/null; then
+    cat "$KEYS" >&2; bad "private key material found"
+  fi
+  if grep -RInE --exclude='*.example' --exclude-dir='.git' -- \
+      '(AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{24,})' . >"$TOKENS" 2>/dev/null; then
+    cat "$TOKENS" >&2; bad "token-like credential found"
+  fi
+fi
+
+# Reject accidental binary blobs outside explicitly expected assets.
+while IFS= read -r f; do
+  [ -f "$f" ] || continue
+  case "$f" in *.png|*.jpg|*.jpeg|*.gif|*.ico|*.woff|*.woff2|*.pdf|*.zip) continue;; esac
+  if [ "$(LC_ALL=C grep -Il . "$f" 2>/dev/null || true)" = "" ] && [ -s "$f" ]; then
+    bad "unexpected binary file: $f"
+  fi
+done < "$FILES"
+
+[ "$fail" -eq 0 ] || exit 1
+echo "secret-scan: passed"
--- a/scripts/status.sh
+++ b/scripts/status.sh
@@ -1,4 +1,17 @@
 #!/bin/sh
 set -eu
-PORT="${CONTROL_HOST_PORT:-8070}"
-curl -fsS "http://127.0.0.1:${PORT}/api/status" | python3 -m json.tool
+ENV_FILE=${ENV_FILE:-.env}
+getv() {
+  [ -f "$ENV_FILE" ] || return 0
+  sed -n "s/^$1=//p" "$ENV_FILE" | tail -n1 | tr -d '\r'
+}
+PORT=${CONTROL_HOST_PORT:-$(getv CONTROL_HOST_PORT)}
+PORT=${PORT:-8070}
+USER_NAME=${CONTROL_BASIC_AUTH_USER:-$(getv CONTROL_BASIC_AUTH_USER)}
+PASSWORD=${CONTROL_BASIC_AUTH_PASSWORD:-$(getv CONTROL_BASIC_AUTH_PASSWORD)}
+if [ -n "$USER_NAME" ] || [ -n "$PASSWORD" ]; then
+  [ -n "$USER_NAME" ] && [ -n "$PASSWORD" ] || { echo "status: incomplete Control Basic Auth" >&2; exit 1; }
+  curl -fsS --user "$USER_NAME:$PASSWORD" "http://127.0.0.1:${PORT}/api/status" | python3 -m json.tool
+else
+  curl -fsS "http://127.0.0.1:${PORT}/api/status" | python3 -m json.tool
+fi