-

2026-09-11 06:14:38 +02:00
parent bf64652300
commit e581949946
161 changed files with 31126 additions and 1 deletions
@@ -0,0 +1,187 @@
+#!/usr/bin/env sh
+set -eu
+
+BASE_URL=${BASE_URL:-http://127.0.0.1:8080}
+MODEL=${MODEL:-qwen3:8b}
+REQUESTS=${REQUESTS:-500}
+WARMUP=${WARMUP:-20}
+CONCURRENCIES=${CONCURRENCIES:-"1 4 16 32 64"}
+OUT_DIR=${OUT_DIR:-"ha-readiness-$(date +%Y%m%d-%H%M%S)"}
+TIMEOUT=${TIMEOUT:-2m}
+STREAM=${STREAM:-false}
+SERVICE_CLASS=${SERVICE_CLASS:-}
+CAPTURE_METRICS=${CAPTURE_METRICS:-true}
+METRICS_URL=${METRICS_URL:-"${BASE_URL%/}/metrics"}
+GATEWAY_PID=${GATEWAY_PID:-}
+CAPTURE_RESOURCES=${CAPTURE_RESOURCES:-auto}
+CAPTURE_SUSTAINED_RESOURCES=${CAPTURE_SUSTAINED_RESOURCES:-auto}
+RESOURCE_SAMPLE_INTERVAL=${RESOURCE_SAMPLE_INTERVAL:-250ms}
+RESOURCE_SAMPLE_MAX_DURATION=${RESOURCE_SAMPLE_MAX_DURATION:-15m}
+
+if [ "$CAPTURE_RESOURCES" = "auto" ]; then
+  if [ -n "$GATEWAY_PID" ]; then
+    CAPTURE_RESOURCES=true
+  else
+    CAPTURE_RESOURCES=false
+  fi
+fi
+if [ "$CAPTURE_RESOURCES" = "true" ] && [ -z "$GATEWAY_PID" ]; then
+  echo "GATEWAY_PID is required when CAPTURE_RESOURCES=true" >&2
+  exit 2
+fi
+
+if [ "$CAPTURE_SUSTAINED_RESOURCES" = "auto" ]; then
+  if [ -n "$GATEWAY_PID" ]; then
+    CAPTURE_SUSTAINED_RESOURCES=true
+  else
+    CAPTURE_SUSTAINED_RESOURCES=false
+  fi
+fi
+if [ "$CAPTURE_SUSTAINED_RESOURCES" = "true" ] && [ -z "$GATEWAY_PID" ]; then
+  echo "GATEWAY_PID is required when CAPTURE_SUSTAINED_RESOURCES=true" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+SAMPLER_BIN=""
+SAMPLER_BG_PID=""
+SAMPLER_STOP_FILE=""
+
+cleanup() {
+  if [ -n "$SAMPLER_STOP_FILE" ]; then
+    : > "$SAMPLER_STOP_FILE" 2>/dev/null || true
+  fi
+  if [ -n "$SAMPLER_BG_PID" ]; then
+    wait "$SAMPLER_BG_PID" 2>/dev/null || true
+  fi
+  if [ -n "$SAMPLER_BIN" ]; then
+    rm -f "$SAMPLER_BIN"
+  fi
+  if [ -n "$SAMPLER_STOP_FILE" ]; then
+    rm -f "$SAMPLER_STOP_FILE"
+  fi
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+if [ "$CAPTURE_SUSTAINED_RESOURCES" = "true" ]; then
+  SAMPLER_BIN="$OUT_DIR/.ha-sampler"
+  CGO_ENABLED=0 go build -trimpath -o "$SAMPLER_BIN" ./cmd/ha-sampler
+fi
+
+echo "HA readiness sweep"
+echo "base_url=$BASE_URL model=$MODEL requests=$REQUESTS warmup=$WARMUP stream=$STREAM"
+echo "concurrency=$CONCURRENCIES"
+echo "capture_metrics=$CAPTURE_METRICS metrics_url=$METRICS_URL"
+echo "capture_resources=$CAPTURE_RESOURCES gateway_pid=${GATEWAY_PID:-unset}"
+echo "capture_sustained_resources=$CAPTURE_SUSTAINED_RESOURCES sample_interval=$RESOURCE_SAMPLE_INTERVAL"
+echo "output=$OUT_DIR"
+
+capture_metrics() {
+  dst=$1
+  if [ "$CAPTURE_METRICS" != "true" ]; then
+    return 0
+  fi
+  if ! command -v curl >/dev/null 2>&1; then
+    echo "curl is required when CAPTURE_METRICS=true" >&2
+    exit 2
+  fi
+  if [ -n "${GATEWAY_BENCH_API_KEY:-}" ]; then
+    curl -fsS --max-time 30 -H "Authorization: Bearer ${GATEWAY_BENCH_API_KEY}" "$METRICS_URL" > "$dst"
+  else
+    curl -fsS --max-time 30 "$METRICS_URL" > "$dst"
+  fi
+}
+
+capture_resources() {
+  dst=$1
+  if [ "$CAPTURE_RESOURCES" != "true" ]; then
+    return 0
+  fi
+  go run ./cmd/ha-snapshot -pid "$GATEWAY_PID" -json-out "$dst"
+}
+
+start_resource_sampler() {
+  c=$1
+  if [ "$CAPTURE_SUSTAINED_RESOURCES" != "true" ]; then
+    return 0
+  fi
+  SAMPLER_STOP_FILE="$OUT_DIR/.resource-sampler-stop-c$c"
+  rm -f "$SAMPLER_STOP_FILE"
+  "$SAMPLER_BIN" \
+    -pid "$GATEWAY_PID" \
+    -interval "$RESOURCE_SAMPLE_INTERVAL" \
+    -max-duration "$RESOURCE_SAMPLE_MAX_DURATION" \
+    -stop-file "$SAMPLER_STOP_FILE" \
+    -json-out "$OUT_DIR/resources-samples-c$c.json" \
+    >"$OUT_DIR/resource-sampler-c$c.log" 2>&1 &
+  SAMPLER_BG_PID=$!
+  # Give the already-built sampler enough time to capture its initial baseline.
+  sleep 0.1
+  if ! kill -0 "$SAMPLER_BG_PID" 2>/dev/null; then
+    wait "$SAMPLER_BG_PID" || true
+    echo "resource sampler failed to start for concurrency $c" >&2
+    cat "$OUT_DIR/resource-sampler-c$c.log" >&2 || true
+    exit 1
+  fi
+}
+
+stop_resource_sampler() {
+  c=$1
+  if [ "$CAPTURE_SUSTAINED_RESOURCES" != "true" ]; then
+    return 0
+  fi
+  : > "$SAMPLER_STOP_FILE"
+  sampler_rc=0
+  wait "$SAMPLER_BG_PID" || sampler_rc=$?
+  rm -f "$SAMPLER_STOP_FILE"
+  SAMPLER_STOP_FILE=""
+  SAMPLER_BG_PID=""
+  if [ "$sampler_rc" -ne 0 ]; then
+    echo "resource sampler failed for concurrency $c (exit $sampler_rc)" >&2
+    cat "$OUT_DIR/resource-sampler-c$c.log" >&2 || true
+    exit "$sampler_rc"
+  fi
+}
+
+for c in $CONCURRENCIES; do
+  echo "== concurrency $c =="
+  capture_metrics "$OUT_DIR/metrics-before-c$c.prom"
+  capture_resources "$OUT_DIR/resources-before-c$c.json"
+  start_resource_sampler "$c"
+
+  set -- go run ./cmd/bench \
+    -base-url "$BASE_URL" \
+    -model "$MODEL" \
+    -concurrency "$c" \
+    -requests "$REQUESTS" \
+    -warmup "$WARMUP" \
+    -timeout "$TIMEOUT" \
+    -stream="$STREAM" \
+    -json-out "$OUT_DIR/concurrency-$c.json"
+  if [ -n "$SERVICE_CLASS" ]; then
+    set -- "$@" -service-class "$SERVICE_CLASS"
+  fi
+  bench_rc=0
+  "$@" || bench_rc=$?
+  stop_resource_sampler "$c"
+  if [ "$bench_rc" -ne 0 ]; then
+    echo "benchmark failed for concurrency $c (exit $bench_rc)" >&2
+    exit "$bench_rc"
+  fi
+
+  capture_resources "$OUT_DIR/resources-after-c$c.json"
+  capture_metrics "$OUT_DIR/metrics-after-c$c.prom"
+done
+
+if [ "$CAPTURE_METRICS" = "true" ]; then
+  go run ./cmd/ha-report \
+    -input "$OUT_DIR" \
+    -json-out "$OUT_DIR/report.json" \
+    -markdown-out "$OUT_DIR/report.md"
+else
+  echo "metrics capture disabled; no reconciled HA evidence report generated"
+fi
+
+echo "completed: $OUT_DIR"
@@ -0,0 +1,9 @@
+#!/bin/sh
+# Starting point for the 256 GB M5 Ultra. Benchmark 1/2/4 parallel requests
+# with your actual model and context length before raising these values.
+export OLLAMA_NUM_PARALLEL="${OLLAMA_NUM_PARALLEL:-4}"
+export OLLAMA_MAX_LOADED_MODELS="${OLLAMA_MAX_LOADED_MODELS:-2}"
+export OLLAMA_MAX_QUEUE="${OLLAMA_MAX_QUEUE:-64}"
+export OLLAMA_CONTEXT_LENGTH="${OLLAMA_CONTEXT_LENGTH:-32768}"
+export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-30m}"
+exec ollama serve
@@ -0,0 +1,54 @@
+#!/bin/sh
+set -eu
+
+CONFIG=${GATEWAY_CONFIG:-./gateway-config.json}
+COMPOSE_FILE=${COMPOSE_FILE:-docker-compose.production.yml}
+PUBLISH_ADDRESS=${GATEWAY_PUBLISH_ADDRESS:-127.0.0.1}
+PUBLISH_PORT=${GATEWAY_PUBLISH_PORT:-9080}
+ALLOW_NON_LOOPBACK_BIND=${ALLOW_NON_LOOPBACK_BIND:-0}
+
+fail() {
+  printf 'production preflight: ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+warn() {
+  printf 'production preflight: WARN: %s\n' "$*" >&2
+}
+
+command -v docker >/dev/null 2>&1 || fail 'docker is not available'
+docker compose version >/dev/null 2>&1 || fail 'docker compose is not available'
+[ -r "$CONFIG" ] || fail "bootstrap config is not readable: $CONFIG"
+
+case "$CONFIG" in
+  *config.example.json|*/config.example.json)
+    fail 'GATEWAY_CONFIG points at config.example.json; select the production bootstrap config explicitly'
+    ;;
+esac
+
+case "$PUBLISH_ADDRESS" in
+  127.0.0.1|::1|localhost)
+    ;;
+  0.0.0.0|::|'[::]')
+    [ "$ALLOW_NON_LOOPBACK_BIND" = 1 ] || fail "gateway would be published on all interfaces ($PUBLISH_ADDRESS:$PUBLISH_PORT); bind to loopback or set ALLOW_NON_LOOPBACK_BIND=1 only with an external firewall/ACL"
+    warn "non-loopback wildcard publishing explicitly allowed: $PUBLISH_ADDRESS:$PUBLISH_PORT"
+    ;;
+  *)
+    [ "$ALLOW_NON_LOOPBACK_BIND" = 1 ] || fail "gateway would be published on non-loopback address $PUBLISH_ADDRESS:$PUBLISH_PORT; use loopback for a host-local reverse proxy, or set ALLOW_NON_LOOPBACK_BIND=1 after restricting the host firewall to the real proxy"
+    warn "non-loopback publishing explicitly allowed: $PUBLISH_ADDRESS:$PUBLISH_PORT"
+    ;;
+esac
+
+export GATEWAY_CONFIG="$CONFIG" GATEWAY_PUBLISH_ADDRESS="$PUBLISH_ADDRESS" GATEWAY_PUBLISH_PORT="$PUBLISH_PORT"
+
+docker compose -f "$COMPOSE_FILE" config >/dev/null
+
+printf 'production preflight: compose syntax OK\n'
+printf 'production preflight: published endpoint %s:%s -> container :8080\n' "$PUBLISH_ADDRESS" "$PUBLISH_PORT"
+
+# Runs the candidate image with the exact bootstrap/state mounts and runtime user.
+# The gateway preflight output is secret-redacted by design.
+docker compose -f "$COMPOSE_FILE" run --rm --no-deps gateway \
+  -config /etc/ollama-gateway/config.json -check-config
+
+printf 'production preflight: OK\n'
@@ -0,0 +1,70 @@
+#!/bin/sh
+set -eu
+
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT HUP INT TERM
+mkdir -p "$TMP/bin"
+
+cat > "$TMP/bin/docker" <<'EOF'
+#!/bin/sh
+set -eu
+[ "$1" = compose ] || exit 64
+shift
+# Accept optional -f <file>.
+if [ "${1:-}" = -f ]; then shift 2; fi
+case "${1:-}" in
+  version) exit 0 ;;
+  config) exit 0 ;;
+  run)
+    cat <<JSON
+{"status":"ok","workers":2,"storage_writable":true}
+JSON
+    exit 0
+    ;;
+  *) exit 65 ;;
+esac
+EOF
+chmod +x "$TMP/bin/docker"
+
+cat > "$TMP/gateway-config.json" <<'EOF'
+{}
+EOF
+cat > "$TMP/config.example.json" <<'EOF'
+{}
+EOF
+
+PATH="$TMP/bin:$PATH" \
+GATEWAY_CONFIG="$TMP/gateway-config.json" \
+GATEWAY_PUBLISH_ADDRESS=127.0.0.1 \
+COMPOSE_FILE="$ROOT/docker-compose.production.yml" \
+"$ROOT/scripts/production-preflight.sh" > "$TMP/ok.out" 2> "$TMP/ok.err"
+grep -q 'production preflight: OK' "$TMP/ok.out"
+
+if PATH="$TMP/bin:$PATH" \
+  GATEWAY_CONFIG="$TMP/config.example.json" \
+  COMPOSE_FILE="$ROOT/docker-compose.production.yml" \
+  "$ROOT/scripts/production-preflight.sh" >/dev/null 2>&1; then
+  echo 'expected config.example.json rejection' >&2
+  exit 1
+fi
+
+if PATH="$TMP/bin:$PATH" \
+  GATEWAY_CONFIG="$TMP/gateway-config.json" \
+  GATEWAY_PUBLISH_ADDRESS=0.0.0.0 \
+  COMPOSE_FILE="$ROOT/docker-compose.production.yml" \
+  "$ROOT/scripts/production-preflight.sh" >/dev/null 2>&1; then
+  echo 'expected wildcard bind rejection' >&2
+  exit 1
+fi
+
+PATH="$TMP/bin:$PATH" \
+GATEWAY_CONFIG="$TMP/gateway-config.json" \
+GATEWAY_PUBLISH_ADDRESS=10.2.10.20 \
+ALLOW_NON_LOOPBACK_BIND=1 \
+COMPOSE_FILE="$ROOT/docker-compose.production.yml" \
+"$ROOT/scripts/production-preflight.sh" > "$TMP/remote.out" 2> "$TMP/remote.err"
+grep -q 'production preflight: OK' "$TMP/remote.out"
+grep -q 'non-loopback publishing explicitly allowed' "$TMP/remote.err"
+
+echo 'production preflight tests: PASS'