[agent-network] Remove e2e shell-script suite from this branch

The end-to-end shell scripts under scripts/e2e/ are maintained in a separate testing suite and are not part of this change set.
2026-10-26 22:39:07 +01:00 · 2026-06-27 01:39:57 +02:00
parent c83e46fbe1
commit 9f485be2f9
11 changed files with 0 additions and 1968 deletions
@@ -1,956 +0,0 @@
 #!/usr/bin/env bash
 # Agent-network full end-to-end driver.
 #
 # One script, many subcommands, so the operator authorizes a single
 # `bash scripts/e2e/agent-network-full/e2e.sh <cmd>` invocation instead of a
 # stream of ad-hoc docker/netbird/curl commands.
 #
 # It joins a Docker NetBird client to the local Tilt management, drives LLM
 # chat-completions through the agent-network proxy over the tunnel, and asserts
 # token/cost/session capture via the REST API and proxy logs.
 #
 # Secrets (PAT, provider keys, setup key) are read from files / a sourced
 # key file and never echoed.
 #
 # Usage:
 #   bash scripts/e2e/agent-network-full/e2e.sh <command>
 #
 # Commands:
 #   key            mint a reusable setup key bound to the Admins group
 #   up             run the Docker NetBird client and join management
 #   status         show netbird status -d inside the client
 #   wait           wait until the proxy peer is Connected (1/1)
 #   diag           dump client + proxy3 + relay diagnostics
 #   chat M P       chat-completion through the proxy: model M, provider-kind P
 #   verify         consumption + recent access-log rows
 #   snapshot       save current agent-network config to the scratch dir
 #   clean          delete all policies, budget-rules, providers (API teardown)
 #   providers      create the five providers from .llm-keys
 #   policy         create one policy: Admins -> all five providers
 #   down           remove the Docker client container
 #   restart-proxy  tilt trigger proxy3 (re-establish its relay link)
 #   all            clean -> providers -> policy -> up -> wait -> chat-all -> verify
 set -uo pipefail
 # --- config ------------------------------------------------------------------
 NB_API="${NB_API:-http://localhost:8080}"
 NB_PAT_FILE="${NB_PAT_FILE:-/Users/maycon/projects/local-dev/nb-pat}"
 LLM_KEYS_FILE="${LLM_KEYS_FILE:-/Users/maycon/.llm-keys}"
 CLIENT="${CLIENT:-nb-e2e-agent}"
 IMAGE="${IMAGE:-netbird:tilt}"
 CURL_IMAGE="${CURL_IMAGE:-curlimages/curl:latest}"
 MGMT_URL="${MGMT_URL:-http://host.docker.internal:8080}"
 PROXY_HOST="${PROXY_HOST:-mitten.proxy.netbird.local}"
 ADMINS_GROUP_NAME="${ADMINS_GROUP_NAME:-Admins}"
 DASH_DIR="${DASH_DIR:-/Users/maycon/projects/dashboard}"
 DASH_URL="${DASH_URL:-http://localhost:3000}"
 DASH_USER="${DASH_USER:-netbird@netbird.io}"
 DASH_PASS="${DASH_PASS:-netbird@netbird.io}"
 STATE_DIR="${STATE_DIR:-/private/tmp/claude-501/-Users-maycon-projects-netbird/a3fe30e4-5777-47d5-b110-ebc228716026/scratchpad/agentnet-snapshot}"
 mkdir -p "$STATE_DIR"
 [ -r "$NB_PAT_FILE" ] || { echo "FAIL: cannot read PAT at $NB_PAT_FILE" >&2; exit 2; }
 PAT="$(tr -d '\n\r ' <"$NB_PAT_FILE")"
 AUTH="Authorization: Token $PAT"
 B="$NB_API/api/agent-network"
 # --- small helpers -----------------------------------------------------------
 log()  { printf '%s\n' "$*" >&2; }
 die()  { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
 api()  { # METHOD PATH [BODY]
  local m="$1" p="$2" body="${3-}"
  if [ -n "$body" ]; then
    curl -fsS -X "$m" -H "$AUTH" -H "Content-Type: application/json" --data "$body" "$NB_API$p"
  else
    curl -fsS -X "$m" -H "$AUTH" "$NB_API$p"
  fi
 }
 # proxy_ip resolves the proxy host to its NetBird IP from inside the client.
 proxy_ip() {
  docker exec "$CLIENT" sh -c "getent hosts $PROXY_HOST" 2>/dev/null | awk '{print $1; exit}'
 }
 admins_group_id() {
  api GET /api/groups | jq -r --arg n "$ADMINS_GROUP_NAME" \
    '.[] | select((.name//"")|ascii_downcase==($n|ascii_downcase)) | .id' | head -n1
 }
 require_llm_keys() {
  [ -r "$LLM_KEYS_FILE" ] || die "cannot read $LLM_KEYS_FILE"
  # shellcheck disable=SC1090
  set -a; . "$LLM_KEYS_FILE"; set +a
 }
 # --- commands ----------------------------------------------------------------
 cmd_snapshot() {
  api GET /settings_unused 2>/dev/null || true
  curl -fsS -H "$AUTH" "$B/settings"      >"$STATE_DIR/settings.json"
  curl -fsS -H "$AUTH" "$B/providers"     >"$STATE_DIR/providers.json"
  curl -fsS -H "$AUTH" "$B/policies"      >"$STATE_DIR/policies.json"
  curl -fsS -H "$AUTH" "$B/budget-rules"  >"$STATE_DIR/budget-rules.json"
  curl -fsS -H "$AUTH" "$NB_API/api/groups" >"$STATE_DIR/groups.json"
  log "snapshot written to $STATE_DIR"
  ls -l "$STATE_DIR" >&2
 }
 cmd_key() {
  local gid kjson
  gid="$(admins_group_id)"
  [ -n "$gid" ] || die "could not resolve Admins group id"
  kjson="$(api POST /api/setup-keys "{\"name\":\"e2e-agentnet-docker\",\"type\":\"reusable\",\"expires_in\":86400,\"usage_limit\":0,\"auto_groups\":[\"$gid\"],\"ephemeral\":false}")"
  echo "$kjson" | jq -r '.key' >"$STATE_DIR/setup-key.txt"
  chmod 600 "$STATE_DIR/setup-key.txt"
  echo "$kjson" | jq '{id,name,type,state,valid,auto_groups}' >&2
  log "setup key saved (value not printed) to $STATE_DIR/setup-key.txt"
 }
 cmd_up() {
  [ -r "$STATE_DIR/setup-key.txt" ] || cmd_key
  local key; key="$(cat "$STATE_DIR/setup-key.txt")"
  docker rm -f "$CLIENT" >/dev/null 2>&1 || true
  docker run -d --name "$CLIENT" \
    --cap-add NET_ADMIN --cap-add SYS_ADMIN --cap-add SYS_RESOURCE \
    --add-host host.docker.internal:host-gateway \
    -e NB_MANAGEMENT_URL="$MGMT_URL" \
    -e NB_SETUP_KEY="$key" \
    -e NB_LOG_LEVEL=info \
    "$IMAGE" >/dev/null
  log "started $CLIENT"
  sleep 6
  docker exec "$CLIENT" netbird status 2>&1 | sed -n '1,14p' >&2
 }
 cmd_status() { docker exec "$CLIENT" netbird status -d 2>&1; }
 cmd_wait() {
  local i=0 line
  while [ "$i" -lt 90 ]; do
    line="$(docker exec "$CLIENT" netbird status 2>/dev/null | grep '^Peers count' || true)"
    log "t=${i}s ${line:-<no status>}"
    case "$line" in *"1/1 Connected"*) log "proxy peer connected"; return 0;; esac
    sleep 5; i=$((i+5))
  done
  log "proxy peer did not connect within ${i}s"
  return 1
 }
 cmd_diag() {
  log "===== client: peers count ====="
  docker exec "$CLIENT" netbird status 2>&1 | grep -E '^(Management|Signal|Relays|Peers)' >&2 || true
  log "===== client: relay/handshake (last 15) ====="
  docker exec "$CLIENT" sh -c 'tail -n 400 /var/log/netbird/client.log' 2>/dev/null \
    | grep -iE 'relay|handshake|offer|answer|error' | tail -15 >&2 || true
  log "===== proxy3: relay (last 15) ====="
  docker logs local-dev-proxy3-1 2>&1 | grep -iE 'relay|signal|handshake|offer|answer' | tail -15 >&2 || true
  log "===== mgmt: proxy peer online ====="
  curl -fsS -H "$AUTH" "$NB_API/api/peers" \
    | jq -r '.[] | select((.name//"")|test("^proxy-")) | "\(.name) connected=\(.connected) last_seen=\(.last_seen)"' >&2 || true
 }
 cmd_restart_proxy() {
  command -v tilt >/dev/null 2>&1 || die "tilt not on PATH"
  tilt trigger proxy3 && log "triggered proxy3"
 }
 # cmd_chat MODEL KIND  — KIND in: chat (POST /v1/chat/completions),
 # messages (POST /v1/messages, anthropic body).
 CHAT_RESP="$STATE_DIR/last-chat.json"
 CHAT_PROMPT_DEFAULT="Reply with exactly: pong"
 # _chat MODEL [KIND] [PROMPT] — POST through the proxy from $CHAT_CLIENT
 # (defaults to $CLIENT). Echoes the HTTP status code on stdout; writes the
 # response body to $CHAT_RESP. Returns 000 if the proxy host won't resolve.
 _chat() {
  local model="$1" kind="${2:-chat}" prompt="${3:-$CHAT_PROMPT_DEFAULT}"
  local client="${CHAT_CLIENT:-$CLIENT}" ip path body out code attempt
  local extra_hdr=()
  case "$kind" in
    messages)
      path="/v1/messages"; extra_hdr=(-H "anthropic-version: 2023-06-01")
      body="$(jq -n --arg m "$model" --arg p "$prompt" '{model:$m,max_tokens:64,messages:[{role:"user",content:$p}]}')" ;;
    *)
      path="/v1/chat/completions"
      body="$(jq -n --arg m "$model" --arg p "$prompt" '{model:$m,messages:[{role:"user",content:$p}]}')" ;;
  esac
  # Re-resolve the proxy IP and retry on connection failure (000): a config
  # change churns the proxy peer's NetBird IP, so a freshly-joined client may
  # need a few seconds for its tunnel + magic DNS to converge.
  code=000
  for attempt in $(seq 1 "${CHAT_RETRIES:-6}"); do
    ip="$(CLIENT="$client" proxy_ip)"
    if [ -n "$ip" ]; then
      out="$(docker run --rm --network "container:$client" "$CURL_IMAGE" \
        -sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
        -w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \
        -H "Content-Type: application/json" ${extra_hdr[@]+"${extra_hdr[@]}"} --data "$body")"
      code="$(printf '%s' "$out" | tail -n1)"
      printf '%s' "$out" | sed '$d' >"$CHAT_RESP"
    fi
    [ "$code" != 000 ] && break
    sleep 4
  done
  echo "$code"
 }
 cmd_chat() {
  local code
  log "POST $PROXY_HOST (model=$1 kind=${2:-chat} client=${CHAT_CLIENT:-$CLIENT})"
  code="$(_chat "$@")"
  cat "$CHAT_RESP" 2>/dev/null
  printf '\n[http %s]\n' "$code"
 }
 cmd_verify() {
  log "===== consumption ====="
  api GET /api/agent-network/consumption | jq -r '.[] | "\(.dimension_kind)/\(.dimension_id) tokens_in=\(.tokens_input) tokens_out=\(.tokens_output) cost=\(.cost_usd // 0)"' >&2 || true
  log "===== last 10 access-log rows ====="
  api GET /api/agent-network/access-logs | jq -r '.data[0:10][] | "\(.timestamp) provider=\(.provider) model=\(.model) status=\(.status_code) decision=\(.decision) src=\(.source_ip) session=\(.session_id // "-") in=\(.input_tokens // 0) out=\(.output_tokens // 0) cost=\(.cost_usd // 0)"' >&2 || true
 }
 cmd_clean() {
  cmd_snapshot
  local id
  for id in $(api GET /api/agent-network/policies | jq -r '.[].id'); do
    api DELETE "/api/agent-network/policies/$id" >/dev/null && log "deleted policy $id"
  done
  for id in $(api GET /api/agent-network/budget-rules | jq -r '.[].id'); do
    api DELETE "/api/agent-network/budget-rules/$id" >/dev/null && log "deleted budget-rule $id"
  done
  for id in $(api GET /api/agent-network/providers | jq -r '.[].id'); do
    api DELETE "/api/agent-network/providers/$id" >/dev/null && log "deleted provider $id"
  done
  log "account cleaned"
 }
 # create_provider NAME PROVIDER_ID UPSTREAM_URL API_KEY
 create_provider() {
  local name="$1" pid="$2" url="$3" key="$4" body resp
  [ -n "$key" ] || { log "skip $name: empty key"; return 0; }
  body="$(jq -n --arg n "$name" --arg p "$pid" --arg u "$url" --arg k "$key" \
    '{name:$n,provider_id:$p,upstream_url:$u,api_key:$k,enabled:true}')"
  resp="$(api POST /api/agent-network/providers "$body")" || { log "create $name FAILED"; return 1; }
  echo "$resp" | jq -r '"created provider \(.name) id=\(.id) provider_id=\(.provider_id)"' >&2
  echo "$resp" | jq -r '.id'
 }
 cmd_providers() {
  require_llm_keys
  : >"$STATE_DIR/provider-ids.txt"
  create_provider "OpenAI API"           openai_api            "https://api.openai.com"     "${OPENAI_TOKEN:-}"     >>"$STATE_DIR/provider-ids.txt"
  create_provider "Anthropic API"        anthropic_api         "https://api.anthropic.com"  "${ANTHROPIC_TOKEN:-}"  >>"$STATE_DIR/provider-ids.txt"
  create_provider "Vercel AI Gateway"    vercel_ai_gateway     "${VERCEL_URL:-}"            "${VERCEL_TOKEN:-}"     >>"$STATE_DIR/provider-ids.txt"
  create_provider "OpenRouter"           openrouter            "${OPENROUTER_URL:-}"        "${OPENROUTER_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
  create_provider "Cloudflare AI Gateway" cloudflare_ai_gateway "${CLOUDFLARE_URL:-}"       "${CLOUDFLARE_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
  log "provider ids:"; cat "$STATE_DIR/provider-ids.txt" >&2
 }
 cmd_policy() {
  local gid ids body
  gid="$(admins_group_id)"; [ -n "$gid" ] || die "no Admins group"
  ids="$(api GET /api/agent-network/providers | jq -c '[.[].id]')"
  body="$(jq -n --arg n "e2e-all-providers" --arg g "$gid" --argjson dst "$ids" '{
    name:$n, description:"e2e: Admins to all providers", enabled:true,
    source_groups:[$g], destination_provider_ids:$dst, guardrail_ids:[],
    limits:{ budget_limit:{enabled:true,group_cap_usd:1000000,user_cap_usd:1000000,window_seconds:2592000},
             token_limit:{enabled:false,group_cap:0,user_cap:0,window_seconds:60} }
  }')"
  api POST /api/agent-network/policies "$body" | jq -r '"created policy \(.name) id=\(.id) dst=\(.destination_provider_ids|length) providers"' >&2
 }
 # set_enabled PROVIDER_ID BOOL — PUT the provider back with enabled toggled,
 # preserving its required fields and keeping the sealed key (api_key omitted).
 set_enabled() {
  local pid="$1" en="$2" cur body
  cur="$(api GET /api/agent-network/providers | jq -c --arg id "$pid" '.[] | select(.id==$id)')"
  [ -n "$cur" ] || { log "no provider $pid"; return 1; }
  body="$(echo "$cur" | jq -c --argjson en "$en" '{name,provider_id,upstream_url,enabled:$en} + (if .extra_values then {extra_values} else {} end)')"
  api PUT "/api/agent-network/providers/$pid" "$body" >/dev/null
 }
 # cmd_isolate NAME — leave only the named provider enabled (sole catch-all),
 # so a request routes to it without first-party-vendor or first-catch-all
 # interference. Matches NAME case-insensitively against provider .name.
 cmd_isolate() {
  local want="$1" id name en
  api GET /api/agent-network/providers | jq -r '.[] | "\(.id)\t\(.name)"' | while IFS=$'\t' read -r id name; do
    case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in
      *"$(echo "$want" | tr '[:upper:]' '[:lower:]')"*) en=true ;;
      *) en=false ;;
    esac
    set_enabled "$id" "$en" && log "$name enabled=$en"
  done
 }
 cmd_enable_all() {
  local id
  for id in $(api GET /api/agent-network/providers | jq -r '.[].id'); do
    set_enabled "$id" true && log "enabled $id"
  done
 }
 # cmd_dashboard — drive the live :3000 dashboard (../dashboard repo) with
 # Playwright, asserting the API-created providers/policy render in the UI.
 cmd_dashboard() {
  command -v node >/dev/null 2>&1 || die "node not on PATH"
  [ -f "$DASH_DIR/e2e/live-agent-network.mjs" ] || die "dashboard script missing in $DASH_DIR/e2e"
  ( cd "$DASH_DIR" && BASE_URL="$DASH_URL" DASH_USER="$DASH_USER" DASH_PASS="$DASH_PASS" node e2e/live-agent-network.mjs )
 }
 cmd_down() { docker rm -f "$CLIENT" >/dev/null 2>&1 && log "removed $CLIENT" || log "no container"; }
 cmd_all() {
  cmd_clean
  cmd_providers
  cmd_policy
  cmd_up
  cmd_wait || { cmd_diag; die "tunnel to proxy not established"; }
  log "===== chat: OpenAI ====="            ; cmd_chat gpt-5.4 chat
  log "===== chat: Anthropic ====="          ; cmd_chat claude-haiku-4-5 messages
  log "===== chat: Vercel (openai/...) ====="; cmd_chat openai/gpt-4o-mini chat
  log "===== chat: OpenRouter (openai/...) =="; cmd_chat openai/gpt-4o-mini chat
  sleep 3
  cmd_verify
 }
 # --- scenario helpers --------------------------------------------------------
 FAILS=0
 ok()   { log "  PASS: $1"; }
 bad()  { log "  FAIL: $1"; FAILS=$((FAILS+1)); }
 expect_code() { # WANT GOT LABEL
  if [ "$2" = "$1" ]; then ok "$3 (http $2)"; else bad "$3 (want $1, got $2)"; fi
 }
 # wait_tunnel [CLIENT] — poll silently until the proxy peer is 1/1 Connected.
 wait_tunnel() {
  local c="${1:-$CLIENT}" i=0
  while [ "$i" -lt 60 ]; do
    docker exec "$c" netbird status 2>/dev/null | grep -q '1/1 Connected' && return 0
    sleep 3; i=$((i+3))
  done
  return 1
 }
 # wait_chat_ready CLIENT KIND MODEL — poll a real request from CLIENT until it
 # returns a non-000 status, i.e. the tunnel + magic DNS + WG handshake to the
 # proxy peer have all converged. Single-attempt probes pace the outer loop.
 # Note: a netbird down/up bounce does NOT help here and is actively harmful —
 # the peer is already in the net-map (status shows 0/1, i.e. known-not-connected),
 # so the blocker is the WG handshake to a freshly-churned proxy peer, and
 # bouncing just resets an in-progress slow handshake. We only wait it out.
 wait_chat_ready() {
  local c="$1" kind="${2:-chat}" model="${3:-gpt-5.4}" rc i=0
  while [ "$i" -lt 30 ]; do
    rc="$(CHAT_CLIENT="$c" CHAT_RETRIES=1 _chat "$model" "$kind")"
    [ "$rc" != 000 ] && { echo "$rc"; return 0; }
    i=$((i + 5))
    sleep 5
  done
  echo 000
  return 1
 }
 # wait_peer_connected CLIENT [TIMEOUT] — poll until the client reports the proxy
 # peer as 1/1 Connected, i.e. the proxy peer has re-stabilised after a churn.
 # Used to settle the peer before joining a fresh client into it.
 wait_peer_connected() {
  local c="$1" timeout="${2:-180}" i=0
  while [ "$i" -lt "$timeout" ]; do
    docker exec "$c" netbird status 2>/dev/null | grep -q '1/1 Connected' && return 0
    sleep 5
    i=$((i + 5))
  done
  return 1
 }
 provider_id_by_name() { api GET /api/agent-network/providers | jq -r --arg n "$1" '.[] | select(.name==$n) | .id' | head -n1; }
 access_log_total()    { api GET /api/agent-network/access-logs | jq -r '.total_records'; }
 access_log_top()      { api GET /api/agent-network/access-logs | jq -r ".data[0].$1 // \"\""; }
 # ensure_group NAME — return the id of group NAME, creating it if absent.
 ensure_group() {
  local name="$1" id
  id="$(api GET /api/groups | jq -r --arg n "$name" '.[] | select(.name==$n) | .id' | head -n1)"
  if [ -z "$id" ] || [ "$id" = null ]; then
    id="$(api POST /api/groups "$(jq -n --arg n "$name" '{name:$n}')" | jq -r '.id')"
  fi
  echo "$id"
 }
 # policy_put_limits LIMITS_JSON — replace the e2e-all-providers policy limits.
 policy_put_limits() {
  local pol id body
  pol="$(api GET /api/agent-network/policies | jq -c '.[] | select(.name=="e2e-all-providers")')"
  id="$(echo "$pol" | jq -r '.id')"
  [ -n "$id" ] && [ "$id" != null ] || die "e2e-all-providers policy missing (run 'policy' first)"
  body="$(echo "$pol" | jq -c --argjson L "$1" '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids,limits:$L}')"
  api PUT "/api/agent-network/policies/$id" "$body" >/dev/null
 }
 settings_put() { api PUT /api/agent-network/settings "$1" >/dev/null; }
 # --- scenarios ---------------------------------------------------------------
 # Scenario 1: policy token-cap enforcement. Seed usage under a high cap, drop
 # the cap to 1 token so the next call is denied, then restore and confirm
 # recovery. Deterministic regardless of prior window usage.
 cmd_scenario_budget() {
  log "### scenario: policy token-cap enforcement (deny + recovery) ###"
  # Seed AND enforce in the same 3600s token window (counters are per-window).
  policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":100000,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat gpt-5.4 chat)" "seed call under high token cap (books usage into 3600s window)"
  # Drop the cap to 1 in the SAME window so the seeded usage exhausts it.
  policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":1,"user_cap":1,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  local code; code="$(_chat gpt-5.4 chat)"
  expect_code 403 "$code" "call denied once token cap (1) is exhausted"
  log "  deny envelope: $(cat "$CHAT_RESP")"
  policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat gpt-5.4 chat)" "call allowed again after cap restored"
 }
 # Scenario 2: account-level budget rule (separate from policy limits). A rule
 # targeting Admins with a 1-token cap must deny via the same gRPC check loop;
 # deleting it restores access.
 cmd_scenario_budget_rule() {
  log "### scenario: account budget-rule enforcement ###"
  local gid rid
  gid="$(admins_group_id)"
  # High cap first so the seed call books usage into the rule's 3600s window.
  rid="$(api POST /api/agent-network/budget-rules "$(jq -n --arg g "$gid" '{name:"e2e-tight-rule",enabled:true,target_groups:[$g],target_users:[],limits:{budget_limit:{enabled:false,group_cap_usd:0,user_cap_usd:0,window_seconds:3600},token_limit:{enabled:true,group_cap:100000,user_cap:100000,window_seconds:3600}}}')" | jq -r '.id')"
  log "  created budget-rule $rid (group token cap=100000)"
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat gpt-5.4 chat)" "seed call under high account cap"
  # Tighten the same rule/window to 1 so the seeded usage exhausts it.
  api PUT "/api/agent-network/budget-rules/$rid" "$(jq -n --arg g "$gid" '{name:"e2e-tight-rule",enabled:true,target_groups:[$g],target_users:[],limits:{budget_limit:{enabled:false,group_cap_usd:0,user_cap_usd:0,window_seconds:3600},token_limit:{enabled:true,group_cap:1,user_cap:1,window_seconds:3600}}}')" >/dev/null
  log "  tightened budget-rule $rid (group token cap=1)"
  wait_tunnel; sleep 2
  expect_code 403 "$(_chat gpt-5.4 chat)" "account budget-rule denies when cap exhausted"
  api DELETE "/api/agent-network/budget-rules/$rid" >/dev/null && log "  deleted budget-rule $rid"
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat gpt-5.4 chat)" "allowed again after budget-rule removed"
 }
 # Scenario 3: multiple groups + policies + destination scoping. A 2nd Docker
 # client joins a 2nd group; each group's policy authorises a different
 # provider. Cross-provider requests must be denied.
 cmd_scenario_multigroup() {
  log "### scenario: multi-group / multi-policy destination scoping ###"
  local gidA gidB oai ant keyB polAllId
  gidA="$(admins_group_id)"; gidB="$(ensure_group e2e-grp-b)"
  oai="$(provider_id_by_name 'OpenAI API')"; ant="$(provider_id_by_name 'Anthropic API')"
  log "  groups: Admins=$gidA grp-b=$gidB ; providers: openai=$oai anthropic=$ant"
  # Disable the broad e2e policy; add two narrow ones.
  polAllId="$(api GET /api/agent-network/policies | jq -r '.[] | select(.name=="e2e-all-providers") | .id')"
  [ -n "$polAllId" ] && api PUT "/api/agent-network/policies/$polAllId" \
    "$(api GET /api/agent-network/policies | jq -c --arg id "$polAllId" '.[] | select(.id==$id) | {name,description,enabled:false,source_groups,destination_provider_ids,guardrail_ids,limits}')" >/dev/null
  local hi='{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":60}}'
  local polA polB
  polA="$(api POST /api/agent-network/policies "$(jq -n --arg g "$gidA" --arg p "$oai" --argjson L "$hi" '{name:"e2e-A-admins-openai",enabled:true,source_groups:[$g],destination_provider_ids:[$p],guardrail_ids:[],limits:$L}')" | jq -r '.id')"
  polB="$(api POST /api/agent-network/policies "$(jq -n --arg g "$gidB" --arg p "$ant" --argjson L "$hi" '{name:"e2e-B-grpb-anthropic",enabled:true,source_groups:[$g],destination_provider_ids:[$p],guardrail_ids:[],limits:$L}')" | jq -r '.id')"
  log "  policy A (Admins->OpenAI)=$polA ; policy B (grp-b->Anthropic)=$polB"
  # The policy swap above re-synthesises the agent-network service and adds a
  # new source group (grp-b), which churns the proxy peer and forces it to
  # absorb the new authorisation. Wait for the peer to re-stabilise (client A
  # reconnects to the new peer) before joining B, so B handshakes into a stable
  # peer instead of racing the churn — otherwise the proxy never answers B's
  # offers and it sits at 0/1 Connected.
  if ! wait_peer_connected "$CLIENT" 30; then
    log "  warning: proxy peer did not re-stabilise for client A within 30s"
  fi
  sleep 5
  # 2nd client bound to grp-b.
  keyB="$(api POST /api/setup-keys "$(jq -n --arg g "$gidB" '{name:"e2e-agentnet-docker-b",type:"reusable",expires_in:86400,usage_limit:0,auto_groups:[$g],ephemeral:false}')" | jq -r '.key')"
  docker rm -f "${CLIENT}-b" >/dev/null 2>&1 || true
  docker run -d --name "${CLIENT}-b" --cap-add NET_ADMIN --cap-add SYS_ADMIN --cap-add SYS_RESOURCE \
    --add-host host.docker.internal:host-gateway -e NB_MANAGEMENT_URL="$MGMT_URL" -e NB_SETUP_KEY="$keyB" \
    -e NB_LOG_LEVEL=info "$IMAGE" >/dev/null
  log "  started ${CLIENT}-b (grp-b)"
  # Gate on real end-to-end connectivity: a freshly-authorised group's client
  # takes ~1-2 min for the proxy to absorb and answer its WG handshake.
  local ra rb
  ra="$(wait_chat_ready "$CLIENT" chat gpt-5.4)"
  rb="$(wait_chat_ready "${CLIENT}-b" messages claude-haiku-4-5)"
  log "  connectivity ready: A(probe=$ra) B(probe=$rb)"
  log "  -- client A (Admins) --"
  CHAT_CLIENT="$CLIENT"   expect_code 200 "$(CHAT_CLIENT="$CLIENT"   _chat gpt-5.4 chat)"          "A->OpenAI authorised"
  CHAT_CLIENT="$CLIENT"   expect_code 403 "$(CHAT_CLIENT="$CLIENT"   _chat claude-haiku-4-5 messages)" "A->Anthropic denied (not in policy A)"
  log "  -- client B (grp-b) --"
  CHAT_CLIENT="${CLIENT}-b" expect_code 200 "$(CHAT_CLIENT="${CLIENT}-b" _chat claude-haiku-4-5 messages)" "B->Anthropic authorised"
  CHAT_CLIENT="${CLIENT}-b" expect_code 403 "$(CHAT_CLIENT="${CLIENT}-b" _chat gpt-5.4 chat)"          "B->OpenAI denied (not in policy B)"
  # Restore: drop A/B, re-enable the broad policy.
  api DELETE "/api/agent-network/policies/$polA" >/dev/null 2>&1 || true
  api DELETE "/api/agent-network/policies/$polB" >/dev/null 2>&1 || true
  [ -n "$polAllId" ] && api PUT "/api/agent-network/policies/$polAllId" \
    "$(api GET /api/agent-network/policies | jq -c --arg id "$polAllId" '.[] | select(.id==$id) | {name,description,enabled:true,source_groups,destination_provider_ids,guardrail_ids,limits}')" >/dev/null
  docker rm -f "${CLIENT}-b" >/dev/null 2>&1 || true
  log "  restored broad policy, removed client B"
 }
 # Scenario 4: log-collection + prompt-collection + redaction settings.
 cmd_scenario_logs() {
  log "### scenario: log/prompt-collection + redaction settings ###"
  settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false}'
  wait_tunnel; sleep 3
  # (a) log collection OFF -> no new access-log row.
  local before after
  before="$(access_log_total)"
  settings_put '{"enable_log_collection":false,"enable_prompt_collection":true,"redact_pii":false}'
  wait_tunnel; sleep 3
  expect_code 200 "$(_chat gpt-5.4 chat)" "request still served with logging off"
  sleep 3; after="$(access_log_total)"
  if [ "$after" = "$before" ]; then ok "log collection OFF -> no new row ($before==$after)"; else bad "log collection OFF still wrote a row ($before->$after)"; fi
  # (b) log ON, prompt OFF -> row present but request_prompt empty.
  settings_put '{"enable_log_collection":true,"enable_prompt_collection":false,"redact_pii":false}'
  wait_tunnel; sleep 3
  expect_code 200 "$(_chat gpt-5.4 chat "this prompt text must NOT be stored")" "served with prompt collection off"
  sleep 3
  local p; p="$(access_log_top request_prompt)"
  if [ -z "$p" ]; then ok "prompt collection OFF -> request_prompt empty"; else bad "prompt stored despite collection off: [$p]"; fi
  # (c) prompt ON + redact ON -> PII scrubbed from stored prompt.
  settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":true}'
  wait_tunnel; sleep 3
  expect_code 200 "$(_chat gpt-5.4 chat "Contact john.doe@example.com, SSN 123-45-6789, phone 555-123-4567")" "served with redaction on"
  sleep 3
  local stored; stored="$(access_log_top request_prompt)"
  log "  stored prompt: [$stored]"
  case "$stored" in
    *john.doe@example.com*) bad "email leaked into stored prompt" ;;
    *) ok "email redacted from stored prompt" ;;
  esac
  case "$stored" in
    *123-45-6789*) bad "SSN leaked into stored prompt" ;;
    *) ok "SSN redacted from stored prompt" ;;
  esac
  settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false}'
  log "  restored settings (log on, prompt on, redact off)"
 }
 # Scenario 5: per-user cap isolated from the group cap. Seed under a high cap,
 # then keep the group cap high but drop the USER cap to 1 — the deny must come
 # from the user dimension, proving per-user accounting is independent.
 cmd_scenario_user_cap() {
  log "### scenario: per-user token cap (isolated from group cap) ###"
  policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":100000,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat gpt-5.4 chat)" "seed call (books user+group usage into 3600s window)"
  # Group cap stays huge; only the user cap is exhausted.
  policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":1,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  expect_code 403 "$(_chat gpt-5.4 chat)" "denied by USER cap=1 while group cap=100000 has headroom"
  log "  deny: $(cat "$CHAT_RESP")"
  policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat gpt-5.4 chat)" "allowed again after user cap lifted"
 }
 # Scenario 6: guardrail model-allowlist blocking. Attach a guardrail allowing
 # only gpt-5.4; a request for any other model must be denied (model_blocked),
 # while the allowed model passes.
 cmd_scenario_guardrail() {
  log "### scenario: guardrail model-allowlist blocking ###"
  local gid polId
  gid="$(api POST /api/agent-network/guardrails "$(jq -n '{name:"e2e-allowlist",description:"e2e: only gpt-5.4",checks:{model_allowlist:{enabled:true,models:["gpt-5.4"]},prompt_capture:{enabled:true,redact_pii:false}}}')" | jq -r '.id')"
  log "  created guardrail $gid (allow only gpt-5.4)"
  local pol; pol="$(api GET /api/agent-network/policies | jq -c '.[] | select(.name=="e2e-all-providers")')"
  polId="$(echo "$pol" | jq -r '.id')"
  api PUT "/api/agent-network/policies/$polId" "$(echo "$pol" | jq -c --arg g "$gid" '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids:[$g],limits}')" >/dev/null
  wait_tunnel; sleep 2
  expect_code 403 "$(_chat gpt-4o-mini chat)" "model not in allowlist is blocked"
  log "  deny: $(cat "$CHAT_RESP")"
  expect_code 200 "$(_chat gpt-5.4 chat)" "allowlisted model passes"
  # Detach + delete.
  api PUT "/api/agent-network/policies/$polId" "$(echo "$pol" | jq -c '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids:[],limits}')" >/dev/null
  api DELETE "/api/agent-network/guardrails/$gid" >/dev/null && log "  detached + deleted guardrail $gid"
 }
 # _chat_stream MODEL — streaming chat completion (SSE) with usage included.
 # Echoes the HTTP status; writes the raw SSE stream to $CHAT_RESP.
 _chat_stream() {
  local model="$1" client="${CHAT_CLIENT:-$CLIENT}" ip body out code
  ip="$(CLIENT="$client" proxy_ip)"; [ -n "$ip" ] || { echo 000; return; }
  body="$(jq -n --arg m "$model" '{model:$m,stream:true,stream_options:{include_usage:true},messages:[{role:"user",content:"Reply with exactly: pong"}]}')"
  out="$(docker run --rm --network "container:$client" "$CURL_IMAGE" \
    -sSkN --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
    -w $'\n%{http_code}' -X POST "https://$PROXY_HOST/v1/chat/completions" \
    -H "Content-Type: application/json" --data "$body")"
  code="$(printf '%s' "$out" | tail -n1)"
  printf '%s' "$out" | sed '$d' >"$CHAT_RESP"
  echo "$code"
 }
 # Scenario 7: streaming (SSE) token capture. The proxy must accumulate token
 # usage from the streamed deltas and persist it on the access-log row.
 cmd_scenario_streaming() {
  log "### scenario: streaming (SSE) token capture ###"
  policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
  wait_tunnel; sleep 2
  expect_code 200 "$(_chat_stream gpt-5.4)" "streaming request served"
  if grep -q 'data: \[DONE\]' "$CHAT_RESP"; then ok "SSE stream terminated with [DONE]"; else bad "no [DONE] terminator in SSE stream"; fi
  if grep -q '"delta"' "$CHAT_RESP"; then ok "SSE carried incremental deltas"; else bad "no delta chunks in SSE stream"; fi
  sleep 3
  local st in out
  st="$(access_log_top stream)"; in="$(access_log_top input_tokens)"; out="$(access_log_top output_tokens)"
  log "  access-log: stream=$st input_tokens=$in output_tokens=$out"
  if [ "$st" = true ]; then ok "access-log row flagged stream=true"; else bad "stream flag not set (got $st)"; fi
  if [ "${in:-0}" -gt 0 ] 2>/dev/null && [ "${out:-0}" -gt 0 ] 2>/dev/null; then ok "streamed token usage captured (in=$in out=$out)"; else bad "streamed token usage not captured (in=$in out=$out)"; fi
 }
 # wait_for_mgmt — poll the management API until it accepts authed requests.
 wait_for_mgmt() {
  local i=0
  while [ "$i" -lt 90 ]; do
    curl -fsS -o /dev/null --max-time 2 -H "$AUTH" "$NB_API/api/users" 2>/dev/null && return 0
    sleep 2; i=$((i+2))
  done
  return 1
 }
 # Scenario 8: access-log retention pruning. The sweep deletes rows older than
 # now - retention_days and runs on management startup. We clone an existing row
 # to a synthetic 2020-dated id, set retention to 365d (so only that far-past
 # row is eligible — real 2026 rows are safe), restart management to trigger the
 # sweep, and assert the synthetic row is gone while real rows survive.
 cmd_scenario_retention() {
  log "### scenario: access-log retention pruning ###"
  local DB="/Users/maycon/projects/wt_testing_data/store.db" acct="d68ag3p31576fp2gmnag" sid="e2e-retention-2020"
  command -v sqlite3 >/dev/null 2>&1 || { bad "sqlite3 not on PATH"; return; }
  [ -w "$DB" ] || { bad "store.db not writable at $DB"; return; }
  local before mid after syn
  before="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")"
  sqlite3 "$DB" "PRAGMA busy_timeout=8000;
    DELETE FROM agent_network_access_log WHERE id='$sid';
    CREATE TEMP TABLE _t AS SELECT * FROM agent_network_access_log WHERE account_id='$acct' LIMIT 1;
    UPDATE _t SET id='$sid', timestamp='2020-01-01 00:00:00.000000000+00:00';
    INSERT INTO agent_network_access_log SELECT * FROM _t;" 2>&1 | sed 's/^/  sqlite: /' >&2 || true
  mid="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")"
  if [ "$mid" = "$((before+1))" ]; then ok "synthetic 2020 row inserted ($before -> $mid)"; else bad "insert failed ($before -> $mid)"; return; fi
  settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false,"access_log_retention_days":365}'
  log "  retention=365d; restarting management to trigger the startup sweep..."
  tilt trigger management >/dev/null 2>&1 || bad "tilt trigger management failed"
  wait_for_mgmt || bad "management did not come back up"
  sleep 4
  syn="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE id='$sid'")"
  after="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")"
  if [ "$syn" = 0 ]; then ok "synthetic 2020 row pruned by retention sweep"; else bad "synthetic row survived (sweep didn't prune)"; fi
  if [ "$after" = "$before" ]; then ok "real rows preserved (count back to $before)"; else bad "real row count changed ($before -> $after)"; fi
  settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false,"access_log_retention_days":0}'
  sqlite3 "$DB" "DELETE FROM agent_network_access_log WHERE id='$sid'" 2>/dev/null || true
  log "  restored retention=0 (keep forever); re-establishing tunnel"
  wait_tunnel >/dev/null 2>&1 || true
 }
 # vertex_request IP PATH BODY — POST a Vertex rawPredict through the client
 # sidecar; echo the HTTP status and stash the response body in VTX_RESP. Shared
 # by the Vertex scenario's initial probe and its live-update propagation checks.
 VTX_RESP=""
 vertex_request() {
  local ip="$1" path="$2" body="$3" out
  out="$(docker run --rm --network "container:$CLIENT" "$CURL_IMAGE" \
    -sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
    -w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \
    -H "Content-Type: application/json" --data "$body")"
  VTX_RESP="$(printf '%s' "$out" | sed '$d')"
  printf '%s' "$out" | tail -n1
 }
 # bedrock_probe LABEL PATH BODY IP — POST a Bedrock request via the client
 # sidecar and assert the pipeline. 200 -> full pass (+ token metering); a
 # Bedrock-origin 404 (account use-case gate / model access) -> pipeline OK
 # (routing + bearer auth reached the model); 401/403 -> fail.
 bedrock_probe() {
  local label="$1" path="$2" body="$3" ip="$4" out code resp
  out="$(docker run --rm --network "container:$CLIENT" "$CURL_IMAGE" \
    -sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
    -w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \
    -H "Content-Type: application/json" --data "$body")"
  code="$(printf '%s' "$out" | tail -n1)"
  resp="$(printf '%s' "$out" | sed '$d')"
  log "  bedrock $label -> http=$code"
  case "$code" in
    200)
      ok "Bedrock $label 200 (path->model parse, routed, bearer accepted)"
      sleep 3
      local vin vout
      vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)"
      if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "Bedrock $label usage metered (in=$vin out=$vout)"; else bad "Bedrock $label usage not metered (in=$vin out=$vout)"; fi
      ;;
    404)
      case "$resp" in
        *"use case"*|*NOT_FOUND*|*bedrock*|*"don't have access"*|*"inference profile"*)
          ok "reached Bedrock ($label, provider+bearer OK); 404 = account use-case gate / model access" ;;
        *) bad "Bedrock $label 404 but not a Bedrock response: $(printf '%s' "$resp" | head -c 160)" ;;
      esac ;;
    401) bad "Bedrock $label rejected the bearer token (401): $(printf '%s' "$resp" | head -c 160)" ;;
    403) bad "Bedrock $label denied before upstream (routing regression): $(printf '%s' "$resp" | head -c 160)" ;;
    000) bad "Bedrock $label no connectivity (tunnel not converged)" ;;
    *)   bad "Bedrock $label unexpected status $code: $(printf '%s' "$resp" | head -c 160)" ;;
  esac
 }
 # Scenario 9: Google Vertex AI (path-routed model + service-account OAuth minting).
 # Vertex carries the model in the URL path and authenticates with a short-lived
 # OAuth2 access token, not an API key. The operator stores a durable GCP
 # service-account key as the provider api_key behind a "keyfile::" prefix; the
 # synthesiser ships the base64 SA JSON to the router, which mints + caches a
 # Bearer token per request. We assert the whole pipeline: path->model parse,
 # route-to-Vertex, token mint + inject, upstream forward. A 200 is a complete
 # pass; a 404 whose body is Vertex's own "Publisher model not found / no access"
 # also passes the pipeline (the token was accepted) and only signals that the
 # test project lacks model access in that region.
 cmd_scenario_vertex() {
  log "### scenario: Google Vertex AI (path-routed + service-account OAuth minting) ###"
  require_llm_keys
  if [ -z "${GOOGLE_VERTEX_SA_BASE64:-}" ] || [ -z "${GOOGLE_VERTEXT_PROJECT:-}" ]; then
    log "  SKIP: GOOGLE_VERTEX_SA_BASE64 / GOOGLE_VERTEXT_PROJECT not set in $LLM_KEYS_FILE"
    return 0
  fi
  local vid pol pid old oldvids region vhost
  # The regional endpoint host is "<region>-aiplatform.googleapis.com"; the
  # "global" location uses the bare "aiplatform.googleapis.com" host.
  region="${GOOGLE_VERTEXT_REGION:-global}"
  if [ "$region" = global ]; then vhost="aiplatform.googleapis.com"; else vhost="${region}-aiplatform.googleapis.com"; fi
  # Remove any pre-existing Vertex providers, de-referencing them from the broad
  # policy first — a provider that is still a policy destination can't be deleted
  # (422), and a survivor leaves a duplicate (possibly stale-host) Vertex route.
  oldvids="$(api GET /api/agent-network/providers | jq -c '[.[]|select(.provider_id=="vertex_ai_api")|.id]')"
  if [ "$oldvids" != "[]" ]; then
    pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
    pid="$(echo "$pol" | jq -r '.id')"
    api PUT "/api/agent-network/policies/$pid" \
      "$(echo "$pol" | jq -c --argjson rm "$oldvids" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-$rm),guardrail_ids,limits}')" >/dev/null 2>&1 || true
    for old in $(echo "$oldvids" | jq -r '.[]'); do
      api DELETE "/api/agent-network/providers/$old" >/dev/null 2>&1 || true
    done
  fi
  vid="$(api POST /api/agent-network/providers "$(jq -n --arg k "keyfile::${GOOGLE_VERTEX_SA_BASE64}" --arg u "https://${vhost}" \
    '{name:"Vertex e2e",provider_id:"vertex_ai_api",upstream_url:$u,api_key:$k,enabled:true}')" \
    | jq -r '.id')"
  [ -n "$vid" ] && [ "$vid" != null ] || { bad "create Vertex provider failed"; return; }
  log "  created Vertex provider $vid (keyfile:: service-account, $vhost)"
  pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
  pid="$(echo "$pol" | jq -r '.id')"
  [ -n "$pid" ] && [ "$pid" != null ] || { bad "e2e-all-providers policy missing (run 'policy' first)"; return; }
  api PUT "/api/agent-network/policies/$pid" \
    "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:((.destination_provider_ids+[$v])|unique),guardrail_ids,limits}')" >/dev/null
  policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
  # Provider create/update propagates to the proxy live via reconcile (the synth
  # config is re-pushed and the middleware chain rebuilt on receipt), so just let
  # the tunnel settle before probing — no restart needed.
  wait_tunnel; sleep 5
  local ip path body code resp
  ip="$(proxy_ip)"; [ -n "$ip" ] || { bad "could not resolve proxy IP"; return; }
  path="/v1/projects/${GOOGLE_VERTEXT_PROJECT}/locations/${region}/publishers/anthropic/models/claude-sonnet-4-5@20250929:rawPredict"
  body="$(jq -n '{anthropic_version:"vertex-2023-10-16",max_tokens:64,messages:[{role:"user",content:"Reply with exactly: pong"}]}')"
  code="$(vertex_request "$ip" "$path" "$body")"; resp="$VTX_RESP"
  log "  vertex rawPredict -> http=$code"
  case "$code" in
    200)
      ok "Vertex 200 (path->model parse, routed, token minted + accepted, model available)"
      sleep 3
      local vin vout
      vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)"
      log "  access-log: model=$(access_log_top model) input_tokens=$vin output_tokens=$vout"
      if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "Vertex usage metered (in=$vin out=$vout)"; else bad "Vertex usage not metered (in=$vin out=$vout)"; fi
      ;;
    404)
      case "$resp" in
        *"Publisher model"*|*aiplatform*|*NOT_FOUND*)
          ok "reached Vertex with a minted token (pipeline OK); 404 = test project lacks model access" ;;
        *) bad "404 but not a Vertex response body: $(printf '%s' "$resp" | head -c 160)" ;;
      esac ;;
    401) bad "Vertex rejected the minted credential (401) — check the SA key/roles: $(printf '%s' "$resp" | head -c 160)" ;;
    403) bad "proxy denied before reaching Vertex (path parse / routing regression): $(printf '%s' "$resp" | head -c 160)" ;;
    000) bad "no connectivity to proxy (tunnel not converged)" ;;
    *)   bad "unexpected status $code: $(printf '%s' "$resp" | head -c 160)" ;;
  esac
  # Live provider-update propagation: a provider change must reach the proxy node
  # without a restart. Disable the provider -> its route disappears and the request
  # is denied (403); re-enable -> served again at the baseline status. Proves
  # reconcile re-pushes the synth config and the proxy rebuilds its chain live.
  case "$code" in
    200|404)
      log "  testing live provider-update propagation (disable/enable, no restart)..."
      local dcode ecode
      set_enabled "$vid" false; sleep 5
      dcode="$(vertex_request "$ip" "$path" "$body")"
      if [ "$dcode" = 403 ]; then ok "provider disable reflected on proxy live (route removed, http 403)"; else bad "provider disable not reflected on proxy (want 403, got $dcode)"; fi
      set_enabled "$vid" true; sleep 5
      ecode="$(vertex_request "$ip" "$path" "$body")"
      if [ "$ecode" = "$code" ]; then ok "provider re-enable reflected on proxy live (served again, http $ecode)"; else bad "provider re-enable not reflected on proxy (want $code, got $ecode)"; fi
      ;;
  esac
  pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
  api PUT "/api/agent-network/policies/$pid" \
    "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-[$v]),guardrail_ids,limits}')" >/dev/null 2>&1 || true
  api DELETE "/api/agent-network/providers/$vid" >/dev/null 2>&1 || true
 }
 # Scenario 10: real-traffic coverage for the OpenAI-compatible gateway providers
 # (Vercel, OpenRouter, Cloudflare). Each is isolated as the sole enabled provider
 # so its model routes to it unambiguously, sent a real chat request, and asserted
 # to return 200 with metered token usage. A gateway without a configured provider
 # is skipped. Restores all providers at the end.
 cmd_scenario_providers() {
  log "### scenario: gateway providers real-traffic (Vercel / OpenRouter / Cloudflare) ###"
  policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
  local spec iso full model kind r pid code vin vout
  for spec in \
    "vercel|Vercel AI Gateway|openai/gpt-4o-mini|chat" \
    "openrouter|OpenRouter|openai/gpt-4o-mini|chat" \
    "cloudflare|Cloudflare AI Gateway|gpt-4o-mini|chat"; do
    iso="${spec%%|*}"; r="${spec#*|}"; full="${r%%|*}"; r="${r#*|}"; model="${r%%|*}"; kind="${r##*|}"
    pid="$(provider_id_by_name "$full")"
    if [ -z "$pid" ] || [ "$pid" = null ]; then log "  SKIP $full: not configured"; continue; fi
    cmd_isolate "$iso" >/dev/null 2>&1
    wait_tunnel; sleep 4
    code="$(_chat "$model" "$kind")"
    if [ "$code" = 200 ]; then ok "$full served real request ($model, http 200)"; else bad "$full request failed ($model, http $code)"; continue; fi
    sleep 3
    vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)"
    if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "$full usage metered (in=$vin out=$vout)"; else bad "$full usage not metered (in=$vin out=$vout)"; fi
  done
  cmd_enable_all >/dev/null 2>&1
  log "  re-enabled all providers"
  wait_tunnel >/dev/null 2>&1 || true
 }
 # Scenario 11: AWS Bedrock (path-routed model + bearer auth) across invoke,
 # converse, and invoke-with-response-stream. Bedrock carries the model in the URL
 # path and authenticates with a static bearer token (Bedrock API key). Asserts
 # the pipeline per endpoint: path->model parse + normalize, route-to-Bedrock,
 # bearer inject, upstream forward (+ token metering on 200). A Bedrock-origin 404
 # (account use-case gate / model access) passes the pipeline; see bedrock_probe.
 cmd_scenario_bedrock() {
  log "### scenario: AWS Bedrock (path-routed + bearer; invoke/converse/stream) ###"
  require_llm_keys
  if [ -z "${AWS_BEARER_TOKEN_BEDROCK:-}" ]; then
    log "  SKIP: AWS_BEARER_TOKEN_BEDROCK not set in $LLM_KEYS_FILE"
    return 0
  fi
  local region model vid pol pid old oldvids
  region="${AWS_BEDROCK_REGION:-eu-central-1}"
  model="${BEDROCK_MODEL:-eu.anthropic.claude-sonnet-4-5-20250929-v1:0}"
  oldvids="$(api GET /api/agent-network/providers | jq -c '[.[]|select(.provider_id=="bedrock_api")|.id]')"
  if [ "$oldvids" != "[]" ]; then
    pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
    pid="$(echo "$pol" | jq -r '.id')"
    api PUT "/api/agent-network/policies/$pid" \
      "$(echo "$pol" | jq -c --argjson rm "$oldvids" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-$rm),guardrail_ids,limits}')" >/dev/null 2>&1 || true
    for old in $(echo "$oldvids" | jq -r '.[]'); do
      api DELETE "/api/agent-network/providers/$old" >/dev/null 2>&1 || true
    done
  fi
  vid="$(api POST /api/agent-network/providers "$(jq -n --arg k "$AWS_BEARER_TOKEN_BEDROCK" --arg u "https://bedrock-runtime.${region}.amazonaws.com" \
    '{name:"Bedrock e2e",provider_id:"bedrock_api",upstream_url:$u,api_key:$k,enabled:true}')" | jq -r '.id')"
  [ -n "$vid" ] && [ "$vid" != null ] || { bad "create Bedrock provider failed"; return; }
  log "  created Bedrock provider $vid (bearer, bedrock-runtime.${region})"
  pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
  pid="$(echo "$pol" | jq -r '.id')"
  [ -n "$pid" ] && [ "$pid" != null ] || { bad "e2e-all-providers policy missing (run 'policy' first)"; return; }
  api PUT "/api/agent-network/policies/$pid" \
    "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:((.destination_provider_ids+[$v])|unique),guardrail_ids,limits}')" >/dev/null
  policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
  wait_tunnel; sleep 5
  local ip invokeBody converseBody
  ip="$(proxy_ip)"; [ -n "$ip" ] || { bad "could not resolve proxy IP"; return; }
  invokeBody='{"anthropic_version":"bedrock-2023-05-31","max_tokens":32,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}'
  converseBody='{"messages":[{"role":"user","content":[{"text":"Reply with exactly: pong"}]}],"inferenceConfig":{"maxTokens":32}}'
  bedrock_probe "invoke" "/model/${model}/invoke" "$invokeBody" "$ip"
  bedrock_probe "converse" "/model/${model}/converse" "$converseBody" "$ip"
  bedrock_probe "invoke-stream" "/model/${model}/invoke-with-response-stream" "$invokeBody" "$ip"
  # Gateway-namespace prefix: /bedrock/... must route the same and be stripped
  # before the upstream call (AWS's native path has no /bedrock prefix).
  bedrock_probe "invoke (/bedrock prefix)" "/bedrock/model/${model}/invoke" "$invokeBody" "$ip"
  pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
  api PUT "/api/agent-network/policies/$pid" \
    "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-[$v]),guardrail_ids,limits}')" >/dev/null 2>&1 || true
  api DELETE "/api/agent-network/providers/$vid" >/dev/null 2>&1 || true
 }
 cmd_scenarios() {
  FAILS=0
  cmd_scenario_budget
  cmd_scenario_budget_rule
  cmd_scenario_user_cap
  cmd_scenario_guardrail
  cmd_scenario_streaming
  cmd_scenario_multigroup
  cmd_scenario_logs
  cmd_scenario_retention
  cmd_scenario_providers
  cmd_scenario_vertex
  cmd_scenario_bedrock
  log "================================================"
  if [ "$FAILS" -eq 0 ]; then log "ALL SCENARIOS PASSED"; else log "SCENARIO FAILURES: $FAILS"; fi
  return "$FAILS"
 }
 # --- dispatch ----------------------------------------------------------------
 cmd="${1:-}"; shift || true
 case "$cmd" in
  snapshot)      cmd_snapshot ;;
  key)           cmd_key ;;
  up)            cmd_up ;;
  status)        cmd_status ;;
  wait)          cmd_wait ;;
  diag)          cmd_diag ;;
  restart-proxy) cmd_restart_proxy ;;
  chat)          cmd_chat "$@" ;;
  verify)        cmd_verify ;;
  clean)         cmd_clean ;;
  providers)     cmd_providers ;;
  policy)        cmd_policy ;;
  isolate)       cmd_isolate "$@" ;;
  enable-all)    cmd_enable_all ;;
  scenario-budget)      cmd_scenario_budget ;;
  scenario-budget-rule) cmd_scenario_budget_rule ;;
  scenario-user-cap)    cmd_scenario_user_cap ;;
  scenario-guardrail)   cmd_scenario_guardrail ;;
  scenario-streaming)   cmd_scenario_streaming ;;
  scenario-multigroup)  cmd_scenario_multigroup ;;
  scenario-logs)        cmd_scenario_logs ;;
  scenario-retention)   cmd_scenario_retention ;;
  scenario-providers)   cmd_scenario_providers ;;
  scenario-vertex)      cmd_scenario_vertex ;;
  scenario-bedrock)     cmd_scenario_bedrock ;;
  dashboard)            cmd_dashboard ;;
  scenarios)            cmd_scenarios ;;
  down)          cmd_down ;;
  all)           cmd_all ;;
  *) cat >&2 <<'USAGE'
 usage: bash scripts/e2e/agent-network-full/e2e.sh <cmd>
  setup/flow : snapshot key up status wait diag restart-proxy clean providers policy isolate NAME enable-all down all
  traffic    : chat MODEL [chat|messages]   verify
  scenarios  : scenarios | scenario-budget | scenario-budget-rule | scenario-user-cap |
               scenario-guardrail | scenario-streaming | scenario-multigroup |
               scenario-logs | scenario-retention | scenario-providers | scenario-vertex |
               scenario-bedrock
  dashboard  : dashboard   (Playwright UI check against the live :3000 dashboard)
 USAGE
     exit 2 ;;
 esac
@@ -1,105 +0,0 @@
 # shellcheck disable=SC2148
 # Sourced helper for the agent-network-policy e2e suite.
 # Verifies the agent-network surface: window_seconds (post-rename)
 # through the API, /api/agent-network/consumption read endpoint, and
 # the CheckLLMPolicyLimits + RecordLLMUsage gRPC RPCs.
 #
 # Do not run directly.
 set -euo pipefail
 : "${NB_API:=http://localhost:8080}"
 : "${NB_PAT_FILE:=/Users/maycon/projects/local-dev/nb-pat}"
 : "${NB_TIMEOUT_SECONDS:=60}"
 : "${NB_POLICY_NAME:=e2e-anpol}"
 : "${NB_PROVIDER_NAME:=e2e-anpol-provider}"
 : "${NB_GROUP_NAME:=e2e-anpol-engineers}"
 : "${NB_GRPC_ADDR:=localhost:8080}"
 # Proxy token shared with the tilt setup. Keep in sync with the
 # NB_PROXY_TOKEN literal in /Users/maycon/projects/local-dev/Tiltfile;
 # the management server accepts it as a registered proxy credential
 # so the e2e binary can reach the proxy_service gRPC surface.
 : "${NB_PROXY_TOKEN:=nbx_MEF9OKRhlLrWkc5TJmM3Eu2rhqigaP2yulHy}"
 : "${NB_STATE_DIR:=/tmp/nb-anpol-e2e-state}"
 mkdir -p "$NB_STATE_DIR"
 if [ ! -r "$NB_PAT_FILE" ]; then
    echo "FAIL: cannot read PAT at $NB_PAT_FILE" >&2
    exit 2
 fi
 NB_PAT=$(tr -d '\n\r ' <"$NB_PAT_FILE")
 if [ ${#NB_PAT} -lt 16 ]; then
    echo "FAIL: PAT at $NB_PAT_FILE is suspiciously short (${#NB_PAT} chars)" >&2
    exit 2
 fi
 if ! command -v jq >/dev/null 2>&1; then
    echo "FAIL: jq is required (brew install jq)" >&2
    exit 2
 fi
 # nb_api METHOD PATH [BODY] — wraps curl with PAT auth and JSON.
 nb_api() {
    local method="$1" path="$2" body="${3-}"
    if [ -n "$body" ]; then
        curl -fsS -X "$method" \
            -H "Authorization: Token $NB_PAT" \
            -H "Content-Type: application/json" \
            --data "$body" \
            "$NB_API$path"
    else
        curl -fsS -X "$method" \
            -H "Authorization: Token $NB_PAT" \
            "$NB_API$path"
    fi
 }
 # nb_api_status METHOD PATH [BODY] — returns the HTTP status code
 # rather than the body. Use for negative-path tests where a 4xx is
 # the expected outcome and curl's default -f would mask the result.
 nb_api_status() {
    local method="$1" path="$2" body="${3-}"
    if [ -n "$body" ]; then
        curl -sS -o /dev/null -w '%{http_code}' -X "$method" \
            -H "Authorization: Token $NB_PAT" \
            -H "Content-Type: application/json" \
            --data "$body" \
            "$NB_API$path"
    else
        curl -sS -o /dev/null -w '%{http_code}' -X "$method" \
            -H "Authorization: Token $NB_PAT" \
            "$NB_API$path"
    fi
 }
 # wait_for COND_CMD TIMEOUT_S — polls every 1s.
 wait_for() {
    local cmd="$1" timeout="${2:-$NB_TIMEOUT_SECONDS}"
    local i=0
    while [ "$i" -lt "$timeout" ]; do
        if eval "$cmd" >/dev/null 2>&1; then
            return 0
        fi
        sleep 1
        i=$((i + 1))
    done
    return 1
 }
 # resolve the absolute path of the netbird repo root from the script
 # location so the Go smoke binary can be invoked via `go run` against
 # the right module regardless of the caller's cwd.
 nb_repo_root() {
    cd "$(dirname "$0")/../../.." && pwd
 }
 pass() {
    printf 'PASS: %s\n' "$1"
    exit 0
 }
 fail() {
    printf 'FAIL: %s — %s\n' "$1" "${2:-}"
    exit 1
 }
@@ -1,44 +0,0 @@
 #!/usr/bin/env bash
 # 01-tilt-restart: re-trigger management + dashboard before each run so
 # we start from a clean process state. The on-disk store survives, so
 # the PAT, account, and any residual config persist across restarts.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 command -v tilt >/dev/null 2>&1 || fail "tilt not on PATH" "brew install tilt-dev/tap/tilt"
 if ! curl -fsS -o /dev/null --max-time 2 http://localhost:10350/ 2>/dev/null; then
    fail "Tilt API not reachable at http://localhost:10350" "is 'tilt up' running in /Users/maycon/projects/local-dev?"
 fi
 restart_one() {
    local name="$1"
    if ! tilt trigger "$name" >/dev/null 2>&1; then
        fail "tilt trigger $name failed" ""
    fi
    echo "triggered $name"
 }
 restart_one management
 restart_one dashboard
 # proxy3 needs to come back too because PR2 wired the new
 # llm_limit_check / llm_limit_record middlewares into the proxy
 # binary; without restarting it, e2e keeps exercising whatever
 # image was last loaded and silently misses any boot-order or
 # wiring regression in the chain. Cheap to restart, expensive to
 # silently miss.
 restart_one proxy3
 echo "waiting for management to accept requests..."
 if ! wait_for "curl -fsS -o /dev/null --max-time 2 $NB_API/oauth2/.well-known/openid-configuration" 60; then
    fail "management did not come back up within 60s" "check 'tilt logs management'"
 fi
 echo "management is up"
 code=$(curl -fsS -o /dev/null -w '%{http_code}' \
    -H "Authorization: Token $NB_PAT" \
    "$NB_API/api/users" 2>&1) || true
 [ "$code" = "200" ] || fail "PAT auth check failed after restart (HTTP $code)" "the PAT may have been revoked"
 pass "Tilt resources restarted: management, dashboard"
@@ -1,132 +0,0 @@
 #!/usr/bin/env bash
 # 10-policy-create: end-to-end round-trip on the new window_seconds
 # field. Creates the prerequisites (group + provider with
 # bootstrap_cluster), POSTs a policy whose Limits.token_limit and
 # budget_limit both carry window_seconds, then re-fetches the policy
 # and asserts the field comes back unchanged.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 # 1. Group — sweep first so re-runs are idempotent, then POST.
 existing_group=$(nb_api GET /api/groups 2>/dev/null \
    | jq -r --arg name "$NB_GROUP_NAME" '.[] | select(.name == $name) | .id // empty' \
    | head -1)
 if [ -n "$existing_group" ]; then
    echo "reusing existing group $existing_group"
    group_id="$existing_group"
 else
    body=$(jq -n --arg name "$NB_GROUP_NAME" '{name:$name}')
    group_resp=$(nb_api POST /api/groups "$body")
    group_id=$(printf '%s' "$group_resp" | jq -r '.id // ""')
    [ -n "$group_id" ] && [ "$group_id" != "null" ] \
        || fail "group create did not return an id" "$group_resp"
    echo "created group $group_id"
 fi
 printf '%s' "$group_id" >"$NB_STATE_DIR/group-id"
 # 2. Provider — also idempotent. Bootstrap cluster pinned to the local
 #    proxy3 cluster so the management settings row resolves and the
 #    create completes (subsequent provider creates in the same account
 #    ignore bootstrap_cluster, but the FIRST one needs it).
 existing_provider=$(nb_api GET /api/agent-network/providers 2>/dev/null \
    | jq -r --arg name "$NB_PROVIDER_NAME" '.[] | select(.name == $name) | .id // empty' \
    | head -1)
 if [ -n "$existing_provider" ]; then
    echo "reusing existing provider $existing_provider"
    provider_id="$existing_provider"
 else
    provider_body=$(jq -n \
        --arg name "$NB_PROVIDER_NAME" \
        '{
            provider_id: "openai_api",
            name: $name,
            upstream_url: "https://api.openai.com",
            api_key: "sk-e2e-placeholder",
            bootstrap_cluster: "proxy.netbird.local",
            models: []
        }')
    prov_resp=$(nb_api POST /api/agent-network/providers "$provider_body")
    provider_id=$(printf '%s' "$prov_resp" | jq -r '.id // ""')
    [ -n "$provider_id" ] && [ "$provider_id" != "null" ] \
        || fail "provider create did not return an id" "$prov_resp"
    echo "created provider $provider_id"
 fi
 printf '%s' "$provider_id" >"$NB_STATE_DIR/provider-id"
 # 3. Policy — drop any prior with the same name, then create with the
 #    NEW window_seconds field on both halves of Limits. 86400s = 24h
 #    on token, 2_592_000s = 30d on budget so the round-trip is
 #    unambiguous (no ambiguous unit-conversion artefact when we read
 #    back).
 existing_policy=$(nb_api GET /api/agent-network/policies 2>/dev/null \
    | jq -r --arg name "$NB_POLICY_NAME" '.[] | select(.name == $name) | .id // empty' \
    | head -1)
 if [ -n "$existing_policy" ]; then
    echo "deleting existing policy $existing_policy"
    nb_api DELETE "/api/agent-network/policies/$existing_policy" >/dev/null 2>&1 || true
 fi
 policy_body=$(jq -n \
    --arg name "$NB_POLICY_NAME" \
    --arg group "$group_id" \
    --arg provider "$provider_id" \
    '{
        name: $name,
        description: "agent-network e2e: window_seconds round-trip",
        enabled: true,
        source_groups: [$group],
        destination_provider_ids: [$provider],
        guardrail_ids: [],
        limits: {
            token_limit: {
                enabled: true,
                group_cap: 10000,
                user_cap: 5000,
                window_seconds: 86400
            },
            budget_limit: {
                enabled: true,
                group_cap_usd: 10.0,
                user_cap_usd: 2.5,
                window_seconds: 2592000
            }
        }
    }')
 resp=$(nb_api POST /api/agent-network/policies "$policy_body")
 policy_id=$(printf '%s' "$resp" | jq -r '.id // ""')
 [ -n "$policy_id" ] && [ "$policy_id" != "null" ] \
    || fail "policy create did not return an id" "$resp"
 printf '%s' "$policy_id" >"$NB_STATE_DIR/policy-id"
 echo "policy id: $policy_id"
 # 4. Round-trip assertions: GET the policy back and verify the
 #    window_seconds values land on both limit halves. The OLD
 #    window_hours / window_days fields must be absent.
 got=$(nb_api GET "/api/agent-network/policies/$policy_id")
 token_window=$(printf '%s' "$got" | jq -r '.limits.token_limit.window_seconds // empty')
 budget_window=$(printf '%s' "$got" | jq -r '.limits.budget_limit.window_seconds // empty')
 [ "$token_window" = "86400" ] \
    || fail "token_limit.window_seconds did not round-trip" \
            "expected=86400 got=$token_window body=$got"
 [ "$budget_window" = "2592000" ] \
    || fail "budget_limit.window_seconds did not round-trip" \
            "expected=2592000 got=$budget_window body=$got"
 # Negative: window_hours / window_days are legacy field names and
 # must not be present in the response at all — their presence would
 # mean the management server is still emitting the legacy shape.
 legacy_h=$(printf '%s' "$got" | jq -r '.limits.token_limit | has("window_hours")')
 legacy_d=$(printf '%s' "$got" | jq -r '.limits.token_limit | has("window_days")')
 [ "$legacy_h" = "false" ] \
    || fail "legacy window_hours field still present in token_limit response" "$got"
 [ "$legacy_d" = "false" ] \
    || fail "legacy window_days field still present in token_limit response" "$got"
 echo "token_limit.window_seconds = $token_window"
 echo "budget_limit.window_seconds = $budget_window"
 pass "policy persisted with window_seconds on both Limits halves"
@@ -1,65 +0,0 @@
 #!/usr/bin/env bash
 # 20-policy-rejects-zero-window: management must reject a policy whose
 # token_limit or budget_limit is enabled but carries window_seconds < 60.
 # Anything below the one-minute floor would either be a zero / negative
 # window with no reset boundary, or a sub-minute window that produces
 # untenable consumption-row volume at scale. The handler's
 # validatePolicyLimits guard owes us a 4xx with a useful message.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 [ -r "$NB_STATE_DIR/group-id" ] && [ -r "$NB_STATE_DIR/provider-id" ] \
    || fail "missing group-id / provider-id state" \
            "run 10-policy-create.sh first to bootstrap prerequisites"
 group_id=$(cat "$NB_STATE_DIR/group-id")
 provider_id=$(cat "$NB_STATE_DIR/provider-id")
 # Build a payload that is well-formed except for window_seconds=30 on
 # token_limit (under the 60s minimum). We mark the field enabled so
 # the validation path actually runs — a disabled limit is allowed to
 # carry zero.
 payload=$(jq -n \
    --arg name "$NB_POLICY_NAME-sub-minute" \
    --arg group "$group_id" \
    --arg provider "$provider_id" \
    '{
        name: $name,
        enabled: true,
        source_groups: [$group],
        destination_provider_ids: [$provider],
        guardrail_ids: [],
        limits: {
            token_limit: {
                enabled: true,
                group_cap: 10000,
                user_cap: 5000,
                window_seconds: 30
            },
            budget_limit: {
                enabled: false,
                group_cap_usd: 0,
                user_cap_usd: 0,
                window_seconds: 0
            }
        }
    }')
 code=$(nb_api_status POST /api/agent-network/policies "$payload")
 [ "$code" = "400" ] || [ "$code" = "422" ] \
    || fail "expected 400/422 on enabled token_limit with window_seconds<60" \
            "got HTTP $code"
 # Sweep any policy that may have been mistakenly persisted (defence
 # against a future bug; today's handler doesn't get there).
 orphan=$(nb_api GET /api/agent-network/policies 2>/dev/null \
    | jq -r --arg name "$NB_POLICY_NAME-sub-minute" '.[] | select(.name == $name) | .id // empty' \
    | head -1)
 if [ -n "$orphan" ]; then
    nb_api DELETE "/api/agent-network/policies/$orphan" >/dev/null 2>&1 || true
    fail "policy was persisted despite sub-minute window_seconds" \
         "id=$orphan — handler validation regression"
 fi
 echo "POST with token_limit.window_seconds=30 rejected with HTTP $code"
 pass "validation rejects sub-minute window_seconds when limit is enabled"
@@ -1,24 +0,0 @@
 #!/usr/bin/env bash
 # 30-consumption-list-empty: GET /api/agent-network/consumption must
 # return a JSON array (possibly empty) — never a 404 / 500. The
 # endpoint is the read side that backs the dashboard's basic counter
 # view and must always be reachable so the page can render an empty
 # state.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 resp=$(nb_api GET /api/agent-network/consumption)
 # Must be a JSON array.
 if ! printf '%s' "$resp" | jq -e 'type == "array"' >/dev/null 2>&1; then
    fail "consumption endpoint did not return a JSON array" "$resp"
 fi
 count=$(printf '%s' "$resp" | jq 'length')
 echo "consumption rows: $count"
 # Stash the baseline count so 40-grpc-record-and-list can compare.
 printf '%s' "$count" >"$NB_STATE_DIR/consumption-baseline"
 pass "consumption read endpoint returns a JSON array (count=$count)"
@@ -1,133 +0,0 @@
 #!/usr/bin/env bash
 # 40-grpc-record-and-list: drives the new RecordLLMUsage and
 # CheckLLMPolicyLimits gRPC RPCs through the e2e usage_smoke helper.
 #
 # Asserts:
 #   - CheckLLMPolicyLimits returns decision=allow, picks the lowest
 #     group id by string sort as attribution, default window of 24h.
 #   - RecordLLMUsage with both user_id and group_id ticks BOTH the
 #     user counter and the group counter exactly once each.
 #   - A second RecordLLMUsage on the same key sums the deltas server
 #     side (database upsert-increment, no read-modify-write race).
 #   - The HTTP /api/agent-network/consumption listing reflects the
 #     post-flight state.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 [ -r "$NB_STATE_DIR/group-id" ] && [ -r "$NB_STATE_DIR/provider-id" ] \
    || fail "missing group-id / provider-id state" \
            "run 10-policy-create.sh first to bootstrap prerequisites"
 group_id=$(cat "$NB_STATE_DIR/group-id")
 provider_id=$(cat "$NB_STATE_DIR/provider-id")
 # Resolve the calling account id. /api/accounts returns NetBird xids,
 # which is what the gRPC service expects (NOT IDP user UUIDs).
 account_id=$(nb_api GET /api/accounts 2>/dev/null | jq -r '.[0].id // empty')
 [ -n "$account_id" ] || fail "could not resolve account id" "GET /api/accounts returned nothing"
 # Pick a stable user id for the test. The dimension is per-user, so we
 # can use any unique value — a synthetic "e2e-..." prefix avoids
 # colliding with real user ids in the consumption listing.
 user_id="e2e-anpol-user-$$"
 window_seconds=86400
 echo "account=$account_id user=$user_id group=$group_id window=${window_seconds}s"
 # ─── 1. CheckLLMPolicyLimits ────────────────────────────────────────
 # Pass 3 group ids out of order; lowest by string sort wins.
 groups_csv="grp-zz,$group_id,grp-aa-z"
 check_resp=$(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke check \
    --account "$account_id" \
    --user "$user_id" \
    --groups "$groups_csv" \
    --provider "$provider_id" \
    --model "gpt-4o" \
    --token "$NB_PROXY_TOKEN" \
    --addr "$NB_GRPC_ADDR" 2>&1) \
    || fail "CheckLLMPolicyLimits gRPC failed" "$check_resp"
 decision=$(printf '%s' "$check_resp" | jq -r '.decision // ""')
 attribution=$(printf '%s' "$check_resp" | jq -r '.attribution_group_id // ""')
 got_window=$(printf '%s' "$check_resp" | jq -r '.window_seconds // ""')
 [ "$decision" = "allow" ] \
    || fail "Check decision must be allow under PR1 stub" "$check_resp"
 # Lowest group id of {grp-zz, $group_id, grp-aa-z} by string sort. The
 # group_id is an xid (lowercase alnum starting with a digit/letter), so
 # the sort answer depends on group_id's prefix. Compute it locally.
 expected_low=$(printf '%s\n%s\n%s' "grp-zz" "$group_id" "grp-aa-z" | sort | head -1)
 [ "$attribution" = "$expected_low" ] \
    || fail "Check did not pick the lowest-by-sort group" \
            "expected=$expected_low got=$attribution"
 [ "$got_window" = "86400" ] \
    || fail "Check window_seconds stub default mismatch" "expected=86400 got=$got_window"
 echo "Check decision=$decision attribution=$attribution window=${got_window}s"
 # ─── 2. RecordLLMUsage — first increment ────────────────────────────
 (cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \
    --account "$account_id" \
    --user "$user_id" \
    --group "$group_id" \
    --window-seconds "$window_seconds" \
    --tokens-in 100 \
    --tokens-out 50 \
    --cost-usd 0.0125 \
    --token "$NB_PROXY_TOKEN" \
    --addr "$NB_GRPC_ADDR") >/dev/null \
    || fail "RecordLLMUsage (first increment) failed" ""
 # ─── 3. RecordLLMUsage — second increment, same key ─────────────────
 (cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \
    --account "$account_id" \
    --user "$user_id" \
    --group "$group_id" \
    --window-seconds "$window_seconds" \
    --tokens-in 50 \
    --tokens-out 25 \
    --cost-usd 0.0025 \
    --token "$NB_PROXY_TOKEN" \
    --addr "$NB_GRPC_ADDR") >/dev/null \
    || fail "RecordLLMUsage (second increment) failed" ""
 # ─── 4. Read-back via HTTP — sums must converge ─────────────────────
 listing=$(nb_api GET /api/agent-network/consumption)
 user_row=$(printf '%s' "$listing" | jq --arg u "$user_id" \
    'map(select(.dimension_kind == "user" and .dimension_id == $u)) | .[0]')
 [ "$(printf '%s' "$user_row" | jq -r '. // empty')" != "" ] \
    || fail "user consumption row missing after RecordLLMUsage" "$listing"
 user_in=$(printf '%s' "$user_row" | jq '.tokens_input')
 user_out=$(printf '%s' "$user_row" | jq '.tokens_output')
 user_cost=$(printf '%s' "$user_row" | jq '.cost_usd')
 user_window=$(printf '%s' "$user_row" | jq '.window_seconds')
 [ "$user_in" = "150" ] && [ "$user_out" = "75" ] \
    || fail "user counter did not sum the two increments" \
            "expected tokens_in=150 tokens_out=75; got in=$user_in out=$user_out"
 # Floating point compare with awk — drift > 1e-9 is a real bug.
 cost_ok=$(awk -v got="$user_cost" 'BEGIN { print (got > 0.0149 && got < 0.0151) ? "y" : "n" }')
 [ "$cost_ok" = "y" ] \
    || fail "user cost did not sum to 0.015" "got=$user_cost"
 [ "$user_window" = "$window_seconds" ] \
    || fail "user counter window_seconds mismatch" \
            "expected=$window_seconds got=$user_window"
 # Group row gets the same deltas because RecordLLMUsage ticks both
 # dimensions on a single call.
 group_row=$(printf '%s' "$listing" | jq --arg g "$group_id" \
    'map(select(.dimension_kind == "group" and .dimension_id == $g)) | .[0]')
 group_in=$(printf '%s' "$group_row" | jq '.tokens_input')
 group_out=$(printf '%s' "$group_row" | jq '.tokens_output')
 [ "$group_in" = "150" ] && [ "$group_out" = "75" ] \
    || fail "group counter did not sum the two increments" \
            "expected tokens_in=150 tokens_out=75; got in=$group_in out=$group_out"
 echo "user counter: $user_in input / $user_out output / \$$user_cost over ${user_window}s"
 echo "group counter: $group_in input / $group_out output"
 pass "gRPC Check + Record round-trip atomically increments user + group counters"
@@ -1,210 +0,0 @@
 #!/usr/bin/env bash
 # 50-grpc-allow-record-deny: drives the full lifecycle of the real
 # selection algorithm landed in PR2 — initial allow, partial fills
 # below the cap, exact-at-cap, then deny once consumption reaches the
 # group_cap. Validates that:
 #
 #   - CheckLLMPolicyLimits picks the test policy as the attribution
 #     when it's the only one authorising the (group, provider) pair.
 #   - The selected_policy_id + window_seconds round-trip on the wire.
 #   - Counter-aware headroom math is wired end-to-end (RecordLLMUsage
 #     ticks counters that the next CheckLLMPolicyLimits call reads).
 #   - At-cap consumption flips decision from allow to deny with the
 #     canonical llm_policy.token_cap_exceeded code.
 #
 # The setup creates a dedicated group + provider + policy with the
 # "-tight" suffix so it's isolated from the e2e-anpol resources
 # 10-policy-create.sh seeded. 99-cleanup's prefix sweep catches both.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 # Per-run suffix so the consumption counter starts at zero on every
 # run. Without this, a successful prior run leaves 200 tokens on the
 # (group, 24h) bucket and the next run's stage 1 check sees a deny
 # from the start. Consumption rows have no delete endpoint today.
 RUN_TAG="$(date +%s)-$$"
 NB_TIGHT_GROUP_NAME="$NB_GROUP_NAME-tight-$RUN_TAG"
 NB_TIGHT_PROVIDER_NAME="$NB_PROVIDER_NAME-tight-$RUN_TAG"
 NB_TIGHT_POLICY_NAME="$NB_POLICY_NAME-tight-$RUN_TAG"
 TIGHT_CAP=200
 TIGHT_WINDOW_SECONDS=86400
 # 1. Group — fresh per-run so the (group, window) consumption counter
 # starts at zero. The 99-cleanup sweep catches it via the
 # `e2e-anpol-engineers-tight-` prefix.
 body=$(jq -n --arg name "$NB_TIGHT_GROUP_NAME" '{name:$name}')
 tight_group_id=$(nb_api POST /api/groups "$body" | jq -r '.id // ""')
 [ -n "$tight_group_id" ] && [ "$tight_group_id" != "null" ] \
    || fail "tight group create failed" ""
 echo "created tight group $tight_group_id"
 printf '%s' "$tight_group_id" >"$NB_STATE_DIR/tight-group-id"
 # 2. Provider — fresh per-run too so the policy targets a provider
 # only this run knows about. Eliminates cross-policy interference if
 # a prior run's tight policy targeted the same provider id.
 body=$(jq -n \
    --arg name "$NB_TIGHT_PROVIDER_NAME" \
    '{
        provider_id: "openai_api",
        name: $name,
        upstream_url: "https://api.openai.com",
        api_key: "sk-e2e-tight-placeholder",
        bootstrap_cluster: "proxy.netbird.local",
        models: []
    }')
 tight_provider_id=$(nb_api POST /api/agent-network/providers "$body" | jq -r '.id // ""')
 [ -n "$tight_provider_id" ] && [ "$tight_provider_id" != "null" ] \
    || fail "tight provider create failed" ""
 echo "created tight provider $tight_provider_id"
 printf '%s' "$tight_provider_id" >"$NB_STATE_DIR/tight-provider-id"
 # 3. Policy with a tight token cap so the test can observe the deny
 # transition without burning thousands of records.
 existing_policy=$(nb_api GET /api/agent-network/policies 2>/dev/null \
    | jq -r --arg name "$NB_TIGHT_POLICY_NAME" '.[] | select(.name == $name) | .id // empty' \
    | head -1)
 if [ -n "$existing_policy" ]; then
    echo "deleting existing tight policy $existing_policy so the run starts clean"
    nb_api DELETE "/api/agent-network/policies/$existing_policy" >/dev/null 2>&1 || true
 fi
 policy_body=$(jq -n \
    --arg name "$NB_TIGHT_POLICY_NAME" \
    --arg group "$tight_group_id" \
    --arg provider "$tight_provider_id" \
    --argjson cap "$TIGHT_CAP" \
    --argjson window "$TIGHT_WINDOW_SECONDS" \
    '{
        name: $name,
        description: "agent-network e2e: cap-exhaust deny test",
        enabled: true,
        source_groups: [$group],
        destination_provider_ids: [$provider],
        guardrail_ids: [],
        limits: {
            token_limit: {
                enabled: true,
                group_cap: $cap,
                user_cap: 0,
                window_seconds: $window
            },
            budget_limit: {
                enabled: false,
                group_cap_usd: 0,
                user_cap_usd: 0,
                window_seconds: $window
            }
        }
    }')
 tight_policy_id=$(nb_api POST /api/agent-network/policies "$policy_body" | jq -r '.id // ""')
 [ -n "$tight_policy_id" ] && [ "$tight_policy_id" != "null" ] \
    || fail "tight policy create failed" ""
 printf '%s' "$tight_policy_id" >"$NB_STATE_DIR/tight-policy-id"
 echo "created tight policy $tight_policy_id (group_cap=$TIGHT_CAP, window=${TIGHT_WINDOW_SECONDS}s)"
 # 4. Resolve the calling account id; the smoke binary stamps it onto
 # every gRPC request the way the real proxy does.
 account_id=$(nb_api GET /api/accounts 2>/dev/null | jq -r '.[0].id // empty')
 [ -n "$account_id" ] || fail "could not resolve account id" ""
 # Test user — synthetic prefix avoids colliding with real users.
 test_user="e2e-anpol-tight-user-$$"
 # Pre-resolve the netbird repo root ONCE so the helpers below can
 # pushd / popd into it without `cd "$(dirname "$0")/.."` re-resolving
 # from a moved cwd between calls. nb_repo_root walks up from $0; if a
 # prior helper left cwd elsewhere the relative walk breaks.
 repo_root=$(nb_repo_root)
 [ -d "$repo_root" ] || fail "could not resolve netbird repo root" "got=$repo_root"
 # Run the smoke binary inside a subshell so its `cd` doesn't pollute
 # the caller's cwd between stages.
 do_check() {
    (
        cd "$repo_root" || exit 1
        go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke check \
            --account "$account_id" \
            --user "$test_user" \
            --groups "$tight_group_id" \
            --provider "$tight_provider_id" \
            --model "gpt-4o" \
            --token "$NB_PROXY_TOKEN" \
            --addr "$NB_GRPC_ADDR"
    )
 }
 do_record() {
    local in="$1" out="$2" cost="$3"
    (
        cd "$repo_root" || exit 1
        go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \
            --account "$account_id" \
            --user "$test_user" \
            --group "$tight_group_id" \
            --window-seconds "$TIGHT_WINDOW_SECONDS" \
            --tokens-in "$in" \
            --tokens-out "$out" \
            --cost-usd "$cost" \
            --token "$NB_PROXY_TOKEN" \
            --addr "$NB_GRPC_ADDR"
    )
 }
 # Stage 1 — fresh state. Selection must allow and pick our tight
 # policy as attribution because it's the ONLY policy authorising the
 # (tight_group, tight_provider) tuple. Also confirms the wire-level
 # selected_policy_id round-trip from manager → grpc → smoke client.
 echo "stage 1: initial check (consumption=0/$TIGHT_CAP)"
 resp=$(do_check) || fail "initial check failed" "$resp"
 decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
 selected=$(printf '%s' "$resp" | jq -r '.selected_policy_id // ""')
 window=$(printf '%s' "$resp" | jq -r '.window_seconds // ""')
 [ "$decision" = "allow" ] \
    || fail "expected allow on fresh state" "$resp"
 [ "$selected" = "$tight_policy_id" ] \
    || fail "selection did not pick the tight policy" \
            "expected=$tight_policy_id got=$selected"
 [ "$window" = "$TIGHT_WINDOW_SECONDS" ] \
    || fail "window_seconds mismatch on the wire" "expected=$TIGHT_WINDOW_SECONDS got=$window"
 echo "  → allow / selected=$selected / window=${window}s"
 # Stage 2 — book half the cap. The next check must still allow.
 echo "stage 2: record 100 input tokens (consumption=100/$TIGHT_CAP)"
 do_record 100 0 0 >/dev/null || fail "record (stage 2) failed" ""
 resp=$(do_check)
 decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
 [ "$decision" = "allow" ] \
    || fail "expected allow at half-cap" "$resp"
 echo "  → allow"
 # Stage 3 — push to one token below cap. Headroom shrinks but
 # decision stays allow.
 echo "stage 3: record 99 more input tokens (consumption=199/$TIGHT_CAP)"
 do_record 99 0 0 >/dev/null || fail "record (stage 3) failed" ""
 resp=$(do_check)
 decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
 [ "$decision" = "allow" ] \
    || fail "expected allow at one-below-cap" "$resp"
 echo "  → allow"
 # Stage 4 — exactly at cap. Selector treats consumed >= cap as
 # exhausted, so the next check must deny with the canonical token cap
 # code. The deny reason names the policy id so operators can debug
 # from the access log.
 echo "stage 4: record 1 final token (consumption=$TIGHT_CAP/$TIGHT_CAP)"
 do_record 1 0 0 >/dev/null || fail "record (stage 4) failed" ""
 resp=$(do_check) || fail "check after cap-exhaust failed" "$resp"
 decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
 deny_code=$(printf '%s' "$resp" | jq -r '.deny_code // ""')
 deny_reason=$(printf '%s' "$resp" | jq -r '.deny_reason // ""')
 [ "$decision" = "deny" ] \
    || fail "expected DENY at cap" "$resp"
 [ "$deny_code" = "llm_policy.token_cap_exceeded" ] \
    || fail "deny_code mismatch" "expected=llm_policy.token_cap_exceeded got=$deny_code resp=$resp"
 echo "  → DENY / deny_code=$deny_code"
 echo "  → deny_reason: $deny_reason"
 [[ "$deny_reason" == *"$tight_policy_id"* ]] \
    || fail "deny_reason must name the policy id for operator debugging" "$deny_reason"
 pass "selection algorithm flips allow → deny at cap-exhaust through the gRPC wire"
@@ -1,58 +0,0 @@
 #!/usr/bin/env bash
 # 99-cleanup: idempotent teardown. Drops the policy, provider, group,
 # and any consumption rows our test created so re-runs start fresh.
 # shellcheck source=00-env.sh
 source "$(dirname "$0")/00-env.sh"
 # Policies first (FK / synth chain owns provider references). Sweep
 # every policy whose name STARTS with the e2e prefix so per-run
 # instances created by 50 (with $RUN_TAG suffixes) get cleaned up.
 nb_api GET /api/agent-network/policies 2>/dev/null \
    | jq -r --arg pfx "$NB_POLICY_NAME" '.[] | select(.name | startswith($pfx)) | .id' \
    | while read -r orphan; do
        [ -n "$orphan" ] || continue
        code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \
            -o /dev/null -w '%{http_code}' \
            "$NB_API/api/agent-network/policies/$orphan" 2>&1) || true
        echo "DELETE policy/$orphan -> $code"
    done
 # Providers — same prefix sweep. Both the 10-policy-create.sh's main
 # provider AND every per-run tight provider 50 minted carry the
 # NB_PROVIDER_NAME prefix.
 nb_api GET /api/agent-network/providers 2>/dev/null \
    | jq -r --arg pfx "$NB_PROVIDER_NAME" '.[] | select(.name | startswith($pfx)) | .id' \
    | while read -r orphan; do
        [ -n "$orphan" ] || continue
        code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \
            -o /dev/null -w '%{http_code}' \
            "$NB_API/api/agent-network/providers/$orphan" 2>&1) || true
        echo "DELETE provider/$orphan -> $code"
    done
 # Groups — prefix sweep. /api/groups doesn't error on a
 # referenced-elsewhere group; if so, the delete is a no-op and we
 # move on.
 nb_api GET /api/groups 2>/dev/null \
    | jq -r --arg pfx "$NB_GROUP_NAME" '.[] | select(.name | startswith($pfx)) | .id' \
    | while read -r orphan; do
        [ -n "$orphan" ] || continue
        code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \
            -o /dev/null -w '%{http_code}' \
            "$NB_API/api/groups/$orphan" 2>&1) || true
        echo "DELETE group/$orphan -> $code"
    done
 # We don't expose a consumption-delete endpoint — the rows survive
 # until the management store is recycled. Document the residue here
 # so anyone debugging "why are there old e2e rows" knows the source.
 remaining=$(nb_api GET /api/agent-network/consumption 2>/dev/null \
    | jq --arg pfx "e2e-anpol-user-" 'map(select(.dimension_id | startswith($pfx))) | length' \
    || echo "0")
 [ "$remaining" = "0" ] || \
    echo "(left $remaining e2e consumption rows in the store — there's no delete endpoint yet)"
 rm -rf "$NB_STATE_DIR"
 pass "tear-down complete (idempotent)"
@@ -1,197 +0,0 @@
 // usage_smoke is the e2e helper that drives the new agent-network
 // gRPC RPCs (CheckLLMPolicyLimits, RecordLLMUsage) against a local
 // management server. Run from the bash suite as a `go run` so the
 // proto types are always in sync with the management binary the
 // suite is exercising.
 //
 // Two subcommands today:
 //   - record: invokes RecordLLMUsage with the supplied tokens / cost
 //     and exits 0 on success.
 //   - check:  invokes CheckLLMPolicyLimits and prints the response as
 //     JSON on stdout so the bash test can assert on it via
 //     jq.
 //
 // Auth uses the same proxy bearer-token shape the real proxy uses
 // (see proxy/internal/grpc/auth.go); the bash suite reads the token
 // from the e2e env (NB_PROXY_TOKEN, defaulted to the Tilt literal).
 package main
 import (
 	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"github.com/netbirdio/netbird/shared/management/proto"
 )
 // proxyTokenCreds mirrors proxy/internal/grpc.WithProxyToken's
 // PerRPCCredentials so the e2e binary can reach the gRPC service
 // using the same bearer-token shape the real proxy uses. Inlined
 // because the production helper lives behind /internal/.
 type proxyTokenCreds struct{ token string }
 func (c proxyTokenCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
 	return map[string]string{"authorization": "Bearer " + c.token}, nil
 }
 // RequireTransportSecurity is false here because Tilt's management is
 // plaintext on localhost — the e2e suite is the *only* caller of this
 // binary, never production.
 func (proxyTokenCreds) RequireTransportSecurity() bool { return false }
 func main() {
 	if len(os.Args) < 2 {
 		usage()
 	}
 	cmd := os.Args[1]
 	os.Args = append(os.Args[:1], os.Args[2:]...)
 	switch cmd {
 	case "record":
 		runRecord()
 	case "check":
 		runCheck()
 	default:
 		usage()
 	}
 }
 func usage() {
 	fmt.Fprintln(os.Stderr, "usage: usage_smoke <record|check> [flags]")
 	os.Exit(2)
 }
 func runRecord() {
 	addr := flag.String("addr", "localhost:8080", "management gRPC address")
 	token := flag.String("token", os.Getenv("NB_PROXY_TOKEN"), "proxy token")
 	accountID := flag.String("account", "", "netbird account id")
 	userID := flag.String("user", "", "netbird user id (optional)")
 	groupID := flag.String("group", "", "netbird policy attribution group id (optional)")
 	groupsCSV := flag.String("groups", "", "CSV of caller group ids (for account-rule fan-out)")
 	windowSeconds := flag.Int64("window-seconds", 86_400, "window length in seconds (0 allowed when only account rules apply)")
 	tokensIn := flag.Int64("tokens-in", 0, "input tokens to add")
 	tokensOut := flag.Int64("tokens-out", 0, "output tokens to add")
 	costUSD := flag.Float64("cost-usd", 0, "USD cost to add")
 	flag.Parse()
 	if strings.TrimSpace(*token) == "" {
 		exitErr("--token is required (or set NB_PROXY_TOKEN)")
 	}
 	if strings.TrimSpace(*accountID) == "" {
 		exitErr("--account is required")
 	}
 	var groupIDs []string
 	for _, g := range strings.Split(*groupsCSV, ",") {
 		g = strings.TrimSpace(g)
 		if g != "" {
 			groupIDs = append(groupIDs, g)
 		}
 	}
 	if *userID == "" && *groupID == "" && len(groupIDs) == 0 {
 		exitErr("at least one of --user, --group, or --groups must be set")
 	}
 	conn := dial(*addr, *token)
 	defer conn.Close()
 	client := proto.NewProxyServiceClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 	_, err := client.RecordLLMUsage(ctx, &proto.RecordLLMUsageRequest{
 		AccountId:     *accountID,
 		UserId:        *userID,
 		GroupId:       *groupID,
 		GroupIds:      groupIDs,
 		WindowSeconds: *windowSeconds,
 		TokensInput:   *tokensIn,
 		TokensOutput:  *tokensOut,
 		CostUsd:       *costUSD,
 	})
 	if err != nil {
 		exitErr(fmt.Sprintf("RecordLLMUsage: %v", err))
 	}
 	//nolint:forbidigo // e2e helper: stdout is the contract with the bash caller
 	fmt.Println("ok")
 }
 func runCheck() {
 	addr := flag.String("addr", "localhost:8080", "management gRPC address")
 	token := flag.String("token", os.Getenv("NB_PROXY_TOKEN"), "proxy token")
 	accountID := flag.String("account", "", "netbird account id")
 	userID := flag.String("user", "", "netbird user id (optional)")
 	groupsCSV := flag.String("groups", "", "CSV of caller group ids")
 	providerID := flag.String("provider", "", "agent-network provider id")
 	model := flag.String("model", "gpt-4o", "upstream model identifier")
 	flag.Parse()
 	if strings.TrimSpace(*token) == "" {
 		exitErr("--token is required (or set NB_PROXY_TOKEN)")
 	}
 	if strings.TrimSpace(*accountID) == "" {
 		exitErr("--account is required")
 	}
 	var groupIDs []string
 	for _, g := range strings.Split(*groupsCSV, ",") {
 		g = strings.TrimSpace(g)
 		if g != "" {
 			groupIDs = append(groupIDs, g)
 		}
 	}
 	conn := dial(*addr, *token)
 	defer conn.Close()
 	client := proto.NewProxyServiceClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 	resp, err := client.CheckLLMPolicyLimits(ctx, &proto.CheckLLMPolicyLimitsRequest{
 		AccountId:  *accountID,
 		UserId:     *userID,
 		GroupIds:   groupIDs,
 		ProviderId: *providerID,
 		Model:      *model,
 	})
 	if err != nil {
 		exitErr(fmt.Sprintf("CheckLLMPolicyLimits: %v", err))
 	}
 	out, _ := json.Marshal(map[string]any{
 		"decision":             resp.GetDecision(),
 		"selected_policy_id":   resp.GetSelectedPolicyId(),
 		"attribution_group_id": resp.GetAttributionGroupId(),
 		"window_seconds":       resp.GetWindowSeconds(),
 		"deny_code":            resp.GetDenyCode(),
 		"deny_reason":          resp.GetDenyReason(),
 	})
 	//nolint:forbidigo // e2e helper: stdout is the contract with the bash caller
 	fmt.Println(string(out))
 }
 // dial connects to the management gRPC over plaintext. The bearer
 // token is sent on every RPC via PerRPCCredentials matching the wire
 // format the production proxy uses.
 func dial(addr, token string) *grpc.ClientConn {
 	conn, err := grpc.NewClient(addr,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.WithPerRPCCredentials(proxyTokenCreds{token: token}),
 	)
 	if err != nil {
 		exitErr(fmt.Sprintf("dial %s: %v", addr, err))
 	}
 	return conn
 }
 func exitErr(msg string) {
 	fmt.Fprintln(os.Stderr, msg)
 	os.Exit(1)
 }
@@ -1,44 +0,0 @@
 #!/usr/bin/env bash
 # run-all: every numbered test in order, halts on first FAIL,
 # prints a `=== PASS x/y ===` summary. -k keeps state (skips
 # 99-cleanup) so you can poke around the persisted policy /
 # consumption rows afterwards.
 set -uo pipefail
 KEEP=0
 [ "${1:-}" = "-k" ] && KEEP=1
 export NB_KEEP_STATE="$KEEP"
 cd "$(dirname "$0")"
 scripts=(
    01-tilt-restart.sh
    10-policy-create.sh
    20-policy-rejects-zero-window.sh
    30-consumption-list-empty.sh
    40-grpc-record-and-list.sh
    50-grpc-allow-record-deny.sh
 )
 [ "$KEEP" -eq 0 ] && scripts+=(99-cleanup.sh)
 passed=0
 total=${#scripts[@]}
 trap '[ "$KEEP" -eq 0 ] && bash ./99-cleanup.sh >/dev/null 2>&1 || true' EXIT
 for s in "${scripts[@]}"; do
    echo
    echo "==================== $s ===================="
    if bash "./$s"; then
        passed=$((passed + 1))
    else
        rc=$?
        echo
        echo "=== FAIL $passed/$total ($s exit=$rc) ==="
        exit "$rc"
    fi
 done
 echo
 echo "=== PASS $passed/$total ==="