From 9f485be2f92272699549fee73b14d29bc4ef56f4 Mon Sep 17 00:00:00 2001 From: mlsmaycon Date: Sat, 27 Jun 2026 01:39:57 +0200 Subject: [PATCH] [agent-network] Remove e2e shell-script suite from this branch The end-to-end shell scripts under scripts/e2e/ are maintained in a separate testing suite and are not part of this change set. --- scripts/e2e/agent-network-full/e2e.sh | 956 ------------------ scripts/e2e/agent-network-policy/00-env.sh | 105 -- .../agent-network-policy/01-tilt-restart.sh | 44 - .../agent-network-policy/10-policy-create.sh | 132 --- .../20-policy-rejects-zero-window.sh | 65 -- .../30-consumption-list-empty.sh | 24 - .../40-grpc-record-and-list.sh | 133 --- .../50-grpc-allow-record-deny.sh | 210 ---- .../e2e/agent-network-policy/99-cleanup.sh | 58 -- .../cmd/usage_smoke/main.go | 197 ---- scripts/e2e/agent-network-policy/run-all.sh | 44 - 11 files changed, 1968 deletions(-) delete mode 100644 scripts/e2e/agent-network-full/e2e.sh delete mode 100755 scripts/e2e/agent-network-policy/00-env.sh delete mode 100755 scripts/e2e/agent-network-policy/01-tilt-restart.sh delete mode 100755 scripts/e2e/agent-network-policy/10-policy-create.sh delete mode 100755 scripts/e2e/agent-network-policy/20-policy-rejects-zero-window.sh delete mode 100755 scripts/e2e/agent-network-policy/30-consumption-list-empty.sh delete mode 100755 scripts/e2e/agent-network-policy/40-grpc-record-and-list.sh delete mode 100755 scripts/e2e/agent-network-policy/50-grpc-allow-record-deny.sh delete mode 100755 scripts/e2e/agent-network-policy/99-cleanup.sh delete mode 100644 scripts/e2e/agent-network-policy/cmd/usage_smoke/main.go delete mode 100755 scripts/e2e/agent-network-policy/run-all.sh diff --git a/scripts/e2e/agent-network-full/e2e.sh b/scripts/e2e/agent-network-full/e2e.sh deleted file mode 100644 index 8d45430ff..000000000 --- a/scripts/e2e/agent-network-full/e2e.sh +++ /dev/null @@ -1,956 +0,0 @@ -#!/usr/bin/env bash -# Agent-network full end-to-end driver. -# -# One script, many subcommands, so the operator authorizes a single -# `bash scripts/e2e/agent-network-full/e2e.sh ` invocation instead of a -# stream of ad-hoc docker/netbird/curl commands. -# -# It joins a Docker NetBird client to the local Tilt management, drives LLM -# chat-completions through the agent-network proxy over the tunnel, and asserts -# token/cost/session capture via the REST API and proxy logs. -# -# Secrets (PAT, provider keys, setup key) are read from files / a sourced -# key file and never echoed. -# -# Usage: -# bash scripts/e2e/agent-network-full/e2e.sh -# -# Commands: -# key mint a reusable setup key bound to the Admins group -# up run the Docker NetBird client and join management -# status show netbird status -d inside the client -# wait wait until the proxy peer is Connected (1/1) -# diag dump client + proxy3 + relay diagnostics -# chat M P chat-completion through the proxy: model M, provider-kind P -# verify consumption + recent access-log rows -# snapshot save current agent-network config to the scratch dir -# clean delete all policies, budget-rules, providers (API teardown) -# providers create the five providers from .llm-keys -# policy create one policy: Admins -> all five providers -# down remove the Docker client container -# restart-proxy tilt trigger proxy3 (re-establish its relay link) -# all clean -> providers -> policy -> up -> wait -> chat-all -> verify - -set -uo pipefail - -# --- config ------------------------------------------------------------------ -NB_API="${NB_API:-http://localhost:8080}" -NB_PAT_FILE="${NB_PAT_FILE:-/Users/maycon/projects/local-dev/nb-pat}" -LLM_KEYS_FILE="${LLM_KEYS_FILE:-/Users/maycon/.llm-keys}" -CLIENT="${CLIENT:-nb-e2e-agent}" -IMAGE="${IMAGE:-netbird:tilt}" -CURL_IMAGE="${CURL_IMAGE:-curlimages/curl:latest}" -MGMT_URL="${MGMT_URL:-http://host.docker.internal:8080}" -PROXY_HOST="${PROXY_HOST:-mitten.proxy.netbird.local}" -ADMINS_GROUP_NAME="${ADMINS_GROUP_NAME:-Admins}" -DASH_DIR="${DASH_DIR:-/Users/maycon/projects/dashboard}" -DASH_URL="${DASH_URL:-http://localhost:3000}" -DASH_USER="${DASH_USER:-netbird@netbird.io}" -DASH_PASS="${DASH_PASS:-netbird@netbird.io}" -STATE_DIR="${STATE_DIR:-/private/tmp/claude-501/-Users-maycon-projects-netbird/a3fe30e4-5777-47d5-b110-ebc228716026/scratchpad/agentnet-snapshot}" - -mkdir -p "$STATE_DIR" - -[ -r "$NB_PAT_FILE" ] || { echo "FAIL: cannot read PAT at $NB_PAT_FILE" >&2; exit 2; } -PAT="$(tr -d '\n\r ' <"$NB_PAT_FILE")" -AUTH="Authorization: Token $PAT" -B="$NB_API/api/agent-network" - -# --- small helpers ----------------------------------------------------------- -log() { printf '%s\n' "$*" >&2; } -die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } - -api() { # METHOD PATH [BODY] - local m="$1" p="$2" body="${3-}" - if [ -n "$body" ]; then - curl -fsS -X "$m" -H "$AUTH" -H "Content-Type: application/json" --data "$body" "$NB_API$p" - else - curl -fsS -X "$m" -H "$AUTH" "$NB_API$p" - fi -} - -# proxy_ip resolves the proxy host to its NetBird IP from inside the client. -proxy_ip() { - docker exec "$CLIENT" sh -c "getent hosts $PROXY_HOST" 2>/dev/null | awk '{print $1; exit}' -} - -admins_group_id() { - api GET /api/groups | jq -r --arg n "$ADMINS_GROUP_NAME" \ - '.[] | select((.name//"")|ascii_downcase==($n|ascii_downcase)) | .id' | head -n1 -} - -require_llm_keys() { - [ -r "$LLM_KEYS_FILE" ] || die "cannot read $LLM_KEYS_FILE" - # shellcheck disable=SC1090 - set -a; . "$LLM_KEYS_FILE"; set +a -} - -# --- commands ---------------------------------------------------------------- -cmd_snapshot() { - api GET /settings_unused 2>/dev/null || true - curl -fsS -H "$AUTH" "$B/settings" >"$STATE_DIR/settings.json" - curl -fsS -H "$AUTH" "$B/providers" >"$STATE_DIR/providers.json" - curl -fsS -H "$AUTH" "$B/policies" >"$STATE_DIR/policies.json" - curl -fsS -H "$AUTH" "$B/budget-rules" >"$STATE_DIR/budget-rules.json" - curl -fsS -H "$AUTH" "$NB_API/api/groups" >"$STATE_DIR/groups.json" - log "snapshot written to $STATE_DIR" - ls -l "$STATE_DIR" >&2 -} - -cmd_key() { - local gid kjson - gid="$(admins_group_id)" - [ -n "$gid" ] || die "could not resolve Admins group id" - kjson="$(api POST /api/setup-keys "{\"name\":\"e2e-agentnet-docker\",\"type\":\"reusable\",\"expires_in\":86400,\"usage_limit\":0,\"auto_groups\":[\"$gid\"],\"ephemeral\":false}")" - echo "$kjson" | jq -r '.key' >"$STATE_DIR/setup-key.txt" - chmod 600 "$STATE_DIR/setup-key.txt" - echo "$kjson" | jq '{id,name,type,state,valid,auto_groups}' >&2 - log "setup key saved (value not printed) to $STATE_DIR/setup-key.txt" -} - -cmd_up() { - [ -r "$STATE_DIR/setup-key.txt" ] || cmd_key - local key; key="$(cat "$STATE_DIR/setup-key.txt")" - docker rm -f "$CLIENT" >/dev/null 2>&1 || true - docker run -d --name "$CLIENT" \ - --cap-add NET_ADMIN --cap-add SYS_ADMIN --cap-add SYS_RESOURCE \ - --add-host host.docker.internal:host-gateway \ - -e NB_MANAGEMENT_URL="$MGMT_URL" \ - -e NB_SETUP_KEY="$key" \ - -e NB_LOG_LEVEL=info \ - "$IMAGE" >/dev/null - log "started $CLIENT" - sleep 6 - docker exec "$CLIENT" netbird status 2>&1 | sed -n '1,14p' >&2 -} - -cmd_status() { docker exec "$CLIENT" netbird status -d 2>&1; } - -cmd_wait() { - local i=0 line - while [ "$i" -lt 90 ]; do - line="$(docker exec "$CLIENT" netbird status 2>/dev/null | grep '^Peers count' || true)" - log "t=${i}s ${line:-}" - case "$line" in *"1/1 Connected"*) log "proxy peer connected"; return 0;; esac - sleep 5; i=$((i+5)) - done - log "proxy peer did not connect within ${i}s" - return 1 -} - -cmd_diag() { - log "===== client: peers count =====" - docker exec "$CLIENT" netbird status 2>&1 | grep -E '^(Management|Signal|Relays|Peers)' >&2 || true - log "===== client: relay/handshake (last 15) =====" - docker exec "$CLIENT" sh -c 'tail -n 400 /var/log/netbird/client.log' 2>/dev/null \ - | grep -iE 'relay|handshake|offer|answer|error' | tail -15 >&2 || true - log "===== proxy3: relay (last 15) =====" - docker logs local-dev-proxy3-1 2>&1 | grep -iE 'relay|signal|handshake|offer|answer' | tail -15 >&2 || true - log "===== mgmt: proxy peer online =====" - curl -fsS -H "$AUTH" "$NB_API/api/peers" \ - | jq -r '.[] | select((.name//"")|test("^proxy-")) | "\(.name) connected=\(.connected) last_seen=\(.last_seen)"' >&2 || true -} - -cmd_restart_proxy() { - command -v tilt >/dev/null 2>&1 || die "tilt not on PATH" - tilt trigger proxy3 && log "triggered proxy3" -} - -# cmd_chat MODEL KIND — KIND in: chat (POST /v1/chat/completions), -# messages (POST /v1/messages, anthropic body). -CHAT_RESP="$STATE_DIR/last-chat.json" -CHAT_PROMPT_DEFAULT="Reply with exactly: pong" - -# _chat MODEL [KIND] [PROMPT] — POST through the proxy from $CHAT_CLIENT -# (defaults to $CLIENT). Echoes the HTTP status code on stdout; writes the -# response body to $CHAT_RESP. Returns 000 if the proxy host won't resolve. -_chat() { - local model="$1" kind="${2:-chat}" prompt="${3:-$CHAT_PROMPT_DEFAULT}" - local client="${CHAT_CLIENT:-$CLIENT}" ip path body out code attempt - local extra_hdr=() - case "$kind" in - messages) - path="/v1/messages"; extra_hdr=(-H "anthropic-version: 2023-06-01") - body="$(jq -n --arg m "$model" --arg p "$prompt" '{model:$m,max_tokens:64,messages:[{role:"user",content:$p}]}')" ;; - *) - path="/v1/chat/completions" - body="$(jq -n --arg m "$model" --arg p "$prompt" '{model:$m,messages:[{role:"user",content:$p}]}')" ;; - esac - # Re-resolve the proxy IP and retry on connection failure (000): a config - # change churns the proxy peer's NetBird IP, so a freshly-joined client may - # need a few seconds for its tunnel + magic DNS to converge. - code=000 - for attempt in $(seq 1 "${CHAT_RETRIES:-6}"); do - ip="$(CLIENT="$client" proxy_ip)" - if [ -n "$ip" ]; then - out="$(docker run --rm --network "container:$client" "$CURL_IMAGE" \ - -sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \ - -w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \ - -H "Content-Type: application/json" ${extra_hdr[@]+"${extra_hdr[@]}"} --data "$body")" - code="$(printf '%s' "$out" | tail -n1)" - printf '%s' "$out" | sed '$d' >"$CHAT_RESP" - fi - [ "$code" != 000 ] && break - sleep 4 - done - echo "$code" -} - -cmd_chat() { - local code - log "POST $PROXY_HOST (model=$1 kind=${2:-chat} client=${CHAT_CLIENT:-$CLIENT})" - code="$(_chat "$@")" - cat "$CHAT_RESP" 2>/dev/null - printf '\n[http %s]\n' "$code" -} - -cmd_verify() { - log "===== consumption =====" - api GET /api/agent-network/consumption | jq -r '.[] | "\(.dimension_kind)/\(.dimension_id) tokens_in=\(.tokens_input) tokens_out=\(.tokens_output) cost=\(.cost_usd // 0)"' >&2 || true - log "===== last 10 access-log rows =====" - api GET /api/agent-network/access-logs | jq -r '.data[0:10][] | "\(.timestamp) provider=\(.provider) model=\(.model) status=\(.status_code) decision=\(.decision) src=\(.source_ip) session=\(.session_id // "-") in=\(.input_tokens // 0) out=\(.output_tokens // 0) cost=\(.cost_usd // 0)"' >&2 || true -} - -cmd_clean() { - cmd_snapshot - local id - for id in $(api GET /api/agent-network/policies | jq -r '.[].id'); do - api DELETE "/api/agent-network/policies/$id" >/dev/null && log "deleted policy $id" - done - for id in $(api GET /api/agent-network/budget-rules | jq -r '.[].id'); do - api DELETE "/api/agent-network/budget-rules/$id" >/dev/null && log "deleted budget-rule $id" - done - for id in $(api GET /api/agent-network/providers | jq -r '.[].id'); do - api DELETE "/api/agent-network/providers/$id" >/dev/null && log "deleted provider $id" - done - log "account cleaned" -} - -# create_provider NAME PROVIDER_ID UPSTREAM_URL API_KEY -create_provider() { - local name="$1" pid="$2" url="$3" key="$4" body resp - [ -n "$key" ] || { log "skip $name: empty key"; return 0; } - body="$(jq -n --arg n "$name" --arg p "$pid" --arg u "$url" --arg k "$key" \ - '{name:$n,provider_id:$p,upstream_url:$u,api_key:$k,enabled:true}')" - resp="$(api POST /api/agent-network/providers "$body")" || { log "create $name FAILED"; return 1; } - echo "$resp" | jq -r '"created provider \(.name) id=\(.id) provider_id=\(.provider_id)"' >&2 - echo "$resp" | jq -r '.id' -} - -cmd_providers() { - require_llm_keys - : >"$STATE_DIR/provider-ids.txt" - create_provider "OpenAI API" openai_api "https://api.openai.com" "${OPENAI_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt" - create_provider "Anthropic API" anthropic_api "https://api.anthropic.com" "${ANTHROPIC_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt" - create_provider "Vercel AI Gateway" vercel_ai_gateway "${VERCEL_URL:-}" "${VERCEL_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt" - create_provider "OpenRouter" openrouter "${OPENROUTER_URL:-}" "${OPENROUTER_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt" - create_provider "Cloudflare AI Gateway" cloudflare_ai_gateway "${CLOUDFLARE_URL:-}" "${CLOUDFLARE_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt" - log "provider ids:"; cat "$STATE_DIR/provider-ids.txt" >&2 -} - -cmd_policy() { - local gid ids body - gid="$(admins_group_id)"; [ -n "$gid" ] || die "no Admins group" - ids="$(api GET /api/agent-network/providers | jq -c '[.[].id]')" - body="$(jq -n --arg n "e2e-all-providers" --arg g "$gid" --argjson dst "$ids" '{ - name:$n, description:"e2e: Admins to all providers", enabled:true, - source_groups:[$g], destination_provider_ids:$dst, guardrail_ids:[], - limits:{ budget_limit:{enabled:true,group_cap_usd:1000000,user_cap_usd:1000000,window_seconds:2592000}, - token_limit:{enabled:false,group_cap:0,user_cap:0,window_seconds:60} } - }')" - api POST /api/agent-network/policies "$body" | jq -r '"created policy \(.name) id=\(.id) dst=\(.destination_provider_ids|length) providers"' >&2 -} - -# set_enabled PROVIDER_ID BOOL — PUT the provider back with enabled toggled, -# preserving its required fields and keeping the sealed key (api_key omitted). -set_enabled() { - local pid="$1" en="$2" cur body - cur="$(api GET /api/agent-network/providers | jq -c --arg id "$pid" '.[] | select(.id==$id)')" - [ -n "$cur" ] || { log "no provider $pid"; return 1; } - body="$(echo "$cur" | jq -c --argjson en "$en" '{name,provider_id,upstream_url,enabled:$en} + (if .extra_values then {extra_values} else {} end)')" - api PUT "/api/agent-network/providers/$pid" "$body" >/dev/null -} - -# cmd_isolate NAME — leave only the named provider enabled (sole catch-all), -# so a request routes to it without first-party-vendor or first-catch-all -# interference. Matches NAME case-insensitively against provider .name. -cmd_isolate() { - local want="$1" id name en - api GET /api/agent-network/providers | jq -r '.[] | "\(.id)\t\(.name)"' | while IFS=$'\t' read -r id name; do - case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in - *"$(echo "$want" | tr '[:upper:]' '[:lower:]')"*) en=true ;; - *) en=false ;; - esac - set_enabled "$id" "$en" && log "$name enabled=$en" - done -} - -cmd_enable_all() { - local id - for id in $(api GET /api/agent-network/providers | jq -r '.[].id'); do - set_enabled "$id" true && log "enabled $id" - done -} - -# cmd_dashboard — drive the live :3000 dashboard (../dashboard repo) with -# Playwright, asserting the API-created providers/policy render in the UI. -cmd_dashboard() { - command -v node >/dev/null 2>&1 || die "node not on PATH" - [ -f "$DASH_DIR/e2e/live-agent-network.mjs" ] || die "dashboard script missing in $DASH_DIR/e2e" - ( cd "$DASH_DIR" && BASE_URL="$DASH_URL" DASH_USER="$DASH_USER" DASH_PASS="$DASH_PASS" node e2e/live-agent-network.mjs ) -} - -cmd_down() { docker rm -f "$CLIENT" >/dev/null 2>&1 && log "removed $CLIENT" || log "no container"; } - -cmd_all() { - cmd_clean - cmd_providers - cmd_policy - cmd_up - cmd_wait || { cmd_diag; die "tunnel to proxy not established"; } - log "===== chat: OpenAI =====" ; cmd_chat gpt-5.4 chat - log "===== chat: Anthropic =====" ; cmd_chat claude-haiku-4-5 messages - log "===== chat: Vercel (openai/...) ====="; cmd_chat openai/gpt-4o-mini chat - log "===== chat: OpenRouter (openai/...) =="; cmd_chat openai/gpt-4o-mini chat - sleep 3 - cmd_verify -} - -# --- scenario helpers -------------------------------------------------------- -FAILS=0 -ok() { log " PASS: $1"; } -bad() { log " FAIL: $1"; FAILS=$((FAILS+1)); } -expect_code() { # WANT GOT LABEL - if [ "$2" = "$1" ]; then ok "$3 (http $2)"; else bad "$3 (want $1, got $2)"; fi -} - -# wait_tunnel [CLIENT] — poll silently until the proxy peer is 1/1 Connected. -wait_tunnel() { - local c="${1:-$CLIENT}" i=0 - while [ "$i" -lt 60 ]; do - docker exec "$c" netbird status 2>/dev/null | grep -q '1/1 Connected' && return 0 - sleep 3; i=$((i+3)) - done - return 1 -} - -# wait_chat_ready CLIENT KIND MODEL — poll a real request from CLIENT until it -# returns a non-000 status, i.e. the tunnel + magic DNS + WG handshake to the -# proxy peer have all converged. Single-attempt probes pace the outer loop. -# Note: a netbird down/up bounce does NOT help here and is actively harmful — -# the peer is already in the net-map (status shows 0/1, i.e. known-not-connected), -# so the blocker is the WG handshake to a freshly-churned proxy peer, and -# bouncing just resets an in-progress slow handshake. We only wait it out. -wait_chat_ready() { - local c="$1" kind="${2:-chat}" model="${3:-gpt-5.4}" rc i=0 - while [ "$i" -lt 30 ]; do - rc="$(CHAT_CLIENT="$c" CHAT_RETRIES=1 _chat "$model" "$kind")" - [ "$rc" != 000 ] && { echo "$rc"; return 0; } - i=$((i + 5)) - sleep 5 - done - echo 000 - return 1 -} - -# wait_peer_connected CLIENT [TIMEOUT] — poll until the client reports the proxy -# peer as 1/1 Connected, i.e. the proxy peer has re-stabilised after a churn. -# Used to settle the peer before joining a fresh client into it. -wait_peer_connected() { - local c="$1" timeout="${2:-180}" i=0 - while [ "$i" -lt "$timeout" ]; do - docker exec "$c" netbird status 2>/dev/null | grep -q '1/1 Connected' && return 0 - sleep 5 - i=$((i + 5)) - done - return 1 -} - -provider_id_by_name() { api GET /api/agent-network/providers | jq -r --arg n "$1" '.[] | select(.name==$n) | .id' | head -n1; } -access_log_total() { api GET /api/agent-network/access-logs | jq -r '.total_records'; } -access_log_top() { api GET /api/agent-network/access-logs | jq -r ".data[0].$1 // \"\""; } - -# ensure_group NAME — return the id of group NAME, creating it if absent. -ensure_group() { - local name="$1" id - id="$(api GET /api/groups | jq -r --arg n "$name" '.[] | select(.name==$n) | .id' | head -n1)" - if [ -z "$id" ] || [ "$id" = null ]; then - id="$(api POST /api/groups "$(jq -n --arg n "$name" '{name:$n}')" | jq -r '.id')" - fi - echo "$id" -} - -# policy_put_limits LIMITS_JSON — replace the e2e-all-providers policy limits. -policy_put_limits() { - local pol id body - pol="$(api GET /api/agent-network/policies | jq -c '.[] | select(.name=="e2e-all-providers")')" - id="$(echo "$pol" | jq -r '.id')" - [ -n "$id" ] && [ "$id" != null ] || die "e2e-all-providers policy missing (run 'policy' first)" - body="$(echo "$pol" | jq -c --argjson L "$1" '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids,limits:$L}')" - api PUT "/api/agent-network/policies/$id" "$body" >/dev/null -} - -settings_put() { api PUT /api/agent-network/settings "$1" >/dev/null; } - -# --- scenarios --------------------------------------------------------------- - -# Scenario 1: policy token-cap enforcement. Seed usage under a high cap, drop -# the cap to 1 token so the next call is denied, then restore and confirm -# recovery. Deterministic regardless of prior window usage. -cmd_scenario_budget() { - log "### scenario: policy token-cap enforcement (deny + recovery) ###" - # Seed AND enforce in the same 3600s token window (counters are per-window). - policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":100000,"window_seconds":3600}}' - wait_tunnel; sleep 2 - expect_code 200 "$(_chat gpt-5.4 chat)" "seed call under high token cap (books usage into 3600s window)" - # Drop the cap to 1 in the SAME window so the seeded usage exhausts it. - policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":1,"user_cap":1,"window_seconds":3600}}' - wait_tunnel; sleep 2 - local code; code="$(_chat gpt-5.4 chat)" - expect_code 403 "$code" "call denied once token cap (1) is exhausted" - log " deny envelope: $(cat "$CHAT_RESP")" - policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}' - wait_tunnel; sleep 2 - expect_code 200 "$(_chat gpt-5.4 chat)" "call allowed again after cap restored" -} - -# Scenario 2: account-level budget rule (separate from policy limits). A rule -# targeting Admins with a 1-token cap must deny via the same gRPC check loop; -# deleting it restores access. -cmd_scenario_budget_rule() { - log "### scenario: account budget-rule enforcement ###" - local gid rid - gid="$(admins_group_id)" - # High cap first so the seed call books usage into the rule's 3600s window. - rid="$(api POST /api/agent-network/budget-rules "$(jq -n --arg g "$gid" '{name:"e2e-tight-rule",enabled:true,target_groups:[$g],target_users:[],limits:{budget_limit:{enabled:false,group_cap_usd:0,user_cap_usd:0,window_seconds:3600},token_limit:{enabled:true,group_cap:100000,user_cap:100000,window_seconds:3600}}}')" | jq -r '.id')" - log " created budget-rule $rid (group token cap=100000)" - wait_tunnel; sleep 2 - expect_code 200 "$(_chat gpt-5.4 chat)" "seed call under high account cap" - # Tighten the same rule/window to 1 so the seeded usage exhausts it. - api PUT "/api/agent-network/budget-rules/$rid" "$(jq -n --arg g "$gid" '{name:"e2e-tight-rule",enabled:true,target_groups:[$g],target_users:[],limits:{budget_limit:{enabled:false,group_cap_usd:0,user_cap_usd:0,window_seconds:3600},token_limit:{enabled:true,group_cap:1,user_cap:1,window_seconds:3600}}}')" >/dev/null - log " tightened budget-rule $rid (group token cap=1)" - wait_tunnel; sleep 2 - expect_code 403 "$(_chat gpt-5.4 chat)" "account budget-rule denies when cap exhausted" - api DELETE "/api/agent-network/budget-rules/$rid" >/dev/null && log " deleted budget-rule $rid" - wait_tunnel; sleep 2 - expect_code 200 "$(_chat gpt-5.4 chat)" "allowed again after budget-rule removed" -} - -# Scenario 3: multiple groups + policies + destination scoping. A 2nd Docker -# client joins a 2nd group; each group's policy authorises a different -# provider. Cross-provider requests must be denied. -cmd_scenario_multigroup() { - log "### scenario: multi-group / multi-policy destination scoping ###" - local gidA gidB oai ant keyB polAllId - gidA="$(admins_group_id)"; gidB="$(ensure_group e2e-grp-b)" - oai="$(provider_id_by_name 'OpenAI API')"; ant="$(provider_id_by_name 'Anthropic API')" - log " groups: Admins=$gidA grp-b=$gidB ; providers: openai=$oai anthropic=$ant" - - # Disable the broad e2e policy; add two narrow ones. - polAllId="$(api GET /api/agent-network/policies | jq -r '.[] | select(.name=="e2e-all-providers") | .id')" - [ -n "$polAllId" ] && api PUT "/api/agent-network/policies/$polAllId" \ - "$(api GET /api/agent-network/policies | jq -c --arg id "$polAllId" '.[] | select(.id==$id) | {name,description,enabled:false,source_groups,destination_provider_ids,guardrail_ids,limits}')" >/dev/null - local hi='{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":60}}' - local polA polB - polA="$(api POST /api/agent-network/policies "$(jq -n --arg g "$gidA" --arg p "$oai" --argjson L "$hi" '{name:"e2e-A-admins-openai",enabled:true,source_groups:[$g],destination_provider_ids:[$p],guardrail_ids:[],limits:$L}')" | jq -r '.id')" - polB="$(api POST /api/agent-network/policies "$(jq -n --arg g "$gidB" --arg p "$ant" --argjson L "$hi" '{name:"e2e-B-grpb-anthropic",enabled:true,source_groups:[$g],destination_provider_ids:[$p],guardrail_ids:[],limits:$L}')" | jq -r '.id')" - log " policy A (Admins->OpenAI)=$polA ; policy B (grp-b->Anthropic)=$polB" - - # The policy swap above re-synthesises the agent-network service and adds a - # new source group (grp-b), which churns the proxy peer and forces it to - # absorb the new authorisation. Wait for the peer to re-stabilise (client A - # reconnects to the new peer) before joining B, so B handshakes into a stable - # peer instead of racing the churn — otherwise the proxy never answers B's - # offers and it sits at 0/1 Connected. - if ! wait_peer_connected "$CLIENT" 30; then - log " warning: proxy peer did not re-stabilise for client A within 30s" - fi - sleep 5 - - # 2nd client bound to grp-b. - keyB="$(api POST /api/setup-keys "$(jq -n --arg g "$gidB" '{name:"e2e-agentnet-docker-b",type:"reusable",expires_in:86400,usage_limit:0,auto_groups:[$g],ephemeral:false}')" | jq -r '.key')" - docker rm -f "${CLIENT}-b" >/dev/null 2>&1 || true - docker run -d --name "${CLIENT}-b" --cap-add NET_ADMIN --cap-add SYS_ADMIN --cap-add SYS_RESOURCE \ - --add-host host.docker.internal:host-gateway -e NB_MANAGEMENT_URL="$MGMT_URL" -e NB_SETUP_KEY="$keyB" \ - -e NB_LOG_LEVEL=info "$IMAGE" >/dev/null - log " started ${CLIENT}-b (grp-b)" - # Gate on real end-to-end connectivity: a freshly-authorised group's client - # takes ~1-2 min for the proxy to absorb and answer its WG handshake. - local ra rb - ra="$(wait_chat_ready "$CLIENT" chat gpt-5.4)" - rb="$(wait_chat_ready "${CLIENT}-b" messages claude-haiku-4-5)" - log " connectivity ready: A(probe=$ra) B(probe=$rb)" - - log " -- client A (Admins) --" - CHAT_CLIENT="$CLIENT" expect_code 200 "$(CHAT_CLIENT="$CLIENT" _chat gpt-5.4 chat)" "A->OpenAI authorised" - CHAT_CLIENT="$CLIENT" expect_code 403 "$(CHAT_CLIENT="$CLIENT" _chat claude-haiku-4-5 messages)" "A->Anthropic denied (not in policy A)" - log " -- client B (grp-b) --" - CHAT_CLIENT="${CLIENT}-b" expect_code 200 "$(CHAT_CLIENT="${CLIENT}-b" _chat claude-haiku-4-5 messages)" "B->Anthropic authorised" - CHAT_CLIENT="${CLIENT}-b" expect_code 403 "$(CHAT_CLIENT="${CLIENT}-b" _chat gpt-5.4 chat)" "B->OpenAI denied (not in policy B)" - - # Restore: drop A/B, re-enable the broad policy. - api DELETE "/api/agent-network/policies/$polA" >/dev/null 2>&1 || true - api DELETE "/api/agent-network/policies/$polB" >/dev/null 2>&1 || true - [ -n "$polAllId" ] && api PUT "/api/agent-network/policies/$polAllId" \ - "$(api GET /api/agent-network/policies | jq -c --arg id "$polAllId" '.[] | select(.id==$id) | {name,description,enabled:true,source_groups,destination_provider_ids,guardrail_ids,limits}')" >/dev/null - docker rm -f "${CLIENT}-b" >/dev/null 2>&1 || true - log " restored broad policy, removed client B" -} - -# Scenario 4: log-collection + prompt-collection + redaction settings. -cmd_scenario_logs() { - log "### scenario: log/prompt-collection + redaction settings ###" - settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false}' - wait_tunnel; sleep 3 - - # (a) log collection OFF -> no new access-log row. - local before after - before="$(access_log_total)" - settings_put '{"enable_log_collection":false,"enable_prompt_collection":true,"redact_pii":false}' - wait_tunnel; sleep 3 - expect_code 200 "$(_chat gpt-5.4 chat)" "request still served with logging off" - sleep 3; after="$(access_log_total)" - if [ "$after" = "$before" ]; then ok "log collection OFF -> no new row ($before==$after)"; else bad "log collection OFF still wrote a row ($before->$after)"; fi - - # (b) log ON, prompt OFF -> row present but request_prompt empty. - settings_put '{"enable_log_collection":true,"enable_prompt_collection":false,"redact_pii":false}' - wait_tunnel; sleep 3 - expect_code 200 "$(_chat gpt-5.4 chat "this prompt text must NOT be stored")" "served with prompt collection off" - sleep 3 - local p; p="$(access_log_top request_prompt)" - if [ -z "$p" ]; then ok "prompt collection OFF -> request_prompt empty"; else bad "prompt stored despite collection off: [$p]"; fi - - # (c) prompt ON + redact ON -> PII scrubbed from stored prompt. - settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":true}' - wait_tunnel; sleep 3 - expect_code 200 "$(_chat gpt-5.4 chat "Contact john.doe@example.com, SSN 123-45-6789, phone 555-123-4567")" "served with redaction on" - sleep 3 - local stored; stored="$(access_log_top request_prompt)" - log " stored prompt: [$stored]" - case "$stored" in - *john.doe@example.com*) bad "email leaked into stored prompt" ;; - *) ok "email redacted from stored prompt" ;; - esac - case "$stored" in - *123-45-6789*) bad "SSN leaked into stored prompt" ;; - *) ok "SSN redacted from stored prompt" ;; - esac - - settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false}' - log " restored settings (log on, prompt on, redact off)" -} - -# Scenario 5: per-user cap isolated from the group cap. Seed under a high cap, -# then keep the group cap high but drop the USER cap to 1 — the deny must come -# from the user dimension, proving per-user accounting is independent. -cmd_scenario_user_cap() { - log "### scenario: per-user token cap (isolated from group cap) ###" - policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":100000,"window_seconds":3600}}' - wait_tunnel; sleep 2 - expect_code 200 "$(_chat gpt-5.4 chat)" "seed call (books user+group usage into 3600s window)" - # Group cap stays huge; only the user cap is exhausted. - policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":1,"window_seconds":3600}}' - wait_tunnel; sleep 2 - expect_code 403 "$(_chat gpt-5.4 chat)" "denied by USER cap=1 while group cap=100000 has headroom" - log " deny: $(cat "$CHAT_RESP")" - policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}' - wait_tunnel; sleep 2 - expect_code 200 "$(_chat gpt-5.4 chat)" "allowed again after user cap lifted" -} - -# Scenario 6: guardrail model-allowlist blocking. Attach a guardrail allowing -# only gpt-5.4; a request for any other model must be denied (model_blocked), -# while the allowed model passes. -cmd_scenario_guardrail() { - log "### scenario: guardrail model-allowlist blocking ###" - local gid polId - gid="$(api POST /api/agent-network/guardrails "$(jq -n '{name:"e2e-allowlist",description:"e2e: only gpt-5.4",checks:{model_allowlist:{enabled:true,models:["gpt-5.4"]},prompt_capture:{enabled:true,redact_pii:false}}}')" | jq -r '.id')" - log " created guardrail $gid (allow only gpt-5.4)" - local pol; pol="$(api GET /api/agent-network/policies | jq -c '.[] | select(.name=="e2e-all-providers")')" - polId="$(echo "$pol" | jq -r '.id')" - api PUT "/api/agent-network/policies/$polId" "$(echo "$pol" | jq -c --arg g "$gid" '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids:[$g],limits}')" >/dev/null - wait_tunnel; sleep 2 - expect_code 403 "$(_chat gpt-4o-mini chat)" "model not in allowlist is blocked" - log " deny: $(cat "$CHAT_RESP")" - expect_code 200 "$(_chat gpt-5.4 chat)" "allowlisted model passes" - # Detach + delete. - api PUT "/api/agent-network/policies/$polId" "$(echo "$pol" | jq -c '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids:[],limits}')" >/dev/null - api DELETE "/api/agent-network/guardrails/$gid" >/dev/null && log " detached + deleted guardrail $gid" -} - -# _chat_stream MODEL — streaming chat completion (SSE) with usage included. -# Echoes the HTTP status; writes the raw SSE stream to $CHAT_RESP. -_chat_stream() { - local model="$1" client="${CHAT_CLIENT:-$CLIENT}" ip body out code - ip="$(CLIENT="$client" proxy_ip)"; [ -n "$ip" ] || { echo 000; return; } - body="$(jq -n --arg m "$model" '{model:$m,stream:true,stream_options:{include_usage:true},messages:[{role:"user",content:"Reply with exactly: pong"}]}')" - out="$(docker run --rm --network "container:$client" "$CURL_IMAGE" \ - -sSkN --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \ - -w $'\n%{http_code}' -X POST "https://$PROXY_HOST/v1/chat/completions" \ - -H "Content-Type: application/json" --data "$body")" - code="$(printf '%s' "$out" | tail -n1)" - printf '%s' "$out" | sed '$d' >"$CHAT_RESP" - echo "$code" -} - -# Scenario 7: streaming (SSE) token capture. The proxy must accumulate token -# usage from the streamed deltas and persist it on the access-log row. -cmd_scenario_streaming() { - log "### scenario: streaming (SSE) token capture ###" - policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}' - wait_tunnel; sleep 2 - expect_code 200 "$(_chat_stream gpt-5.4)" "streaming request served" - if grep -q 'data: \[DONE\]' "$CHAT_RESP"; then ok "SSE stream terminated with [DONE]"; else bad "no [DONE] terminator in SSE stream"; fi - if grep -q '"delta"' "$CHAT_RESP"; then ok "SSE carried incremental deltas"; else bad "no delta chunks in SSE stream"; fi - sleep 3 - local st in out - st="$(access_log_top stream)"; in="$(access_log_top input_tokens)"; out="$(access_log_top output_tokens)" - log " access-log: stream=$st input_tokens=$in output_tokens=$out" - if [ "$st" = true ]; then ok "access-log row flagged stream=true"; else bad "stream flag not set (got $st)"; fi - if [ "${in:-0}" -gt 0 ] 2>/dev/null && [ "${out:-0}" -gt 0 ] 2>/dev/null; then ok "streamed token usage captured (in=$in out=$out)"; else bad "streamed token usage not captured (in=$in out=$out)"; fi -} - -# wait_for_mgmt — poll the management API until it accepts authed requests. -wait_for_mgmt() { - local i=0 - while [ "$i" -lt 90 ]; do - curl -fsS -o /dev/null --max-time 2 -H "$AUTH" "$NB_API/api/users" 2>/dev/null && return 0 - sleep 2; i=$((i+2)) - done - return 1 -} - -# Scenario 8: access-log retention pruning. The sweep deletes rows older than -# now - retention_days and runs on management startup. We clone an existing row -# to a synthetic 2020-dated id, set retention to 365d (so only that far-past -# row is eligible — real 2026 rows are safe), restart management to trigger the -# sweep, and assert the synthetic row is gone while real rows survive. -cmd_scenario_retention() { - log "### scenario: access-log retention pruning ###" - local DB="/Users/maycon/projects/wt_testing_data/store.db" acct="d68ag3p31576fp2gmnag" sid="e2e-retention-2020" - command -v sqlite3 >/dev/null 2>&1 || { bad "sqlite3 not on PATH"; return; } - [ -w "$DB" ] || { bad "store.db not writable at $DB"; return; } - local before mid after syn - before="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")" - sqlite3 "$DB" "PRAGMA busy_timeout=8000; - DELETE FROM agent_network_access_log WHERE id='$sid'; - CREATE TEMP TABLE _t AS SELECT * FROM agent_network_access_log WHERE account_id='$acct' LIMIT 1; - UPDATE _t SET id='$sid', timestamp='2020-01-01 00:00:00.000000000+00:00'; - INSERT INTO agent_network_access_log SELECT * FROM _t;" 2>&1 | sed 's/^/ sqlite: /' >&2 || true - mid="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")" - if [ "$mid" = "$((before+1))" ]; then ok "synthetic 2020 row inserted ($before -> $mid)"; else bad "insert failed ($before -> $mid)"; return; fi - - settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false,"access_log_retention_days":365}' - log " retention=365d; restarting management to trigger the startup sweep..." - tilt trigger management >/dev/null 2>&1 || bad "tilt trigger management failed" - wait_for_mgmt || bad "management did not come back up" - sleep 4 - - syn="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE id='$sid'")" - after="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")" - if [ "$syn" = 0 ]; then ok "synthetic 2020 row pruned by retention sweep"; else bad "synthetic row survived (sweep didn't prune)"; fi - if [ "$after" = "$before" ]; then ok "real rows preserved (count back to $before)"; else bad "real row count changed ($before -> $after)"; fi - - settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false,"access_log_retention_days":0}' - sqlite3 "$DB" "DELETE FROM agent_network_access_log WHERE id='$sid'" 2>/dev/null || true - log " restored retention=0 (keep forever); re-establishing tunnel" - wait_tunnel >/dev/null 2>&1 || true -} - -# vertex_request IP PATH BODY — POST a Vertex rawPredict through the client -# sidecar; echo the HTTP status and stash the response body in VTX_RESP. Shared -# by the Vertex scenario's initial probe and its live-update propagation checks. -VTX_RESP="" -vertex_request() { - local ip="$1" path="$2" body="$3" out - out="$(docker run --rm --network "container:$CLIENT" "$CURL_IMAGE" \ - -sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \ - -w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \ - -H "Content-Type: application/json" --data "$body")" - VTX_RESP="$(printf '%s' "$out" | sed '$d')" - printf '%s' "$out" | tail -n1 -} - -# bedrock_probe LABEL PATH BODY IP — POST a Bedrock request via the client -# sidecar and assert the pipeline. 200 -> full pass (+ token metering); a -# Bedrock-origin 404 (account use-case gate / model access) -> pipeline OK -# (routing + bearer auth reached the model); 401/403 -> fail. -bedrock_probe() { - local label="$1" path="$2" body="$3" ip="$4" out code resp - out="$(docker run --rm --network "container:$CLIENT" "$CURL_IMAGE" \ - -sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \ - -w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \ - -H "Content-Type: application/json" --data "$body")" - code="$(printf '%s' "$out" | tail -n1)" - resp="$(printf '%s' "$out" | sed '$d')" - log " bedrock $label -> http=$code" - case "$code" in - 200) - ok "Bedrock $label 200 (path->model parse, routed, bearer accepted)" - sleep 3 - local vin vout - vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)" - if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "Bedrock $label usage metered (in=$vin out=$vout)"; else bad "Bedrock $label usage not metered (in=$vin out=$vout)"; fi - ;; - 404) - case "$resp" in - *"use case"*|*NOT_FOUND*|*bedrock*|*"don't have access"*|*"inference profile"*) - ok "reached Bedrock ($label, provider+bearer OK); 404 = account use-case gate / model access" ;; - *) bad "Bedrock $label 404 but not a Bedrock response: $(printf '%s' "$resp" | head -c 160)" ;; - esac ;; - 401) bad "Bedrock $label rejected the bearer token (401): $(printf '%s' "$resp" | head -c 160)" ;; - 403) bad "Bedrock $label denied before upstream (routing regression): $(printf '%s' "$resp" | head -c 160)" ;; - 000) bad "Bedrock $label no connectivity (tunnel not converged)" ;; - *) bad "Bedrock $label unexpected status $code: $(printf '%s' "$resp" | head -c 160)" ;; - esac -} - -# Scenario 9: Google Vertex AI (path-routed model + service-account OAuth minting). -# Vertex carries the model in the URL path and authenticates with a short-lived -# OAuth2 access token, not an API key. The operator stores a durable GCP -# service-account key as the provider api_key behind a "keyfile::" prefix; the -# synthesiser ships the base64 SA JSON to the router, which mints + caches a -# Bearer token per request. We assert the whole pipeline: path->model parse, -# route-to-Vertex, token mint + inject, upstream forward. A 200 is a complete -# pass; a 404 whose body is Vertex's own "Publisher model not found / no access" -# also passes the pipeline (the token was accepted) and only signals that the -# test project lacks model access in that region. -cmd_scenario_vertex() { - log "### scenario: Google Vertex AI (path-routed + service-account OAuth minting) ###" - require_llm_keys - if [ -z "${GOOGLE_VERTEX_SA_BASE64:-}" ] || [ -z "${GOOGLE_VERTEXT_PROJECT:-}" ]; then - log " SKIP: GOOGLE_VERTEX_SA_BASE64 / GOOGLE_VERTEXT_PROJECT not set in $LLM_KEYS_FILE" - return 0 - fi - local vid pol pid old oldvids region vhost - # The regional endpoint host is "-aiplatform.googleapis.com"; the - # "global" location uses the bare "aiplatform.googleapis.com" host. - region="${GOOGLE_VERTEXT_REGION:-global}" - if [ "$region" = global ]; then vhost="aiplatform.googleapis.com"; else vhost="${region}-aiplatform.googleapis.com"; fi - # Remove any pre-existing Vertex providers, de-referencing them from the broad - # policy first — a provider that is still a policy destination can't be deleted - # (422), and a survivor leaves a duplicate (possibly stale-host) Vertex route. - oldvids="$(api GET /api/agent-network/providers | jq -c '[.[]|select(.provider_id=="vertex_ai_api")|.id]')" - if [ "$oldvids" != "[]" ]; then - pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')" - pid="$(echo "$pol" | jq -r '.id')" - api PUT "/api/agent-network/policies/$pid" \ - "$(echo "$pol" | jq -c --argjson rm "$oldvids" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-$rm),guardrail_ids,limits}')" >/dev/null 2>&1 || true - for old in $(echo "$oldvids" | jq -r '.[]'); do - api DELETE "/api/agent-network/providers/$old" >/dev/null 2>&1 || true - done - fi - vid="$(api POST /api/agent-network/providers "$(jq -n --arg k "keyfile::${GOOGLE_VERTEX_SA_BASE64}" --arg u "https://${vhost}" \ - '{name:"Vertex e2e",provider_id:"vertex_ai_api",upstream_url:$u,api_key:$k,enabled:true}')" \ - | jq -r '.id')" - [ -n "$vid" ] && [ "$vid" != null ] || { bad "create Vertex provider failed"; return; } - log " created Vertex provider $vid (keyfile:: service-account, $vhost)" - - pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')" - pid="$(echo "$pol" | jq -r '.id')" - [ -n "$pid" ] && [ "$pid" != null ] || { bad "e2e-all-providers policy missing (run 'policy' first)"; return; } - api PUT "/api/agent-network/policies/$pid" \ - "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:((.destination_provider_ids+[$v])|unique),guardrail_ids,limits}')" >/dev/null - policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}' - # Provider create/update propagates to the proxy live via reconcile (the synth - # config is re-pushed and the middleware chain rebuilt on receipt), so just let - # the tunnel settle before probing — no restart needed. - wait_tunnel; sleep 5 - - local ip path body code resp - ip="$(proxy_ip)"; [ -n "$ip" ] || { bad "could not resolve proxy IP"; return; } - path="/v1/projects/${GOOGLE_VERTEXT_PROJECT}/locations/${region}/publishers/anthropic/models/claude-sonnet-4-5@20250929:rawPredict" - body="$(jq -n '{anthropic_version:"vertex-2023-10-16",max_tokens:64,messages:[{role:"user",content:"Reply with exactly: pong"}]}')" - code="$(vertex_request "$ip" "$path" "$body")"; resp="$VTX_RESP" - log " vertex rawPredict -> http=$code" - case "$code" in - 200) - ok "Vertex 200 (path->model parse, routed, token minted + accepted, model available)" - sleep 3 - local vin vout - vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)" - log " access-log: model=$(access_log_top model) input_tokens=$vin output_tokens=$vout" - if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "Vertex usage metered (in=$vin out=$vout)"; else bad "Vertex usage not metered (in=$vin out=$vout)"; fi - ;; - 404) - case "$resp" in - *"Publisher model"*|*aiplatform*|*NOT_FOUND*) - ok "reached Vertex with a minted token (pipeline OK); 404 = test project lacks model access" ;; - *) bad "404 but not a Vertex response body: $(printf '%s' "$resp" | head -c 160)" ;; - esac ;; - 401) bad "Vertex rejected the minted credential (401) — check the SA key/roles: $(printf '%s' "$resp" | head -c 160)" ;; - 403) bad "proxy denied before reaching Vertex (path parse / routing regression): $(printf '%s' "$resp" | head -c 160)" ;; - 000) bad "no connectivity to proxy (tunnel not converged)" ;; - *) bad "unexpected status $code: $(printf '%s' "$resp" | head -c 160)" ;; - esac - - # Live provider-update propagation: a provider change must reach the proxy node - # without a restart. Disable the provider -> its route disappears and the request - # is denied (403); re-enable -> served again at the baseline status. Proves - # reconcile re-pushes the synth config and the proxy rebuilds its chain live. - case "$code" in - 200|404) - log " testing live provider-update propagation (disable/enable, no restart)..." - local dcode ecode - set_enabled "$vid" false; sleep 5 - dcode="$(vertex_request "$ip" "$path" "$body")" - if [ "$dcode" = 403 ]; then ok "provider disable reflected on proxy live (route removed, http 403)"; else bad "provider disable not reflected on proxy (want 403, got $dcode)"; fi - set_enabled "$vid" true; sleep 5 - ecode="$(vertex_request "$ip" "$path" "$body")" - if [ "$ecode" = "$code" ]; then ok "provider re-enable reflected on proxy live (served again, http $ecode)"; else bad "provider re-enable not reflected on proxy (want $code, got $ecode)"; fi - ;; - esac - - pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')" - api PUT "/api/agent-network/policies/$pid" \ - "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-[$v]),guardrail_ids,limits}')" >/dev/null 2>&1 || true - api DELETE "/api/agent-network/providers/$vid" >/dev/null 2>&1 || true -} - -# Scenario 10: real-traffic coverage for the OpenAI-compatible gateway providers -# (Vercel, OpenRouter, Cloudflare). Each is isolated as the sole enabled provider -# so its model routes to it unambiguously, sent a real chat request, and asserted -# to return 200 with metered token usage. A gateway without a configured provider -# is skipped. Restores all providers at the end. -cmd_scenario_providers() { - log "### scenario: gateway providers real-traffic (Vercel / OpenRouter / Cloudflare) ###" - policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}' - local spec iso full model kind r pid code vin vout - for spec in \ - "vercel|Vercel AI Gateway|openai/gpt-4o-mini|chat" \ - "openrouter|OpenRouter|openai/gpt-4o-mini|chat" \ - "cloudflare|Cloudflare AI Gateway|gpt-4o-mini|chat"; do - iso="${spec%%|*}"; r="${spec#*|}"; full="${r%%|*}"; r="${r#*|}"; model="${r%%|*}"; kind="${r##*|}" - pid="$(provider_id_by_name "$full")" - if [ -z "$pid" ] || [ "$pid" = null ]; then log " SKIP $full: not configured"; continue; fi - cmd_isolate "$iso" >/dev/null 2>&1 - wait_tunnel; sleep 4 - code="$(_chat "$model" "$kind")" - if [ "$code" = 200 ]; then ok "$full served real request ($model, http 200)"; else bad "$full request failed ($model, http $code)"; continue; fi - sleep 3 - vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)" - if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "$full usage metered (in=$vin out=$vout)"; else bad "$full usage not metered (in=$vin out=$vout)"; fi - done - cmd_enable_all >/dev/null 2>&1 - log " re-enabled all providers" - wait_tunnel >/dev/null 2>&1 || true -} - -# Scenario 11: AWS Bedrock (path-routed model + bearer auth) across invoke, -# converse, and invoke-with-response-stream. Bedrock carries the model in the URL -# path and authenticates with a static bearer token (Bedrock API key). Asserts -# the pipeline per endpoint: path->model parse + normalize, route-to-Bedrock, -# bearer inject, upstream forward (+ token metering on 200). A Bedrock-origin 404 -# (account use-case gate / model access) passes the pipeline; see bedrock_probe. -cmd_scenario_bedrock() { - log "### scenario: AWS Bedrock (path-routed + bearer; invoke/converse/stream) ###" - require_llm_keys - if [ -z "${AWS_BEARER_TOKEN_BEDROCK:-}" ]; then - log " SKIP: AWS_BEARER_TOKEN_BEDROCK not set in $LLM_KEYS_FILE" - return 0 - fi - local region model vid pol pid old oldvids - region="${AWS_BEDROCK_REGION:-eu-central-1}" - model="${BEDROCK_MODEL:-eu.anthropic.claude-sonnet-4-5-20250929-v1:0}" - oldvids="$(api GET /api/agent-network/providers | jq -c '[.[]|select(.provider_id=="bedrock_api")|.id]')" - if [ "$oldvids" != "[]" ]; then - pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')" - pid="$(echo "$pol" | jq -r '.id')" - api PUT "/api/agent-network/policies/$pid" \ - "$(echo "$pol" | jq -c --argjson rm "$oldvids" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-$rm),guardrail_ids,limits}')" >/dev/null 2>&1 || true - for old in $(echo "$oldvids" | jq -r '.[]'); do - api DELETE "/api/agent-network/providers/$old" >/dev/null 2>&1 || true - done - fi - vid="$(api POST /api/agent-network/providers "$(jq -n --arg k "$AWS_BEARER_TOKEN_BEDROCK" --arg u "https://bedrock-runtime.${region}.amazonaws.com" \ - '{name:"Bedrock e2e",provider_id:"bedrock_api",upstream_url:$u,api_key:$k,enabled:true}')" | jq -r '.id')" - [ -n "$vid" ] && [ "$vid" != null ] || { bad "create Bedrock provider failed"; return; } - log " created Bedrock provider $vid (bearer, bedrock-runtime.${region})" - - pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')" - pid="$(echo "$pol" | jq -r '.id')" - [ -n "$pid" ] && [ "$pid" != null ] || { bad "e2e-all-providers policy missing (run 'policy' first)"; return; } - api PUT "/api/agent-network/policies/$pid" \ - "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:((.destination_provider_ids+[$v])|unique),guardrail_ids,limits}')" >/dev/null - policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}' - wait_tunnel; sleep 5 - - local ip invokeBody converseBody - ip="$(proxy_ip)"; [ -n "$ip" ] || { bad "could not resolve proxy IP"; return; } - invokeBody='{"anthropic_version":"bedrock-2023-05-31","max_tokens":32,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}' - converseBody='{"messages":[{"role":"user","content":[{"text":"Reply with exactly: pong"}]}],"inferenceConfig":{"maxTokens":32}}' - bedrock_probe "invoke" "/model/${model}/invoke" "$invokeBody" "$ip" - bedrock_probe "converse" "/model/${model}/converse" "$converseBody" "$ip" - bedrock_probe "invoke-stream" "/model/${model}/invoke-with-response-stream" "$invokeBody" "$ip" - # Gateway-namespace prefix: /bedrock/... must route the same and be stripped - # before the upstream call (AWS's native path has no /bedrock prefix). - bedrock_probe "invoke (/bedrock prefix)" "/bedrock/model/${model}/invoke" "$invokeBody" "$ip" - - pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')" - api PUT "/api/agent-network/policies/$pid" \ - "$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-[$v]),guardrail_ids,limits}')" >/dev/null 2>&1 || true - api DELETE "/api/agent-network/providers/$vid" >/dev/null 2>&1 || true -} - -cmd_scenarios() { - FAILS=0 - cmd_scenario_budget - cmd_scenario_budget_rule - cmd_scenario_user_cap - cmd_scenario_guardrail - cmd_scenario_streaming - cmd_scenario_multigroup - cmd_scenario_logs - cmd_scenario_retention - cmd_scenario_providers - cmd_scenario_vertex - cmd_scenario_bedrock - log "================================================" - if [ "$FAILS" -eq 0 ]; then log "ALL SCENARIOS PASSED"; else log "SCENARIO FAILURES: $FAILS"; fi - return "$FAILS" -} - -# --- dispatch ---------------------------------------------------------------- -cmd="${1:-}"; shift || true -case "$cmd" in - snapshot) cmd_snapshot ;; - key) cmd_key ;; - up) cmd_up ;; - status) cmd_status ;; - wait) cmd_wait ;; - diag) cmd_diag ;; - restart-proxy) cmd_restart_proxy ;; - chat) cmd_chat "$@" ;; - verify) cmd_verify ;; - clean) cmd_clean ;; - providers) cmd_providers ;; - policy) cmd_policy ;; - isolate) cmd_isolate "$@" ;; - enable-all) cmd_enable_all ;; - scenario-budget) cmd_scenario_budget ;; - scenario-budget-rule) cmd_scenario_budget_rule ;; - scenario-user-cap) cmd_scenario_user_cap ;; - scenario-guardrail) cmd_scenario_guardrail ;; - scenario-streaming) cmd_scenario_streaming ;; - scenario-multigroup) cmd_scenario_multigroup ;; - scenario-logs) cmd_scenario_logs ;; - scenario-retention) cmd_scenario_retention ;; - scenario-providers) cmd_scenario_providers ;; - scenario-vertex) cmd_scenario_vertex ;; - scenario-bedrock) cmd_scenario_bedrock ;; - dashboard) cmd_dashboard ;; - scenarios) cmd_scenarios ;; - down) cmd_down ;; - all) cmd_all ;; - *) cat >&2 <<'USAGE' -usage: bash scripts/e2e/agent-network-full/e2e.sh - setup/flow : snapshot key up status wait diag restart-proxy clean providers policy isolate NAME enable-all down all - traffic : chat MODEL [chat|messages] verify - scenarios : scenarios | scenario-budget | scenario-budget-rule | scenario-user-cap | - scenario-guardrail | scenario-streaming | scenario-multigroup | - scenario-logs | scenario-retention | scenario-providers | scenario-vertex | - scenario-bedrock - dashboard : dashboard (Playwright UI check against the live :3000 dashboard) -USAGE - exit 2 ;; -esac diff --git a/scripts/e2e/agent-network-policy/00-env.sh b/scripts/e2e/agent-network-policy/00-env.sh deleted file mode 100755 index a26bb167d..000000000 --- a/scripts/e2e/agent-network-policy/00-env.sh +++ /dev/null @@ -1,105 +0,0 @@ -# shellcheck disable=SC2148 -# Sourced helper for the agent-network-policy e2e suite. -# Verifies the agent-network surface: window_seconds (post-rename) -# through the API, /api/agent-network/consumption read endpoint, and -# the CheckLLMPolicyLimits + RecordLLMUsage gRPC RPCs. -# -# Do not run directly. - -set -euo pipefail - -: "${NB_API:=http://localhost:8080}" -: "${NB_PAT_FILE:=/Users/maycon/projects/local-dev/nb-pat}" -: "${NB_TIMEOUT_SECONDS:=60}" -: "${NB_POLICY_NAME:=e2e-anpol}" -: "${NB_PROVIDER_NAME:=e2e-anpol-provider}" -: "${NB_GROUP_NAME:=e2e-anpol-engineers}" -: "${NB_GRPC_ADDR:=localhost:8080}" -# Proxy token shared with the tilt setup. Keep in sync with the -# NB_PROXY_TOKEN literal in /Users/maycon/projects/local-dev/Tiltfile; -# the management server accepts it as a registered proxy credential -# so the e2e binary can reach the proxy_service gRPC surface. -: "${NB_PROXY_TOKEN:=nbx_MEF9OKRhlLrWkc5TJmM3Eu2rhqigaP2yulHy}" -: "${NB_STATE_DIR:=/tmp/nb-anpol-e2e-state}" - -mkdir -p "$NB_STATE_DIR" - -if [ ! -r "$NB_PAT_FILE" ]; then - echo "FAIL: cannot read PAT at $NB_PAT_FILE" >&2 - exit 2 -fi -NB_PAT=$(tr -d '\n\r ' <"$NB_PAT_FILE") -if [ ${#NB_PAT} -lt 16 ]; then - echo "FAIL: PAT at $NB_PAT_FILE is suspiciously short (${#NB_PAT} chars)" >&2 - exit 2 -fi - -if ! command -v jq >/dev/null 2>&1; then - echo "FAIL: jq is required (brew install jq)" >&2 - exit 2 -fi - -# nb_api METHOD PATH [BODY] — wraps curl with PAT auth and JSON. -nb_api() { - local method="$1" path="$2" body="${3-}" - if [ -n "$body" ]; then - curl -fsS -X "$method" \ - -H "Authorization: Token $NB_PAT" \ - -H "Content-Type: application/json" \ - --data "$body" \ - "$NB_API$path" - else - curl -fsS -X "$method" \ - -H "Authorization: Token $NB_PAT" \ - "$NB_API$path" - fi -} - -# nb_api_status METHOD PATH [BODY] — returns the HTTP status code -# rather than the body. Use for negative-path tests where a 4xx is -# the expected outcome and curl's default -f would mask the result. -nb_api_status() { - local method="$1" path="$2" body="${3-}" - if [ -n "$body" ]; then - curl -sS -o /dev/null -w '%{http_code}' -X "$method" \ - -H "Authorization: Token $NB_PAT" \ - -H "Content-Type: application/json" \ - --data "$body" \ - "$NB_API$path" - else - curl -sS -o /dev/null -w '%{http_code}' -X "$method" \ - -H "Authorization: Token $NB_PAT" \ - "$NB_API$path" - fi -} - -# wait_for COND_CMD TIMEOUT_S — polls every 1s. -wait_for() { - local cmd="$1" timeout="${2:-$NB_TIMEOUT_SECONDS}" - local i=0 - while [ "$i" -lt "$timeout" ]; do - if eval "$cmd" >/dev/null 2>&1; then - return 0 - fi - sleep 1 - i=$((i + 1)) - done - return 1 -} - -# resolve the absolute path of the netbird repo root from the script -# location so the Go smoke binary can be invoked via `go run` against -# the right module regardless of the caller's cwd. -nb_repo_root() { - cd "$(dirname "$0")/../../.." && pwd -} - -pass() { - printf 'PASS: %s\n' "$1" - exit 0 -} - -fail() { - printf 'FAIL: %s — %s\n' "$1" "${2:-}" - exit 1 -} diff --git a/scripts/e2e/agent-network-policy/01-tilt-restart.sh b/scripts/e2e/agent-network-policy/01-tilt-restart.sh deleted file mode 100755 index 041a6c022..000000000 --- a/scripts/e2e/agent-network-policy/01-tilt-restart.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# 01-tilt-restart: re-trigger management + dashboard before each run so -# we start from a clean process state. The on-disk store survives, so -# the PAT, account, and any residual config persist across restarts. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -command -v tilt >/dev/null 2>&1 || fail "tilt not on PATH" "brew install tilt-dev/tap/tilt" - -if ! curl -fsS -o /dev/null --max-time 2 http://localhost:10350/ 2>/dev/null; then - fail "Tilt API not reachable at http://localhost:10350" "is 'tilt up' running in /Users/maycon/projects/local-dev?" -fi - -restart_one() { - local name="$1" - if ! tilt trigger "$name" >/dev/null 2>&1; then - fail "tilt trigger $name failed" "" - fi - echo "triggered $name" -} - -restart_one management -restart_one dashboard -# proxy3 needs to come back too because PR2 wired the new -# llm_limit_check / llm_limit_record middlewares into the proxy -# binary; without restarting it, e2e keeps exercising whatever -# image was last loaded and silently misses any boot-order or -# wiring regression in the chain. Cheap to restart, expensive to -# silently miss. -restart_one proxy3 - -echo "waiting for management to accept requests..." -if ! wait_for "curl -fsS -o /dev/null --max-time 2 $NB_API/oauth2/.well-known/openid-configuration" 60; then - fail "management did not come back up within 60s" "check 'tilt logs management'" -fi -echo "management is up" - -code=$(curl -fsS -o /dev/null -w '%{http_code}' \ - -H "Authorization: Token $NB_PAT" \ - "$NB_API/api/users" 2>&1) || true -[ "$code" = "200" ] || fail "PAT auth check failed after restart (HTTP $code)" "the PAT may have been revoked" - -pass "Tilt resources restarted: management, dashboard" diff --git a/scripts/e2e/agent-network-policy/10-policy-create.sh b/scripts/e2e/agent-network-policy/10-policy-create.sh deleted file mode 100755 index 1f3d1a460..000000000 --- a/scripts/e2e/agent-network-policy/10-policy-create.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# 10-policy-create: end-to-end round-trip on the new window_seconds -# field. Creates the prerequisites (group + provider with -# bootstrap_cluster), POSTs a policy whose Limits.token_limit and -# budget_limit both carry window_seconds, then re-fetches the policy -# and asserts the field comes back unchanged. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -# 1. Group — sweep first so re-runs are idempotent, then POST. -existing_group=$(nb_api GET /api/groups 2>/dev/null \ - | jq -r --arg name "$NB_GROUP_NAME" '.[] | select(.name == $name) | .id // empty' \ - | head -1) -if [ -n "$existing_group" ]; then - echo "reusing existing group $existing_group" - group_id="$existing_group" -else - body=$(jq -n --arg name "$NB_GROUP_NAME" '{name:$name}') - group_resp=$(nb_api POST /api/groups "$body") - group_id=$(printf '%s' "$group_resp" | jq -r '.id // ""') - [ -n "$group_id" ] && [ "$group_id" != "null" ] \ - || fail "group create did not return an id" "$group_resp" - echo "created group $group_id" -fi -printf '%s' "$group_id" >"$NB_STATE_DIR/group-id" - -# 2. Provider — also idempotent. Bootstrap cluster pinned to the local -# proxy3 cluster so the management settings row resolves and the -# create completes (subsequent provider creates in the same account -# ignore bootstrap_cluster, but the FIRST one needs it). -existing_provider=$(nb_api GET /api/agent-network/providers 2>/dev/null \ - | jq -r --arg name "$NB_PROVIDER_NAME" '.[] | select(.name == $name) | .id // empty' \ - | head -1) -if [ -n "$existing_provider" ]; then - echo "reusing existing provider $existing_provider" - provider_id="$existing_provider" -else - provider_body=$(jq -n \ - --arg name "$NB_PROVIDER_NAME" \ - '{ - provider_id: "openai_api", - name: $name, - upstream_url: "https://api.openai.com", - api_key: "sk-e2e-placeholder", - bootstrap_cluster: "proxy.netbird.local", - models: [] - }') - prov_resp=$(nb_api POST /api/agent-network/providers "$provider_body") - provider_id=$(printf '%s' "$prov_resp" | jq -r '.id // ""') - [ -n "$provider_id" ] && [ "$provider_id" != "null" ] \ - || fail "provider create did not return an id" "$prov_resp" - echo "created provider $provider_id" -fi -printf '%s' "$provider_id" >"$NB_STATE_DIR/provider-id" - -# 3. Policy — drop any prior with the same name, then create with the -# NEW window_seconds field on both halves of Limits. 86400s = 24h -# on token, 2_592_000s = 30d on budget so the round-trip is -# unambiguous (no ambiguous unit-conversion artefact when we read -# back). -existing_policy=$(nb_api GET /api/agent-network/policies 2>/dev/null \ - | jq -r --arg name "$NB_POLICY_NAME" '.[] | select(.name == $name) | .id // empty' \ - | head -1) -if [ -n "$existing_policy" ]; then - echo "deleting existing policy $existing_policy" - nb_api DELETE "/api/agent-network/policies/$existing_policy" >/dev/null 2>&1 || true -fi - -policy_body=$(jq -n \ - --arg name "$NB_POLICY_NAME" \ - --arg group "$group_id" \ - --arg provider "$provider_id" \ - '{ - name: $name, - description: "agent-network e2e: window_seconds round-trip", - enabled: true, - source_groups: [$group], - destination_provider_ids: [$provider], - guardrail_ids: [], - limits: { - token_limit: { - enabled: true, - group_cap: 10000, - user_cap: 5000, - window_seconds: 86400 - }, - budget_limit: { - enabled: true, - group_cap_usd: 10.0, - user_cap_usd: 2.5, - window_seconds: 2592000 - } - } - }') - -resp=$(nb_api POST /api/agent-network/policies "$policy_body") -policy_id=$(printf '%s' "$resp" | jq -r '.id // ""') -[ -n "$policy_id" ] && [ "$policy_id" != "null" ] \ - || fail "policy create did not return an id" "$resp" -printf '%s' "$policy_id" >"$NB_STATE_DIR/policy-id" -echo "policy id: $policy_id" - -# 4. Round-trip assertions: GET the policy back and verify the -# window_seconds values land on both limit halves. The OLD -# window_hours / window_days fields must be absent. -got=$(nb_api GET "/api/agent-network/policies/$policy_id") - -token_window=$(printf '%s' "$got" | jq -r '.limits.token_limit.window_seconds // empty') -budget_window=$(printf '%s' "$got" | jq -r '.limits.budget_limit.window_seconds // empty') - -[ "$token_window" = "86400" ] \ - || fail "token_limit.window_seconds did not round-trip" \ - "expected=86400 got=$token_window body=$got" -[ "$budget_window" = "2592000" ] \ - || fail "budget_limit.window_seconds did not round-trip" \ - "expected=2592000 got=$budget_window body=$got" - -# Negative: window_hours / window_days are legacy field names and -# must not be present in the response at all — their presence would -# mean the management server is still emitting the legacy shape. -legacy_h=$(printf '%s' "$got" | jq -r '.limits.token_limit | has("window_hours")') -legacy_d=$(printf '%s' "$got" | jq -r '.limits.token_limit | has("window_days")') -[ "$legacy_h" = "false" ] \ - || fail "legacy window_hours field still present in token_limit response" "$got" -[ "$legacy_d" = "false" ] \ - || fail "legacy window_days field still present in token_limit response" "$got" - -echo "token_limit.window_seconds = $token_window" -echo "budget_limit.window_seconds = $budget_window" - -pass "policy persisted with window_seconds on both Limits halves" diff --git a/scripts/e2e/agent-network-policy/20-policy-rejects-zero-window.sh b/scripts/e2e/agent-network-policy/20-policy-rejects-zero-window.sh deleted file mode 100755 index c9a59ce48..000000000 --- a/scripts/e2e/agent-network-policy/20-policy-rejects-zero-window.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# 20-policy-rejects-zero-window: management must reject a policy whose -# token_limit or budget_limit is enabled but carries window_seconds < 60. -# Anything below the one-minute floor would either be a zero / negative -# window with no reset boundary, or a sub-minute window that produces -# untenable consumption-row volume at scale. The handler's -# validatePolicyLimits guard owes us a 4xx with a useful message. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -[ -r "$NB_STATE_DIR/group-id" ] && [ -r "$NB_STATE_DIR/provider-id" ] \ - || fail "missing group-id / provider-id state" \ - "run 10-policy-create.sh first to bootstrap prerequisites" -group_id=$(cat "$NB_STATE_DIR/group-id") -provider_id=$(cat "$NB_STATE_DIR/provider-id") - -# Build a payload that is well-formed except for window_seconds=30 on -# token_limit (under the 60s minimum). We mark the field enabled so -# the validation path actually runs — a disabled limit is allowed to -# carry zero. -payload=$(jq -n \ - --arg name "$NB_POLICY_NAME-sub-minute" \ - --arg group "$group_id" \ - --arg provider "$provider_id" \ - '{ - name: $name, - enabled: true, - source_groups: [$group], - destination_provider_ids: [$provider], - guardrail_ids: [], - limits: { - token_limit: { - enabled: true, - group_cap: 10000, - user_cap: 5000, - window_seconds: 30 - }, - budget_limit: { - enabled: false, - group_cap_usd: 0, - user_cap_usd: 0, - window_seconds: 0 - } - } - }') - -code=$(nb_api_status POST /api/agent-network/policies "$payload") -[ "$code" = "400" ] || [ "$code" = "422" ] \ - || fail "expected 400/422 on enabled token_limit with window_seconds<60" \ - "got HTTP $code" - -# Sweep any policy that may have been mistakenly persisted (defence -# against a future bug; today's handler doesn't get there). -orphan=$(nb_api GET /api/agent-network/policies 2>/dev/null \ - | jq -r --arg name "$NB_POLICY_NAME-sub-minute" '.[] | select(.name == $name) | .id // empty' \ - | head -1) -if [ -n "$orphan" ]; then - nb_api DELETE "/api/agent-network/policies/$orphan" >/dev/null 2>&1 || true - fail "policy was persisted despite sub-minute window_seconds" \ - "id=$orphan — handler validation regression" -fi - -echo "POST with token_limit.window_seconds=30 rejected with HTTP $code" -pass "validation rejects sub-minute window_seconds when limit is enabled" diff --git a/scripts/e2e/agent-network-policy/30-consumption-list-empty.sh b/scripts/e2e/agent-network-policy/30-consumption-list-empty.sh deleted file mode 100755 index 5a5ffc3ec..000000000 --- a/scripts/e2e/agent-network-policy/30-consumption-list-empty.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# 30-consumption-list-empty: GET /api/agent-network/consumption must -# return a JSON array (possibly empty) — never a 404 / 500. The -# endpoint is the read side that backs the dashboard's basic counter -# view and must always be reachable so the page can render an empty -# state. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -resp=$(nb_api GET /api/agent-network/consumption) - -# Must be a JSON array. -if ! printf '%s' "$resp" | jq -e 'type == "array"' >/dev/null 2>&1; then - fail "consumption endpoint did not return a JSON array" "$resp" -fi - -count=$(printf '%s' "$resp" | jq 'length') -echo "consumption rows: $count" - -# Stash the baseline count so 40-grpc-record-and-list can compare. -printf '%s' "$count" >"$NB_STATE_DIR/consumption-baseline" - -pass "consumption read endpoint returns a JSON array (count=$count)" diff --git a/scripts/e2e/agent-network-policy/40-grpc-record-and-list.sh b/scripts/e2e/agent-network-policy/40-grpc-record-and-list.sh deleted file mode 100755 index d295d6670..000000000 --- a/scripts/e2e/agent-network-policy/40-grpc-record-and-list.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# 40-grpc-record-and-list: drives the new RecordLLMUsage and -# CheckLLMPolicyLimits gRPC RPCs through the e2e usage_smoke helper. -# -# Asserts: -# - CheckLLMPolicyLimits returns decision=allow, picks the lowest -# group id by string sort as attribution, default window of 24h. -# - RecordLLMUsage with both user_id and group_id ticks BOTH the -# user counter and the group counter exactly once each. -# - A second RecordLLMUsage on the same key sums the deltas server -# side (database upsert-increment, no read-modify-write race). -# - The HTTP /api/agent-network/consumption listing reflects the -# post-flight state. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -[ -r "$NB_STATE_DIR/group-id" ] && [ -r "$NB_STATE_DIR/provider-id" ] \ - || fail "missing group-id / provider-id state" \ - "run 10-policy-create.sh first to bootstrap prerequisites" -group_id=$(cat "$NB_STATE_DIR/group-id") -provider_id=$(cat "$NB_STATE_DIR/provider-id") - -# Resolve the calling account id. /api/accounts returns NetBird xids, -# which is what the gRPC service expects (NOT IDP user UUIDs). -account_id=$(nb_api GET /api/accounts 2>/dev/null | jq -r '.[0].id // empty') -[ -n "$account_id" ] || fail "could not resolve account id" "GET /api/accounts returned nothing" - -# Pick a stable user id for the test. The dimension is per-user, so we -# can use any unique value — a synthetic "e2e-..." prefix avoids -# colliding with real user ids in the consumption listing. -user_id="e2e-anpol-user-$$" -window_seconds=86400 -echo "account=$account_id user=$user_id group=$group_id window=${window_seconds}s" - -# ─── 1. CheckLLMPolicyLimits ──────────────────────────────────────── -# Pass 3 group ids out of order; lowest by string sort wins. -groups_csv="grp-zz,$group_id,grp-aa-z" -check_resp=$(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke check \ - --account "$account_id" \ - --user "$user_id" \ - --groups "$groups_csv" \ - --provider "$provider_id" \ - --model "gpt-4o" \ - --token "$NB_PROXY_TOKEN" \ - --addr "$NB_GRPC_ADDR" 2>&1) \ - || fail "CheckLLMPolicyLimits gRPC failed" "$check_resp" - -decision=$(printf '%s' "$check_resp" | jq -r '.decision // ""') -attribution=$(printf '%s' "$check_resp" | jq -r '.attribution_group_id // ""') -got_window=$(printf '%s' "$check_resp" | jq -r '.window_seconds // ""') - -[ "$decision" = "allow" ] \ - || fail "Check decision must be allow under PR1 stub" "$check_resp" -# Lowest group id of {grp-zz, $group_id, grp-aa-z} by string sort. The -# group_id is an xid (lowercase alnum starting with a digit/letter), so -# the sort answer depends on group_id's prefix. Compute it locally. -expected_low=$(printf '%s\n%s\n%s' "grp-zz" "$group_id" "grp-aa-z" | sort | head -1) -[ "$attribution" = "$expected_low" ] \ - || fail "Check did not pick the lowest-by-sort group" \ - "expected=$expected_low got=$attribution" -[ "$got_window" = "86400" ] \ - || fail "Check window_seconds stub default mismatch" "expected=86400 got=$got_window" - -echo "Check decision=$decision attribution=$attribution window=${got_window}s" - -# ─── 2. RecordLLMUsage — first increment ──────────────────────────── -(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \ - --account "$account_id" \ - --user "$user_id" \ - --group "$group_id" \ - --window-seconds "$window_seconds" \ - --tokens-in 100 \ - --tokens-out 50 \ - --cost-usd 0.0125 \ - --token "$NB_PROXY_TOKEN" \ - --addr "$NB_GRPC_ADDR") >/dev/null \ - || fail "RecordLLMUsage (first increment) failed" "" - -# ─── 3. RecordLLMUsage — second increment, same key ───────────────── -(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \ - --account "$account_id" \ - --user "$user_id" \ - --group "$group_id" \ - --window-seconds "$window_seconds" \ - --tokens-in 50 \ - --tokens-out 25 \ - --cost-usd 0.0025 \ - --token "$NB_PROXY_TOKEN" \ - --addr "$NB_GRPC_ADDR") >/dev/null \ - || fail "RecordLLMUsage (second increment) failed" "" - -# ─── 4. Read-back via HTTP — sums must converge ───────────────────── -listing=$(nb_api GET /api/agent-network/consumption) - -user_row=$(printf '%s' "$listing" | jq --arg u "$user_id" \ - 'map(select(.dimension_kind == "user" and .dimension_id == $u)) | .[0]') -[ "$(printf '%s' "$user_row" | jq -r '. // empty')" != "" ] \ - || fail "user consumption row missing after RecordLLMUsage" "$listing" - -user_in=$(printf '%s' "$user_row" | jq '.tokens_input') -user_out=$(printf '%s' "$user_row" | jq '.tokens_output') -user_cost=$(printf '%s' "$user_row" | jq '.cost_usd') -user_window=$(printf '%s' "$user_row" | jq '.window_seconds') - -[ "$user_in" = "150" ] && [ "$user_out" = "75" ] \ - || fail "user counter did not sum the two increments" \ - "expected tokens_in=150 tokens_out=75; got in=$user_in out=$user_out" - -# Floating point compare with awk — drift > 1e-9 is a real bug. -cost_ok=$(awk -v got="$user_cost" 'BEGIN { print (got > 0.0149 && got < 0.0151) ? "y" : "n" }') -[ "$cost_ok" = "y" ] \ - || fail "user cost did not sum to 0.015" "got=$user_cost" - -[ "$user_window" = "$window_seconds" ] \ - || fail "user counter window_seconds mismatch" \ - "expected=$window_seconds got=$user_window" - -# Group row gets the same deltas because RecordLLMUsage ticks both -# dimensions on a single call. -group_row=$(printf '%s' "$listing" | jq --arg g "$group_id" \ - 'map(select(.dimension_kind == "group" and .dimension_id == $g)) | .[0]') -group_in=$(printf '%s' "$group_row" | jq '.tokens_input') -group_out=$(printf '%s' "$group_row" | jq '.tokens_output') - -[ "$group_in" = "150" ] && [ "$group_out" = "75" ] \ - || fail "group counter did not sum the two increments" \ - "expected tokens_in=150 tokens_out=75; got in=$group_in out=$group_out" - -echo "user counter: $user_in input / $user_out output / \$$user_cost over ${user_window}s" -echo "group counter: $group_in input / $group_out output" - -pass "gRPC Check + Record round-trip atomically increments user + group counters" diff --git a/scripts/e2e/agent-network-policy/50-grpc-allow-record-deny.sh b/scripts/e2e/agent-network-policy/50-grpc-allow-record-deny.sh deleted file mode 100755 index 5a5a41e0f..000000000 --- a/scripts/e2e/agent-network-policy/50-grpc-allow-record-deny.sh +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env bash -# 50-grpc-allow-record-deny: drives the full lifecycle of the real -# selection algorithm landed in PR2 — initial allow, partial fills -# below the cap, exact-at-cap, then deny once consumption reaches the -# group_cap. Validates that: -# -# - CheckLLMPolicyLimits picks the test policy as the attribution -# when it's the only one authorising the (group, provider) pair. -# - The selected_policy_id + window_seconds round-trip on the wire. -# - Counter-aware headroom math is wired end-to-end (RecordLLMUsage -# ticks counters that the next CheckLLMPolicyLimits call reads). -# - At-cap consumption flips decision from allow to deny with the -# canonical llm_policy.token_cap_exceeded code. -# -# The setup creates a dedicated group + provider + policy with the -# "-tight" suffix so it's isolated from the e2e-anpol resources -# 10-policy-create.sh seeded. 99-cleanup's prefix sweep catches both. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -# Per-run suffix so the consumption counter starts at zero on every -# run. Without this, a successful prior run leaves 200 tokens on the -# (group, 24h) bucket and the next run's stage 1 check sees a deny -# from the start. Consumption rows have no delete endpoint today. -RUN_TAG="$(date +%s)-$$" -NB_TIGHT_GROUP_NAME="$NB_GROUP_NAME-tight-$RUN_TAG" -NB_TIGHT_PROVIDER_NAME="$NB_PROVIDER_NAME-tight-$RUN_TAG" -NB_TIGHT_POLICY_NAME="$NB_POLICY_NAME-tight-$RUN_TAG" -TIGHT_CAP=200 -TIGHT_WINDOW_SECONDS=86400 - -# 1. Group — fresh per-run so the (group, window) consumption counter -# starts at zero. The 99-cleanup sweep catches it via the -# `e2e-anpol-engineers-tight-` prefix. -body=$(jq -n --arg name "$NB_TIGHT_GROUP_NAME" '{name:$name}') -tight_group_id=$(nb_api POST /api/groups "$body" | jq -r '.id // ""') -[ -n "$tight_group_id" ] && [ "$tight_group_id" != "null" ] \ - || fail "tight group create failed" "" -echo "created tight group $tight_group_id" -printf '%s' "$tight_group_id" >"$NB_STATE_DIR/tight-group-id" - -# 2. Provider — fresh per-run too so the policy targets a provider -# only this run knows about. Eliminates cross-policy interference if -# a prior run's tight policy targeted the same provider id. -body=$(jq -n \ - --arg name "$NB_TIGHT_PROVIDER_NAME" \ - '{ - provider_id: "openai_api", - name: $name, - upstream_url: "https://api.openai.com", - api_key: "sk-e2e-tight-placeholder", - bootstrap_cluster: "proxy.netbird.local", - models: [] - }') -tight_provider_id=$(nb_api POST /api/agent-network/providers "$body" | jq -r '.id // ""') -[ -n "$tight_provider_id" ] && [ "$tight_provider_id" != "null" ] \ - || fail "tight provider create failed" "" -echo "created tight provider $tight_provider_id" -printf '%s' "$tight_provider_id" >"$NB_STATE_DIR/tight-provider-id" - -# 3. Policy with a tight token cap so the test can observe the deny -# transition without burning thousands of records. -existing_policy=$(nb_api GET /api/agent-network/policies 2>/dev/null \ - | jq -r --arg name "$NB_TIGHT_POLICY_NAME" '.[] | select(.name == $name) | .id // empty' \ - | head -1) -if [ -n "$existing_policy" ]; then - echo "deleting existing tight policy $existing_policy so the run starts clean" - nb_api DELETE "/api/agent-network/policies/$existing_policy" >/dev/null 2>&1 || true -fi - -policy_body=$(jq -n \ - --arg name "$NB_TIGHT_POLICY_NAME" \ - --arg group "$tight_group_id" \ - --arg provider "$tight_provider_id" \ - --argjson cap "$TIGHT_CAP" \ - --argjson window "$TIGHT_WINDOW_SECONDS" \ - '{ - name: $name, - description: "agent-network e2e: cap-exhaust deny test", - enabled: true, - source_groups: [$group], - destination_provider_ids: [$provider], - guardrail_ids: [], - limits: { - token_limit: { - enabled: true, - group_cap: $cap, - user_cap: 0, - window_seconds: $window - }, - budget_limit: { - enabled: false, - group_cap_usd: 0, - user_cap_usd: 0, - window_seconds: $window - } - } - }') -tight_policy_id=$(nb_api POST /api/agent-network/policies "$policy_body" | jq -r '.id // ""') -[ -n "$tight_policy_id" ] && [ "$tight_policy_id" != "null" ] \ - || fail "tight policy create failed" "" -printf '%s' "$tight_policy_id" >"$NB_STATE_DIR/tight-policy-id" -echo "created tight policy $tight_policy_id (group_cap=$TIGHT_CAP, window=${TIGHT_WINDOW_SECONDS}s)" - -# 4. Resolve the calling account id; the smoke binary stamps it onto -# every gRPC request the way the real proxy does. -account_id=$(nb_api GET /api/accounts 2>/dev/null | jq -r '.[0].id // empty') -[ -n "$account_id" ] || fail "could not resolve account id" "" - -# Test user — synthetic prefix avoids colliding with real users. -test_user="e2e-anpol-tight-user-$$" - -# Pre-resolve the netbird repo root ONCE so the helpers below can -# pushd / popd into it without `cd "$(dirname "$0")/.."` re-resolving -# from a moved cwd between calls. nb_repo_root walks up from $0; if a -# prior helper left cwd elsewhere the relative walk breaks. -repo_root=$(nb_repo_root) -[ -d "$repo_root" ] || fail "could not resolve netbird repo root" "got=$repo_root" - -# Run the smoke binary inside a subshell so its `cd` doesn't pollute -# the caller's cwd between stages. -do_check() { - ( - cd "$repo_root" || exit 1 - go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke check \ - --account "$account_id" \ - --user "$test_user" \ - --groups "$tight_group_id" \ - --provider "$tight_provider_id" \ - --model "gpt-4o" \ - --token "$NB_PROXY_TOKEN" \ - --addr "$NB_GRPC_ADDR" - ) -} - -do_record() { - local in="$1" out="$2" cost="$3" - ( - cd "$repo_root" || exit 1 - go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \ - --account "$account_id" \ - --user "$test_user" \ - --group "$tight_group_id" \ - --window-seconds "$TIGHT_WINDOW_SECONDS" \ - --tokens-in "$in" \ - --tokens-out "$out" \ - --cost-usd "$cost" \ - --token "$NB_PROXY_TOKEN" \ - --addr "$NB_GRPC_ADDR" - ) -} - -# Stage 1 — fresh state. Selection must allow and pick our tight -# policy as attribution because it's the ONLY policy authorising the -# (tight_group, tight_provider) tuple. Also confirms the wire-level -# selected_policy_id round-trip from manager → grpc → smoke client. -echo "stage 1: initial check (consumption=0/$TIGHT_CAP)" -resp=$(do_check) || fail "initial check failed" "$resp" -decision=$(printf '%s' "$resp" | jq -r '.decision // ""') -selected=$(printf '%s' "$resp" | jq -r '.selected_policy_id // ""') -window=$(printf '%s' "$resp" | jq -r '.window_seconds // ""') -[ "$decision" = "allow" ] \ - || fail "expected allow on fresh state" "$resp" -[ "$selected" = "$tight_policy_id" ] \ - || fail "selection did not pick the tight policy" \ - "expected=$tight_policy_id got=$selected" -[ "$window" = "$TIGHT_WINDOW_SECONDS" ] \ - || fail "window_seconds mismatch on the wire" "expected=$TIGHT_WINDOW_SECONDS got=$window" -echo " → allow / selected=$selected / window=${window}s" - -# Stage 2 — book half the cap. The next check must still allow. -echo "stage 2: record 100 input tokens (consumption=100/$TIGHT_CAP)" -do_record 100 0 0 >/dev/null || fail "record (stage 2) failed" "" -resp=$(do_check) -decision=$(printf '%s' "$resp" | jq -r '.decision // ""') -[ "$decision" = "allow" ] \ - || fail "expected allow at half-cap" "$resp" -echo " → allow" - -# Stage 3 — push to one token below cap. Headroom shrinks but -# decision stays allow. -echo "stage 3: record 99 more input tokens (consumption=199/$TIGHT_CAP)" -do_record 99 0 0 >/dev/null || fail "record (stage 3) failed" "" -resp=$(do_check) -decision=$(printf '%s' "$resp" | jq -r '.decision // ""') -[ "$decision" = "allow" ] \ - || fail "expected allow at one-below-cap" "$resp" -echo " → allow" - -# Stage 4 — exactly at cap. Selector treats consumed >= cap as -# exhausted, so the next check must deny with the canonical token cap -# code. The deny reason names the policy id so operators can debug -# from the access log. -echo "stage 4: record 1 final token (consumption=$TIGHT_CAP/$TIGHT_CAP)" -do_record 1 0 0 >/dev/null || fail "record (stage 4) failed" "" -resp=$(do_check) || fail "check after cap-exhaust failed" "$resp" -decision=$(printf '%s' "$resp" | jq -r '.decision // ""') -deny_code=$(printf '%s' "$resp" | jq -r '.deny_code // ""') -deny_reason=$(printf '%s' "$resp" | jq -r '.deny_reason // ""') -[ "$decision" = "deny" ] \ - || fail "expected DENY at cap" "$resp" -[ "$deny_code" = "llm_policy.token_cap_exceeded" ] \ - || fail "deny_code mismatch" "expected=llm_policy.token_cap_exceeded got=$deny_code resp=$resp" -echo " → DENY / deny_code=$deny_code" -echo " → deny_reason: $deny_reason" -[[ "$deny_reason" == *"$tight_policy_id"* ]] \ - || fail "deny_reason must name the policy id for operator debugging" "$deny_reason" - -pass "selection algorithm flips allow → deny at cap-exhaust through the gRPC wire" diff --git a/scripts/e2e/agent-network-policy/99-cleanup.sh b/scripts/e2e/agent-network-policy/99-cleanup.sh deleted file mode 100755 index e30ae97e2..000000000 --- a/scripts/e2e/agent-network-policy/99-cleanup.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# 99-cleanup: idempotent teardown. Drops the policy, provider, group, -# and any consumption rows our test created so re-runs start fresh. - -# shellcheck source=00-env.sh -source "$(dirname "$0")/00-env.sh" - -# Policies first (FK / synth chain owns provider references). Sweep -# every policy whose name STARTS with the e2e prefix so per-run -# instances created by 50 (with $RUN_TAG suffixes) get cleaned up. -nb_api GET /api/agent-network/policies 2>/dev/null \ - | jq -r --arg pfx "$NB_POLICY_NAME" '.[] | select(.name | startswith($pfx)) | .id' \ - | while read -r orphan; do - [ -n "$orphan" ] || continue - code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \ - -o /dev/null -w '%{http_code}' \ - "$NB_API/api/agent-network/policies/$orphan" 2>&1) || true - echo "DELETE policy/$orphan -> $code" - done - -# Providers — same prefix sweep. Both the 10-policy-create.sh's main -# provider AND every per-run tight provider 50 minted carry the -# NB_PROVIDER_NAME prefix. -nb_api GET /api/agent-network/providers 2>/dev/null \ - | jq -r --arg pfx "$NB_PROVIDER_NAME" '.[] | select(.name | startswith($pfx)) | .id' \ - | while read -r orphan; do - [ -n "$orphan" ] || continue - code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \ - -o /dev/null -w '%{http_code}' \ - "$NB_API/api/agent-network/providers/$orphan" 2>&1) || true - echo "DELETE provider/$orphan -> $code" - done - -# Groups — prefix sweep. /api/groups doesn't error on a -# referenced-elsewhere group; if so, the delete is a no-op and we -# move on. -nb_api GET /api/groups 2>/dev/null \ - | jq -r --arg pfx "$NB_GROUP_NAME" '.[] | select(.name | startswith($pfx)) | .id' \ - | while read -r orphan; do - [ -n "$orphan" ] || continue - code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \ - -o /dev/null -w '%{http_code}' \ - "$NB_API/api/groups/$orphan" 2>&1) || true - echo "DELETE group/$orphan -> $code" - done - -# We don't expose a consumption-delete endpoint — the rows survive -# until the management store is recycled. Document the residue here -# so anyone debugging "why are there old e2e rows" knows the source. -remaining=$(nb_api GET /api/agent-network/consumption 2>/dev/null \ - | jq --arg pfx "e2e-anpol-user-" 'map(select(.dimension_id | startswith($pfx))) | length' \ - || echo "0") -[ "$remaining" = "0" ] || \ - echo "(left $remaining e2e consumption rows in the store — there's no delete endpoint yet)" - -rm -rf "$NB_STATE_DIR" - -pass "tear-down complete (idempotent)" diff --git a/scripts/e2e/agent-network-policy/cmd/usage_smoke/main.go b/scripts/e2e/agent-network-policy/cmd/usage_smoke/main.go deleted file mode 100644 index e203d83e8..000000000 --- a/scripts/e2e/agent-network-policy/cmd/usage_smoke/main.go +++ /dev/null @@ -1,197 +0,0 @@ -// usage_smoke is the e2e helper that drives the new agent-network -// gRPC RPCs (CheckLLMPolicyLimits, RecordLLMUsage) against a local -// management server. Run from the bash suite as a `go run` so the -// proto types are always in sync with the management binary the -// suite is exercising. -// -// Two subcommands today: -// - record: invokes RecordLLMUsage with the supplied tokens / cost -// and exits 0 on success. -// - check: invokes CheckLLMPolicyLimits and prints the response as -// JSON on stdout so the bash test can assert on it via -// jq. -// -// Auth uses the same proxy bearer-token shape the real proxy uses -// (see proxy/internal/grpc/auth.go); the bash suite reads the token -// from the e2e env (NB_PROXY_TOKEN, defaulted to the Tilt literal). -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "strings" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - - "github.com/netbirdio/netbird/shared/management/proto" -) - -// proxyTokenCreds mirrors proxy/internal/grpc.WithProxyToken's -// PerRPCCredentials so the e2e binary can reach the gRPC service -// using the same bearer-token shape the real proxy uses. Inlined -// because the production helper lives behind /internal/. -type proxyTokenCreds struct{ token string } - -func (c proxyTokenCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { - return map[string]string{"authorization": "Bearer " + c.token}, nil -} - -// RequireTransportSecurity is false here because Tilt's management is -// plaintext on localhost — the e2e suite is the *only* caller of this -// binary, never production. -func (proxyTokenCreds) RequireTransportSecurity() bool { return false } - -func main() { - if len(os.Args) < 2 { - usage() - } - cmd := os.Args[1] - os.Args = append(os.Args[:1], os.Args[2:]...) - - switch cmd { - case "record": - runRecord() - case "check": - runCheck() - default: - usage() - } -} - -func usage() { - fmt.Fprintln(os.Stderr, "usage: usage_smoke [flags]") - os.Exit(2) -} - -func runRecord() { - addr := flag.String("addr", "localhost:8080", "management gRPC address") - token := flag.String("token", os.Getenv("NB_PROXY_TOKEN"), "proxy token") - accountID := flag.String("account", "", "netbird account id") - userID := flag.String("user", "", "netbird user id (optional)") - groupID := flag.String("group", "", "netbird policy attribution group id (optional)") - groupsCSV := flag.String("groups", "", "CSV of caller group ids (for account-rule fan-out)") - windowSeconds := flag.Int64("window-seconds", 86_400, "window length in seconds (0 allowed when only account rules apply)") - tokensIn := flag.Int64("tokens-in", 0, "input tokens to add") - tokensOut := flag.Int64("tokens-out", 0, "output tokens to add") - costUSD := flag.Float64("cost-usd", 0, "USD cost to add") - flag.Parse() - - if strings.TrimSpace(*token) == "" { - exitErr("--token is required (or set NB_PROXY_TOKEN)") - } - if strings.TrimSpace(*accountID) == "" { - exitErr("--account is required") - } - var groupIDs []string - for _, g := range strings.Split(*groupsCSV, ",") { - g = strings.TrimSpace(g) - if g != "" { - groupIDs = append(groupIDs, g) - } - } - if *userID == "" && *groupID == "" && len(groupIDs) == 0 { - exitErr("at least one of --user, --group, or --groups must be set") - } - - conn := dial(*addr, *token) - defer conn.Close() - client := proto.NewProxyServiceClient(conn) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - _, err := client.RecordLLMUsage(ctx, &proto.RecordLLMUsageRequest{ - AccountId: *accountID, - UserId: *userID, - GroupId: *groupID, - GroupIds: groupIDs, - WindowSeconds: *windowSeconds, - TokensInput: *tokensIn, - TokensOutput: *tokensOut, - CostUsd: *costUSD, - }) - if err != nil { - exitErr(fmt.Sprintf("RecordLLMUsage: %v", err)) - } - //nolint:forbidigo // e2e helper: stdout is the contract with the bash caller - fmt.Println("ok") -} - -func runCheck() { - addr := flag.String("addr", "localhost:8080", "management gRPC address") - token := flag.String("token", os.Getenv("NB_PROXY_TOKEN"), "proxy token") - accountID := flag.String("account", "", "netbird account id") - userID := flag.String("user", "", "netbird user id (optional)") - groupsCSV := flag.String("groups", "", "CSV of caller group ids") - providerID := flag.String("provider", "", "agent-network provider id") - model := flag.String("model", "gpt-4o", "upstream model identifier") - flag.Parse() - - if strings.TrimSpace(*token) == "" { - exitErr("--token is required (or set NB_PROXY_TOKEN)") - } - if strings.TrimSpace(*accountID) == "" { - exitErr("--account is required") - } - - var groupIDs []string - for _, g := range strings.Split(*groupsCSV, ",") { - g = strings.TrimSpace(g) - if g != "" { - groupIDs = append(groupIDs, g) - } - } - - conn := dial(*addr, *token) - defer conn.Close() - client := proto.NewProxyServiceClient(conn) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - resp, err := client.CheckLLMPolicyLimits(ctx, &proto.CheckLLMPolicyLimitsRequest{ - AccountId: *accountID, - UserId: *userID, - GroupIds: groupIDs, - ProviderId: *providerID, - Model: *model, - }) - if err != nil { - exitErr(fmt.Sprintf("CheckLLMPolicyLimits: %v", err)) - } - - out, _ := json.Marshal(map[string]any{ - "decision": resp.GetDecision(), - "selected_policy_id": resp.GetSelectedPolicyId(), - "attribution_group_id": resp.GetAttributionGroupId(), - "window_seconds": resp.GetWindowSeconds(), - "deny_code": resp.GetDenyCode(), - "deny_reason": resp.GetDenyReason(), - }) - //nolint:forbidigo // e2e helper: stdout is the contract with the bash caller - fmt.Println(string(out)) -} - -// dial connects to the management gRPC over plaintext. The bearer -// token is sent on every RPC via PerRPCCredentials matching the wire -// format the production proxy uses. -func dial(addr, token string) *grpc.ClientConn { - conn, err := grpc.NewClient(addr, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithPerRPCCredentials(proxyTokenCreds{token: token}), - ) - if err != nil { - exitErr(fmt.Sprintf("dial %s: %v", addr, err)) - } - return conn -} - -func exitErr(msg string) { - fmt.Fprintln(os.Stderr, msg) - os.Exit(1) -} diff --git a/scripts/e2e/agent-network-policy/run-all.sh b/scripts/e2e/agent-network-policy/run-all.sh deleted file mode 100755 index 4598680ca..000000000 --- a/scripts/e2e/agent-network-policy/run-all.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# run-all: every numbered test in order, halts on first FAIL, -# prints a `=== PASS x/y ===` summary. -k keeps state (skips -# 99-cleanup) so you can poke around the persisted policy / -# consumption rows afterwards. - -set -uo pipefail - -KEEP=0 -[ "${1:-}" = "-k" ] && KEEP=1 -export NB_KEEP_STATE="$KEEP" - -cd "$(dirname "$0")" - -scripts=( - 01-tilt-restart.sh - 10-policy-create.sh - 20-policy-rejects-zero-window.sh - 30-consumption-list-empty.sh - 40-grpc-record-and-list.sh - 50-grpc-allow-record-deny.sh -) -[ "$KEEP" -eq 0 ] && scripts+=(99-cleanup.sh) - -passed=0 -total=${#scripts[@]} - -trap '[ "$KEEP" -eq 0 ] && bash ./99-cleanup.sh >/dev/null 2>&1 || true' EXIT - -for s in "${scripts[@]}"; do - echo - echo "==================== $s ====================" - if bash "./$s"; then - passed=$((passed + 1)) - else - rc=$? - echo - echo "=== FAIL $passed/$total ($s exit=$rc) ===" - exit "$rc" - fi -done - -echo -echo "=== PASS $passed/$total ==="