sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-07-18 04:29:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

[agent-network] Remove e2e shell-script suite from this branch

Browse Source 
The end-to-end shell scripts under scripts/e2e/ are maintained in a separate
testing suite and are not part of this change set.
This commit is contained in:
mlsmaycon
2026-06-27 01:39:57 +02:00
parent c83e46fbe1
commit 9f485be2f9
11 changed files with 0 additions and 1968 deletions
Download Patch File Download Diff File Expand all files Collapse all files

956
scripts/e2e/agent-network-full/e2e.sh
View File

@@ -1,956 +0,0 @@
#!/usr/bin/env bash
# Agent-network full end-to-end driver.
#
# One script, many subcommands, so the operator authorizes a single
# `bash scripts/e2e/agent-network-full/e2e.sh <cmd>` invocation instead of a
# stream of ad-hoc docker/netbird/curl commands.
#
# It joins a Docker NetBird client to the local Tilt management, drives LLM
# chat-completions through the agent-network proxy over the tunnel, and asserts
# token/cost/session capture via the REST API and proxy logs.
#
# Secrets (PAT, provider keys, setup key) are read from files / a sourced
# key file and never echoed.
#
# Usage:
# bash scripts/e2e/agent-network-full/e2e.sh <command>
#
# Commands:
# key mint a reusable setup key bound to the Admins group
# up run the Docker NetBird client and join management
# status show netbird status -d inside the client
# wait wait until the proxy peer is Connected (1/1)
# diag dump client + proxy3 + relay diagnostics
# chat M P chat-completion through the proxy: model M, provider-kind P
# verify consumption + recent access-log rows
# snapshot save current agent-network config to the scratch dir
# clean delete all policies, budget-rules, providers (API teardown)
# providers create the five providers from .llm-keys
# policy create one policy: Admins -> all five providers
# down remove the Docker client container
# restart-proxy tilt trigger proxy3 (re-establish its relay link)
# all clean -> providers -> policy -> up -> wait -> chat-all -> verify
set -uo pipefail
# --- config ------------------------------------------------------------------
NB_API="${NB_API:-http://localhost:8080}"
NB_PAT_FILE="${NB_PAT_FILE:-/Users/maycon/projects/local-dev/nb-pat}"
LLM_KEYS_FILE="${LLM_KEYS_FILE:-/Users/maycon/.llm-keys}"
CLIENT="${CLIENT:-nb-e2e-agent}"
IMAGE="${IMAGE:-netbird:tilt}"
CURL_IMAGE="${CURL_IMAGE:-curlimages/curl:latest}"
MGMT_URL="${MGMT_URL:-http://host.docker.internal:8080}"
PROXY_HOST="${PROXY_HOST:-mitten.proxy.netbird.local}"
ADMINS_GROUP_NAME="${ADMINS_GROUP_NAME:-Admins}"
DASH_DIR="${DASH_DIR:-/Users/maycon/projects/dashboard}"
DASH_URL="${DASH_URL:-http://localhost:3000}"
DASH_USER="${DASH_USER:-netbird@netbird.io}"
DASH_PASS="${DASH_PASS:-netbird@netbird.io}"
STATE_DIR="${STATE_DIR:-/private/tmp/claude-501/-Users-maycon-projects-netbird/a3fe30e4-5777-47d5-b110-ebc228716026/scratchpad/agentnet-snapshot}"
mkdir -p "$STATE_DIR"
[ -r "$NB_PAT_FILE" ] || { echo "FAIL: cannot read PAT at $NB_PAT_FILE" >&2; exit 2; }
PAT="$(tr -d '\n\r ' <"$NB_PAT_FILE")"
AUTH="Authorization: Token $PAT"
B="$NB_API/api/agent-network"
# --- small helpers -----------------------------------------------------------
log() { printf '%s\n' "$*" >&2; }
die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
api() { # METHOD PATH [BODY]
local m="$1" p="$2" body="${3-}"
if [ -n "$body" ]; then
curl -fsS -X "$m" -H "$AUTH" -H "Content-Type: application/json" --data "$body" "$NB_API$p"
else
curl -fsS -X "$m" -H "$AUTH" "$NB_API$p"
fi
}
# proxy_ip resolves the proxy host to its NetBird IP from inside the client.
proxy_ip() {
docker exec "$CLIENT" sh -c "getent hosts $PROXY_HOST" 2>/dev/null | awk '{print $1; exit}'
}
admins_group_id() {
api GET /api/groups | jq -r --arg n "$ADMINS_GROUP_NAME" \
'.[] | select((.name//"")|ascii_downcase==($n|ascii_downcase)) | .id' | head -n1
}
require_llm_keys() {
[ -r "$LLM_KEYS_FILE" ] || die "cannot read $LLM_KEYS_FILE"
# shellcheck disable=SC1090
set -a; . "$LLM_KEYS_FILE"; set +a
}
# --- commands ----------------------------------------------------------------
cmd_snapshot() {
api GET /settings_unused 2>/dev/null || true
curl -fsS -H "$AUTH" "$B/settings" >"$STATE_DIR/settings.json"
curl -fsS -H "$AUTH" "$B/providers" >"$STATE_DIR/providers.json"
curl -fsS -H "$AUTH" "$B/policies" >"$STATE_DIR/policies.json"
curl -fsS -H "$AUTH" "$B/budget-rules" >"$STATE_DIR/budget-rules.json"
curl -fsS -H "$AUTH" "$NB_API/api/groups" >"$STATE_DIR/groups.json"
log "snapshot written to $STATE_DIR"
ls -l "$STATE_DIR" >&2
}
cmd_key() {
local gid kjson
gid="$(admins_group_id)"
[ -n "$gid" ] || die "could not resolve Admins group id"
kjson="$(api POST /api/setup-keys "{\"name\":\"e2e-agentnet-docker\",\"type\":\"reusable\",\"expires_in\":86400,\"usage_limit\":0,\"auto_groups\":[\"$gid\"],\"ephemeral\":false}")"
echo "$kjson" | jq -r '.key' >"$STATE_DIR/setup-key.txt"
chmod 600 "$STATE_DIR/setup-key.txt"
echo "$kjson" | jq '{id,name,type,state,valid,auto_groups}' >&2
log "setup key saved (value not printed) to $STATE_DIR/setup-key.txt"
}
cmd_up() {
[ -r "$STATE_DIR/setup-key.txt" ] || cmd_key
local key; key="$(cat "$STATE_DIR/setup-key.txt")"
docker rm -f "$CLIENT" >/dev/null 2>&1 || true
docker run -d --name "$CLIENT" \
--cap-add NET_ADMIN --cap-add SYS_ADMIN --cap-add SYS_RESOURCE \
--add-host host.docker.internal:host-gateway \
-e NB_MANAGEMENT_URL="$MGMT_URL" \
-e NB_SETUP_KEY="$key" \
-e NB_LOG_LEVEL=info \
"$IMAGE" >/dev/null
log "started $CLIENT"
sleep 6
docker exec "$CLIENT" netbird status 2>&1 | sed -n '1,14p' >&2
}
cmd_status() { docker exec "$CLIENT" netbird status -d 2>&1; }
cmd_wait() {
local i=0 line
while [ "$i" -lt 90 ]; do
line="$(docker exec "$CLIENT" netbird status 2>/dev/null | grep '^Peers count' || true)"
log "t=${i}s ${line:-<no status>}"
case "$line" in *"1/1 Connected"*) log "proxy peer connected"; return 0;; esac
sleep 5; i=$((i+5))
done
log "proxy peer did not connect within ${i}s"
return 1
}
cmd_diag() {
log "===== client: peers count ====="
docker exec "$CLIENT" netbird status 2>&1 | grep -E '^(Management|Signal|Relays|Peers)' >&2 || true
log "===== client: relay/handshake (last 15) ====="
docker exec "$CLIENT" sh -c 'tail -n 400 /var/log/netbird/client.log' 2>/dev/null \
| grep -iE 'relay|handshake|offer|answer|error' | tail -15 >&2 || true
log "===== proxy3: relay (last 15) ====="
docker logs local-dev-proxy3-1 2>&1 | grep -iE 'relay|signal|handshake|offer|answer' | tail -15 >&2 || true
log "===== mgmt: proxy peer online ====="
curl -fsS -H "$AUTH" "$NB_API/api/peers" \
| jq -r '.[] | select((.name//"")|test("^proxy-")) | "\(.name) connected=\(.connected) last_seen=\(.last_seen)"' >&2 || true
}
cmd_restart_proxy() {
command -v tilt >/dev/null 2>&1 || die "tilt not on PATH"
tilt trigger proxy3 && log "triggered proxy3"
}
# cmd_chat MODEL KIND — KIND in: chat (POST /v1/chat/completions),
# messages (POST /v1/messages, anthropic body).
CHAT_RESP="$STATE_DIR/last-chat.json"
CHAT_PROMPT_DEFAULT="Reply with exactly: pong"
# _chat MODEL [KIND] [PROMPT] — POST through the proxy from $CHAT_CLIENT
# (defaults to $CLIENT). Echoes the HTTP status code on stdout; writes the
# response body to $CHAT_RESP. Returns 000 if the proxy host won't resolve.
_chat() {
local model="$1" kind="${2:-chat}" prompt="${3:-$CHAT_PROMPT_DEFAULT}"
local client="${CHAT_CLIENT:-$CLIENT}" ip path body out code attempt
local extra_hdr=()
case "$kind" in
messages)
path="/v1/messages"; extra_hdr=(-H "anthropic-version: 2023-06-01")
body="$(jq -n --arg m "$model" --arg p "$prompt" '{model:$m,max_tokens:64,messages:[{role:"user",content:$p}]}')" ;;
*)
path="/v1/chat/completions"
body="$(jq -n --arg m "$model" --arg p "$prompt" '{model:$m,messages:[{role:"user",content:$p}]}')" ;;
esac
# Re-resolve the proxy IP and retry on connection failure (000): a config
# change churns the proxy peer's NetBird IP, so a freshly-joined client may
# need a few seconds for its tunnel + magic DNS to converge.
code=000
for attempt in $(seq 1 "${CHAT_RETRIES:-6}"); do
ip="$(CLIENT="$client" proxy_ip)"
if [ -n "$ip" ]; then
out="$(docker run --rm --network "container:$client" "$CURL_IMAGE" \
-sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
-w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \
-H "Content-Type: application/json" ${extra_hdr[@]+"${extra_hdr[@]}"} --data "$body")"
code="$(printf '%s' "$out" | tail -n1)"
printf '%s' "$out" | sed '$d' >"$CHAT_RESP"
fi
[ "$code" != 000 ] && break
sleep 4
done
echo "$code"
}
cmd_chat() {
local code
log "POST $PROXY_HOST (model=$1 kind=${2:-chat} client=${CHAT_CLIENT:-$CLIENT})"
code="$(_chat "$@")"
cat "$CHAT_RESP" 2>/dev/null
printf '\n[http %s]\n' "$code"
}
cmd_verify() {
log "===== consumption ====="
api GET /api/agent-network/consumption | jq -r '.[] | "\(.dimension_kind)/\(.dimension_id) tokens_in=\(.tokens_input) tokens_out=\(.tokens_output) cost=\(.cost_usd // 0)"' >&2 || true
log "===== last 10 access-log rows ====="
api GET /api/agent-network/access-logs | jq -r '.data[0:10][] | "\(.timestamp) provider=\(.provider) model=\(.model) status=\(.status_code) decision=\(.decision) src=\(.source_ip) session=\(.session_id // "-") in=\(.input_tokens // 0) out=\(.output_tokens // 0) cost=\(.cost_usd // 0)"' >&2 || true
}
cmd_clean() {
cmd_snapshot
local id
for id in $(api GET /api/agent-network/policies | jq -r '.[].id'); do
api DELETE "/api/agent-network/policies/$id" >/dev/null && log "deleted policy $id"
done
for id in $(api GET /api/agent-network/budget-rules | jq -r '.[].id'); do
api DELETE "/api/agent-network/budget-rules/$id" >/dev/null && log "deleted budget-rule $id"
done
for id in $(api GET /api/agent-network/providers | jq -r '.[].id'); do
api DELETE "/api/agent-network/providers/$id" >/dev/null && log "deleted provider $id"
done
log "account cleaned"
}
# create_provider NAME PROVIDER_ID UPSTREAM_URL API_KEY
create_provider() {
local name="$1" pid="$2" url="$3" key="$4" body resp
[ -n "$key" ] || { log "skip $name: empty key"; return 0; }
body="$(jq -n --arg n "$name" --arg p "$pid" --arg u "$url" --arg k "$key" \
'{name:$n,provider_id:$p,upstream_url:$u,api_key:$k,enabled:true}')"
resp="$(api POST /api/agent-network/providers "$body")" || { log "create $name FAILED"; return 1; }
echo "$resp" | jq -r '"created provider \(.name) id=\(.id) provider_id=\(.provider_id)"' >&2
echo "$resp" | jq -r '.id'
}
cmd_providers() {
require_llm_keys
: >"$STATE_DIR/provider-ids.txt"
create_provider "OpenAI API" openai_api "https://api.openai.com" "${OPENAI_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
create_provider "Anthropic API" anthropic_api "https://api.anthropic.com" "${ANTHROPIC_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
create_provider "Vercel AI Gateway" vercel_ai_gateway "${VERCEL_URL:-}" "${VERCEL_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
create_provider "OpenRouter" openrouter "${OPENROUTER_URL:-}" "${OPENROUTER_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
create_provider "Cloudflare AI Gateway" cloudflare_ai_gateway "${CLOUDFLARE_URL:-}" "${CLOUDFLARE_TOKEN:-}" >>"$STATE_DIR/provider-ids.txt"
log "provider ids:"; cat "$STATE_DIR/provider-ids.txt" >&2
}
cmd_policy() {
local gid ids body
gid="$(admins_group_id)"; [ -n "$gid" ] || die "no Admins group"
ids="$(api GET /api/agent-network/providers | jq -c '[.[].id]')"
body="$(jq -n --arg n "e2e-all-providers" --arg g "$gid" --argjson dst "$ids" '{
name:$n, description:"e2e: Admins to all providers", enabled:true,
source_groups:[$g], destination_provider_ids:$dst, guardrail_ids:[],
limits:{ budget_limit:{enabled:true,group_cap_usd:1000000,user_cap_usd:1000000,window_seconds:2592000},
token_limit:{enabled:false,group_cap:0,user_cap:0,window_seconds:60} }
}')"
api POST /api/agent-network/policies "$body" | jq -r '"created policy \(.name) id=\(.id) dst=\(.destination_provider_ids|length) providers"' >&2
}
# set_enabled PROVIDER_ID BOOL — PUT the provider back with enabled toggled,
# preserving its required fields and keeping the sealed key (api_key omitted).
set_enabled() {
local pid="$1" en="$2" cur body
cur="$(api GET /api/agent-network/providers | jq -c --arg id "$pid" '.[] | select(.id==$id)')"
[ -n "$cur" ] || { log "no provider $pid"; return 1; }
body="$(echo "$cur" | jq -c --argjson en "$en" '{name,provider_id,upstream_url,enabled:$en} + (if .extra_values then {extra_values} else {} end)')"
api PUT "/api/agent-network/providers/$pid" "$body" >/dev/null
}
# cmd_isolate NAME — leave only the named provider enabled (sole catch-all),
# so a request routes to it without first-party-vendor or first-catch-all
# interference. Matches NAME case-insensitively against provider .name.
cmd_isolate() {
local want="$1" id name en
api GET /api/agent-network/providers | jq -r '.[] | "\(.id)\t\(.name)"' | while IFS=$'\t' read -r id name; do
case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in
*"$(echo "$want" | tr '[:upper:]' '[:lower:]')"*) en=true ;;
*) en=false ;;
esac
set_enabled "$id" "$en" && log "$name enabled=$en"
done
}
cmd_enable_all() {
local id
for id in $(api GET /api/agent-network/providers | jq -r '.[].id'); do
set_enabled "$id" true && log "enabled $id"
done
}
# cmd_dashboard — drive the live :3000 dashboard (../dashboard repo) with
# Playwright, asserting the API-created providers/policy render in the UI.
cmd_dashboard() {
command -v node >/dev/null 2>&1 || die "node not on PATH"
[ -f "$DASH_DIR/e2e/live-agent-network.mjs" ] || die "dashboard script missing in $DASH_DIR/e2e"
( cd "$DASH_DIR" && BASE_URL="$DASH_URL" DASH_USER="$DASH_USER" DASH_PASS="$DASH_PASS" node e2e/live-agent-network.mjs )
}
cmd_down() { docker rm -f "$CLIENT" >/dev/null 2>&1 && log "removed $CLIENT" || log "no container"; }
cmd_all() {
cmd_clean
cmd_providers
cmd_policy
cmd_up
cmd_wait || { cmd_diag; die "tunnel to proxy not established"; }
log "===== chat: OpenAI =====" ; cmd_chat gpt-5.4 chat
log "===== chat: Anthropic =====" ; cmd_chat claude-haiku-4-5 messages
log "===== chat: Vercel (openai/...) ====="; cmd_chat openai/gpt-4o-mini chat
log "===== chat: OpenRouter (openai/...) =="; cmd_chat openai/gpt-4o-mini chat
sleep 3
cmd_verify
}
# --- scenario helpers --------------------------------------------------------
FAILS=0
ok() { log " PASS: $1"; }
bad() { log " FAIL: $1"; FAILS=$((FAILS+1)); }
expect_code() { # WANT GOT LABEL
if [ "$2" = "$1" ]; then ok "$3 (http $2)"; else bad "$3 (want $1, got $2)"; fi
}
# wait_tunnel [CLIENT] — poll silently until the proxy peer is 1/1 Connected.
wait_tunnel() {
local c="${1:-$CLIENT}" i=0
while [ "$i" -lt 60 ]; do
docker exec "$c" netbird status 2>/dev/null | grep -q '1/1 Connected' && return 0
sleep 3; i=$((i+3))
done
return 1
}
# wait_chat_ready CLIENT KIND MODEL — poll a real request from CLIENT until it
# returns a non-000 status, i.e. the tunnel + magic DNS + WG handshake to the
# proxy peer have all converged. Single-attempt probes pace the outer loop.
# Note: a netbird down/up bounce does NOT help here and is actively harmful —
# the peer is already in the net-map (status shows 0/1, i.e. known-not-connected),
# so the blocker is the WG handshake to a freshly-churned proxy peer, and
# bouncing just resets an in-progress slow handshake. We only wait it out.
wait_chat_ready() {
local c="$1" kind="${2:-chat}" model="${3:-gpt-5.4}" rc i=0
while [ "$i" -lt 30 ]; do
rc="$(CHAT_CLIENT="$c" CHAT_RETRIES=1 _chat "$model" "$kind")"
[ "$rc" != 000 ] && { echo "$rc"; return 0; }
i=$((i + 5))
sleep 5
done
echo 000
return 1
}
# wait_peer_connected CLIENT [TIMEOUT] — poll until the client reports the proxy
# peer as 1/1 Connected, i.e. the proxy peer has re-stabilised after a churn.
# Used to settle the peer before joining a fresh client into it.
wait_peer_connected() {
local c="$1" timeout="${2:-180}" i=0
while [ "$i" -lt "$timeout" ]; do
docker exec "$c" netbird status 2>/dev/null | grep -q '1/1 Connected' && return 0
sleep 5
i=$((i + 5))
done
return 1
}
provider_id_by_name() { api GET /api/agent-network/providers | jq -r --arg n "$1" '.[] | select(.name==$n) | .id' | head -n1; }
access_log_total() { api GET /api/agent-network/access-logs | jq -r '.total_records'; }
access_log_top() { api GET /api/agent-network/access-logs | jq -r ".data[0].$1 // \"\""; }
# ensure_group NAME — return the id of group NAME, creating it if absent.
ensure_group() {
local name="$1" id
id="$(api GET /api/groups | jq -r --arg n "$name" '.[] | select(.name==$n) | .id' | head -n1)"
if [ -z "$id" ] || [ "$id" = null ]; then
id="$(api POST /api/groups "$(jq -n --arg n "$name" '{name:$n}')" | jq -r '.id')"
fi
echo "$id"
}
# policy_put_limits LIMITS_JSON — replace the e2e-all-providers policy limits.
policy_put_limits() {
local pol id body
pol="$(api GET /api/agent-network/policies | jq -c '.[] | select(.name=="e2e-all-providers")')"
id="$(echo "$pol" | jq -r '.id')"
[ -n "$id" ] && [ "$id" != null ] || die "e2e-all-providers policy missing (run 'policy' first)"
body="$(echo "$pol" | jq -c --argjson L "$1" '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids,limits:$L}')"
api PUT "/api/agent-network/policies/$id" "$body" >/dev/null
}
settings_put() { api PUT /api/agent-network/settings "$1" >/dev/null; }
# --- scenarios ---------------------------------------------------------------
# Scenario 1: policy token-cap enforcement. Seed usage under a high cap, drop
# the cap to 1 token so the next call is denied, then restore and confirm
# recovery. Deterministic regardless of prior window usage.
cmd_scenario_budget() {
log "### scenario: policy token-cap enforcement (deny + recovery) ###"
# Seed AND enforce in the same 3600s token window (counters are per-window).
policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":100000,"window_seconds":3600}}'
wait_tunnel; sleep 2
expect_code 200 "$(_chat gpt-5.4 chat)" "seed call under high token cap (books usage into 3600s window)"
# Drop the cap to 1 in the SAME window so the seeded usage exhausts it.
policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":1,"user_cap":1,"window_seconds":3600}}'
wait_tunnel; sleep 2
local code; code="$(_chat gpt-5.4 chat)"
expect_code 403 "$code" "call denied once token cap (1) is exhausted"
log " deny envelope: $(cat "$CHAT_RESP")"
policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
wait_tunnel; sleep 2
expect_code 200 "$(_chat gpt-5.4 chat)" "call allowed again after cap restored"
}
# Scenario 2: account-level budget rule (separate from policy limits). A rule
# targeting Admins with a 1-token cap must deny via the same gRPC check loop;
# deleting it restores access.
cmd_scenario_budget_rule() {
log "### scenario: account budget-rule enforcement ###"
local gid rid
gid="$(admins_group_id)"
# High cap first so the seed call books usage into the rule's 3600s window.
rid="$(api POST /api/agent-network/budget-rules "$(jq -n --arg g "$gid" '{name:"e2e-tight-rule",enabled:true,target_groups:[$g],target_users:[],limits:{budget_limit:{enabled:false,group_cap_usd:0,user_cap_usd:0,window_seconds:3600},token_limit:{enabled:true,group_cap:100000,user_cap:100000,window_seconds:3600}}}')" | jq -r '.id')"
log " created budget-rule $rid (group token cap=100000)"
wait_tunnel; sleep 2
expect_code 200 "$(_chat gpt-5.4 chat)" "seed call under high account cap"
# Tighten the same rule/window to 1 so the seeded usage exhausts it.
api PUT "/api/agent-network/budget-rules/$rid" "$(jq -n --arg g "$gid" '{name:"e2e-tight-rule",enabled:true,target_groups:[$g],target_users:[],limits:{budget_limit:{enabled:false,group_cap_usd:0,user_cap_usd:0,window_seconds:3600},token_limit:{enabled:true,group_cap:1,user_cap:1,window_seconds:3600}}}')" >/dev/null
log " tightened budget-rule $rid (group token cap=1)"
wait_tunnel; sleep 2
expect_code 403 "$(_chat gpt-5.4 chat)" "account budget-rule denies when cap exhausted"
api DELETE "/api/agent-network/budget-rules/$rid" >/dev/null && log " deleted budget-rule $rid"
wait_tunnel; sleep 2
expect_code 200 "$(_chat gpt-5.4 chat)" "allowed again after budget-rule removed"
}
# Scenario 3: multiple groups + policies + destination scoping. A 2nd Docker
# client joins a 2nd group; each group's policy authorises a different
# provider. Cross-provider requests must be denied.
cmd_scenario_multigroup() {
log "### scenario: multi-group / multi-policy destination scoping ###"
local gidA gidB oai ant keyB polAllId
gidA="$(admins_group_id)"; gidB="$(ensure_group e2e-grp-b)"
oai="$(provider_id_by_name 'OpenAI API')"; ant="$(provider_id_by_name 'Anthropic API')"
log " groups: Admins=$gidA grp-b=$gidB ; providers: openai=$oai anthropic=$ant"
# Disable the broad e2e policy; add two narrow ones.
polAllId="$(api GET /api/agent-network/policies | jq -r '.[] | select(.name=="e2e-all-providers") | .id')"
[ -n "$polAllId" ] && api PUT "/api/agent-network/policies/$polAllId" \
"$(api GET /api/agent-network/policies | jq -c --arg id "$polAllId" '.[] | select(.id==$id) | {name,description,enabled:false,source_groups,destination_provider_ids,guardrail_ids,limits}')" >/dev/null
local hi='{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":60}}'
local polA polB
polA="$(api POST /api/agent-network/policies "$(jq -n --arg g "$gidA" --arg p "$oai" --argjson L "$hi" '{name:"e2e-A-admins-openai",enabled:true,source_groups:[$g],destination_provider_ids:[$p],guardrail_ids:[],limits:$L}')" | jq -r '.id')"
polB="$(api POST /api/agent-network/policies "$(jq -n --arg g "$gidB" --arg p "$ant" --argjson L "$hi" '{name:"e2e-B-grpb-anthropic",enabled:true,source_groups:[$g],destination_provider_ids:[$p],guardrail_ids:[],limits:$L}')" | jq -r '.id')"
log " policy A (Admins->OpenAI)=$polA ; policy B (grp-b->Anthropic)=$polB"
# The policy swap above re-synthesises the agent-network service and adds a
# new source group (grp-b), which churns the proxy peer and forces it to
# absorb the new authorisation. Wait for the peer to re-stabilise (client A
# reconnects to the new peer) before joining B, so B handshakes into a stable
# peer instead of racing the churn — otherwise the proxy never answers B's
# offers and it sits at 0/1 Connected.
if ! wait_peer_connected "$CLIENT" 30; then
log " warning: proxy peer did not re-stabilise for client A within 30s"
fi
sleep 5
# 2nd client bound to grp-b.
keyB="$(api POST /api/setup-keys "$(jq -n --arg g "$gidB" '{name:"e2e-agentnet-docker-b",type:"reusable",expires_in:86400,usage_limit:0,auto_groups:[$g],ephemeral:false}')" | jq -r '.key')"
docker rm -f "${CLIENT}-b" >/dev/null 2>&1 || true
docker run -d --name "${CLIENT}-b" --cap-add NET_ADMIN --cap-add SYS_ADMIN --cap-add SYS_RESOURCE \
--add-host host.docker.internal:host-gateway -e NB_MANAGEMENT_URL="$MGMT_URL" -e NB_SETUP_KEY="$keyB" \
-e NB_LOG_LEVEL=info "$IMAGE" >/dev/null
log " started ${CLIENT}-b (grp-b)"
# Gate on real end-to-end connectivity: a freshly-authorised group's client
# takes ~1-2 min for the proxy to absorb and answer its WG handshake.
local ra rb
ra="$(wait_chat_ready "$CLIENT" chat gpt-5.4)"
rb="$(wait_chat_ready "${CLIENT}-b" messages claude-haiku-4-5)"
log " connectivity ready: A(probe=$ra) B(probe=$rb)"
log " -- client A (Admins) --"
CHAT_CLIENT="$CLIENT" expect_code 200 "$(CHAT_CLIENT="$CLIENT" _chat gpt-5.4 chat)" "A->OpenAI authorised"
CHAT_CLIENT="$CLIENT" expect_code 403 "$(CHAT_CLIENT="$CLIENT" _chat claude-haiku-4-5 messages)" "A->Anthropic denied (not in policy A)"
log " -- client B (grp-b) --"
CHAT_CLIENT="${CLIENT}-b" expect_code 200 "$(CHAT_CLIENT="${CLIENT}-b" _chat claude-haiku-4-5 messages)" "B->Anthropic authorised"
CHAT_CLIENT="${CLIENT}-b" expect_code 403 "$(CHAT_CLIENT="${CLIENT}-b" _chat gpt-5.4 chat)" "B->OpenAI denied (not in policy B)"
# Restore: drop A/B, re-enable the broad policy.
api DELETE "/api/agent-network/policies/$polA" >/dev/null 2>&1 || true
api DELETE "/api/agent-network/policies/$polB" >/dev/null 2>&1 || true
[ -n "$polAllId" ] && api PUT "/api/agent-network/policies/$polAllId" \
"$(api GET /api/agent-network/policies | jq -c --arg id "$polAllId" '.[] | select(.id==$id) | {name,description,enabled:true,source_groups,destination_provider_ids,guardrail_ids,limits}')" >/dev/null
docker rm -f "${CLIENT}-b" >/dev/null 2>&1 || true
log " restored broad policy, removed client B"
}
# Scenario 4: log-collection + prompt-collection + redaction settings.
cmd_scenario_logs() {
log "### scenario: log/prompt-collection + redaction settings ###"
settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false}'
wait_tunnel; sleep 3
# (a) log collection OFF -> no new access-log row.
local before after
before="$(access_log_total)"
settings_put '{"enable_log_collection":false,"enable_prompt_collection":true,"redact_pii":false}'
wait_tunnel; sleep 3
expect_code 200 "$(_chat gpt-5.4 chat)" "request still served with logging off"
sleep 3; after="$(access_log_total)"
if [ "$after" = "$before" ]; then ok "log collection OFF -> no new row ($before==$after)"; else bad "log collection OFF still wrote a row ($before->$after)"; fi
# (b) log ON, prompt OFF -> row present but request_prompt empty.
settings_put '{"enable_log_collection":true,"enable_prompt_collection":false,"redact_pii":false}'
wait_tunnel; sleep 3
expect_code 200 "$(_chat gpt-5.4 chat "this prompt text must NOT be stored")" "served with prompt collection off"
sleep 3
local p; p="$(access_log_top request_prompt)"
if [ -z "$p" ]; then ok "prompt collection OFF -> request_prompt empty"; else bad "prompt stored despite collection off: [$p]"; fi
# (c) prompt ON + redact ON -> PII scrubbed from stored prompt.
settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":true}'
wait_tunnel; sleep 3
expect_code 200 "$(_chat gpt-5.4 chat "Contact john.doe@example.com, SSN 123-45-6789, phone 555-123-4567")" "served with redaction on"
sleep 3
local stored; stored="$(access_log_top request_prompt)"
log " stored prompt: [$stored]"
case "$stored" in
*john.doe@example.com*) bad "email leaked into stored prompt" ;;
*) ok "email redacted from stored prompt" ;;
esac
case "$stored" in
*123-45-6789*) bad "SSN leaked into stored prompt" ;;
*) ok "SSN redacted from stored prompt" ;;
esac
settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false}'
log " restored settings (log on, prompt on, redact off)"
}
# Scenario 5: per-user cap isolated from the group cap. Seed under a high cap,
# then keep the group cap high but drop the USER cap to 1 — the deny must come
# from the user dimension, proving per-user accounting is independent.
cmd_scenario_user_cap() {
log "### scenario: per-user token cap (isolated from group cap) ###"
policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":100000,"window_seconds":3600}}'
wait_tunnel; sleep 2
expect_code 200 "$(_chat gpt-5.4 chat)" "seed call (books user+group usage into 3600s window)"
# Group cap stays huge; only the user cap is exhausted.
policy_put_limits '{"budget_limit":{"enabled":false,"group_cap_usd":0,"user_cap_usd":0,"window_seconds":3600},"token_limit":{"enabled":true,"group_cap":100000,"user_cap":1,"window_seconds":3600}}'
wait_tunnel; sleep 2
expect_code 403 "$(_chat gpt-5.4 chat)" "denied by USER cap=1 while group cap=100000 has headroom"
log " deny: $(cat "$CHAT_RESP")"
policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
wait_tunnel; sleep 2
expect_code 200 "$(_chat gpt-5.4 chat)" "allowed again after user cap lifted"
}
# Scenario 6: guardrail model-allowlist blocking. Attach a guardrail allowing
# only gpt-5.4; a request for any other model must be denied (model_blocked),
# while the allowed model passes.
cmd_scenario_guardrail() {
log "### scenario: guardrail model-allowlist blocking ###"
local gid polId
gid="$(api POST /api/agent-network/guardrails "$(jq -n '{name:"e2e-allowlist",description:"e2e: only gpt-5.4",checks:{model_allowlist:{enabled:true,models:["gpt-5.4"]},prompt_capture:{enabled:true,redact_pii:false}}}')" | jq -r '.id')"
log " created guardrail $gid (allow only gpt-5.4)"
local pol; pol="$(api GET /api/agent-network/policies | jq -c '.[] | select(.name=="e2e-all-providers")')"
polId="$(echo "$pol" | jq -r '.id')"
api PUT "/api/agent-network/policies/$polId" "$(echo "$pol" | jq -c --arg g "$gid" '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids:[$g],limits}')" >/dev/null
wait_tunnel; sleep 2
expect_code 403 "$(_chat gpt-4o-mini chat)" "model not in allowlist is blocked"
log " deny: $(cat "$CHAT_RESP")"
expect_code 200 "$(_chat gpt-5.4 chat)" "allowlisted model passes"
# Detach + delete.
api PUT "/api/agent-network/policies/$polId" "$(echo "$pol" | jq -c '{name,description,enabled,source_groups,destination_provider_ids,guardrail_ids:[],limits}')" >/dev/null
api DELETE "/api/agent-network/guardrails/$gid" >/dev/null && log " detached + deleted guardrail $gid"
}
# _chat_stream MODEL — streaming chat completion (SSE) with usage included.
# Echoes the HTTP status; writes the raw SSE stream to $CHAT_RESP.
_chat_stream() {
local model="$1" client="${CHAT_CLIENT:-$CLIENT}" ip body out code
ip="$(CLIENT="$client" proxy_ip)"; [ -n "$ip" ] || { echo 000; return; }
body="$(jq -n --arg m "$model" '{model:$m,stream:true,stream_options:{include_usage:true},messages:[{role:"user",content:"Reply with exactly: pong"}]}')"
out="$(docker run --rm --network "container:$client" "$CURL_IMAGE" \
-sSkN --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
-w $'\n%{http_code}' -X POST "https://$PROXY_HOST/v1/chat/completions" \
-H "Content-Type: application/json" --data "$body")"
code="$(printf '%s' "$out" | tail -n1)"
printf '%s' "$out" | sed '$d' >"$CHAT_RESP"
echo "$code"
}
# Scenario 7: streaming (SSE) token capture. The proxy must accumulate token
# usage from the streamed deltas and persist it on the access-log row.
cmd_scenario_streaming() {
log "### scenario: streaming (SSE) token capture ###"
policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
wait_tunnel; sleep 2
expect_code 200 "$(_chat_stream gpt-5.4)" "streaming request served"
if grep -q 'data: \[DONE\]' "$CHAT_RESP"; then ok "SSE stream terminated with [DONE]"; else bad "no [DONE] terminator in SSE stream"; fi
if grep -q '"delta"' "$CHAT_RESP"; then ok "SSE carried incremental deltas"; else bad "no delta chunks in SSE stream"; fi
sleep 3
local st in out
st="$(access_log_top stream)"; in="$(access_log_top input_tokens)"; out="$(access_log_top output_tokens)"
log " access-log: stream=$st input_tokens=$in output_tokens=$out"
if [ "$st" = true ]; then ok "access-log row flagged stream=true"; else bad "stream flag not set (got $st)"; fi
if [ "${in:-0}" -gt 0 ] 2>/dev/null && [ "${out:-0}" -gt 0 ] 2>/dev/null; then ok "streamed token usage captured (in=$in out=$out)"; else bad "streamed token usage not captured (in=$in out=$out)"; fi
}
# wait_for_mgmt — poll the management API until it accepts authed requests.
wait_for_mgmt() {
local i=0
while [ "$i" -lt 90 ]; do
curl -fsS -o /dev/null --max-time 2 -H "$AUTH" "$NB_API/api/users" 2>/dev/null && return 0
sleep 2; i=$((i+2))
done
return 1
}
# Scenario 8: access-log retention pruning. The sweep deletes rows older than
# now - retention_days and runs on management startup. We clone an existing row
# to a synthetic 2020-dated id, set retention to 365d (so only that far-past
# row is eligible — real 2026 rows are safe), restart management to trigger the
# sweep, and assert the synthetic row is gone while real rows survive.
cmd_scenario_retention() {
log "### scenario: access-log retention pruning ###"
local DB="/Users/maycon/projects/wt_testing_data/store.db" acct="d68ag3p31576fp2gmnag" sid="e2e-retention-2020"
command -v sqlite3 >/dev/null 2>&1 || { bad "sqlite3 not on PATH"; return; }
[ -w "$DB" ] || { bad "store.db not writable at $DB"; return; }
local before mid after syn
before="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")"
sqlite3 "$DB" "PRAGMA busy_timeout=8000;
DELETE FROM agent_network_access_log WHERE id='$sid';
CREATE TEMP TABLE _t AS SELECT * FROM agent_network_access_log WHERE account_id='$acct' LIMIT 1;
UPDATE _t SET id='$sid', timestamp='2020-01-01 00:00:00.000000000+00:00';
INSERT INTO agent_network_access_log SELECT * FROM _t;" 2>&1 | sed 's/^/ sqlite: /' >&2 || true
mid="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")"
if [ "$mid" = "$((before+1))" ]; then ok "synthetic 2020 row inserted ($before -> $mid)"; else bad "insert failed ($before -> $mid)"; return; fi
settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false,"access_log_retention_days":365}'
log " retention=365d; restarting management to trigger the startup sweep..."
tilt trigger management >/dev/null 2>&1 || bad "tilt trigger management failed"
wait_for_mgmt || bad "management did not come back up"
sleep 4
syn="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE id='$sid'")"
after="$(sqlite3 "$DB" "SELECT count(*) FROM agent_network_access_log WHERE account_id='$acct'")"
if [ "$syn" = 0 ]; then ok "synthetic 2020 row pruned by retention sweep"; else bad "synthetic row survived (sweep didn't prune)"; fi
if [ "$after" = "$before" ]; then ok "real rows preserved (count back to $before)"; else bad "real row count changed ($before -> $after)"; fi
settings_put '{"enable_log_collection":true,"enable_prompt_collection":true,"redact_pii":false,"access_log_retention_days":0}'
sqlite3 "$DB" "DELETE FROM agent_network_access_log WHERE id='$sid'" 2>/dev/null || true
log " restored retention=0 (keep forever); re-establishing tunnel"
wait_tunnel >/dev/null 2>&1 || true
}
# vertex_request IP PATH BODY — POST a Vertex rawPredict through the client
# sidecar; echo the HTTP status and stash the response body in VTX_RESP. Shared
# by the Vertex scenario's initial probe and its live-update propagation checks.
VTX_RESP=""
vertex_request() {
local ip="$1" path="$2" body="$3" out
out="$(docker run --rm --network "container:$CLIENT" "$CURL_IMAGE" \
-sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
-w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \
-H "Content-Type: application/json" --data "$body")"
VTX_RESP="$(printf '%s' "$out" | sed '$d')"
printf '%s' "$out" | tail -n1
}
# bedrock_probe LABEL PATH BODY IP — POST a Bedrock request via the client
# sidecar and assert the pipeline. 200 -> full pass (+ token metering); a
# Bedrock-origin 404 (account use-case gate / model access) -> pipeline OK
# (routing + bearer auth reached the model); 401/403 -> fail.
bedrock_probe() {
local label="$1" path="$2" body="$3" ip="$4" out code resp
out="$(docker run --rm --network "container:$CLIENT" "$CURL_IMAGE" \
-sSk --connect-timeout 5 --max-time 90 --resolve "$PROXY_HOST:443:$ip" \
-w $'\n%{http_code}' -X POST "https://$PROXY_HOST$path" \
-H "Content-Type: application/json" --data "$body")"
code="$(printf '%s' "$out" | tail -n1)"
resp="$(printf '%s' "$out" | sed '$d')"
log " bedrock $label -> http=$code"
case "$code" in
200)
ok "Bedrock $label 200 (path->model parse, routed, bearer accepted)"
sleep 3
local vin vout
vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)"
if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "Bedrock $label usage metered (in=$vin out=$vout)"; else bad "Bedrock $label usage not metered (in=$vin out=$vout)"; fi
;;
404)
case "$resp" in
*"use case"*|*NOT_FOUND*|*bedrock*|*"don't have access"*|*"inference profile"*)
ok "reached Bedrock ($label, provider+bearer OK); 404 = account use-case gate / model access" ;;
*) bad "Bedrock $label 404 but not a Bedrock response: $(printf '%s' "$resp" | head -c 160)" ;;
esac ;;
401) bad "Bedrock $label rejected the bearer token (401): $(printf '%s' "$resp" | head -c 160)" ;;
403) bad "Bedrock $label denied before upstream (routing regression): $(printf '%s' "$resp" | head -c 160)" ;;
000) bad "Bedrock $label no connectivity (tunnel not converged)" ;;
*) bad "Bedrock $label unexpected status $code: $(printf '%s' "$resp" | head -c 160)" ;;
esac
}
# Scenario 9: Google Vertex AI (path-routed model + service-account OAuth minting).
# Vertex carries the model in the URL path and authenticates with a short-lived
# OAuth2 access token, not an API key. The operator stores a durable GCP
# service-account key as the provider api_key behind a "keyfile::" prefix; the
# synthesiser ships the base64 SA JSON to the router, which mints + caches a
# Bearer token per request. We assert the whole pipeline: path->model parse,
# route-to-Vertex, token mint + inject, upstream forward. A 200 is a complete
# pass; a 404 whose body is Vertex's own "Publisher model not found / no access"
# also passes the pipeline (the token was accepted) and only signals that the
# test project lacks model access in that region.
cmd_scenario_vertex() {
log "### scenario: Google Vertex AI (path-routed + service-account OAuth minting) ###"
require_llm_keys
if [ -z "${GOOGLE_VERTEX_SA_BASE64:-}" ] || [ -z "${GOOGLE_VERTEXT_PROJECT:-}" ]; then
log " SKIP: GOOGLE_VERTEX_SA_BASE64 / GOOGLE_VERTEXT_PROJECT not set in $LLM_KEYS_FILE"
return 0
fi
local vid pol pid old oldvids region vhost
# The regional endpoint host is "<region>-aiplatform.googleapis.com"; the
# "global" location uses the bare "aiplatform.googleapis.com" host.
region="${GOOGLE_VERTEXT_REGION:-global}"
if [ "$region" = global ]; then vhost="aiplatform.googleapis.com"; else vhost="${region}-aiplatform.googleapis.com"; fi
# Remove any pre-existing Vertex providers, de-referencing them from the broad
# policy first — a provider that is still a policy destination can't be deleted
# (422), and a survivor leaves a duplicate (possibly stale-host) Vertex route.
oldvids="$(api GET /api/agent-network/providers | jq -c '[.[]|select(.provider_id=="vertex_ai_api")|.id]')"
if [ "$oldvids" != "[]" ]; then
pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
pid="$(echo "$pol" | jq -r '.id')"
api PUT "/api/agent-network/policies/$pid" \
"$(echo "$pol" | jq -c --argjson rm "$oldvids" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-$rm),guardrail_ids,limits}')" >/dev/null 2>&1 || true
for old in $(echo "$oldvids" | jq -r '.[]'); do
api DELETE "/api/agent-network/providers/$old" >/dev/null 2>&1 || true
done
fi
vid="$(api POST /api/agent-network/providers "$(jq -n --arg k "keyfile::${GOOGLE_VERTEX_SA_BASE64}" --arg u "https://${vhost}" \
'{name:"Vertex e2e",provider_id:"vertex_ai_api",upstream_url:$u,api_key:$k,enabled:true}')" \
| jq -r '.id')"
[ -n "$vid" ] && [ "$vid" != null ] || { bad "create Vertex provider failed"; return; }
log " created Vertex provider $vid (keyfile:: service-account, $vhost)"
pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
pid="$(echo "$pol" | jq -r '.id')"
[ -n "$pid" ] && [ "$pid" != null ] || { bad "e2e-all-providers policy missing (run 'policy' first)"; return; }
api PUT "/api/agent-network/policies/$pid" \
"$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:((.destination_provider_ids+[$v])|unique),guardrail_ids,limits}')" >/dev/null
policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
# Provider create/update propagates to the proxy live via reconcile (the synth
# config is re-pushed and the middleware chain rebuilt on receipt), so just let
# the tunnel settle before probing — no restart needed.
wait_tunnel; sleep 5
local ip path body code resp
ip="$(proxy_ip)"; [ -n "$ip" ] || { bad "could not resolve proxy IP"; return; }
path="/v1/projects/${GOOGLE_VERTEXT_PROJECT}/locations/${region}/publishers/anthropic/models/claude-sonnet-4-5@20250929:rawPredict"
body="$(jq -n '{anthropic_version:"vertex-2023-10-16",max_tokens:64,messages:[{role:"user",content:"Reply with exactly: pong"}]}')"
code="$(vertex_request "$ip" "$path" "$body")"; resp="$VTX_RESP"
log " vertex rawPredict -> http=$code"
case "$code" in
200)
ok "Vertex 200 (path->model parse, routed, token minted + accepted, model available)"
sleep 3
local vin vout
vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)"
log " access-log: model=$(access_log_top model) input_tokens=$vin output_tokens=$vout"
if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "Vertex usage metered (in=$vin out=$vout)"; else bad "Vertex usage not metered (in=$vin out=$vout)"; fi
;;
404)
case "$resp" in
*"Publisher model"*|*aiplatform*|*NOT_FOUND*)
ok "reached Vertex with a minted token (pipeline OK); 404 = test project lacks model access" ;;
*) bad "404 but not a Vertex response body: $(printf '%s' "$resp" | head -c 160)" ;;
esac ;;
401) bad "Vertex rejected the minted credential (401) — check the SA key/roles: $(printf '%s' "$resp" | head -c 160)" ;;
403) bad "proxy denied before reaching Vertex (path parse / routing regression): $(printf '%s' "$resp" | head -c 160)" ;;
000) bad "no connectivity to proxy (tunnel not converged)" ;;
*) bad "unexpected status $code: $(printf '%s' "$resp" | head -c 160)" ;;
esac
# Live provider-update propagation: a provider change must reach the proxy node
# without a restart. Disable the provider -> its route disappears and the request
# is denied (403); re-enable -> served again at the baseline status. Proves
# reconcile re-pushes the synth config and the proxy rebuilds its chain live.
case "$code" in
200|404)
log " testing live provider-update propagation (disable/enable, no restart)..."
local dcode ecode
set_enabled "$vid" false; sleep 5
dcode="$(vertex_request "$ip" "$path" "$body")"
if [ "$dcode" = 403 ]; then ok "provider disable reflected on proxy live (route removed, http 403)"; else bad "provider disable not reflected on proxy (want 403, got $dcode)"; fi
set_enabled "$vid" true; sleep 5
ecode="$(vertex_request "$ip" "$path" "$body")"
if [ "$ecode" = "$code" ]; then ok "provider re-enable reflected on proxy live (served again, http $ecode)"; else bad "provider re-enable not reflected on proxy (want $code, got $ecode)"; fi
;;
esac
pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
api PUT "/api/agent-network/policies/$pid" \
"$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-[$v]),guardrail_ids,limits}')" >/dev/null 2>&1 || true
api DELETE "/api/agent-network/providers/$vid" >/dev/null 2>&1 || true
}
# Scenario 10: real-traffic coverage for the OpenAI-compatible gateway providers
# (Vercel, OpenRouter, Cloudflare). Each is isolated as the sole enabled provider
# so its model routes to it unambiguously, sent a real chat request, and asserted
# to return 200 with metered token usage. A gateway without a configured provider
# is skipped. Restores all providers at the end.
cmd_scenario_providers() {
log "### scenario: gateway providers real-traffic (Vercel / OpenRouter / Cloudflare) ###"
policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
local spec iso full model kind r pid code vin vout
for spec in \
"vercel|Vercel AI Gateway|openai/gpt-4o-mini|chat" \
"openrouter|OpenRouter|openai/gpt-4o-mini|chat" \
"cloudflare|Cloudflare AI Gateway|gpt-4o-mini|chat"; do
iso="${spec%%|*}"; r="${spec#*|}"; full="${r%%|*}"; r="${r#*|}"; model="${r%%|*}"; kind="${r##*|}"
pid="$(provider_id_by_name "$full")"
if [ -z "$pid" ] || [ "$pid" = null ]; then log " SKIP $full: not configured"; continue; fi
cmd_isolate "$iso" >/dev/null 2>&1
wait_tunnel; sleep 4
code="$(_chat "$model" "$kind")"
if [ "$code" = 200 ]; then ok "$full served real request ($model, http 200)"; else bad "$full request failed ($model, http $code)"; continue; fi
sleep 3
vin="$(access_log_top input_tokens)"; vout="$(access_log_top output_tokens)"
if [ "${vin:-0}" -gt 0 ] 2>/dev/null && [ "${vout:-0}" -gt 0 ] 2>/dev/null; then ok "$full usage metered (in=$vin out=$vout)"; else bad "$full usage not metered (in=$vin out=$vout)"; fi
done
cmd_enable_all >/dev/null 2>&1
log " re-enabled all providers"
wait_tunnel >/dev/null 2>&1 || true
}
# Scenario 11: AWS Bedrock (path-routed model + bearer auth) across invoke,
# converse, and invoke-with-response-stream. Bedrock carries the model in the URL
# path and authenticates with a static bearer token (Bedrock API key). Asserts
# the pipeline per endpoint: path->model parse + normalize, route-to-Bedrock,
# bearer inject, upstream forward (+ token metering on 200). A Bedrock-origin 404
# (account use-case gate / model access) passes the pipeline; see bedrock_probe.
cmd_scenario_bedrock() {
log "### scenario: AWS Bedrock (path-routed + bearer; invoke/converse/stream) ###"
require_llm_keys
if [ -z "${AWS_BEARER_TOKEN_BEDROCK:-}" ]; then
log " SKIP: AWS_BEARER_TOKEN_BEDROCK not set in $LLM_KEYS_FILE"
return 0
fi
local region model vid pol pid old oldvids
region="${AWS_BEDROCK_REGION:-eu-central-1}"
model="${BEDROCK_MODEL:-eu.anthropic.claude-sonnet-4-5-20250929-v1:0}"
oldvids="$(api GET /api/agent-network/providers | jq -c '[.[]|select(.provider_id=="bedrock_api")|.id]')"
if [ "$oldvids" != "[]" ]; then
pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
pid="$(echo "$pol" | jq -r '.id')"
api PUT "/api/agent-network/policies/$pid" \
"$(echo "$pol" | jq -c --argjson rm "$oldvids" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-$rm),guardrail_ids,limits}')" >/dev/null 2>&1 || true
for old in $(echo "$oldvids" | jq -r '.[]'); do
api DELETE "/api/agent-network/providers/$old" >/dev/null 2>&1 || true
done
fi
vid="$(api POST /api/agent-network/providers "$(jq -n --arg k "$AWS_BEARER_TOKEN_BEDROCK" --arg u "https://bedrock-runtime.${region}.amazonaws.com" \
'{name:"Bedrock e2e",provider_id:"bedrock_api",upstream_url:$u,api_key:$k,enabled:true}')" | jq -r '.id')"
[ -n "$vid" ] && [ "$vid" != null ] || { bad "create Bedrock provider failed"; return; }
log " created Bedrock provider $vid (bearer, bedrock-runtime.${region})"
pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
pid="$(echo "$pol" | jq -r '.id')"
[ -n "$pid" ] && [ "$pid" != null ] || { bad "e2e-all-providers policy missing (run 'policy' first)"; return; }
api PUT "/api/agent-network/policies/$pid" \
"$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:((.destination_provider_ids+[$v])|unique),guardrail_ids,limits}')" >/dev/null
policy_put_limits '{"budget_limit":{"enabled":true,"group_cap_usd":1000000,"user_cap_usd":1000000,"window_seconds":2592000},"token_limit":{"enabled":false,"group_cap":0,"user_cap":0,"window_seconds":3600}}'
wait_tunnel; sleep 5
local ip invokeBody converseBody
ip="$(proxy_ip)"; [ -n "$ip" ] || { bad "could not resolve proxy IP"; return; }
invokeBody='{"anthropic_version":"bedrock-2023-05-31","max_tokens":32,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}'
converseBody='{"messages":[{"role":"user","content":[{"text":"Reply with exactly: pong"}]}],"inferenceConfig":{"maxTokens":32}}'
bedrock_probe "invoke" "/model/${model}/invoke" "$invokeBody" "$ip"
bedrock_probe "converse" "/model/${model}/converse" "$converseBody" "$ip"
bedrock_probe "invoke-stream" "/model/${model}/invoke-with-response-stream" "$invokeBody" "$ip"
# Gateway-namespace prefix: /bedrock/... must route the same and be stripped
# before the upstream call (AWS's native path has no /bedrock prefix).
bedrock_probe "invoke (/bedrock prefix)" "/bedrock/model/${model}/invoke" "$invokeBody" "$ip"
pol="$(api GET /api/agent-network/policies | jq -c '.[]|select(.name=="e2e-all-providers")')"
api PUT "/api/agent-network/policies/$pid" \
"$(echo "$pol" | jq -c --arg v "$vid" '{name,description,enabled,source_groups,destination_provider_ids:(.destination_provider_ids-[$v]),guardrail_ids,limits}')" >/dev/null 2>&1 || true
api DELETE "/api/agent-network/providers/$vid" >/dev/null 2>&1 || true
}
cmd_scenarios() {
FAILS=0
cmd_scenario_budget
cmd_scenario_budget_rule
cmd_scenario_user_cap
cmd_scenario_guardrail
cmd_scenario_streaming
cmd_scenario_multigroup
cmd_scenario_logs
cmd_scenario_retention
cmd_scenario_providers
cmd_scenario_vertex
cmd_scenario_bedrock
log "================================================"
if [ "$FAILS" -eq 0 ]; then log "ALL SCENARIOS PASSED"; else log "SCENARIO FAILURES: $FAILS"; fi
return "$FAILS"
}
# --- dispatch ----------------------------------------------------------------
cmd="${1:-}"; shift || true
case "$cmd" in
snapshot) cmd_snapshot ;;
key) cmd_key ;;
up) cmd_up ;;
status) cmd_status ;;
wait) cmd_wait ;;
diag) cmd_diag ;;
restart-proxy) cmd_restart_proxy ;;
chat) cmd_chat "$@" ;;
verify) cmd_verify ;;
clean) cmd_clean ;;
providers) cmd_providers ;;
policy) cmd_policy ;;
isolate) cmd_isolate "$@" ;;
enable-all) cmd_enable_all ;;
scenario-budget) cmd_scenario_budget ;;
scenario-budget-rule) cmd_scenario_budget_rule ;;
scenario-user-cap) cmd_scenario_user_cap ;;
scenario-guardrail) cmd_scenario_guardrail ;;
scenario-streaming) cmd_scenario_streaming ;;
scenario-multigroup) cmd_scenario_multigroup ;;
scenario-logs) cmd_scenario_logs ;;
scenario-retention) cmd_scenario_retention ;;
scenario-providers) cmd_scenario_providers ;;
scenario-vertex) cmd_scenario_vertex ;;
scenario-bedrock) cmd_scenario_bedrock ;;
dashboard) cmd_dashboard ;;
scenarios) cmd_scenarios ;;
down) cmd_down ;;
all) cmd_all ;;
*) cat >&2 <<'USAGE'
usage: bash scripts/e2e/agent-network-full/e2e.sh <cmd>
setup/flow : snapshot key up status wait diag restart-proxy clean providers policy isolate NAME enable-all down all
traffic : chat MODEL [chat|messages] verify
scenarios : scenarios | scenario-budget | scenario-budget-rule | scenario-user-cap |
scenario-guardrail | scenario-streaming | scenario-multigroup |
scenario-logs | scenario-retention | scenario-providers | scenario-vertex |
scenario-bedrock
dashboard : dashboard (Playwright UI check against the live :3000 dashboard)
USAGE
exit 2 ;;
esac

105
scripts/e2e/agent-network-policy/00-env.sh
View File

@@ -1,105 +0,0 @@
# shellcheck disable=SC2148
# Sourced helper for the agent-network-policy e2e suite.
# Verifies the agent-network surface: window_seconds (post-rename)
# through the API, /api/agent-network/consumption read endpoint, and
# the CheckLLMPolicyLimits + RecordLLMUsage gRPC RPCs.
#
# Do not run directly.
set -euo pipefail
: "${NB_API:=http://localhost:8080}"
: "${NB_PAT_FILE:=/Users/maycon/projects/local-dev/nb-pat}"
: "${NB_TIMEOUT_SECONDS:=60}"
: "${NB_POLICY_NAME:=e2e-anpol}"
: "${NB_PROVIDER_NAME:=e2e-anpol-provider}"
: "${NB_GROUP_NAME:=e2e-anpol-engineers}"
: "${NB_GRPC_ADDR:=localhost:8080}"
# Proxy token shared with the tilt setup. Keep in sync with the
# NB_PROXY_TOKEN literal in /Users/maycon/projects/local-dev/Tiltfile;
# the management server accepts it as a registered proxy credential
# so the e2e binary can reach the proxy_service gRPC surface.
: "${NB_PROXY_TOKEN:=nbx_MEF9OKRhlLrWkc5TJmM3Eu2rhqigaP2yulHy}"
: "${NB_STATE_DIR:=/tmp/nb-anpol-e2e-state}"
mkdir -p "$NB_STATE_DIR"
if [ ! -r "$NB_PAT_FILE" ]; then
echo "FAIL: cannot read PAT at $NB_PAT_FILE" >&2
exit 2
fi
NB_PAT=$(tr -d '\n\r ' <"$NB_PAT_FILE")
if [ ${#NB_PAT} -lt 16 ]; then
echo "FAIL: PAT at $NB_PAT_FILE is suspiciously short (${#NB_PAT} chars)" >&2
exit 2
fi
if ! command -v jq >/dev/null 2>&1; then
echo "FAIL: jq is required (brew install jq)" >&2
exit 2
fi
# nb_api METHOD PATH [BODY] — wraps curl with PAT auth and JSON.
nb_api() {
local method="$1" path="$2" body="${3-}"
if [ -n "$body" ]; then
curl -fsS -X "$method" \
-H "Authorization: Token $NB_PAT" \
-H "Content-Type: application/json" \
--data "$body" \
"$NB_API$path"
else
curl -fsS -X "$method" \
-H "Authorization: Token $NB_PAT" \
"$NB_API$path"
fi
}
# nb_api_status METHOD PATH [BODY] — returns the HTTP status code
# rather than the body. Use for negative-path tests where a 4xx is
# the expected outcome and curl's default -f would mask the result.
nb_api_status() {
local method="$1" path="$2" body="${3-}"
if [ -n "$body" ]; then
curl -sS -o /dev/null -w '%{http_code}' -X "$method" \
-H "Authorization: Token $NB_PAT" \
-H "Content-Type: application/json" \
--data "$body" \
"$NB_API$path"
else
curl -sS -o /dev/null -w '%{http_code}' -X "$method" \
-H "Authorization: Token $NB_PAT" \
"$NB_API$path"
fi
}
# wait_for COND_CMD TIMEOUT_S — polls every 1s.
wait_for() {
local cmd="$1" timeout="${2:-$NB_TIMEOUT_SECONDS}"
local i=0
while [ "$i" -lt "$timeout" ]; do
if eval "$cmd" >/dev/null 2>&1; then
return 0
fi
sleep 1
i=$((i + 1))
done
return 1
}
# resolve the absolute path of the netbird repo root from the script
# location so the Go smoke binary can be invoked via `go run` against
# the right module regardless of the caller's cwd.
nb_repo_root() {
cd "$(dirname "$0")/../../.." && pwd
}
pass() {
printf 'PASS: %s\n' "$1"
exit 0
}
fail() {
printf 'FAIL: %s — %s\n' "$1" "${2:-}"
exit 1
}

44
scripts/e2e/agent-network-policy/01-tilt-restart.sh
View File

@@ -1,44 +0,0 @@
#!/usr/bin/env bash
# 01-tilt-restart: re-trigger management + dashboard before each run so
# we start from a clean process state. The on-disk store survives, so
# the PAT, account, and any residual config persist across restarts.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
command -v tilt >/dev/null 2>&1 || fail "tilt not on PATH" "brew install tilt-dev/tap/tilt"
if ! curl -fsS -o /dev/null --max-time 2 http://localhost:10350/ 2>/dev/null; then
fail "Tilt API not reachable at http://localhost:10350" "is 'tilt up' running in /Users/maycon/projects/local-dev?"
fi
restart_one() {
local name="$1"
if ! tilt trigger "$name" >/dev/null 2>&1; then
fail "tilt trigger $name failed" ""
fi
echo "triggered $name"
}
restart_one management
restart_one dashboard
# proxy3 needs to come back too because PR2 wired the new
# llm_limit_check / llm_limit_record middlewares into the proxy
# binary; without restarting it, e2e keeps exercising whatever
# image was last loaded and silently misses any boot-order or
# wiring regression in the chain. Cheap to restart, expensive to
# silently miss.
restart_one proxy3
echo "waiting for management to accept requests..."
if ! wait_for "curl -fsS -o /dev/null --max-time 2 $NB_API/oauth2/.well-known/openid-configuration" 60; then
fail "management did not come back up within 60s" "check 'tilt logs management'"
fi
echo "management is up"
code=$(curl -fsS -o /dev/null -w '%{http_code}' \
-H "Authorization: Token $NB_PAT" \
"$NB_API/api/users" 2>&1) || true
[ "$code" = "200" ] || fail "PAT auth check failed after restart (HTTP $code)" "the PAT may have been revoked"
pass "Tilt resources restarted: management, dashboard"

132
scripts/e2e/agent-network-policy/10-policy-create.sh
View File

@@ -1,132 +0,0 @@
#!/usr/bin/env bash
# 10-policy-create: end-to-end round-trip on the new window_seconds
# field. Creates the prerequisites (group + provider with
# bootstrap_cluster), POSTs a policy whose Limits.token_limit and
# budget_limit both carry window_seconds, then re-fetches the policy
# and asserts the field comes back unchanged.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
# 1. Group — sweep first so re-runs are idempotent, then POST.
existing_group=$(nb_api GET /api/groups 2>/dev/null \
| jq -r --arg name "$NB_GROUP_NAME" '.[] | select(.name == $name) | .id // empty' \
| head -1)
if [ -n "$existing_group" ]; then
echo "reusing existing group $existing_group"
group_id="$existing_group"
else
body=$(jq -n --arg name "$NB_GROUP_NAME" '{name:$name}')
group_resp=$(nb_api POST /api/groups "$body")
group_id=$(printf '%s' "$group_resp" | jq -r '.id // ""')
[ -n "$group_id" ] && [ "$group_id" != "null" ] \
|| fail "group create did not return an id" "$group_resp"
echo "created group $group_id"
fi
printf '%s' "$group_id" >"$NB_STATE_DIR/group-id"
# 2. Provider — also idempotent. Bootstrap cluster pinned to the local
# proxy3 cluster so the management settings row resolves and the
# create completes (subsequent provider creates in the same account
# ignore bootstrap_cluster, but the FIRST one needs it).
existing_provider=$(nb_api GET /api/agent-network/providers 2>/dev/null \
| jq -r --arg name "$NB_PROVIDER_NAME" '.[] | select(.name == $name) | .id // empty' \
| head -1)
if [ -n "$existing_provider" ]; then
echo "reusing existing provider $existing_provider"
provider_id="$existing_provider"
else
provider_body=$(jq -n \
--arg name "$NB_PROVIDER_NAME" \
'{
provider_id: "openai_api",
name: $name,
upstream_url: "https://api.openai.com",
api_key: "sk-e2e-placeholder",
bootstrap_cluster: "proxy.netbird.local",
models: []
}')
prov_resp=$(nb_api POST /api/agent-network/providers "$provider_body")
provider_id=$(printf '%s' "$prov_resp" | jq -r '.id // ""')
[ -n "$provider_id" ] && [ "$provider_id" != "null" ] \
|| fail "provider create did not return an id" "$prov_resp"
echo "created provider $provider_id"
fi
printf '%s' "$provider_id" >"$NB_STATE_DIR/provider-id"
# 3. Policy — drop any prior with the same name, then create with the
# NEW window_seconds field on both halves of Limits. 86400s = 24h
# on token, 2_592_000s = 30d on budget so the round-trip is
# unambiguous (no ambiguous unit-conversion artefact when we read
# back).
existing_policy=$(nb_api GET /api/agent-network/policies 2>/dev/null \
| jq -r --arg name "$NB_POLICY_NAME" '.[] | select(.name == $name) | .id // empty' \
| head -1)
if [ -n "$existing_policy" ]; then
echo "deleting existing policy $existing_policy"
nb_api DELETE "/api/agent-network/policies/$existing_policy" >/dev/null 2>&1 || true
fi
policy_body=$(jq -n \
--arg name "$NB_POLICY_NAME" \
--arg group "$group_id" \
--arg provider "$provider_id" \
'{
name: $name,
description: "agent-network e2e: window_seconds round-trip",
enabled: true,
source_groups: [$group],
destination_provider_ids: [$provider],
guardrail_ids: [],
limits: {
token_limit: {
enabled: true,
group_cap: 10000,
user_cap: 5000,
window_seconds: 86400
},
budget_limit: {
enabled: true,
group_cap_usd: 10.0,
user_cap_usd: 2.5,
window_seconds: 2592000
}
}
}')
resp=$(nb_api POST /api/agent-network/policies "$policy_body")
policy_id=$(printf '%s' "$resp" | jq -r '.id // ""')
[ -n "$policy_id" ] && [ "$policy_id" != "null" ] \
|| fail "policy create did not return an id" "$resp"
printf '%s' "$policy_id" >"$NB_STATE_DIR/policy-id"
echo "policy id: $policy_id"
# 4. Round-trip assertions: GET the policy back and verify the
# window_seconds values land on both limit halves. The OLD
# window_hours / window_days fields must be absent.
got=$(nb_api GET "/api/agent-network/policies/$policy_id")
token_window=$(printf '%s' "$got" | jq -r '.limits.token_limit.window_seconds // empty')
budget_window=$(printf '%s' "$got" | jq -r '.limits.budget_limit.window_seconds // empty')
[ "$token_window" = "86400" ] \
|| fail "token_limit.window_seconds did not round-trip" \
"expected=86400 got=$token_window body=$got"
[ "$budget_window" = "2592000" ] \
|| fail "budget_limit.window_seconds did not round-trip" \
"expected=2592000 got=$budget_window body=$got"
# Negative: window_hours / window_days are legacy field names and
# must not be present in the response at all — their presence would
# mean the management server is still emitting the legacy shape.
legacy_h=$(printf '%s' "$got" | jq -r '.limits.token_limit | has("window_hours")')
legacy_d=$(printf '%s' "$got" | jq -r '.limits.token_limit | has("window_days")')
[ "$legacy_h" = "false" ] \
|| fail "legacy window_hours field still present in token_limit response" "$got"
[ "$legacy_d" = "false" ] \
|| fail "legacy window_days field still present in token_limit response" "$got"
echo "token_limit.window_seconds = $token_window"
echo "budget_limit.window_seconds = $budget_window"
pass "policy persisted with window_seconds on both Limits halves"

65
scripts/e2e/agent-network-policy/20-policy-rejects-zero-window.sh
View File

@@ -1,65 +0,0 @@
#!/usr/bin/env bash
# 20-policy-rejects-zero-window: management must reject a policy whose
# token_limit or budget_limit is enabled but carries window_seconds < 60.
# Anything below the one-minute floor would either be a zero / negative
# window with no reset boundary, or a sub-minute window that produces
# untenable consumption-row volume at scale. The handler's
# validatePolicyLimits guard owes us a 4xx with a useful message.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
[ -r "$NB_STATE_DIR/group-id" ] && [ -r "$NB_STATE_DIR/provider-id" ] \
|| fail "missing group-id / provider-id state" \
"run 10-policy-create.sh first to bootstrap prerequisites"
group_id=$(cat "$NB_STATE_DIR/group-id")
provider_id=$(cat "$NB_STATE_DIR/provider-id")
# Build a payload that is well-formed except for window_seconds=30 on
# token_limit (under the 60s minimum). We mark the field enabled so
# the validation path actually runs — a disabled limit is allowed to
# carry zero.
payload=$(jq -n \
--arg name "$NB_POLICY_NAME-sub-minute" \
--arg group "$group_id" \
--arg provider "$provider_id" \
'{
name: $name,
enabled: true,
source_groups: [$group],
destination_provider_ids: [$provider],
guardrail_ids: [],
limits: {
token_limit: {
enabled: true,
group_cap: 10000,
user_cap: 5000,
window_seconds: 30
},
budget_limit: {
enabled: false,
group_cap_usd: 0,
user_cap_usd: 0,
window_seconds: 0
}
}
}')
code=$(nb_api_status POST /api/agent-network/policies "$payload")
[ "$code" = "400" ] || [ "$code" = "422" ] \
|| fail "expected 400/422 on enabled token_limit with window_seconds<60" \
"got HTTP $code"
# Sweep any policy that may have been mistakenly persisted (defence
# against a future bug; today's handler doesn't get there).
orphan=$(nb_api GET /api/agent-network/policies 2>/dev/null \
| jq -r --arg name "$NB_POLICY_NAME-sub-minute" '.[] | select(.name == $name) | .id // empty' \
| head -1)
if [ -n "$orphan" ]; then
nb_api DELETE "/api/agent-network/policies/$orphan" >/dev/null 2>&1 || true
fail "policy was persisted despite sub-minute window_seconds" \
"id=$orphan — handler validation regression"
fi
echo "POST with token_limit.window_seconds=30 rejected with HTTP $code"
pass "validation rejects sub-minute window_seconds when limit is enabled"

24
scripts/e2e/agent-network-policy/30-consumption-list-empty.sh
View File

@@ -1,24 +0,0 @@
#!/usr/bin/env bash
# 30-consumption-list-empty: GET /api/agent-network/consumption must
# return a JSON array (possibly empty) — never a 404 / 500. The
# endpoint is the read side that backs the dashboard's basic counter
# view and must always be reachable so the page can render an empty
# state.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
resp=$(nb_api GET /api/agent-network/consumption)
# Must be a JSON array.
if ! printf '%s' "$resp" | jq -e 'type == "array"' >/dev/null 2>&1; then
fail "consumption endpoint did not return a JSON array" "$resp"
fi
count=$(printf '%s' "$resp" | jq 'length')
echo "consumption rows: $count"
# Stash the baseline count so 40-grpc-record-and-list can compare.
printf '%s' "$count" >"$NB_STATE_DIR/consumption-baseline"
pass "consumption read endpoint returns a JSON array (count=$count)"

133
scripts/e2e/agent-network-policy/40-grpc-record-and-list.sh
View File

@@ -1,133 +0,0 @@
#!/usr/bin/env bash
# 40-grpc-record-and-list: drives the new RecordLLMUsage and
# CheckLLMPolicyLimits gRPC RPCs through the e2e usage_smoke helper.
#
# Asserts:
# - CheckLLMPolicyLimits returns decision=allow, picks the lowest
# group id by string sort as attribution, default window of 24h.
# - RecordLLMUsage with both user_id and group_id ticks BOTH the
# user counter and the group counter exactly once each.
# - A second RecordLLMUsage on the same key sums the deltas server
# side (database upsert-increment, no read-modify-write race).
# - The HTTP /api/agent-network/consumption listing reflects the
# post-flight state.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
[ -r "$NB_STATE_DIR/group-id" ] && [ -r "$NB_STATE_DIR/provider-id" ] \
|| fail "missing group-id / provider-id state" \
"run 10-policy-create.sh first to bootstrap prerequisites"
group_id=$(cat "$NB_STATE_DIR/group-id")
provider_id=$(cat "$NB_STATE_DIR/provider-id")
# Resolve the calling account id. /api/accounts returns NetBird xids,
# which is what the gRPC service expects (NOT IDP user UUIDs).
account_id=$(nb_api GET /api/accounts 2>/dev/null | jq -r '.[0].id // empty')
[ -n "$account_id" ] || fail "could not resolve account id" "GET /api/accounts returned nothing"
# Pick a stable user id for the test. The dimension is per-user, so we
# can use any unique value — a synthetic "e2e-..." prefix avoids
# colliding with real user ids in the consumption listing.
user_id="e2e-anpol-user-$$"
window_seconds=86400
echo "account=$account_id user=$user_id group=$group_id window=${window_seconds}s"
# ─── 1. CheckLLMPolicyLimits ────────────────────────────────────────
# Pass 3 group ids out of order; lowest by string sort wins.
groups_csv="grp-zz,$group_id,grp-aa-z"
check_resp=$(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke check \
--account "$account_id" \
--user "$user_id" \
--groups "$groups_csv" \
--provider "$provider_id" \
--model "gpt-4o" \
--token "$NB_PROXY_TOKEN" \
--addr "$NB_GRPC_ADDR" 2>&1) \
|| fail "CheckLLMPolicyLimits gRPC failed" "$check_resp"
decision=$(printf '%s' "$check_resp" | jq -r '.decision // ""')
attribution=$(printf '%s' "$check_resp" | jq -r '.attribution_group_id // ""')
got_window=$(printf '%s' "$check_resp" | jq -r '.window_seconds // ""')
[ "$decision" = "allow" ] \
|| fail "Check decision must be allow under PR1 stub" "$check_resp"
# Lowest group id of {grp-zz, $group_id, grp-aa-z} by string sort. The
# group_id is an xid (lowercase alnum starting with a digit/letter), so
# the sort answer depends on group_id's prefix. Compute it locally.
expected_low=$(printf '%s\n%s\n%s' "grp-zz" "$group_id" "grp-aa-z" | sort | head -1)
[ "$attribution" = "$expected_low" ] \
|| fail "Check did not pick the lowest-by-sort group" \
"expected=$expected_low got=$attribution"
[ "$got_window" = "86400" ] \
|| fail "Check window_seconds stub default mismatch" "expected=86400 got=$got_window"
echo "Check decision=$decision attribution=$attribution window=${got_window}s"
# ─── 2. RecordLLMUsage — first increment ────────────────────────────
(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \
--account "$account_id" \
--user "$user_id" \
--group "$group_id" \
--window-seconds "$window_seconds" \
--tokens-in 100 \
--tokens-out 50 \
--cost-usd 0.0125 \
--token "$NB_PROXY_TOKEN" \
--addr "$NB_GRPC_ADDR") >/dev/null \
|| fail "RecordLLMUsage (first increment) failed" ""
# ─── 3. RecordLLMUsage — second increment, same key ─────────────────
(cd "$(nb_repo_root)" && go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \
--account "$account_id" \
--user "$user_id" \
--group "$group_id" \
--window-seconds "$window_seconds" \
--tokens-in 50 \
--tokens-out 25 \
--cost-usd 0.0025 \
--token "$NB_PROXY_TOKEN" \
--addr "$NB_GRPC_ADDR") >/dev/null \
|| fail "RecordLLMUsage (second increment) failed" ""
# ─── 4. Read-back via HTTP — sums must converge ─────────────────────
listing=$(nb_api GET /api/agent-network/consumption)
user_row=$(printf '%s' "$listing" | jq --arg u "$user_id" \
'map(select(.dimension_kind == "user" and .dimension_id == $u)) | .[0]')
[ "$(printf '%s' "$user_row" | jq -r '. // empty')" != "" ] \
|| fail "user consumption row missing after RecordLLMUsage" "$listing"
user_in=$(printf '%s' "$user_row" | jq '.tokens_input')
user_out=$(printf '%s' "$user_row" | jq '.tokens_output')
user_cost=$(printf '%s' "$user_row" | jq '.cost_usd')
user_window=$(printf '%s' "$user_row" | jq '.window_seconds')
[ "$user_in" = "150" ] && [ "$user_out" = "75" ] \
|| fail "user counter did not sum the two increments" \
"expected tokens_in=150 tokens_out=75; got in=$user_in out=$user_out"
# Floating point compare with awk — drift > 1e-9 is a real bug.
cost_ok=$(awk -v got="$user_cost" 'BEGIN { print (got > 0.0149 && got < 0.0151) ? "y" : "n" }')
[ "$cost_ok" = "y" ] \
|| fail "user cost did not sum to 0.015" "got=$user_cost"
[ "$user_window" = "$window_seconds" ] \
|| fail "user counter window_seconds mismatch" \
"expected=$window_seconds got=$user_window"
# Group row gets the same deltas because RecordLLMUsage ticks both
# dimensions on a single call.
group_row=$(printf '%s' "$listing" | jq --arg g "$group_id" \
'map(select(.dimension_kind == "group" and .dimension_id == $g)) | .[0]')
group_in=$(printf '%s' "$group_row" | jq '.tokens_input')
group_out=$(printf '%s' "$group_row" | jq '.tokens_output')
[ "$group_in" = "150" ] && [ "$group_out" = "75" ] \
|| fail "group counter did not sum the two increments" \
"expected tokens_in=150 tokens_out=75; got in=$group_in out=$group_out"
echo "user counter: $user_in input / $user_out output / \$$user_cost over ${user_window}s"
echo "group counter: $group_in input / $group_out output"
pass "gRPC Check + Record round-trip atomically increments user + group counters"

210
scripts/e2e/agent-network-policy/50-grpc-allow-record-deny.sh
View File

@@ -1,210 +0,0 @@
#!/usr/bin/env bash
# 50-grpc-allow-record-deny: drives the full lifecycle of the real
# selection algorithm landed in PR2 — initial allow, partial fills
# below the cap, exact-at-cap, then deny once consumption reaches the
# group_cap. Validates that:
#
# - CheckLLMPolicyLimits picks the test policy as the attribution
# when it's the only one authorising the (group, provider) pair.
# - The selected_policy_id + window_seconds round-trip on the wire.
# - Counter-aware headroom math is wired end-to-end (RecordLLMUsage
# ticks counters that the next CheckLLMPolicyLimits call reads).
# - At-cap consumption flips decision from allow to deny with the
# canonical llm_policy.token_cap_exceeded code.
#
# The setup creates a dedicated group + provider + policy with the
# "-tight" suffix so it's isolated from the e2e-anpol resources
# 10-policy-create.sh seeded. 99-cleanup's prefix sweep catches both.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
# Per-run suffix so the consumption counter starts at zero on every
# run. Without this, a successful prior run leaves 200 tokens on the
# (group, 24h) bucket and the next run's stage 1 check sees a deny
# from the start. Consumption rows have no delete endpoint today.
RUN_TAG="$(date +%s)-$$"
NB_TIGHT_GROUP_NAME="$NB_GROUP_NAME-tight-$RUN_TAG"
NB_TIGHT_PROVIDER_NAME="$NB_PROVIDER_NAME-tight-$RUN_TAG"
NB_TIGHT_POLICY_NAME="$NB_POLICY_NAME-tight-$RUN_TAG"
TIGHT_CAP=200
TIGHT_WINDOW_SECONDS=86400
# 1. Group — fresh per-run so the (group, window) consumption counter
# starts at zero. The 99-cleanup sweep catches it via the
# `e2e-anpol-engineers-tight-` prefix.
body=$(jq -n --arg name "$NB_TIGHT_GROUP_NAME" '{name:$name}')
tight_group_id=$(nb_api POST /api/groups "$body" | jq -r '.id // ""')
[ -n "$tight_group_id" ] && [ "$tight_group_id" != "null" ] \
|| fail "tight group create failed" ""
echo "created tight group $tight_group_id"
printf '%s' "$tight_group_id" >"$NB_STATE_DIR/tight-group-id"
# 2. Provider — fresh per-run too so the policy targets a provider
# only this run knows about. Eliminates cross-policy interference if
# a prior run's tight policy targeted the same provider id.
body=$(jq -n \
--arg name "$NB_TIGHT_PROVIDER_NAME" \
'{
provider_id: "openai_api",
name: $name,
upstream_url: "https://api.openai.com",
api_key: "sk-e2e-tight-placeholder",
bootstrap_cluster: "proxy.netbird.local",
models: []
}')
tight_provider_id=$(nb_api POST /api/agent-network/providers "$body" | jq -r '.id // ""')
[ -n "$tight_provider_id" ] && [ "$tight_provider_id" != "null" ] \
|| fail "tight provider create failed" ""
echo "created tight provider $tight_provider_id"
printf '%s' "$tight_provider_id" >"$NB_STATE_DIR/tight-provider-id"
# 3. Policy with a tight token cap so the test can observe the deny
# transition without burning thousands of records.
existing_policy=$(nb_api GET /api/agent-network/policies 2>/dev/null \
| jq -r --arg name "$NB_TIGHT_POLICY_NAME" '.[] | select(.name == $name) | .id // empty' \
| head -1)
if [ -n "$existing_policy" ]; then
echo "deleting existing tight policy $existing_policy so the run starts clean"
nb_api DELETE "/api/agent-network/policies/$existing_policy" >/dev/null 2>&1 || true
fi
policy_body=$(jq -n \
--arg name "$NB_TIGHT_POLICY_NAME" \
--arg group "$tight_group_id" \
--arg provider "$tight_provider_id" \
--argjson cap "$TIGHT_CAP" \
--argjson window "$TIGHT_WINDOW_SECONDS" \
'{
name: $name,
description: "agent-network e2e: cap-exhaust deny test",
enabled: true,
source_groups: [$group],
destination_provider_ids: [$provider],
guardrail_ids: [],
limits: {
token_limit: {
enabled: true,
group_cap: $cap,
user_cap: 0,
window_seconds: $window
},
budget_limit: {
enabled: false,
group_cap_usd: 0,
user_cap_usd: 0,
window_seconds: $window
}
}
}')
tight_policy_id=$(nb_api POST /api/agent-network/policies "$policy_body" | jq -r '.id // ""')
[ -n "$tight_policy_id" ] && [ "$tight_policy_id" != "null" ] \
|| fail "tight policy create failed" ""
printf '%s' "$tight_policy_id" >"$NB_STATE_DIR/tight-policy-id"
echo "created tight policy $tight_policy_id (group_cap=$TIGHT_CAP, window=${TIGHT_WINDOW_SECONDS}s)"
# 4. Resolve the calling account id; the smoke binary stamps it onto
# every gRPC request the way the real proxy does.
account_id=$(nb_api GET /api/accounts 2>/dev/null | jq -r '.[0].id // empty')
[ -n "$account_id" ] || fail "could not resolve account id" ""
# Test user — synthetic prefix avoids colliding with real users.
test_user="e2e-anpol-tight-user-$$"
# Pre-resolve the netbird repo root ONCE so the helpers below can
# pushd / popd into it without `cd "$(dirname "$0")/.."` re-resolving
# from a moved cwd between calls. nb_repo_root walks up from $0; if a
# prior helper left cwd elsewhere the relative walk breaks.
repo_root=$(nb_repo_root)
[ -d "$repo_root" ] || fail "could not resolve netbird repo root" "got=$repo_root"
# Run the smoke binary inside a subshell so its `cd` doesn't pollute
# the caller's cwd between stages.
do_check() {
(
cd "$repo_root" || exit 1
go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke check \
--account "$account_id" \
--user "$test_user" \
--groups "$tight_group_id" \
--provider "$tight_provider_id" \
--model "gpt-4o" \
--token "$NB_PROXY_TOKEN" \
--addr "$NB_GRPC_ADDR"
)
}
do_record() {
local in="$1" out="$2" cost="$3"
(
cd "$repo_root" || exit 1
go run ./scripts/e2e/agent-network-policy/cmd/usage_smoke record \
--account "$account_id" \
--user "$test_user" \
--group "$tight_group_id" \
--window-seconds "$TIGHT_WINDOW_SECONDS" \
--tokens-in "$in" \
--tokens-out "$out" \
--cost-usd "$cost" \
--token "$NB_PROXY_TOKEN" \
--addr "$NB_GRPC_ADDR"
)
}
# Stage 1 — fresh state. Selection must allow and pick our tight
# policy as attribution because it's the ONLY policy authorising the
# (tight_group, tight_provider) tuple. Also confirms the wire-level
# selected_policy_id round-trip from manager → grpc → smoke client.
echo "stage 1: initial check (consumption=0/$TIGHT_CAP)"
resp=$(do_check) || fail "initial check failed" "$resp"
decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
selected=$(printf '%s' "$resp" | jq -r '.selected_policy_id // ""')
window=$(printf '%s' "$resp" | jq -r '.window_seconds // ""')
[ "$decision" = "allow" ] \
|| fail "expected allow on fresh state" "$resp"
[ "$selected" = "$tight_policy_id" ] \
|| fail "selection did not pick the tight policy" \
"expected=$tight_policy_id got=$selected"
[ "$window" = "$TIGHT_WINDOW_SECONDS" ] \
|| fail "window_seconds mismatch on the wire" "expected=$TIGHT_WINDOW_SECONDS got=$window"
echo " → allow / selected=$selected / window=${window}s"
# Stage 2 — book half the cap. The next check must still allow.
echo "stage 2: record 100 input tokens (consumption=100/$TIGHT_CAP)"
do_record 100 0 0 >/dev/null || fail "record (stage 2) failed" ""
resp=$(do_check)
decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
[ "$decision" = "allow" ] \
|| fail "expected allow at half-cap" "$resp"
echo " → allow"
# Stage 3 — push to one token below cap. Headroom shrinks but
# decision stays allow.
echo "stage 3: record 99 more input tokens (consumption=199/$TIGHT_CAP)"
do_record 99 0 0 >/dev/null || fail "record (stage 3) failed" ""
resp=$(do_check)
decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
[ "$decision" = "allow" ] \
|| fail "expected allow at one-below-cap" "$resp"
echo " → allow"
# Stage 4 — exactly at cap. Selector treats consumed >= cap as
# exhausted, so the next check must deny with the canonical token cap
# code. The deny reason names the policy id so operators can debug
# from the access log.
echo "stage 4: record 1 final token (consumption=$TIGHT_CAP/$TIGHT_CAP)"
do_record 1 0 0 >/dev/null || fail "record (stage 4) failed" ""
resp=$(do_check) || fail "check after cap-exhaust failed" "$resp"
decision=$(printf '%s' "$resp" | jq -r '.decision // ""')
deny_code=$(printf '%s' "$resp" | jq -r '.deny_code // ""')
deny_reason=$(printf '%s' "$resp" | jq -r '.deny_reason // ""')
[ "$decision" = "deny" ] \
|| fail "expected DENY at cap" "$resp"
[ "$deny_code" = "llm_policy.token_cap_exceeded" ] \
|| fail "deny_code mismatch" "expected=llm_policy.token_cap_exceeded got=$deny_code resp=$resp"
echo " → DENY / deny_code=$deny_code"
echo " → deny_reason: $deny_reason"
[[ "$deny_reason" == *"$tight_policy_id"* ]] \
|| fail "deny_reason must name the policy id for operator debugging" "$deny_reason"
pass "selection algorithm flips allow → deny at cap-exhaust through the gRPC wire"

58
scripts/e2e/agent-network-policy/99-cleanup.sh
View File

@@ -1,58 +0,0 @@
#!/usr/bin/env bash
# 99-cleanup: idempotent teardown. Drops the policy, provider, group,
# and any consumption rows our test created so re-runs start fresh.
# shellcheck source=00-env.sh
source "$(dirname "$0")/00-env.sh"
# Policies first (FK / synth chain owns provider references). Sweep
# every policy whose name STARTS with the e2e prefix so per-run
# instances created by 50 (with $RUN_TAG suffixes) get cleaned up.
nb_api GET /api/agent-network/policies 2>/dev/null \
| jq -r --arg pfx "$NB_POLICY_NAME" '.[] | select(.name | startswith($pfx)) | .id' \
| while read -r orphan; do
[ -n "$orphan" ] || continue
code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \
-o /dev/null -w '%{http_code}' \
"$NB_API/api/agent-network/policies/$orphan" 2>&1) || true
echo "DELETE policy/$orphan -> $code"
done
# Providers — same prefix sweep. Both the 10-policy-create.sh's main
# provider AND every per-run tight provider 50 minted carry the
# NB_PROVIDER_NAME prefix.
nb_api GET /api/agent-network/providers 2>/dev/null \
| jq -r --arg pfx "$NB_PROVIDER_NAME" '.[] | select(.name | startswith($pfx)) | .id' \
| while read -r orphan; do
[ -n "$orphan" ] || continue
code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \
-o /dev/null -w '%{http_code}' \
"$NB_API/api/agent-network/providers/$orphan" 2>&1) || true
echo "DELETE provider/$orphan -> $code"
done
# Groups — prefix sweep. /api/groups doesn't error on a
# referenced-elsewhere group; if so, the delete is a no-op and we
# move on.
nb_api GET /api/groups 2>/dev/null \
| jq -r --arg pfx "$NB_GROUP_NAME" '.[] | select(.name | startswith($pfx)) | .id' \
| while read -r orphan; do
[ -n "$orphan" ] || continue
code=$(curl -fsS -X DELETE -H "Authorization: Token $NB_PAT" \
-o /dev/null -w '%{http_code}' \
"$NB_API/api/groups/$orphan" 2>&1) || true
echo "DELETE group/$orphan -> $code"
done
# We don't expose a consumption-delete endpoint — the rows survive
# until the management store is recycled. Document the residue here
# so anyone debugging "why are there old e2e rows" knows the source.
remaining=$(nb_api GET /api/agent-network/consumption 2>/dev/null \
| jq --arg pfx "e2e-anpol-user-" 'map(select(.dimension_id | startswith($pfx))) | length' \
|| echo "0")
[ "$remaining" = "0" ] || \
echo "(left $remaining e2e consumption rows in the store — there's no delete endpoint yet)"
rm -rf "$NB_STATE_DIR"
pass "tear-down complete (idempotent)"

197
scripts/e2e/agent-network-policy/cmd/usage_smoke/main.go
View File

@@ -1,197 +0,0 @@
// usage_smoke is the e2e helper that drives the new agent-network
// gRPC RPCs (CheckLLMPolicyLimits, RecordLLMUsage) against a local
// management server. Run from the bash suite as a `go run` so the
// proto types are always in sync with the management binary the
// suite is exercising.
//
// Two subcommands today:
// - record: invokes RecordLLMUsage with the supplied tokens / cost
// and exits 0 on success.
// - check: invokes CheckLLMPolicyLimits and prints the response as
// JSON on stdout so the bash test can assert on it via
// jq.
//
// Auth uses the same proxy bearer-token shape the real proxy uses
// (see proxy/internal/grpc/auth.go); the bash suite reads the token
// from the e2e env (NB_PROXY_TOKEN, defaulted to the Tilt literal).
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"strings"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/netbirdio/netbird/shared/management/proto"
)
// proxyTokenCreds mirrors proxy/internal/grpc.WithProxyToken's
// PerRPCCredentials so the e2e binary can reach the gRPC service
// using the same bearer-token shape the real proxy uses. Inlined
// because the production helper lives behind /internal/.
type proxyTokenCreds struct{ token string }
func (c proxyTokenCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
return map[string]string{"authorization": "Bearer " + c.token}, nil
}
// RequireTransportSecurity is false here because Tilt's management is
// plaintext on localhost — the e2e suite is the *only* caller of this
// binary, never production.
func (proxyTokenCreds) RequireTransportSecurity() bool { return false }
func main() {
if len(os.Args) < 2 {
usage()
}
cmd := os.Args[1]
os.Args = append(os.Args[:1], os.Args[2:]...)
switch cmd {
case "record":
runRecord()
case "check":
runCheck()
default:
usage()
}
}
func usage() {
fmt.Fprintln(os.Stderr, "usage: usage_smoke <record|check> [flags]")
os.Exit(2)
}
func runRecord() {
addr := flag.String("addr", "localhost:8080", "management gRPC address")
token := flag.String("token", os.Getenv("NB_PROXY_TOKEN"), "proxy token")
accountID := flag.String("account", "", "netbird account id")
userID := flag.String("user", "", "netbird user id (optional)")
groupID := flag.String("group", "", "netbird policy attribution group id (optional)")
groupsCSV := flag.String("groups", "", "CSV of caller group ids (for account-rule fan-out)")
windowSeconds := flag.Int64("window-seconds", 86_400, "window length in seconds (0 allowed when only account rules apply)")
tokensIn := flag.Int64("tokens-in", 0, "input tokens to add")
tokensOut := flag.Int64("tokens-out", 0, "output tokens to add")
costUSD := flag.Float64("cost-usd", 0, "USD cost to add")
flag.Parse()
if strings.TrimSpace(*token) == "" {
exitErr("--token is required (or set NB_PROXY_TOKEN)")
}
if strings.TrimSpace(*accountID) == "" {
exitErr("--account is required")
}
var groupIDs []string
for _, g := range strings.Split(*groupsCSV, ",") {
g = strings.TrimSpace(g)
if g != "" {
groupIDs = append(groupIDs, g)
}
}
if *userID == "" && *groupID == "" && len(groupIDs) == 0 {
exitErr("at least one of --user, --group, or --groups must be set")
}
conn := dial(*addr, *token)
defer conn.Close()
client := proto.NewProxyServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := client.RecordLLMUsage(ctx, &proto.RecordLLMUsageRequest{
AccountId: *accountID,
UserId: *userID,
GroupId: *groupID,
GroupIds: groupIDs,
WindowSeconds: *windowSeconds,
TokensInput: *tokensIn,
TokensOutput: *tokensOut,
CostUsd: *costUSD,
})
if err != nil {
exitErr(fmt.Sprintf("RecordLLMUsage: %v", err))
}
//nolint:forbidigo // e2e helper: stdout is the contract with the bash caller
fmt.Println("ok")
}
func runCheck() {
addr := flag.String("addr", "localhost:8080", "management gRPC address")
token := flag.String("token", os.Getenv("NB_PROXY_TOKEN"), "proxy token")
accountID := flag.String("account", "", "netbird account id")
userID := flag.String("user", "", "netbird user id (optional)")
groupsCSV := flag.String("groups", "", "CSV of caller group ids")
providerID := flag.String("provider", "", "agent-network provider id")
model := flag.String("model", "gpt-4o", "upstream model identifier")
flag.Parse()
if strings.TrimSpace(*token) == "" {
exitErr("--token is required (or set NB_PROXY_TOKEN)")
}
if strings.TrimSpace(*accountID) == "" {
exitErr("--account is required")
}
var groupIDs []string
for _, g := range strings.Split(*groupsCSV, ",") {
g = strings.TrimSpace(g)
if g != "" {
groupIDs = append(groupIDs, g)
}
}
conn := dial(*addr, *token)
defer conn.Close()
client := proto.NewProxyServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp, err := client.CheckLLMPolicyLimits(ctx, &proto.CheckLLMPolicyLimitsRequest{
AccountId: *accountID,
UserId: *userID,
GroupIds: groupIDs,
ProviderId: *providerID,
Model: *model,
})
if err != nil {
exitErr(fmt.Sprintf("CheckLLMPolicyLimits: %v", err))
}
out, _ := json.Marshal(map[string]any{
"decision": resp.GetDecision(),
"selected_policy_id": resp.GetSelectedPolicyId(),
"attribution_group_id": resp.GetAttributionGroupId(),
"window_seconds": resp.GetWindowSeconds(),
"deny_code": resp.GetDenyCode(),
"deny_reason": resp.GetDenyReason(),
})
//nolint:forbidigo // e2e helper: stdout is the contract with the bash caller
fmt.Println(string(out))
}
// dial connects to the management gRPC over plaintext. The bearer
// token is sent on every RPC via PerRPCCredentials matching the wire
// format the production proxy uses.
func dial(addr, token string) *grpc.ClientConn {
conn, err := grpc.NewClient(addr,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithPerRPCCredentials(proxyTokenCreds{token: token}),
)
if err != nil {
exitErr(fmt.Sprintf("dial %s: %v", addr, err))
}
return conn
}
func exitErr(msg string) {
fmt.Fprintln(os.Stderr, msg)
os.Exit(1)
}

44
scripts/e2e/agent-network-policy/run-all.sh
View File

@@ -1,44 +0,0 @@
#!/usr/bin/env bash
# run-all: every numbered test in order, halts on first FAIL,
# prints a `=== PASS x/y ===` summary. -k keeps state (skips
# 99-cleanup) so you can poke around the persisted policy /
# consumption rows afterwards.
set -uo pipefail
KEEP=0
[ "${1:-}" = "-k" ] && KEEP=1
export NB_KEEP_STATE="$KEEP"
cd "$(dirname "$0")"
scripts=(
01-tilt-restart.sh
10-policy-create.sh
20-policy-rejects-zero-window.sh
30-consumption-list-empty.sh
40-grpc-record-and-list.sh
50-grpc-allow-record-deny.sh
)
[ "$KEEP" -eq 0 ] && scripts+=(99-cleanup.sh)
passed=0
total=${#scripts[@]}
trap '[ "$KEEP" -eq 0 ] && bash ./99-cleanup.sh >/dev/null 2>&1 || true' EXIT
for s in "${scripts[@]}"; do
echo
echo "==================== $s ===================="
if bash "./$s"; then
passed=$((passed + 1))
else
rc=$?
echo
echo "=== FAIL $passed/$total ($s exit=$rc) ==="
exit "$rc"
fi
done
echo
echo "=== PASS $passed/$total ==="