mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-19 14:11:28 +02:00
Compare commits
47 Commits
feature/io
...
agent-netw
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
494b22848b | ||
|
|
a33fab103d | ||
|
|
3439500cc6 | ||
|
|
aa72df1654 | ||
|
|
da155bd60f | ||
|
|
52bed3ec26 | ||
|
|
378fe13257 | ||
|
|
520c912c07 | ||
|
|
9cd7e27027 | ||
|
|
0783605690 | ||
|
|
16544dbc58 | ||
|
|
f458c1f265 | ||
|
|
ec6f1b8c27 | ||
|
|
2cfe14d7ec | ||
|
|
85dd335836 | ||
|
|
5544761b47 | ||
|
|
1d372bb634 | ||
|
|
e290769df1 | ||
|
|
58c09ead21 | ||
|
|
c5503fdc7f | ||
|
|
6b69f5c05d | ||
|
|
db9fcf39ef | ||
|
|
52faa202b2 | ||
|
|
f5ce0bc65a | ||
|
|
dc91325ac1 | ||
|
|
1796b2a1d8 | ||
|
|
dd87760f3d | ||
|
|
63d16a7cd4 | ||
|
|
fa691721d0 | ||
|
|
d46e2574a7 | ||
|
|
649a867cd3 | ||
|
|
a39b3c4af4 | ||
|
|
b1337f09d0 | ||
|
|
08e187ccb7 | ||
|
|
93cdf64a19 | ||
|
|
c04fb1c388 | ||
|
|
5353cab54f | ||
|
|
a64a417ea5 | ||
|
|
652c5c8b68 | ||
|
|
4d2b8b407b | ||
|
|
03e02c86ce | ||
|
|
789d416215 | ||
|
|
6415215126 | ||
|
|
1ae352a08d | ||
|
|
d928bcb630 | ||
|
|
47b2667653 | ||
|
|
875dda1708 |
@@ -3,7 +3,7 @@
|
||||
[branches]
|
||||
main = "main"
|
||||
perennials = []
|
||||
perennial-regex = ""
|
||||
perennial-regex = "^release-"
|
||||
|
||||
[create]
|
||||
new-branch-type = "feature"
|
||||
|
||||
@@ -2,7 +2,7 @@ name: Check License Dependencies
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, "release-*"]
|
||||
paths:
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
|
||||
1
.github/workflows/frontend-ui.yml
vendored
1
.github/workflows/frontend-ui.yml
vendored
@@ -10,6 +10,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
paths:
|
||||
- "client/ui/frontend/**"
|
||||
- "client/ui/i18n/**"
|
||||
|
||||
1
.github/workflows/golang-test-darwin.yml
vendored
1
.github/workflows/golang-test-darwin.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
|
||||
1
.github/workflows/golang-test-freebsd.yml
vendored
1
.github/workflows/golang-test-freebsd.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
|
||||
1
.github/workflows/golang-test-linux.yml
vendored
1
.github/workflows/golang-test-linux.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
|
||||
1
.github/workflows/golang-test-windows.yml
vendored
1
.github/workflows/golang-test-windows.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
|
||||
1
.github/workflows/install-script-test.yml
vendored
1
.github/workflows/install-script-test.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
paths:
|
||||
- "release_files/install.sh"
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
|
||||
15
.github/workflows/release.yml
vendored
15
.github/workflows/release.yml
vendored
@@ -6,6 +6,7 @@ on:
|
||||
- "v*"
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
@@ -254,15 +255,23 @@ jobs:
|
||||
id: tag_and_push_images
|
||||
if: |
|
||||
(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
(github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-')))
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# $GITHUB_REF / $GITHUB_EVENT_NAME are read from the runner
|
||||
# environment rather than substituted into this script with the
|
||||
# workflow expression syntax: branch names may legally contain
|
||||
# $(…), and interpolating github.ref would execute it.
|
||||
resolve_tags() {
|
||||
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
||||
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
|
||||
echo "pr-${{ github.event.pull_request.number }}"
|
||||
else
|
||||
elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then
|
||||
echo "main sha-$(git rev-parse --short HEAD)"
|
||||
else
|
||||
# Release branches get an immutable sha-* tag only — the floating
|
||||
# "main" tag must never move from a release branch.
|
||||
echo "sha-$(git rev-parse --short HEAD)"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
16
.github/workflows/sync-tag.yml
vendored
16
.github/workflows/sync-tag.yml
vendored
@@ -9,21 +9,9 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# Receiving workflows (cloud sync-tag, mobile bump-netbird) expect the short
|
||||
# tag form (e.g. v0.30.0), not refs/tags/v0.30.0 — github.ref_name, not github.ref.
|
||||
# The receiving bump-netbird workflows expect the short tag form
|
||||
# (e.g. v0.30.0), not refs/tags/v0.30.0 — github.ref_name, not github.ref.
|
||||
jobs:
|
||||
trigger_sync_tag:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Trigger release tag sync
|
||||
uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2
|
||||
with:
|
||||
workflow: sync-tag.yml
|
||||
ref: main
|
||||
repo: ${{ secrets.UPSTREAM_REPO }}
|
||||
token: ${{ secrets.NC_GITHUB_TOKEN }}
|
||||
inputs: '{ "tag": "${{ github.ref_name }}" }'
|
||||
|
||||
trigger_android_bump:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.created && !github.event.deleted && startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
paths:
|
||||
- "infrastructure_files/**"
|
||||
|
||||
42
.github/workflows/ui-translations.yml
vendored
Normal file
42
.github/workflows/ui-translations.yml
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
name: UI Translations
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "client/ui/i18n/locales/**"
|
||||
- "client/ui/i18n/check-translations.mjs"
|
||||
- ".github/workflows/ui-translations.yml"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "client/ui/i18n/locales/**"
|
||||
- "client/ui/i18n/check-translations.mjs"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-translations:
|
||||
name: Check translation key parity
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
|
||||
# English (en) is the source of truth for translation keys; every other
|
||||
# locale declared in _index.json must carry the exact same key set.
|
||||
- name: Check translation key parity
|
||||
run: node client/ui/i18n/check-translations.mjs
|
||||
1
.github/workflows/wasm-build-validation.yml
vendored
1
.github/workflows/wasm-build-validation.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "release-*"
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -468,6 +468,13 @@ checksum:
|
||||
- glob: ./infrastructure_files/migrate-to-enterprise.sh
|
||||
|
||||
release:
|
||||
# The signing pipeline (netbirdio/sign-pipelines, dispatched by
|
||||
# trigger_signer) marks the release latest once the Windows and macOS
|
||||
# artifacts are signed. Without this override goreleaser marks it latest
|
||||
# at publish time, while those artifacts are still unsigned.
|
||||
make_latest: false
|
||||
# Mark x.y.z-rc.* and other prerelease tags as prereleases on GitHub.
|
||||
prerelease: auto
|
||||
extra_files:
|
||||
- glob: ./infrastructure_files/getting-started-with-zitadel.sh
|
||||
- glob: ./release_files/install.sh
|
||||
|
||||
@@ -144,3 +144,11 @@ uploads:
|
||||
target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }}
|
||||
username: dev@wiretrustee.com
|
||||
method: PUT
|
||||
|
||||
release:
|
||||
# Uploads into the release created by the main .goreleaser.yaml run.
|
||||
# make_latest stays false everywhere: the signing pipeline
|
||||
# (netbirdio/sign-pipelines) marks the release latest after the Windows
|
||||
# and macOS artifacts are signed.
|
||||
make_latest: false
|
||||
prerelease: auto
|
||||
|
||||
@@ -43,3 +43,11 @@ checksum:
|
||||
name_template: "{{ .ProjectName }}_darwin_checksums.txt"
|
||||
changelog:
|
||||
disable: true
|
||||
|
||||
release:
|
||||
# Uploads into the release created by the main .goreleaser.yaml run.
|
||||
# make_latest stays false everywhere: the signing pipeline
|
||||
# (netbirdio/sign-pipelines) marks the release latest after the Windows
|
||||
# and macOS artifacts are signed.
|
||||
make_latest: false
|
||||
prerelease: auto
|
||||
|
||||
@@ -134,3 +134,11 @@ uploads:
|
||||
target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }}
|
||||
username: dev@wiretrustee.com
|
||||
method: PUT
|
||||
|
||||
release:
|
||||
# Uploads into the release created by the main .goreleaser.yaml run.
|
||||
# make_latest stays false everywhere: the signing pipeline
|
||||
# (netbirdio/sign-pipelines) marks the release latest after the Windows
|
||||
# and macOS artifacts are signed.
|
||||
make_latest: false
|
||||
prerelease: auto
|
||||
|
||||
@@ -40,6 +40,35 @@ You can then use this private endpoint to configure your AI agents, whether that
|
||||
Full step-by-step setup:
|
||||
**https://docs.netbird.io/agent-network/quickstart**
|
||||
|
||||
## Client settings that don't follow the endpoint
|
||||
|
||||
Most of an agent's traffic follows the base URL you hand it, but a few
|
||||
client-side checks call their vendor directly and never reach the proxy. On a
|
||||
network that blocks direct egress they fail even though inference works, so
|
||||
they are worth setting once when you roll the endpoint out.
|
||||
|
||||
For Claude Code:
|
||||
|
||||
- **Fast mode** checks availability against `api.anthropic.com` rather than the
|
||||
configured base URL. Set `CLAUDE_CODE_SKIP_FAST_MODE_ORG_CHECK=1` when the
|
||||
agent authenticates with `ANTHROPIC_AUTH_TOKEN` alone (the usual shape when
|
||||
the proxy injects the real provider key) or when a TLS-inspecting proxy
|
||||
answers the check itself. Set
|
||||
`CLAUDE_CODE_SKIP_FAST_MODE_NETWORK_ERRORS=1` when the network refuses the
|
||||
connection outright. Fast mode is an Anthropic-API feature, so it is
|
||||
unavailable on a Bedrock- or Vertex-backed endpoint whatever you set.
|
||||
- **Model discovery** is off by default. Set
|
||||
`CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` for the picker to list the
|
||||
models your policies authorise; the proxy filters the response to that set.
|
||||
The client gives discovery a three-second budget and treats any redirect as
|
||||
a failure, so the endpoint must serve `/v1/models` directly.
|
||||
- **The WebFetch domain safety check** also calls `api.anthropic.com` directly
|
||||
and is unaffected by the variables above.
|
||||
|
||||
Allowing direct egress to `api.anthropic.com` covers the network cases but not
|
||||
the credential one, where the check reaches Anthropic and is rejected because
|
||||
the agent presents a proxy-issued key.
|
||||
|
||||
## Architecture
|
||||
|
||||
Agent Network is built on two existing NetBird capabilities:
|
||||
|
||||
@@ -204,8 +204,9 @@ func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener
|
||||
return nil, fmt.Errorf("failed to get OAuth flow: %v", err)
|
||||
}
|
||||
|
||||
// An empty hint is deliberate, not a fallback: a fresh or logged-out profile
|
||||
// leaves the choice to the IdP, which is how accounts get switched.
|
||||
// An empty hint is deliberate, not a fallback: a fresh profile leaves the
|
||||
// choice to the IdP. Switching accounts is done by switching or removing
|
||||
// profiles, not by logging out — logout keeps the email.
|
||||
if a.cfgPath != "" {
|
||||
if hint := readProfileEmail(a.cfgPath); hint != "" {
|
||||
if setter, ok := oAuthFlow.(loginHintSetter); ok {
|
||||
|
||||
@@ -22,7 +22,8 @@ type Profile struct {
|
||||
ID string
|
||||
Name string
|
||||
// Email is the account this profile last logged in with, "" if it never
|
||||
// completed an SSO login or was logged out. See profile_state.go.
|
||||
// completed an SSO login. Kept across logouts; cleared when the profile is
|
||||
// removed. See profile_state.go.
|
||||
Email string
|
||||
IsActive bool
|
||||
}
|
||||
@@ -200,11 +201,9 @@ func (pm *ProfileManager) LogoutProfile(id string) error {
|
||||
return fmt.Errorf("failed to save config: %w", err)
|
||||
}
|
||||
|
||||
// Not fatal: a stale hint costs an account switch, not the logout itself.
|
||||
if err := removeProfileEmail(configPath); err != nil {
|
||||
log.Warnf("failed to clear stored account email for profile %s: %v", id, err)
|
||||
}
|
||||
|
||||
// The stored account email is kept on purpose, matching the desktop and CLI
|
||||
// logout semantics: the next login passes it as the login_hint so the IdP
|
||||
// preselects the account. Removing the profile is what deletes it.
|
||||
log.Infof("logged out from profile: %s", id)
|
||||
return nil
|
||||
}
|
||||
@@ -224,11 +223,24 @@ func (pm *ProfileManager) RenameProfile(id string, newName string) error {
|
||||
|
||||
// RemoveProfile deletes a profile
|
||||
func (pm *ProfileManager) RemoveProfile(id string) error {
|
||||
configPath, err := pm.getProfileConfigPath(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use ServiceManager (removes profile from profiles/ directory)
|
||||
if err := pm.serviceMgr.RemoveProfile(profilemanager.ID(id), androidUsername); err != nil {
|
||||
return fmt.Errorf("failed to remove profile: %w", err)
|
||||
}
|
||||
|
||||
// The account file is this package's, not the ServiceManager's, so it must
|
||||
// go here. The default profile has a fixed filename, so a recreated one
|
||||
// would otherwise inherit the deleted profile's email as its login_hint.
|
||||
// Not fatal: the profile itself is gone.
|
||||
if err := removeProfileEmail(configPath); err != nil {
|
||||
log.Warnf("failed to remove stored account email for profile %s: %v", id, err)
|
||||
}
|
||||
|
||||
log.Infof("removed profile: %s", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -90,10 +90,10 @@ func writeProfileEmail(configPath string, email string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeProfileEmail drops the stored account email. Called on logout: while the
|
||||
// email is on disk it goes out as a login_hint, which would steer the next login
|
||||
// straight back into the account just logged out of. Mirrors the desktop UI's
|
||||
// RemoveProfileState call.
|
||||
// removeProfileEmail drops the stored account email. Called on profile removal,
|
||||
// not on logout: a logged-out profile keeps its email so the next login passes
|
||||
// it as the login_hint, matching the desktop and CLI semantics. Mirrors the
|
||||
// desktop UI's RemoveProfileState call.
|
||||
func removeProfileEmail(configPath string) error {
|
||||
accountPath, err := profileAccountPathFor(configPath)
|
||||
if err != nil {
|
||||
|
||||
@@ -127,10 +127,10 @@ func TestWriteThenReadProfileEmail(t *testing.T) {
|
||||
t.Fatalf("remove: %v", err)
|
||||
}
|
||||
if got := readProfileEmail(configPath); got != "" {
|
||||
t.Errorf("expected no email after logout, got %q", got)
|
||||
t.Errorf("expected no email after removal, got %q", got)
|
||||
}
|
||||
|
||||
// Logout may run on a never-logged-in profile, so a second remove must pass.
|
||||
// Removal may run on a never-logged-in profile, so a second remove must pass.
|
||||
if err := removeProfileEmail(configPath); err != nil {
|
||||
t.Fatalf("second remove should be a no-op: %v", err)
|
||||
}
|
||||
|
||||
@@ -305,6 +305,12 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string {
|
||||
return domain
|
||||
}
|
||||
|
||||
// A reverse zone names an address prefix, so it follows the address rules,
|
||||
// which also keeps its digit labels intact.
|
||||
if zone, ok := a.anonymizeReverseZone(baseDomain); ok {
|
||||
return withTrailingDot(zone, hasDot)
|
||||
}
|
||||
|
||||
if suffix := protectedSuffix(baseDomain); suffix != "" {
|
||||
if a.level < LevelStrict || baseDomain == suffix || suffix == infraDomain {
|
||||
return domain
|
||||
@@ -405,6 +411,10 @@ func (a *Anonymizer) AnonymizeString(str string) string {
|
||||
ipv4Regex := regexp.MustCompile(`\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b`)
|
||||
ipv6Regex := regexp.MustCompile(`\b([0-9a-fA-F:]+:+[0-9a-fA-F]{0,4})(?:%[0-9a-zA-Z]+)?(?:\/[0-9]{1,3})?(?::[0-9]{1,5})?\b`)
|
||||
|
||||
// Reverse zones go first and are then held out of the passes below: their
|
||||
// labels are digits, which the address patterns would otherwise consume.
|
||||
str, restoreZones := a.replaceReverseZones(str)
|
||||
|
||||
str = ipv4Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString)
|
||||
str = ipv6Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString)
|
||||
|
||||
@@ -425,7 +435,7 @@ func (a *Anonymizer) AnonymizeString(str string) string {
|
||||
str = wgKeyRegex.ReplaceAllStringFunc(str, a.AnonymizeWGKey)
|
||||
}
|
||||
|
||||
return str
|
||||
return restoreZones(str)
|
||||
}
|
||||
|
||||
// sortedDomains returns the domain mappings longest-first, so a full-FQDN
|
||||
|
||||
174
client/anonymize/reverse_zone.go
Normal file
174
client/anonymize/reverse_zone.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package anonymize
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
reverseZoneSuffixV4 = ".in-addr.arpa"
|
||||
reverseZoneSuffixV6 = ".ip6.arpa"
|
||||
|
||||
v6Nibbles = 32
|
||||
v4Octets = 4
|
||||
)
|
||||
|
||||
// reverseZoneRegexes match a reverse zone or a full reverse name in free text.
|
||||
// They are applied before the address passes of AnonymizeString, whose IPv4
|
||||
// pattern would otherwise consume the digit labels of a zone and replace parts
|
||||
// of it with unrelated addresses.
|
||||
var reverseZoneRegexes = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?:[0-9]{1,3}\.){1,4}in-addr\.arpa\b`),
|
||||
regexp.MustCompile(`(?:[0-9a-fA-F]\.){1,32}ip6\.arpa\b`),
|
||||
}
|
||||
|
||||
// anonymizeReverseZone maps a reverse zone to the zone of the anonymized form
|
||||
// of the prefix it encodes, so it follows the address rules rather than the
|
||||
// domain ones: the zone of an address that is preserved is preserved too, and
|
||||
// the zone of one that is replaced names the replacement. This keeps a reverse
|
||||
// zone recognizable as such, and consistent with the addresses it belongs to
|
||||
// elsewhere in the same output. It reports false for anything that is not a
|
||||
// reverse zone.
|
||||
func (a *Anonymizer) anonymizeReverseZone(domain string) (string, bool) {
|
||||
prefix, labelCount, suffix, ok := parseReverseZone(domain)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
anonymized := a.AnonymizeIP(prefix)
|
||||
if anonymized == prefix {
|
||||
return domain, true
|
||||
}
|
||||
|
||||
return reverseZoneName(anonymized, labelCount) + suffix, true
|
||||
}
|
||||
|
||||
// replaceReverseZones anonymizes every reverse zone in str and swaps each one
|
||||
// for a placeholder, returning a function that puts the anonymized zones back.
|
||||
// The placeholders carry no dots, digits or colons, so no later pass matches
|
||||
// them.
|
||||
func (a *Anonymizer) replaceReverseZones(str string) (string, func(string) string) {
|
||||
var zones []string
|
||||
|
||||
for _, re := range reverseZoneRegexes {
|
||||
str = re.ReplaceAllStringFunc(str, func(match string) string {
|
||||
zone, ok := a.anonymizeReverseZone(match)
|
||||
if !ok {
|
||||
return match
|
||||
}
|
||||
|
||||
zones = append(zones, zone)
|
||||
return reverseZonePlaceholder(len(zones) - 1)
|
||||
})
|
||||
}
|
||||
|
||||
if len(zones) == 0 {
|
||||
return str, func(s string) string { return s }
|
||||
}
|
||||
|
||||
return str, func(s string) string {
|
||||
for i, zone := range zones {
|
||||
s = strings.ReplaceAll(s, reverseZonePlaceholder(i), zone)
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
func reverseZonePlaceholder(index int) string {
|
||||
return "\x00reversezone" + strconv.Itoa(index) + "\x00"
|
||||
}
|
||||
|
||||
// parseReverseZone turns a reverse zone into the address of the prefix its
|
||||
// labels spell backwards, padding the absent low-order part with zeroes, and
|
||||
// returns the label count and zone suffix so the name can be rebuilt.
|
||||
func parseReverseZone(domain string) (netip.Addr, int, string, bool) {
|
||||
lower := strings.ToLower(domain)
|
||||
|
||||
switch {
|
||||
case strings.HasSuffix(lower, reverseZoneSuffixV4):
|
||||
labels := strings.Split(strings.TrimSuffix(lower, reverseZoneSuffixV4), ".")
|
||||
addr, ok := reverseZoneAddrV4(labels)
|
||||
return addr, len(labels), reverseZoneSuffixV4, ok
|
||||
case strings.HasSuffix(lower, reverseZoneSuffixV6):
|
||||
labels := strings.Split(strings.TrimSuffix(lower, reverseZoneSuffixV6), ".")
|
||||
addr, ok := reverseZoneAddrV6(labels)
|
||||
return addr, len(labels), reverseZoneSuffixV6, ok
|
||||
default:
|
||||
return netip.Addr{}, 0, "", false
|
||||
}
|
||||
}
|
||||
|
||||
func reverseZoneAddrV4(labels []string) (netip.Addr, bool) {
|
||||
if len(labels) == 0 || len(labels) > v4Octets {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
|
||||
var octets [v4Octets]byte
|
||||
for i, label := range labels {
|
||||
octet, err := strconv.ParseUint(label, 10, 8)
|
||||
if err != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
octets[len(labels)-1-i] = byte(octet)
|
||||
}
|
||||
|
||||
return netip.AddrFrom4(octets), true
|
||||
}
|
||||
|
||||
func reverseZoneAddrV6(labels []string) (netip.Addr, bool) {
|
||||
if len(labels) == 0 || len(labels) > v6Nibbles {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
|
||||
nibbles := make([]byte, 0, v6Nibbles)
|
||||
for i := len(labels) - 1; i >= 0; i-- {
|
||||
if len(labels[i]) != 1 || !isHexDigit(labels[i][0]) {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
nibbles = append(nibbles, labels[i][0])
|
||||
}
|
||||
for len(nibbles) < v6Nibbles {
|
||||
nibbles = append(nibbles, '0')
|
||||
}
|
||||
|
||||
var groups []string
|
||||
for i := 0; i < len(nibbles); i += 4 {
|
||||
groups = append(groups, string(nibbles[i:i+4]))
|
||||
}
|
||||
|
||||
addr, err := netip.ParseAddr(strings.Join(groups, ":"))
|
||||
if err != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
|
||||
return addr, true
|
||||
}
|
||||
|
||||
// reverseZoneName spells the first labelCount labels of addr backwards, the
|
||||
// inverse of parseReverseZone, without the zone suffix.
|
||||
func reverseZoneName(addr netip.Addr, labelCount int) string {
|
||||
labels := make([]string, 0, labelCount)
|
||||
|
||||
if addr.Is4() {
|
||||
octets := addr.As4()
|
||||
for i := labelCount - 1; i >= 0; i-- {
|
||||
labels = append(labels, strconv.Itoa(int(octets[i])))
|
||||
}
|
||||
return strings.Join(labels, ".")
|
||||
}
|
||||
|
||||
address := addr.As16()
|
||||
nibbles := hex.EncodeToString(address[:])
|
||||
for i := labelCount - 1; i >= 0; i-- {
|
||||
labels = append(labels, string(nibbles[i]))
|
||||
}
|
||||
|
||||
return strings.Join(labels, ".")
|
||||
}
|
||||
|
||||
func isHexDigit(c byte) bool {
|
||||
return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'
|
||||
}
|
||||
171
client/anonymize/reverse_zone_test.go
Normal file
171
client/anonymize/reverse_zone_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package anonymize
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newLeveledAnonymizer(level Level) *Anonymizer {
|
||||
a := NewAnonymizer(DefaultAddresses())
|
||||
a.SetLevel(level)
|
||||
return a
|
||||
}
|
||||
|
||||
// TestAnonymizeDomainReverseZone covers reverse zones going through the address
|
||||
// rules instead of the domain ones, so a zone stays a zone and an address that
|
||||
// is preserved keeps the zone that names it.
|
||||
func TestAnonymizeDomainReverseZone(t *testing.T) {
|
||||
// 100.64.0.0/10 is the overlay range, which is CGNAT: preserved at the
|
||||
// default level and replaced from the internal pool at the strict one
|
||||
const overlayZone = "64.100.in-addr.arpa"
|
||||
|
||||
t.Run("overlay zone preserved at the default level", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
assert.Equal(t, overlayZone, a.AnonymizeDomain(overlayZone), "should keep the zone of a preserved address")
|
||||
})
|
||||
|
||||
t.Run("private zone preserved at the default level", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
assert.Equal(t, "168.192.in-addr.arpa", a.AnonymizeDomain("168.192.in-addr.arpa"), "should keep the zone of a private address")
|
||||
})
|
||||
|
||||
t.Run("overlay zone replaced at the strict level", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelStrict)
|
||||
|
||||
got := a.AnonymizeDomain(overlayZone)
|
||||
require.True(t, strings.HasSuffix(got, reverseZoneSuffixV4), "should stay a reverse zone, got %q", got)
|
||||
assert.NotEqual(t, overlayZone, got, "should replace the encoded prefix")
|
||||
assert.Len(t, strings.Split(strings.TrimSuffix(got, reverseZoneSuffixV4), "."), 2,
|
||||
"should keep the label count, got %q", got)
|
||||
})
|
||||
|
||||
t.Run("public zone replaced at the default level", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
got := a.AnonymizeDomain("113.0.203.in-addr.arpa")
|
||||
require.True(t, strings.HasSuffix(got, reverseZoneSuffixV4), "should stay a reverse zone, got %q", got)
|
||||
assert.NotEqual(t, "113.0.203.in-addr.arpa", got, "should replace a public prefix")
|
||||
})
|
||||
|
||||
t.Run("zone of an address keeps that address mapping", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
anonymizedAddr := a.AnonymizeIPString("203.0.113.7")
|
||||
got := a.AnonymizeDomain("7.113.0.203.in-addr.arpa")
|
||||
|
||||
octets := strings.Split(anonymizedAddr, ".")
|
||||
want := octets[3] + "." + octets[2] + "." + octets[1] + "." + octets[0] + reverseZoneSuffixV4
|
||||
assert.Equal(t, want, got, "should name the same replacement as the address itself")
|
||||
})
|
||||
|
||||
t.Run("ipv6 nibble labels stay single digits", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
zone := "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6
|
||||
got := a.AnonymizeDomain(zone)
|
||||
|
||||
require.True(t, strings.HasSuffix(got, reverseZoneSuffixV6), "should stay a reverse zone, got %q", got)
|
||||
labels := strings.Split(strings.TrimSuffix(got, reverseZoneSuffixV6), ".")
|
||||
assert.Len(t, labels, 28, "should keep every nibble label, got %q", got)
|
||||
for _, label := range labels {
|
||||
assert.Len(t, label, 1, "nibble label %q should stay a single digit", label)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trailing dot is kept", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
assert.Equal(t, "64.100.in-addr.arpa.", a.AnonymizeDomain("64.100.in-addr.arpa."), "should keep the trailing dot")
|
||||
})
|
||||
|
||||
t.Run("a domain that only looks like a zone is anonymized as a domain", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
got := a.AnonymizeDomain("not-a-zone.in-addr.arpa")
|
||||
assert.NotContains(t, got, "in-addr.arpa", "should fall back to domain anonymization")
|
||||
})
|
||||
}
|
||||
|
||||
// TestAnonymizeStringReverseZone verifies that a zone inside free text, such as
|
||||
// a DNS log line, is not chewed up by the address passes. The IPv4 pattern
|
||||
// matches any run of dotted digits, which a reverse zone is made of.
|
||||
func TestAnonymizeStringReverseZone(t *testing.T) {
|
||||
t.Run("ipv6 zone survives the address passes", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
zone := "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6
|
||||
got := a.AnonymizeString("question: domain=" + zone + " type=PTR")
|
||||
|
||||
assert.Contains(t, got, "type=PTR", "should keep the rest of the line")
|
||||
assert.NotContains(t, got, "198.51.100", "should not rewrite nibble labels as an address")
|
||||
|
||||
labels := strings.Split(strings.TrimSuffix(strings.TrimPrefix(got, "question: domain="), reverseZoneSuffixV6+" type=PTR"), ".")
|
||||
assert.Len(t, labels, 28, "should keep every nibble label, got %q", got)
|
||||
})
|
||||
|
||||
t.Run("preserved ipv4 zone is untouched", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
line := "reverse zone 64.100.in-addr.arpa registered"
|
||||
assert.Equal(t, line, a.AnonymizeString(line), "should keep the zone of a preserved address")
|
||||
})
|
||||
|
||||
t.Run("public ipv4 zone is replaced consistently", func(t *testing.T) {
|
||||
a := newLeveledAnonymizer(LevelDefault)
|
||||
|
||||
got := a.AnonymizeString("zone 113.0.203.in-addr.arpa and address 203.0.113.7")
|
||||
assert.NotContains(t, got, "113.0.203.in-addr.arpa", "should replace the zone")
|
||||
assert.NotContains(t, got, "203.0.113.7", "should replace the address")
|
||||
assert.Contains(t, got, reverseZoneSuffixV4, "should keep the zone suffix")
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseReverseZone(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
zone string
|
||||
addr string
|
||||
labels int
|
||||
}{
|
||||
{name: "v4 two labels", zone: "0.100" + reverseZoneSuffixV4, addr: "100.0.0.0", labels: 2},
|
||||
{name: "v4 three labels", zone: "1.168.192" + reverseZoneSuffixV4, addr: "192.168.1.0", labels: 3},
|
||||
{name: "v4 full address", zone: "7.113.0.203" + reverseZoneSuffixV4, addr: "203.0.113.7", labels: 4},
|
||||
{
|
||||
name: "v6 prefix",
|
||||
zone: "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6,
|
||||
addr: "2::",
|
||||
labels: 28,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
addr, labels, suffix, ok := parseReverseZone(tc.zone)
|
||||
require.True(t, ok, "should decode the reverse zone")
|
||||
assert.Equal(t, tc.addr, addr.String(), "should decode to the encoded prefix")
|
||||
assert.Equal(t, tc.labels, labels, "should count the labels")
|
||||
assert.Equal(t, tc.zone, reverseZoneName(addr, labels)+suffix, "should re-encode to the original zone")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReverseZoneRejectsNonZones(t *testing.T) {
|
||||
tests := []string{
|
||||
"example.com",
|
||||
"in-addr.arpa",
|
||||
"x.100" + reverseZoneSuffixV4,
|
||||
"256" + reverseZoneSuffixV4,
|
||||
"1.2.3.4.5" + reverseZoneSuffixV4,
|
||||
"ab" + reverseZoneSuffixV6,
|
||||
"g" + reverseZoneSuffixV6,
|
||||
}
|
||||
|
||||
for _, zone := range tests {
|
||||
t.Run(zone, func(t *testing.T) {
|
||||
_, _, _, ok := parseReverseZone(zone)
|
||||
assert.False(t, ok, "should reject %q", zone)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -121,7 +120,7 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str
|
||||
loginRequest := proto.LoginRequest{
|
||||
SetupKey: providedSetupKey,
|
||||
ManagementUrl: managementURL,
|
||||
IsUnixDesktopClient: isUnixRunningDesktop(),
|
||||
IsUnixDesktopClient: util.HasGraphicalSession(),
|
||||
Hostname: hostName,
|
||||
DnsLabels: dnsLabelsReq,
|
||||
ProfileName: &handle,
|
||||
@@ -189,7 +188,8 @@ func doExtendSession(ctx context.Context, cmd *cobra.Command) error {
|
||||
|
||||
client := proto.NewDaemonServiceClient(conn)
|
||||
|
||||
req := &proto.RequestExtendAuthSessionRequest{}
|
||||
// the CLI runs in the user's session, the daemon does not: tell it what we can see
|
||||
req := &proto.RequestExtendAuthSessionRequest{HasGraphicalSession: util.HasGraphicalSession()}
|
||||
// Pre-fill the IdP login hint from the active profile so the user
|
||||
// doesn't have to retype their email. Best-effort: we still proceed
|
||||
// without a hint if the lookup fails.
|
||||
@@ -408,7 +408,7 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *pro
|
||||
hint = profileState.Email
|
||||
}
|
||||
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isUnixRunningDesktop(), false, hint)
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -458,14 +458,6 @@ func openURL(cmd *cobra.Command, verificationURIComplete, userCode string, noBro
|
||||
}
|
||||
}
|
||||
|
||||
// isUnixRunningDesktop checks if a Linux OS is running desktop environment
|
||||
func isUnixRunningDesktop() bool {
|
||||
if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" {
|
||||
return false
|
||||
}
|
||||
return os.Getenv("DESKTOP_SESSION") != "" || os.Getenv("XDG_CURRENT_DESKTOP") != ""
|
||||
}
|
||||
|
||||
func setEnvAndFlags(cmd *cobra.Command) error {
|
||||
SetFlagsFromEnvVars(rootCmd)
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal"
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
nbnet "github.com/netbirdio/netbird/client/net"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
"github.com/netbirdio/netbird/client/server"
|
||||
"github.com/netbirdio/netbird/client/system"
|
||||
"github.com/netbirdio/netbird/shared/management/domain"
|
||||
@@ -626,7 +626,7 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte
|
||||
NatExternalIPs: natExternalIPs,
|
||||
CleanNATExternalIPs: natExternalIPs != nil && len(natExternalIPs) == 0,
|
||||
CustomDNSAddress: customDNSAddressConverted,
|
||||
IsUnixDesktopClient: isUnixRunningDesktop(),
|
||||
IsUnixDesktopClient: util.HasGraphicalSession(),
|
||||
Hostname: hostName,
|
||||
ExtraIFaceBlacklist: extraIFaceBlackList,
|
||||
DnsLabels: dnsLabels,
|
||||
|
||||
@@ -42,6 +42,7 @@ type aclManager struct {
|
||||
optionalEntries map[string][]entry
|
||||
ipsetStore *ipsetStore
|
||||
v6 bool
|
||||
ipsetSupported bool
|
||||
|
||||
stateManager *statemanager.Manager
|
||||
}
|
||||
@@ -60,6 +61,8 @@ func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*acl
|
||||
func (m *aclManager) init(stateManager *statemanager.Manager) error {
|
||||
m.stateManager = stateManager
|
||||
|
||||
m.ipsetSupported = m.probeIPSetSupport()
|
||||
|
||||
m.seedInitialEntries()
|
||||
m.seedInitialOptionalEntries()
|
||||
|
||||
@@ -91,6 +94,12 @@ func (m *aclManager) AddPeerFiltering(
|
||||
if m.v6 && ipsetName != "" {
|
||||
ipsetName += "-v6"
|
||||
}
|
||||
// When the kernel lacks the required ipset hash module, fall back to
|
||||
// per-IP iptables rules (pre-0.68 behavior) so ACLs keep working instead
|
||||
// of silently leaving the chain empty.
|
||||
if ipsetName != "" && !m.ipsetSupported {
|
||||
ipsetName = ""
|
||||
}
|
||||
proto := protoForFamily(protocol, m.v6)
|
||||
specs := filterRuleSpecs(ip, proto, sPort, dPort, action, ipsetName)
|
||||
|
||||
@@ -498,6 +507,40 @@ func transformIPsetName(ipsetName string, sPort, dPort *firewall.Port, action fi
|
||||
}
|
||||
}
|
||||
|
||||
// probeIPSetSupport checks whether the kernel can create the ipset type used for
|
||||
// ACL rules. On kernels lacking the required ipset hash module, ipset creation
|
||||
// fails (e.g. "invalid argument"), which would otherwise leave the ACL chain
|
||||
// empty and silently drop all policy-permitted inbound traffic. When unsupported,
|
||||
// the manager falls back to per-IP iptables rules.
|
||||
func (m *aclManager) probeIPSetSupport() bool {
|
||||
// Use a unique name so concurrent processes don't collide and we only ever
|
||||
// destroy the set we created ourselves. ipset names are limited to 31 chars,
|
||||
// so use a short random suffix.
|
||||
probeName := "nb-probe-" + uuid.New().String()[:8]
|
||||
|
||||
opts := ipset.CreateOptions{
|
||||
Replace: true,
|
||||
}
|
||||
if m.v6 {
|
||||
opts.Family = ipset.FamilyIPV6
|
||||
}
|
||||
|
||||
if err := ipset.Create(probeName, ipset.TypeHashNet, opts); err != nil {
|
||||
log.Warnf("ipset is not available (failed to create probe set: %v); "+
|
||||
"falling back to per-IP iptables ACL rules. Ensure the kernel provides "+
|
||||
"the ipset hash:net module (ip_set_hash_net) for better performance with large rule sets", err)
|
||||
return false
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := ipset.Destroy(probeName); err != nil {
|
||||
log.Debugf("destroy ipset probe set %q: %v", probeName, err)
|
||||
}
|
||||
}()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *aclManager) createIPSet(name string) error {
|
||||
opts := ipset.CreateOptions{
|
||||
Replace: true,
|
||||
|
||||
240
client/firewall/iptables/dnat_refcount_linux_test.go
Normal file
240
client/firewall/iptables/dnat_refcount_linux_test.go
Normal file
@@ -0,0 +1,240 @@
|
||||
//go:build privileged
|
||||
|
||||
package iptables
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
fw "github.com/netbirdio/netbird/client/firewall/manager"
|
||||
"github.com/netbirdio/netbird/client/iface"
|
||||
"github.com/netbirdio/netbird/client/iface/wgaddr"
|
||||
)
|
||||
|
||||
func iptRefcountIfaceV4() *iFaceMock {
|
||||
return &iFaceMock{
|
||||
NameFunc: func() string { return "wt-refcount" },
|
||||
AddressFunc: func() wgaddr.Address {
|
||||
return wgaddr.Address{
|
||||
IP: netip.MustParseAddr("10.20.0.1"),
|
||||
Network: netip.MustParsePrefix("10.20.0.0/24"),
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func iptRefcountIfaceDual() *iFaceMock {
|
||||
return &iFaceMock{
|
||||
NameFunc: func() string { return "wt-refcount" },
|
||||
AddressFunc: func() wgaddr.Address {
|
||||
return wgaddr.Address{
|
||||
IP: netip.MustParseAddr("10.20.0.1"),
|
||||
Network: netip.MustParsePrefix("10.20.0.0/24"),
|
||||
IPv6: netip.MustParseAddr("fd00::1"),
|
||||
IPv6Net: netip.MustParsePrefix("fd00::/64"),
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newIptRefcountManager(t *testing.T, dual bool) *Manager {
|
||||
t.Helper()
|
||||
var ifMock *iFaceMock
|
||||
if dual {
|
||||
ifMock = iptRefcountIfaceDual()
|
||||
} else {
|
||||
ifMock = iptRefcountIfaceV4()
|
||||
}
|
||||
m, err := Create(ifMock, iface.DefaultMTU)
|
||||
require.NoError(t, err, "create manager")
|
||||
require.NoError(t, m.Init(nil), "init manager")
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, m.Close(nil), "close manager")
|
||||
})
|
||||
return m
|
||||
}
|
||||
|
||||
func iptDnatV4(port uint16) fw.ForwardRule {
|
||||
return fw.ForwardRule{
|
||||
Protocol: fw.ProtocolTCP,
|
||||
DestinationPort: fw.Port{Values: []uint16{port}},
|
||||
TranslatedAddress: netip.MustParseAddr("10.20.0.2"),
|
||||
TranslatedPort: fw.Port{Values: []uint16{80}},
|
||||
}
|
||||
}
|
||||
|
||||
func iptDnatV6(port uint16) fw.ForwardRule {
|
||||
return fw.ForwardRule{
|
||||
Protocol: fw.ProtocolTCP,
|
||||
DestinationPort: fw.Port{Values: []uint16{port}},
|
||||
TranslatedAddress: netip.MustParseAddr("fd00::2"),
|
||||
TranslatedPort: fw.Port{Values: []uint16{80}},
|
||||
}
|
||||
}
|
||||
|
||||
// TestIptablesRouting_RepeatedEnableSingleReference verifies that EnableRouting
|
||||
// (called on every network-map update) holds at most one reference per family
|
||||
// and a single DisableRouting drops both back to zero.
|
||||
func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) {
|
||||
m := newIptRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
require.NoError(t, m.EnableRouting(), "first enable")
|
||||
require.NoError(t, m.EnableRouting(), "second enable")
|
||||
require.NoError(t, m.EnableRouting(), "third enable")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference")
|
||||
assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference")
|
||||
|
||||
require.NoError(t, m.DisableRouting(), "disable")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "single disable releases the v4 reference")
|
||||
assert.Equal(t, 0, v6, "single disable releases the v6 reference")
|
||||
}
|
||||
|
||||
// TestIptablesRouting_DisableKeepsDNATReference verifies that an unpaired
|
||||
// DisableRouting does not release references held by active DNAT rules.
|
||||
func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) {
|
||||
m := newIptRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(iptDnatV6(9095))
|
||||
require.NoError(t, err, "add v6 dnat")
|
||||
|
||||
require.NoError(t, m.DisableRouting(), "unpaired disable")
|
||||
_, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat")
|
||||
_, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v6, "delete releases the DNAT reference")
|
||||
}
|
||||
|
||||
// TestIptablesDNAT_RefcountBalancedV4 covers a Balanced Add/Delete pair on v4.
|
||||
func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) {
|
||||
m := newIptRefcountManager(t, false)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(iptDnatV4(7081))
|
||||
require.NoError(t, err, "add v4 dnat 1")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v4, "v4 refcount after first add")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
|
||||
r2, err := m.AddDNATRule(iptDnatV4(7082))
|
||||
require.NoError(t, err, "add v4 dnat 2")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 2, v4, "v4 refcount after second add")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1))
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 1, v4, "v4 refcount after first delete")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r2))
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount after second delete")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
}
|
||||
|
||||
// TestIptablesDNAT_RefcountBalancedV6 checks the v6 path increments v6 only and
|
||||
// decrements back to zero.
|
||||
func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) {
|
||||
m := newIptRefcountManager(t, true)
|
||||
require.NotNil(t, m.router6, "v6 router")
|
||||
require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state")
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(iptDnatV6(9081))
|
||||
require.NoError(t, err, "add v6 dnat 1")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 1, v6, "v6 refcount after first add")
|
||||
|
||||
r2, err := m.AddDNATRule(iptDnatV6(9082))
|
||||
require.NoError(t, err, "add v6 dnat 2")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount unchanged")
|
||||
assert.Equal(t, 2, v6, "v6 refcount after second add")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1))
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount unchanged")
|
||||
assert.Equal(t, 1, v6, "v6 refcount after first delete")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r2))
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 0, v6, "v6 refcount after second delete")
|
||||
}
|
||||
|
||||
// TestIptablesDNAT_DuplicateAddNoLeak verifies the duplicate-rule path returns
|
||||
// without bumping the refcount.
|
||||
func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) {
|
||||
m := newIptRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
rule := iptDnatV4(7083)
|
||||
r1, err := m.AddDNATRule(rule)
|
||||
require.NoError(t, err)
|
||||
v4, _ := state.Counts()
|
||||
assert.Equal(t, 1, v4)
|
||||
|
||||
_, err = m.AddDNATRule(rule)
|
||||
require.NoError(t, err, "duplicate add")
|
||||
v4, _ = state.Counts()
|
||||
assert.Equal(t, 1, v4, "duplicate add must not increment")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1))
|
||||
v4, _ = state.Counts()
|
||||
assert.Equal(t, 0, v4, "single delete must drop to zero")
|
||||
}
|
||||
|
||||
// TestIptablesDNAT_DeleteMissingNoUnderflow verifies Delete on an unknown rule
|
||||
// neither errors nor releases the refcount.
|
||||
func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) {
|
||||
m := newIptRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
phantom := iptDnatV4(7099)
|
||||
require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 0, v6)
|
||||
|
||||
phantom6 := iptDnatV6(9099)
|
||||
require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 0, v6)
|
||||
|
||||
r1, err := m.AddDNATRule(iptDnatV4(7100))
|
||||
require.NoError(t, err)
|
||||
v4, _ = state.Counts()
|
||||
assert.Equal(t, 1, v4, "real add still increments after phantom delete")
|
||||
require.NoError(t, m.DeleteDNATRule(r1))
|
||||
}
|
||||
|
||||
// TestIptablesDNAT_DoubleDeleteNoUnderflow verifies a second Delete on the same
|
||||
// rule is a no-op.
|
||||
func TestIptablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) {
|
||||
m := newIptRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(iptDnatV6(9083))
|
||||
require.NoError(t, err)
|
||||
_, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v6)
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "first delete")
|
||||
_, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v6)
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op")
|
||||
_, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v6, "double delete must not underflow")
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func (m *Manager) createIPv6Components(wgIface iFaceMapper, mtu uint16) error {
|
||||
}
|
||||
|
||||
// Share the same IP forwarding state with the v4 router, since
|
||||
// EnableIPForwarding controls both v4 and v6 sysctls.
|
||||
// Forwarding refcounter is per-family but shared between v4 and v6 routers.
|
||||
m.router6.ipFwdState = m.router.ipFwdState
|
||||
|
||||
m.aclMgr6, err = newAclManager(ip6Client, wgIface)
|
||||
@@ -402,17 +402,12 @@ func (m *Manager) SetLogLevel(log.Level) {
|
||||
}
|
||||
|
||||
func (m *Manager) EnableRouting() error {
|
||||
if err := m.router.ipFwdState.RequestForwarding(); err != nil {
|
||||
return fmt.Errorf("enable IP forwarding: %w", err)
|
||||
}
|
||||
return nil
|
||||
// v6 only when the overlay actually has v6.
|
||||
return m.router.ipFwdState.RequestRouting(m.router6 != nil)
|
||||
}
|
||||
|
||||
func (m *Manager) DisableRouting() error {
|
||||
if err := m.router.ipFwdState.ReleaseForwarding(); err != nil {
|
||||
return fmt.Errorf("disable IP forwarding: %w", err)
|
||||
}
|
||||
return nil
|
||||
return m.router.ipFwdState.ReleaseRouting()
|
||||
}
|
||||
|
||||
// AddDNATRule adds a DNAT rule
|
||||
|
||||
@@ -291,3 +291,40 @@ func TestIptablesCreatePerformance(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIptablesACLIPSetFallback verifies that when the kernel lacks ipset support,
|
||||
// the ACL manager falls back to per-IP iptables rules (-s <ip>) instead of
|
||||
// silently leaving the chain empty. See discussion #6125.
|
||||
func TestIptablesACLIPSetFallback(t *testing.T) {
|
||||
ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Use Create()/Init() so the router-owned chains (chainRTFWDIN/OUT) are
|
||||
// created before the ACL manager's createDefaultChains() references them.
|
||||
manager, err := Create(ifaceMock, iface.DefaultMTU)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, manager.Init(nil))
|
||||
|
||||
aclMgr := manager.aclMgr
|
||||
// Simulate a kernel without the ipset hash module.
|
||||
aclMgr.ipsetSupported = false
|
||||
|
||||
defer func() {
|
||||
require.NoError(t, manager.Close(nil))
|
||||
}()
|
||||
|
||||
ip := netip.MustParseAddr("10.20.0.42")
|
||||
port := &fw.Port{Values: []uint16{22}}
|
||||
|
||||
rules, err := aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "nb0000001")
|
||||
require.NoError(t, err, "AddPeerFiltering should succeed via fallback")
|
||||
require.NotEmpty(t, rules)
|
||||
|
||||
rule := rules[0].(*Rule)
|
||||
require.Empty(t, rule.ipsetName, "fallback rule must not reference an ipset")
|
||||
require.Contains(t, strings.Join(rule.specs, " "), "-s 10.20.0.42", "fallback rule must match by source IP")
|
||||
require.NotContains(t, strings.Join(rule.specs, " "), "--match-set", "fallback rule must not use ipset matching")
|
||||
|
||||
// The rule must actually be present in the ACL chain (not silently dropped).
|
||||
checkRuleSpecs(t, ipv4Client, rule.chain, true, rule.specs...)
|
||||
}
|
||||
|
||||
@@ -102,7 +102,7 @@ func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint1
|
||||
wgIface: wgIface,
|
||||
mtu: mtu,
|
||||
v6: iptablesClient.Proto() == iptables.ProtocolIPv6,
|
||||
ipFwdState: ipfwdstate.NewIPForwardingState(),
|
||||
ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()),
|
||||
}
|
||||
|
||||
r.ipsetCounter = refcounter.New(
|
||||
@@ -770,10 +770,6 @@ func (r *router) updateState() {
|
||||
}
|
||||
|
||||
func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
|
||||
if err := r.ipFwdState.RequestForwarding(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ruleKey := rule.ID()
|
||||
if _, exists := r.rules[ruleKey+dnatSuffix]; exists {
|
||||
return rule, nil
|
||||
@@ -840,18 +836,34 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
|
||||
|
||||
for key, ruleInfo := range rules {
|
||||
if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil {
|
||||
if rollbackErr := r.rollbackRules(rules); rollbackErr != nil {
|
||||
log.Errorf("rollback failed: %v", rollbackErr)
|
||||
}
|
||||
r.cleanupFailedDNATAdd(rules)
|
||||
return nil, fmt.Errorf("add rule %s: %w", key, err)
|
||||
}
|
||||
r.rules[key] = ruleInfo.rule
|
||||
}
|
||||
|
||||
if err := r.ipFwdState.RequestForwarding(r.v6); err != nil {
|
||||
r.cleanupFailedDNATAdd(rules)
|
||||
return nil, fmt.Errorf("enable forwarding: %w", err)
|
||||
}
|
||||
|
||||
r.updateState()
|
||||
return rule, nil
|
||||
}
|
||||
|
||||
// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied
|
||||
// AddDNATRule before rolling back the kernel rules, so no entries remain that
|
||||
// never got a forwarding refcount. rollbackRules re-adds entries it failed to
|
||||
// remove from the kernel.
|
||||
func (r *router) cleanupFailedDNATAdd(rules map[string]ruleInfo) {
|
||||
for key := range rules {
|
||||
delete(r.rules, key)
|
||||
}
|
||||
if err := r.rollbackRules(rules); err != nil {
|
||||
log.Errorf("rollback failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *router) rollbackRules(rules map[string]ruleInfo) error {
|
||||
var merr *multierror.Error
|
||||
for key, ruleInfo := range rules {
|
||||
@@ -868,32 +880,47 @@ func (r *router) rollbackRules(rules map[string]ruleInfo) error {
|
||||
}
|
||||
|
||||
func (r *router) DeleteDNATRule(rule firewall.Rule) error {
|
||||
if err := r.ipFwdState.ReleaseForwarding(); err != nil {
|
||||
log.Errorf("%v", err)
|
||||
}
|
||||
|
||||
ruleKey := rule.ID()
|
||||
|
||||
_, hadDNAT := r.rules[ruleKey+dnatSuffix]
|
||||
_, hadSNAT := r.rules[ruleKey+snatSuffix]
|
||||
_, hadFWD := r.rules[ruleKey+fwdSuffix]
|
||||
if !hadDNAT && !hadSNAT && !hadFWD {
|
||||
return nil
|
||||
}
|
||||
|
||||
var merr *multierror.Error
|
||||
if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists {
|
||||
if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err))
|
||||
} else {
|
||||
delete(r.rules, ruleKey+dnatSuffix)
|
||||
}
|
||||
delete(r.rules, ruleKey+dnatSuffix)
|
||||
}
|
||||
|
||||
if snatRule, exists := r.rules[ruleKey+snatSuffix]; exists {
|
||||
if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err))
|
||||
} else {
|
||||
delete(r.rules, ruleKey+snatSuffix)
|
||||
}
|
||||
delete(r.rules, ruleKey+snatSuffix)
|
||||
}
|
||||
|
||||
if fwdRule, exists := r.rules[ruleKey+fwdSuffix]; exists {
|
||||
if err := r.iptablesClient.Delete(tableFilter, chainRTFWDOUT, fwdRule...); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err))
|
||||
} else {
|
||||
delete(r.rules, ruleKey+fwdSuffix)
|
||||
}
|
||||
}
|
||||
|
||||
// Release the refcount only once all rules are gone from the kernel. On
|
||||
// partial failure the failed entries stay in r.rules so a retry can remove
|
||||
// them and release then.
|
||||
if merr == nil {
|
||||
if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil {
|
||||
log.Errorf("%v", err)
|
||||
}
|
||||
delete(r.rules, ruleKey+fwdSuffix)
|
||||
}
|
||||
|
||||
r.updateState()
|
||||
|
||||
249
client/firewall/nftables/dnat_refcount_linux_test.go
Normal file
249
client/firewall/nftables/dnat_refcount_linux_test.go
Normal file
@@ -0,0 +1,249 @@
|
||||
//go:build privileged
|
||||
|
||||
package nftables
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
fw "github.com/netbirdio/netbird/client/firewall/manager"
|
||||
"github.com/netbirdio/netbird/client/iface"
|
||||
"github.com/netbirdio/netbird/client/iface/wgaddr"
|
||||
)
|
||||
|
||||
func nftRefcountIfaceV4() *iFaceMock {
|
||||
return &iFaceMock{
|
||||
NameFunc: func() string { return "wt-refcount" },
|
||||
AddressFunc: func() wgaddr.Address {
|
||||
return wgaddr.Address{
|
||||
IP: netip.MustParseAddr("100.96.0.1"),
|
||||
Network: netip.MustParsePrefix("100.96.0.0/16"),
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func nftRefcountIfaceDual() *iFaceMock {
|
||||
return &iFaceMock{
|
||||
NameFunc: func() string { return "wt-refcount" },
|
||||
AddressFunc: func() wgaddr.Address {
|
||||
return wgaddr.Address{
|
||||
IP: netip.MustParseAddr("100.96.0.1"),
|
||||
Network: netip.MustParsePrefix("100.96.0.0/16"),
|
||||
IPv6: netip.MustParseAddr("fd00::1"),
|
||||
IPv6Net: netip.MustParsePrefix("fd00::/64"),
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newNftRefcountManager(t *testing.T, dual bool) *Manager {
|
||||
t.Helper()
|
||||
if check() != NFTABLES {
|
||||
t.Skip("nftables not supported on this system")
|
||||
}
|
||||
var ifMock *iFaceMock
|
||||
if dual {
|
||||
ifMock = nftRefcountIfaceDual()
|
||||
} else {
|
||||
ifMock = nftRefcountIfaceV4()
|
||||
}
|
||||
m, err := Create(ifMock, iface.DefaultMTU)
|
||||
require.NoError(t, err, "create manager")
|
||||
require.NoError(t, m.Init(nil), "init manager")
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, m.Close(nil), "close manager")
|
||||
})
|
||||
return m
|
||||
}
|
||||
|
||||
func dnatV4(port uint16) fw.ForwardRule {
|
||||
return fw.ForwardRule{
|
||||
Protocol: fw.ProtocolTCP,
|
||||
DestinationPort: fw.Port{Values: []uint16{port}},
|
||||
TranslatedAddress: netip.MustParseAddr("100.96.0.2"),
|
||||
TranslatedPort: fw.Port{Values: []uint16{80}},
|
||||
}
|
||||
}
|
||||
|
||||
func dnatV6(port uint16) fw.ForwardRule {
|
||||
return fw.ForwardRule{
|
||||
Protocol: fw.ProtocolTCP,
|
||||
DestinationPort: fw.Port{Values: []uint16{port}},
|
||||
TranslatedAddress: netip.MustParseAddr("fd00::2"),
|
||||
TranslatedPort: fw.Port{Values: []uint16{80}},
|
||||
}
|
||||
}
|
||||
|
||||
// TestNftablesDNAT_RefcountBalancedV4 verifies that Add/Delete pairs leave the
|
||||
// v4 refcount at zero.
|
||||
func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) {
|
||||
m := newNftRefcountManager(t, false)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(dnatV4(8081))
|
||||
require.NoError(t, err, "add v4 dnat 1")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v4, "v4 refcount after first add")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
|
||||
r2, err := m.AddDNATRule(dnatV4(8082))
|
||||
require.NoError(t, err, "add v4 dnat 2")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 2, v4, "v4 refcount after second add")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat 1")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 1, v4, "v4 refcount after first delete")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r2), "delete v4 dnat 2")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount after second delete")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unchanged")
|
||||
}
|
||||
|
||||
// TestNftablesDNAT_RefcountBalancedV6 verifies the v6 path increments v6 only
|
||||
// and decrements back to zero on Delete.
|
||||
func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) {
|
||||
m := newNftRefcountManager(t, true)
|
||||
require.NotNil(t, m.router6, "v6 router")
|
||||
require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state")
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(dnatV6(9091))
|
||||
require.NoError(t, err, "add v6 dnat 1")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount unchanged")
|
||||
assert.Equal(t, 1, v6, "v6 refcount after first add")
|
||||
|
||||
r2, err := m.AddDNATRule(dnatV6(9092))
|
||||
require.NoError(t, err, "add v6 dnat 2")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 2, v6, "v6 refcount after second add")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat 1")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount unchanged")
|
||||
assert.Equal(t, 1, v6, "v6 refcount after first delete")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r2), "delete v6 dnat 2")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 0, v6, "v6 refcount after second delete")
|
||||
}
|
||||
|
||||
// TestNftablesDNAT_DuplicateAddNoLeak verifies that a duplicate Add (same
|
||||
// ForwardRule) does not double-increment the refcount.
|
||||
func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) {
|
||||
m := newNftRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
rule := dnatV4(8083)
|
||||
r1, err := m.AddDNATRule(rule)
|
||||
require.NoError(t, err, "add v4 dnat")
|
||||
v4, _ := state.Counts()
|
||||
assert.Equal(t, 1, v4)
|
||||
|
||||
// duplicate add: same rule ID, must be a no-op for the refcount.
|
||||
_, err = m.AddDNATRule(rule)
|
||||
require.NoError(t, err, "duplicate add")
|
||||
v4, _ = state.Counts()
|
||||
assert.Equal(t, 1, v4, "duplicate add must not increment")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat")
|
||||
v4, _ = state.Counts()
|
||||
assert.Equal(t, 0, v4, "single delete must drop to zero")
|
||||
}
|
||||
|
||||
// TestNftablesDNAT_DeleteMissingNoUnderflow verifies deleting a rule that was
|
||||
// never added does not underflow the refcount.
|
||||
func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) {
|
||||
m := newNftRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
// Construct a Rule reference for something never added. The router stores
|
||||
// rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing
|
||||
// entry must be a no-op rather than calling Release.
|
||||
phantom := dnatV4(8099)
|
||||
require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4 dnat")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 0, v4, "v4 refcount unaffected by missing delete")
|
||||
assert.Equal(t, 0, v6, "v6 refcount unaffected")
|
||||
|
||||
phantom6 := dnatV6(9099)
|
||||
require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6 dnat")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4)
|
||||
assert.Equal(t, 0, v6, "v6 refcount unaffected by missing delete")
|
||||
|
||||
// And after a phantom delete, a real add still results in count=1.
|
||||
r1, err := m.AddDNATRule(dnatV4(8100))
|
||||
require.NoError(t, err, "add v4 dnat after phantom delete")
|
||||
v4, _ = state.Counts()
|
||||
assert.Equal(t, 1, v4, "real add still increments after phantom delete")
|
||||
require.NoError(t, m.DeleteDNATRule(r1))
|
||||
}
|
||||
|
||||
// TestNftablesRouting_RepeatedEnableSingleReference verifies that EnableRouting
|
||||
// (called on every network-map update) holds at most one reference per family
|
||||
// and a single DisableRouting drops both back to zero.
|
||||
func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) {
|
||||
m := newNftRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
require.NoError(t, m.EnableRouting(), "first enable")
|
||||
require.NoError(t, m.EnableRouting(), "second enable")
|
||||
require.NoError(t, m.EnableRouting(), "third enable")
|
||||
v4, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference")
|
||||
assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference")
|
||||
|
||||
require.NoError(t, m.DisableRouting(), "disable")
|
||||
v4, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v4, "single disable releases the v4 reference")
|
||||
assert.Equal(t, 0, v6, "single disable releases the v6 reference")
|
||||
}
|
||||
|
||||
// TestNftablesRouting_DisableKeepsDNATReference verifies that an unpaired
|
||||
// DisableRouting does not release references held by active DNAT rules.
|
||||
func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) {
|
||||
m := newNftRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(dnatV6(9095))
|
||||
require.NoError(t, err, "add v6 dnat")
|
||||
|
||||
require.NoError(t, m.DisableRouting(), "unpaired disable")
|
||||
_, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting")
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat")
|
||||
_, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v6, "delete releases the DNAT reference")
|
||||
}
|
||||
|
||||
// TestNftablesDNAT_DoubleDeleteNoUnderflow verifies that deleting the same rule
|
||||
// twice does not underflow the refcount (the second delete is a no-op).
|
||||
func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) {
|
||||
m := newNftRefcountManager(t, true)
|
||||
state := m.router.ipFwdState
|
||||
|
||||
r1, err := m.AddDNATRule(dnatV6(9093))
|
||||
require.NoError(t, err)
|
||||
_, v6 := state.Counts()
|
||||
assert.Equal(t, 1, v6)
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "first delete")
|
||||
_, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v6)
|
||||
|
||||
require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op")
|
||||
_, v6 = state.Counts()
|
||||
assert.Equal(t, 0, v6, "double delete must not underflow")
|
||||
}
|
||||
@@ -105,8 +105,8 @@ func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mt
|
||||
return fmt.Errorf("create v6 router: %w", err)
|
||||
}
|
||||
|
||||
// Share the same IP forwarding state with the v4 router, since
|
||||
// EnableIPForwarding controls both v4 and v6 sysctls.
|
||||
// Share the per-family forwarding refcounter with the v4 router so a v4
|
||||
// rule and a v6 rule against the same state machine cooperate cleanly.
|
||||
m.router6.ipFwdState = m.router.ipFwdState
|
||||
|
||||
m.aclManager6, err = newAclManager(workTable6, wgIface, chainNameRoutingFw)
|
||||
@@ -530,17 +530,12 @@ func (m *Manager) SetLogLevel(log.Level) {
|
||||
}
|
||||
|
||||
func (m *Manager) EnableRouting() error {
|
||||
if err := m.router.ipFwdState.RequestForwarding(); err != nil {
|
||||
return fmt.Errorf("enable IP forwarding: %w", err)
|
||||
}
|
||||
return nil
|
||||
// v6 only when the overlay actually has v6.
|
||||
return m.router.ipFwdState.RequestRouting(m.router6 != nil)
|
||||
}
|
||||
|
||||
func (m *Manager) DisableRouting() error {
|
||||
if err := m.router.ipFwdState.ReleaseForwarding(); err != nil {
|
||||
return fmt.Errorf("disable IP forwarding: %w", err)
|
||||
}
|
||||
return nil
|
||||
return m.router.ipFwdState.ReleaseRouting()
|
||||
}
|
||||
|
||||
// Flush rule/chain/set operations from the buffer
|
||||
|
||||
@@ -93,7 +93,7 @@ func newRouter(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) (*rou
|
||||
rules: make(map[string]*nftables.Rule),
|
||||
af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4),
|
||||
wgIface: wgIface,
|
||||
ipFwdState: ipfwdstate.NewIPForwardingState(),
|
||||
ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()),
|
||||
mtu: mtu,
|
||||
}
|
||||
|
||||
@@ -1553,10 +1553,6 @@ func (r *router) refreshRulesMap() error {
|
||||
}
|
||||
|
||||
func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
|
||||
if err := r.ipFwdState.RequestForwarding(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ruleKey := rule.ID()
|
||||
if _, exists := r.rules[ruleKey+dnatSuffix]; exists {
|
||||
return rule, nil
|
||||
@@ -1567,7 +1563,18 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
|
||||
return nil, fmt.Errorf("convert protocol to number: %w", err)
|
||||
}
|
||||
|
||||
// Request forwarding before queueing rules: addDnatRedirect/addDnatMasq
|
||||
// buffer netlink messages on r.conn that the next caller's Flush would
|
||||
// commit if we returned without flushing them ourselves.
|
||||
v6 := r.af.tableFamily == nftables.TableFamilyIPv6
|
||||
if err := r.ipFwdState.RequestForwarding(v6); err != nil {
|
||||
return nil, fmt.Errorf("enable forwarding: %w", err)
|
||||
}
|
||||
|
||||
if err := r.addDnatRedirect(rule, protoNum, ruleKey); err != nil {
|
||||
if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil {
|
||||
log.Warnf("rollback forwarding refcount: %v", rerr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1579,6 +1586,11 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
|
||||
// TODO: find chains with drop policies and add rules there
|
||||
|
||||
if err := r.conn.Flush(); err != nil {
|
||||
if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil {
|
||||
log.Warnf("rollback forwarding refcount: %v", rerr)
|
||||
}
|
||||
delete(r.rules, ruleKey+dnatSuffix)
|
||||
delete(r.rules, ruleKey+snatSuffix)
|
||||
return nil, fmt.Errorf("flush rules: %w", err)
|
||||
}
|
||||
|
||||
@@ -1781,16 +1793,18 @@ func (r *router) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleKey
|
||||
}
|
||||
|
||||
func (r *router) DeleteDNATRule(rule firewall.Rule) error {
|
||||
if err := r.ipFwdState.ReleaseForwarding(); err != nil {
|
||||
log.Errorf("%v", err)
|
||||
}
|
||||
|
||||
ruleKey := rule.ID()
|
||||
|
||||
if err := r.refreshRulesMap(); err != nil {
|
||||
return fmt.Errorf(refreshRulesMapError, err)
|
||||
}
|
||||
|
||||
_, hadDNAT := r.rules[ruleKey+dnatSuffix]
|
||||
_, hadSNAT := r.rules[ruleKey+snatSuffix]
|
||||
if !hadDNAT && !hadSNAT {
|
||||
return nil
|
||||
}
|
||||
|
||||
var merr *multierror.Error
|
||||
var needsFlush bool
|
||||
|
||||
@@ -1822,9 +1836,16 @@ func (r *router) DeleteDNATRule(rule firewall.Rule) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Release the refcount only once the rules are gone from the kernel. On
|
||||
// failure (including the refreshRulesMap error above) the rules and their
|
||||
// map entries remain, keeping forwarding on until a retry removes them.
|
||||
if merr == nil {
|
||||
delete(r.rules, ruleKey+dnatSuffix)
|
||||
delete(r.rules, ruleKey+snatSuffix)
|
||||
|
||||
if err := r.ipFwdState.ReleaseForwarding(r.af.tableFamily == nftables.TableFamilyIPv6); err != nil {
|
||||
log.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nberrors.FormatErrorOrNil(merr)
|
||||
|
||||
@@ -22,8 +22,6 @@
|
||||
!define UI_REG_APP_PATH "Software\Microsoft\Windows\CurrentVersion\App Paths\${UI_APP_EXE}"
|
||||
!define UI_UNINSTALL_PATH "Software\Microsoft\Windows\CurrentVersion\Uninstall\${UI_APP_NAME}"
|
||||
|
||||
!define AUTOSTART_REG_KEY "Software\Microsoft\Windows\CurrentVersion\Run"
|
||||
|
||||
!define NETBIRD_DATA_DIR "$COMMONPROGRAMDATA\Netbird"
|
||||
|
||||
Unicode True
|
||||
@@ -228,13 +226,6 @@ WriteRegStr ${REG_ROOT} "${UNINSTALL_PATH}" "Publisher" "${COMP_NAME}"
|
||||
|
||||
WriteRegStr ${REG_ROOT} "${UI_REG_APP_PATH}" "" "$INSTDIR\${UI_APP_EXE}"
|
||||
|
||||
; Autostart is owned by the UI's per-user setting (HKCU\...\Run via Wails),
|
||||
; not the installer. Drop the machine-wide entry older installers wrote so the
|
||||
; toggle is the single source of truth. HKCU is left untouched -- it may hold
|
||||
; the user's own toggle state, which must survive upgrades.
|
||||
DetailPrint "Removing installer-managed autostart registry entry if present..."
|
||||
DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}"
|
||||
|
||||
EnVar::SetHKLM
|
||||
EnVar::AddValueEx "path" "$INSTDIR"
|
||||
|
||||
@@ -299,15 +290,6 @@ ExecWait '"$INSTDIR\${MAIN_APP_EXE}" service uninstall'
|
||||
DetailPrint "Terminating Netbird UI process..."
|
||||
ExecWait `taskkill /im ${UI_APP_EXE}.exe /f`
|
||||
|
||||
; Remove autostart registry entries
|
||||
DetailPrint "Removing autostart registry entries if they exist..."
|
||||
; Legacy machine-wide entry written by older installers.
|
||||
DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}"
|
||||
; Per-user entry the UI toggle writes via Wails (value name is the lowercase
|
||||
; app-name slug). Uninstall removes the app, so drop it too.
|
||||
DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}"
|
||||
DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "netbird"
|
||||
|
||||
; Handle data deletion based on checkbox
|
||||
DetailPrint "Checking if user requested data deletion..."
|
||||
${If} $DeleteDataEnabled == "1"
|
||||
|
||||
@@ -51,6 +51,7 @@ nftables.txt: Anonymized nftables rules with packet counters across all families
|
||||
sysctls.txt: Forwarding, reverse-path filter, source-validation, and conntrack accounting sysctl values that the NetBird client may read or modify, if --system-info flag was provided (Linux only).
|
||||
resolv.conf: DNS resolver configuration from /etc/resolv.conf (Unix systems only), if --system-info flag was provided.
|
||||
scutil_dns.txt: DNS configuration from scutil --dns (macOS only), if --system-info flag was provided.
|
||||
dns_windows.txt: Anonymized NRPT rules and policy table in effect, DNS client policy, and per-interface and per-adapter DNS configuration (Windows only), if --system-info flag was provided.
|
||||
resolved_domains.txt: Anonymized resolved domain IP addresses from the status recorder.
|
||||
config.txt: Anonymized configuration information of the NetBird client.
|
||||
network_map.json: Anonymized sync response containing peer configurations, routes, DNS settings, and firewall rules.
|
||||
@@ -237,6 +238,13 @@ scutil_dns.txt (macOS only):
|
||||
- Shows DNS configuration for all network interfaces
|
||||
- Includes search domains, nameservers, and DNS resolver settings
|
||||
- All IP addresses and domain names are anonymized
|
||||
|
||||
dns_windows.txt (Windows only):
|
||||
- Lists the NRPT rules of both policy stores, the local one and the group policy one, marking the rules the client created
|
||||
- Follows them with the policy table the resolver has loaded, which differs from the rules while a change has not been picked up yet
|
||||
- Includes the DNS client group policy, the global TCP/IP and Dnscache parameters, and the DNS values of every interface that has any
|
||||
- Ends with the resolver configuration in effect per adapter, from GetAdaptersAddresses
|
||||
- All IP addresses and domain names are anonymized
|
||||
`
|
||||
|
||||
const (
|
||||
|
||||
@@ -844,6 +844,10 @@ func collectSysctls() string {
|
||||
[]string{"net.ipv4.conf.all.src_valid_mark", "net.ipv4.conf.default.src_valid_mark"},
|
||||
listInterfaceSysctls("ipv4", "src_valid_mark")...,
|
||||
))
|
||||
writeSysctlGroup(&builder, "accept_ra", append(
|
||||
[]string{"net.ipv6.conf.all.accept_ra", "net.ipv6.conf.default.accept_ra"},
|
||||
listInterfaceSysctls("ipv6", "accept_ra")...,
|
||||
))
|
||||
writeSysctlGroup(&builder, "conntrack", []string{
|
||||
"net.netfilter.nf_conntrack_acct",
|
||||
"net.netfilter.nf_conntrack_tcp_loose",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !unix
|
||||
//go:build !unix && !windows
|
||||
|
||||
package debug
|
||||
|
||||
|
||||
443
client/internal/debug/debug_windows.go
Normal file
443
client/internal/debug/debug_windows.go
Normal file
@@ -0,0 +1,443 @@
|
||||
//go:build windows
|
||||
|
||||
package debug
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"unsafe"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sys/windows"
|
||||
"golang.org/x/sys/windows/registry"
|
||||
|
||||
nbdns "github.com/netbirdio/netbird/client/internal/dns"
|
||||
)
|
||||
|
||||
const dnsInfoFileName = "dns_windows.txt"
|
||||
|
||||
const (
|
||||
gpoDNSClientRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient`
|
||||
tcpipParamsPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters`
|
||||
dnscacheParams = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters`
|
||||
)
|
||||
|
||||
// interfaceDNSValues are the per-interface values that decide how a name is
|
||||
// resolved and registered. Everything the DNS host manager writes is in here,
|
||||
// so a bundle shows both what we set and what it replaced.
|
||||
var interfaceDNSValues = []string{
|
||||
"NameServer",
|
||||
"DhcpNameServer",
|
||||
"Domain",
|
||||
"DhcpDomain",
|
||||
"SearchList",
|
||||
"RegistrationEnabled",
|
||||
"DisableDynamicUpdate",
|
||||
"MaxNumberOfAddressesToRegister",
|
||||
"EnableDHCP",
|
||||
}
|
||||
|
||||
// addDNSInfo collects and adds DNS configuration information to the archive
|
||||
func (g *BundleGenerator) addDNSInfo() error {
|
||||
if err := g.addFileToZip(strings.NewReader(g.collectDNSInfo()), dnsInfoFileName); err != nil {
|
||||
return fmt.Errorf("add DNS info to zip: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// collectDNSInfo renders the report. Everything below it reaches the platform
|
||||
// through COM and through lazily resolved procedures, which panic when a
|
||||
// procedure is missing rather than returning an error, and a debug bundle is not
|
||||
// allowed to take the daemon down. The panic is contained here, and whatever was
|
||||
// collected before it is kept and reported with it.
|
||||
func (g *BundleGenerator) collectDNSInfo() (content string) {
|
||||
var sb strings.Builder
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Errorf("collecting Windows DNS configuration panicked: %v", r)
|
||||
fmt.Fprintf(&sb, "\nerror: collection stopped: %v\n", r)
|
||||
}
|
||||
content = sb.String()
|
||||
}()
|
||||
|
||||
sb.WriteString("Windows DNS configuration\n")
|
||||
sb.WriteString("=========================\n")
|
||||
|
||||
adapters, adaptersErr := adapterAddresses()
|
||||
|
||||
g.writeNRPTRules(&sb, "NRPT rules, local policy store", nbdns.DNSPolicyConfigRoot)
|
||||
g.writeNRPTRules(&sb, "NRPT rules, group policy store", nbdns.GPODNSPolicyConfigRoot)
|
||||
g.writeEffectiveNRPTPolicies(&sb)
|
||||
g.writeRegistryKey(&sb, "DNS client group policy", gpoDNSClientRoot)
|
||||
g.writeRegistryKey(&sb, "Global TCP/IP parameters", tcpipParamsPath)
|
||||
g.writeRegistryKey(&sb, "Dnscache parameters", dnscacheParams)
|
||||
g.writeInterfaceDNS(&sb, "Per-interface DNS, IPv4", nbdns.InterfaceConfigPath, adapterNames(adapters))
|
||||
g.writeInterfaceDNS(&sb, "Per-interface DNS, IPv6", nbdns.InterfaceConfigPathV6, adapterNames(adapters))
|
||||
g.writeAdapterDNS(&sb, adapters, adaptersErr)
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// writeNRPTRules lists every rule in a policy store, ours and any other
|
||||
// product's, since a foreign rule for the same namespace decides resolution
|
||||
// just as ours does. Rules the client wrote are marked.
|
||||
func (g *BundleGenerator) writeNRPTRules(sb *strings.Builder, title, root string) {
|
||||
writeSection(sb, title, root)
|
||||
|
||||
names, err := subKeyNames(root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(sb, "error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(names) == 0 {
|
||||
sb.WriteString("no rules\n")
|
||||
return
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
owner := ""
|
||||
if strings.HasPrefix(strings.ToLower(name), strings.ToLower(nbdns.NRPTKeyPrefix)) {
|
||||
owner = " (netbird)"
|
||||
}
|
||||
fmt.Fprintf(sb, "%s%s\n", name, owner)
|
||||
g.writeValues(sb, root+`\`+name, nil, " ")
|
||||
}
|
||||
}
|
||||
|
||||
// writeEffectiveNRPTPolicies reports the table the resolver answers from, which
|
||||
// the registry cannot show: a rule is written before it is loaded, and it keeps
|
||||
// being enforced after its key is gone until the resolver reloads its policy.
|
||||
func (g *BundleGenerator) writeEffectiveNRPTPolicies(sb *strings.Builder) {
|
||||
writeSection(sb, "NRPT policy table in effect", nrptPolicyClass+"."+nrptPolicyMethod+" in "+nrptPolicyNamespace)
|
||||
|
||||
entries, err := effectiveNRPTPolicies()
|
||||
if err != nil {
|
||||
fmt.Fprintf(sb, "error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
sb.WriteString("no policies\n")
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fmt.Fprintf(sb, "%s\n", g.anonymizeValue("Namespace", entry.namespace))
|
||||
for _, value := range entry.values {
|
||||
fmt.Fprintf(sb, " %s: %s\n", value.name, g.anonymizeValue(value.name, value.value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeInterfaceDNS reports the DNS values of every interface that has any, so
|
||||
// the netbird interface can be compared against the physical ones. The registry
|
||||
// keys the values by GUID, so each is named from the adapter list; a GUID with
|
||||
// no adapter is a leftover key of an interface that no longer exists.
|
||||
func (g *BundleGenerator) writeInterfaceDNS(sb *strings.Builder, title, root string, names map[string]string) {
|
||||
writeSection(sb, title, root)
|
||||
|
||||
guids, err := subKeyNames(root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(sb, "error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var reported int
|
||||
for _, guid := range guids {
|
||||
var iface strings.Builder
|
||||
g.writeValues(&iface, root+`\`+guid, interfaceDNSValues, " ")
|
||||
if iface.Len() == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
name, ok := names[strings.ToLower(guid)]
|
||||
if !ok {
|
||||
name = "no adapter with this GUID"
|
||||
}
|
||||
|
||||
reported++
|
||||
fmt.Fprintf(sb, "%s (%s)\n%s", guid, name, iface.String())
|
||||
}
|
||||
|
||||
if reported == 0 {
|
||||
sb.WriteString("no interface holds DNS values\n")
|
||||
}
|
||||
}
|
||||
|
||||
// writeRegistryKey reports the values of a single key, without its subkeys.
|
||||
func (g *BundleGenerator) writeRegistryKey(sb *strings.Builder, title, path string) {
|
||||
writeSection(sb, title, path)
|
||||
|
||||
var values strings.Builder
|
||||
g.writeValues(&values, path, nil, "")
|
||||
if values.Len() == 0 {
|
||||
sb.WriteString("no values\n")
|
||||
return
|
||||
}
|
||||
|
||||
sb.WriteString(values.String())
|
||||
}
|
||||
|
||||
// writeValues renders the values of a key. A nil names list reports every
|
||||
// value, otherwise only those named and present.
|
||||
func (g *BundleGenerator) writeValues(sb *strings.Builder, path string, names []string, indent string) {
|
||||
k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE)
|
||||
switch {
|
||||
case errors.Is(err, registry.ErrNotExist), errors.Is(err, windows.ERROR_PATH_NOT_FOUND):
|
||||
// an absent key is the normal state for the GPO store and for
|
||||
// interfaces without DNS settings
|
||||
log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", path)
|
||||
return
|
||||
case err != nil:
|
||||
fmt.Fprintf(sb, "%serror: open HKEY_LOCAL_MACHINE\\%s: %v\n", indent, path, err)
|
||||
return
|
||||
}
|
||||
defer closeKey(k)
|
||||
|
||||
if names == nil {
|
||||
names, err = k.ReadValueNames(-1)
|
||||
if err != nil {
|
||||
fmt.Fprintf(sb, "%serror: read value names: %v\n", indent, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
value, err := readRegistryValue(k, name)
|
||||
switch {
|
||||
case errors.Is(err, registry.ErrNotExist):
|
||||
// the caller asks for a fixed set of values, most of which a
|
||||
// given interface does not carry
|
||||
continue
|
||||
case err != nil:
|
||||
// report rather than omit: a value that is there but cannot be
|
||||
// read reads as unset otherwise
|
||||
fmt.Fprintf(sb, "%s%s: error: %v\n", indent, name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Fprintf(sb, "%s%s: %s\n", indent, name, g.anonymizeValue(name, value))
|
||||
}
|
||||
}
|
||||
|
||||
// anonymizeValue redacts a registry value according to what its name says it
|
||||
// holds. Domains and addresses are handled per entry rather than by the string
|
||||
// pass: the pass only replaces domains something else in the bundle already
|
||||
// seeded, and its address regex would eat the digit labels of a reverse zone.
|
||||
func (g *BundleGenerator) anonymizeValue(name, value string) string {
|
||||
if !g.anonymize || value == "" {
|
||||
return value
|
||||
}
|
||||
|
||||
switch {
|
||||
case holdsDomains(name):
|
||||
return joinValueEntries(splitValueEntries(value), g.anonymizeDomain)
|
||||
case holdsAddresses(name):
|
||||
return joinValueEntries(splitValueEntries(value), g.anonymizer.AnonymizeIPString)
|
||||
default:
|
||||
return g.anonymizer.AnonymizeString(value)
|
||||
}
|
||||
}
|
||||
|
||||
// holdsDomains reports whether a value name holds domains: the domain list of
|
||||
// an NRPT rule (Name) or of the policy table (Namespace), a search list, the
|
||||
// DNS suffix values of the TCP/IP and policy keys, which all end in "Domain"
|
||||
// (Domain, DhcpDomain, NV Domain, ICSDomain), and a proxy host name.
|
||||
func holdsDomains(name string) bool {
|
||||
lower := strings.ToLower(name)
|
||||
return lower == "name" || lower == "namespace" || lower == "searchlist" ||
|
||||
strings.HasSuffix(lower, "domain") || strings.HasSuffix(lower, "proxyname")
|
||||
}
|
||||
|
||||
// holdsAddresses reports whether a value name holds DNS server addresses
|
||||
// (NameServer, DhcpNameServer, GenericDNSServers, NameServers).
|
||||
func holdsAddresses(name string) bool {
|
||||
lower := strings.ToLower(name)
|
||||
return strings.Contains(lower, "nameserver") || strings.Contains(lower, "dnsserver")
|
||||
}
|
||||
|
||||
// adapterNames maps adapter GUIDs, as the registry keys the interfaces, to the
|
||||
// names an operator sees.
|
||||
func adapterNames(adapters []*windows.IpAdapterAddresses) map[string]string {
|
||||
names := make(map[string]string, len(adapters))
|
||||
for _, adapter := range adapters {
|
||||
guid := windows.BytePtrToString(adapter.AdapterName)
|
||||
names[strings.ToLower(guid)] = windows.UTF16PtrToString(adapter.FriendlyName)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// writeAdapterDNS reports the resolver configuration in effect per adapter,
|
||||
// which is what the resolver uses for a name no NRPT rule matches.
|
||||
func (g *BundleGenerator) writeAdapterDNS(sb *strings.Builder, adapters []*windows.IpAdapterAddresses, err error) {
|
||||
writeSection(sb, "Adapter DNS configuration", "GetAdaptersAddresses")
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintf(sb, "error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, adapter := range adapters {
|
||||
name := windows.UTF16PtrToString(adapter.FriendlyName)
|
||||
suffix := g.anonymizeDomain(windows.UTF16PtrToString(adapter.DnsSuffix))
|
||||
|
||||
fmt.Fprintf(sb, "%s (index %d, oper status %d)\n", name, adapter.IfIndex, adapter.OperStatus)
|
||||
fmt.Fprintf(sb, " DNS suffix: %s\n", suffix)
|
||||
|
||||
var servers []string
|
||||
for server := adapter.FirstDnsServerAddress; server != nil; server = server.Next {
|
||||
addr, ok := netip.AddrFromSlice(server.Address.IP())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
addr = addr.Unmap()
|
||||
if g.anonymize {
|
||||
addr = g.anonymizer.AnonymizeIP(addr)
|
||||
}
|
||||
servers = append(servers, addr.String())
|
||||
}
|
||||
|
||||
fmt.Fprintf(sb, " DNS servers: %s\n", strings.Join(servers, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// anonymizeDomain anonymizes a single domain, keeping the leading dot an NRPT
|
||||
// match domain carries.
|
||||
func (g *BundleGenerator) anonymizeDomain(entry string) string {
|
||||
if !g.anonymize {
|
||||
return entry
|
||||
}
|
||||
|
||||
domain, dot := strings.CutPrefix(entry, ".")
|
||||
if domain == "" {
|
||||
return entry
|
||||
}
|
||||
|
||||
anonymized := g.anonymizer.AnonymizeDomain(domain)
|
||||
if dot {
|
||||
anonymized = "." + anonymized
|
||||
}
|
||||
return anonymized
|
||||
}
|
||||
|
||||
// splitValueEntries splits a registry value that holds a list. The separator
|
||||
// differs per value: a REG_MULTI_SZ arrives joined with ", ", a SearchList is
|
||||
// comma separated and a NameServer may use commas or spaces.
|
||||
func splitValueEntries(value string) []string {
|
||||
return strings.FieldsFunc(value, func(r rune) bool {
|
||||
return r == ',' || r == ';' || r == ' ' || r == '\t'
|
||||
})
|
||||
}
|
||||
|
||||
func joinValueEntries(entries []string, anonymize func(string) string) string {
|
||||
for i, entry := range entries {
|
||||
entries[i] = anonymize(entry)
|
||||
}
|
||||
return strings.Join(entries, ", ")
|
||||
}
|
||||
|
||||
func writeSection(sb *strings.Builder, title, source string) {
|
||||
fmt.Fprintf(sb, "\n%s\n%s\n%s\n", title, strings.Repeat("-", len(title)), source)
|
||||
}
|
||||
|
||||
func subKeyNames(root string) ([]string, error) {
|
||||
k, err := registry.OpenKey(registry.LOCAL_MACHINE, root, registry.ENUMERATE_SUB_KEYS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", root, err)
|
||||
}
|
||||
defer closeKey(k)
|
||||
|
||||
names, err := k.ReadSubKeyNames(-1)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read subkey names: %w", err)
|
||||
}
|
||||
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// readRegistryValue renders a value as text regardless of its type, so an
|
||||
// unexpected type in a policy key still shows up instead of being dropped.
|
||||
func readRegistryValue(k registry.Key, name string) (string, error) {
|
||||
_, valueType, err := k.GetValue(name, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get value %s: %w", name, err)
|
||||
}
|
||||
|
||||
switch valueType {
|
||||
case registry.SZ, registry.EXPAND_SZ:
|
||||
value, _, err := k.GetStringValue(name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get string value %s: %w", name, err)
|
||||
}
|
||||
return value, nil
|
||||
case registry.MULTI_SZ:
|
||||
values, _, err := k.GetStringsValue(name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get strings value %s: %w", name, err)
|
||||
}
|
||||
return strings.Join(values, ", "), nil
|
||||
case registry.DWORD, registry.QWORD:
|
||||
value, _, err := k.GetIntegerValue(name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get integer value %s: %w", name, err)
|
||||
}
|
||||
return fmt.Sprintf("%d (0x%x)", value, value), nil
|
||||
case registry.BINARY:
|
||||
value, _, err := k.GetBinaryValue(name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get binary value %s: %w", name, err)
|
||||
}
|
||||
return hex.EncodeToString(value), nil
|
||||
default:
|
||||
return fmt.Sprintf("<unhandled registry type %d>", valueType), nil
|
||||
}
|
||||
}
|
||||
|
||||
// adapterAddresses returns the adapter list including DNS servers. The call
|
||||
// reports the size it needs, so grow the buffer and retry until it fits.
|
||||
func adapterAddresses() (adapters []*windows.IpAdapterAddresses, err error) {
|
||||
// GetAdaptersAddresses is resolved on first use and panics when it is
|
||||
// missing, so this reports it as an error and leaves the rest of the
|
||||
// report intact.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
adapters, err = nil, fmt.Errorf("GetAdaptersAddresses: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
const flags = windows.GAA_FLAG_SKIP_ANYCAST | windows.GAA_FLAG_SKIP_MULTICAST
|
||||
|
||||
size := uint32(15000)
|
||||
for range 3 {
|
||||
buf := make([]byte, size)
|
||||
first := (*windows.IpAdapterAddresses)(unsafe.Pointer(&buf[0]))
|
||||
|
||||
err := windows.GetAdaptersAddresses(windows.AF_UNSPEC, flags, 0, first, &size)
|
||||
if errors.Is(err, windows.ERROR_BUFFER_OVERFLOW) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetAdaptersAddresses: %w", err)
|
||||
}
|
||||
|
||||
for adapter := first; adapter != nil; adapter = adapter.Next {
|
||||
adapters = append(adapters, adapter)
|
||||
}
|
||||
return adapters, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("GetAdaptersAddresses: buffer kept growing")
|
||||
}
|
||||
|
||||
func closeKey(k registry.Key) {
|
||||
if err := k.Close(); err != nil {
|
||||
log.Debugf("close registry key: %v", err)
|
||||
}
|
||||
}
|
||||
146
client/internal/debug/debug_windows_test.go
Normal file
146
client/internal/debug/debug_windows_test.go
Normal file
@@ -0,0 +1,146 @@
|
||||
//go:build windows
|
||||
|
||||
package debug
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/client/anonymize"
|
||||
)
|
||||
|
||||
func newDNSValueGenerator(level anonymize.Level) *BundleGenerator {
|
||||
anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses())
|
||||
anonymizer.SetLevel(level)
|
||||
|
||||
return &BundleGenerator{
|
||||
anonymize: true,
|
||||
anonymizeLevel: level,
|
||||
anonymizer: anonymizer,
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnonymizeValueByName covers the value kinds of the DNS registry keys. The
|
||||
// names decide the treatment, because the string pass alone replaces only
|
||||
// domains another part of the bundle already seeded.
|
||||
func TestAnonymizeValueByName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
valueName string
|
||||
value string
|
||||
assert func(t *testing.T, got string)
|
||||
}{
|
||||
{
|
||||
name: "NRPT match domains keep the leading dot",
|
||||
valueName: "Name",
|
||||
value: ".internal.example.com, .corp.example.org",
|
||||
assert: func(t *testing.T, got string) {
|
||||
t.Helper()
|
||||
for _, entry := range strings.Split(got, ", ") {
|
||||
assert.True(t, strings.HasPrefix(entry, "."), "entry %q should keep its leading dot", entry)
|
||||
assert.NotContains(t, entry, "example", "entry %q should not keep the original domain", entry)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "any value name ending in Domain is treated as a domain",
|
||||
valueName: "ICSDomain",
|
||||
value: "mshome.net",
|
||||
assert: func(t *testing.T, got string) {
|
||||
t.Helper()
|
||||
assert.NotContains(t, got, "mshome", "should anonymize a domain suffix value")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "search list is a comma separated domain list",
|
||||
valueName: "SearchList",
|
||||
value: "corp.example.com,branch.example.com",
|
||||
assert: func(t *testing.T, got string) {
|
||||
t.Helper()
|
||||
assert.NotContains(t, got, "example", "should anonymize every search domain")
|
||||
assert.Len(t, strings.Split(got, ", "), 2, "should keep both search domains")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "name servers are anonymized as addresses",
|
||||
valueName: "DhcpNameServer",
|
||||
value: "203.0.113.10 8.8.8.8",
|
||||
assert: func(t *testing.T, got string) {
|
||||
t.Helper()
|
||||
assert.NotContains(t, got, "203.0.113.10", "should anonymize a public resolver address")
|
||||
// well-known resolvers stay readable at every level
|
||||
assert.Contains(t, got, "8.8.8.8", "should keep a well-known resolver address")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "opaque values are left to the string pass",
|
||||
valueName: "DataBasePath",
|
||||
value: `%SystemRoot%\System32\drivers\etc`,
|
||||
assert: func(t *testing.T, got string) {
|
||||
t.Helper()
|
||||
assert.Equal(t, `%SystemRoot%\System32\drivers\etc`, got, "should not alter a path")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := newDNSValueGenerator(anonymize.LevelDefault)
|
||||
tc.assert(t, g.anonymizeValue(tc.valueName, tc.value))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseNRPTPolicyTable parses the MOF text of the policy table out
|
||||
// parameters, as the provider on a client with one NRPT rule renders it.
|
||||
func TestParseNRPTPolicyTable(t *testing.T) {
|
||||
const text = `[abstract]
|
||||
class __PARAMETERS
|
||||
{
|
||||
[Out, EmbeddedInstance("DnsClientPolicyConfiguration"): ToSubClass, ID(2): DisableOverride ToInstance] DnsClientPolicyConfiguration cmdletOutput[] = {
|
||||
instance of DnsClientPolicyConfiguration
|
||||
{
|
||||
DirectAccessProxyType = "NoProxy";
|
||||
DirectAccessQueryIPsecRequired = FALSE;
|
||||
NameEncoding = "Utf8WithoutMapping";
|
||||
Namespace = ".0.100.in-addr.arpa";
|
||||
},
|
||||
instance of DnsClientPolicyConfiguration
|
||||
{
|
||||
DirectAccessProxyType = "NoProxy";
|
||||
NameEncoding = "Utf8WithoutMapping";
|
||||
NameServers = {"100.0.255.254", "100.0.255.253"};
|
||||
Namespace = ".nb.internal";
|
||||
}};
|
||||
[in] boolean Effective;
|
||||
[out] uint32 ReturnValue = 0;
|
||||
};
|
||||
`
|
||||
|
||||
entries := parseNRPTPolicyTable(text)
|
||||
require.Len(t, entries, 2, "should parse both embedded instances")
|
||||
|
||||
assert.Equal(t, ".0.100.in-addr.arpa", entries[0].namespace, "should read the namespace of the first instance")
|
||||
assert.Equal(t, ".nb.internal", entries[1].namespace, "should read the namespace of the second instance")
|
||||
|
||||
assert.Equal(t, []registryValue{
|
||||
{name: "DirectAccessProxyType", value: "NoProxy"},
|
||||
{name: "DirectAccessQueryIPsecRequired", value: "FALSE"},
|
||||
{name: "NameEncoding", value: "Utf8WithoutMapping"},
|
||||
}, entries[0].values, "should keep the remaining values in order")
|
||||
|
||||
assert.Contains(t, entries[1].values, registryValue{name: "NameServers", value: "100.0.255.254, 100.0.255.253"},
|
||||
"should flatten a MOF array")
|
||||
|
||||
for _, value := range entries[1].values {
|
||||
assert.NotContains(t, value.name, "ReturnValue", "should not read the class level parameters as values")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNRPTPolicyTableEmpty(t *testing.T) {
|
||||
assert.Empty(t, parseNRPTPolicyTable(""), "should parse no entries from empty text")
|
||||
assert.Empty(t, parseNRPTPolicyTable("class __PARAMETERS\n{\n};\n"), "should parse no entries from a table with no instances")
|
||||
}
|
||||
317
client/internal/debug/nrpt_windows.go
Normal file
317
client/internal/debug/nrpt_windows.go
Normal file
@@ -0,0 +1,317 @@
|
||||
//go:build windows
|
||||
|
||||
package debug
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-ole/go-ole"
|
||||
"github.com/go-ole/go-ole/oleutil"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// The NRPT policy table is reachable through the CIM class that backs
|
||||
// Get-DnsClientNrptPolicy. Unlike the rules in the registry, the table is
|
||||
// what the resolver currently has loaded, which is the only way to tell an
|
||||
// applied rule from one that is merely written, in either direction.
|
||||
nrptPolicyNamespace = `root\Microsoft\Windows\DNS`
|
||||
nrptPolicyClass = "PS_DnsClientNrptPolicy"
|
||||
nrptPolicyMethod = "Get"
|
||||
|
||||
// The class has no instances, so the table comes from the out parameters
|
||||
// of a static method call, rendered as MOF text: the embedded instances
|
||||
// arrive as a safe array of objects, which cannot be read back through the
|
||||
// COM bindings, and the text form carries all of them.
|
||||
nrptPolicyInstanceKeyword = "instance of DnsClientPolicyConfiguration"
|
||||
|
||||
nrptPolicyTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// COM initialization results that leave the calling thread usable: S_FALSE for
|
||||
// a thread this process already initialized, RPC_E_CHANGED_MODE for one that
|
||||
// belongs to another apartment.
|
||||
const (
|
||||
sFalse = 0x00000001
|
||||
rpcEChangedMode = 0x80010106
|
||||
)
|
||||
|
||||
// nrptQueryInFlight admits one read of the policy table at a time. A provider
|
||||
// that stops answering keeps its goroutine and the OS thread that goroutine
|
||||
// pinned, so a later bundle reports that instead of pinning another one.
|
||||
var nrptQueryInFlight = make(chan struct{}, 1)
|
||||
|
||||
// nrptPolicyEntry is one namespace of the effective policy table, holding the
|
||||
// values of an embedded DnsClientPolicyConfiguration instance in the order the
|
||||
// provider reported them.
|
||||
type nrptPolicyEntry struct {
|
||||
namespace string
|
||||
values []registryValue
|
||||
}
|
||||
|
||||
// registryValue is a name and its rendered value, shared by the registry and
|
||||
// policy table readers so both anonymize by value name the same way.
|
||||
type registryValue struct {
|
||||
name string
|
||||
value string
|
||||
}
|
||||
|
||||
// effectiveNRPTPolicies reads the effective NRPT table. The call is bounded
|
||||
// because a WMI provider can block indefinitely and a debug bundle must not.
|
||||
func effectiveNRPTPolicies() ([]nrptPolicyEntry, error) {
|
||||
type result struct {
|
||||
text string
|
||||
err error
|
||||
}
|
||||
|
||||
select {
|
||||
case nrptQueryInFlight <- struct{}{}:
|
||||
default:
|
||||
return nil, errors.New("an earlier read of the policy table has not returned")
|
||||
}
|
||||
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
// the slot is released here rather than by the caller, so a read that
|
||||
// outlives the timeout holds it until the provider answers
|
||||
defer func() { <-nrptQueryInFlight }()
|
||||
|
||||
text, err := nrptPolicyTableText()
|
||||
done <- result{text: text, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case res := <-done:
|
||||
if res.err != nil {
|
||||
return nil, res.err
|
||||
}
|
||||
return parseNRPTPolicyTable(res.text), nil
|
||||
case <-time.After(nrptPolicyTimeout):
|
||||
return nil, errors.New("read of the policy table timed out")
|
||||
}
|
||||
}
|
||||
|
||||
// nrptPolicyTableText calls the policy table method and returns the MOF text of
|
||||
// its out parameters.
|
||||
func nrptPolicyTableText() (text string, err error) {
|
||||
// COM is per thread, and the collection is short lived, so the thread is
|
||||
// pinned for the duration rather than initialized for the process.
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
defer func() {
|
||||
// The COM call chain is dynamically typed, so a provider that answers
|
||||
// with an unexpected shape must not take the daemon down with it.
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("read NRPT policy table: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
owns, err := coInitialize()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if owns {
|
||||
defer ole.CoUninitialize()
|
||||
}
|
||||
|
||||
locator, err := oleutil.CreateObject("WbemScripting.SWbemLocator")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create WMI locator: %w", err)
|
||||
}
|
||||
defer locator.Release()
|
||||
|
||||
dispatch, err := locator.QueryInterface(ole.IID_IDispatch)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("query WMI locator interface: %w", err)
|
||||
}
|
||||
defer dispatch.Release()
|
||||
|
||||
service, err := dispatchCall(dispatch, "ConnectServer", nil, nrptPolicyNamespace)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connect to %s: %w", nrptPolicyNamespace, err)
|
||||
}
|
||||
defer service.Release()
|
||||
|
||||
inParams, err := spawnMethodInParams(service)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer inParams.Release()
|
||||
|
||||
// The effective table is the merge of the local and the group policy
|
||||
// store, which is what the resolver answers from.
|
||||
if _, err := oleutil.PutProperty(inParams, "Effective", true); err != nil {
|
||||
return "", fmt.Errorf("set Effective parameter: %w", err)
|
||||
}
|
||||
|
||||
outParams, err := dispatchCall(service, "ExecMethod", nrptPolicyClass, nrptPolicyMethod, inParams)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("call %s.%s: %w", nrptPolicyClass, nrptPolicyMethod, err)
|
||||
}
|
||||
defer outParams.Release()
|
||||
|
||||
textVariant, err := oleutil.CallMethod(outParams, "GetObjectText_")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("render policy table: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := textVariant.Clear(); err != nil {
|
||||
log.Debugf("clear policy table variant: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return textVariant.ToString(), nil
|
||||
}
|
||||
|
||||
// spawnMethodInParams builds the in parameters instance the method needs. The
|
||||
// provider rejects the call without one, even when every parameter is optional.
|
||||
func spawnMethodInParams(service *ole.IDispatch) (*ole.IDispatch, error) {
|
||||
class, err := dispatchCall(service, "Get", nrptPolicyClass)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get class %s: %w", nrptPolicyClass, err)
|
||||
}
|
||||
defer class.Release()
|
||||
|
||||
methods, err := dispatchProperty(class, "Methods_")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get class methods: %w", err)
|
||||
}
|
||||
defer methods.Release()
|
||||
|
||||
method, err := dispatchCall(methods, "Item", nrptPolicyMethod)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get method %s: %w", nrptPolicyMethod, err)
|
||||
}
|
||||
defer method.Release()
|
||||
|
||||
params, err := dispatchProperty(method, "InParameters")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get method parameters: %w", err)
|
||||
}
|
||||
defer params.Release()
|
||||
|
||||
inParams, err := dispatchCall(params, "SpawnInstance_")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("spawn parameter instance: %w", err)
|
||||
}
|
||||
|
||||
return inParams, nil
|
||||
}
|
||||
|
||||
// parseNRPTPolicyTable pulls the embedded instances out of the MOF text. Each
|
||||
// instance is a namespace of the table, with one name and value per line.
|
||||
func parseNRPTPolicyTable(text string) []nrptPolicyEntry {
|
||||
var entries []nrptPolicyEntry
|
||||
var current *nrptPolicyEntry
|
||||
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(line), ";"))
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(line, nrptPolicyInstanceKeyword):
|
||||
entries = append(entries, nrptPolicyEntry{})
|
||||
current = &entries[len(entries)-1]
|
||||
continue
|
||||
case strings.HasPrefix(line, "}"):
|
||||
// closes an instance, and the array with the last one, so the
|
||||
// class level parameters that follow are not read as values
|
||||
current = nil
|
||||
continue
|
||||
case current == nil, line == "{":
|
||||
continue
|
||||
}
|
||||
|
||||
name, value, ok := strings.Cut(line, " = ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
value = unquoteMOFValue(value)
|
||||
if name == "Namespace" {
|
||||
current.namespace = value
|
||||
continue
|
||||
}
|
||||
|
||||
current.values = append(current.values, registryValue{name: name, value: value})
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
// unquoteMOFValue renders a MOF scalar or array as plain text: "a" becomes a,
|
||||
// and {"a", "b"} becomes a, b.
|
||||
func unquoteMOFValue(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
|
||||
if inner, ok := strings.CutPrefix(value, "{"); ok {
|
||||
value = strings.TrimSuffix(inner, "}")
|
||||
|
||||
entries := strings.Split(value, ",")
|
||||
for i, entry := range entries {
|
||||
entries[i] = strings.Trim(strings.TrimSpace(entry), `"`)
|
||||
}
|
||||
return strings.Join(entries, ", ")
|
||||
}
|
||||
|
||||
return strings.Trim(value, `"`)
|
||||
}
|
||||
|
||||
// coInitialize prepares the calling thread for COM and reports whether this
|
||||
// call owns the initialization, which decides whether it may be balanced with
|
||||
// CoUninitialize. S_FALSE took a reference on a thread this process had already
|
||||
// initialized and so has to be released, while RPC_E_CHANGED_MODE took none:
|
||||
// the thread belongs to another apartment, which is usable but is not ours to
|
||||
// uninitialize.
|
||||
func coInitialize() (bool, error) {
|
||||
err := ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var oleErr *ole.OleError
|
||||
if errors.As(err, &oleErr) {
|
||||
switch oleErr.Code() {
|
||||
case sFalse:
|
||||
return true, nil
|
||||
case rpcEChangedMode:
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("initialize COM: %w", err)
|
||||
}
|
||||
|
||||
// dispatchCall calls a COM method that returns an object.
|
||||
func dispatchCall(dispatch *ole.IDispatch, method string, params ...any) (*ole.IDispatch, error) {
|
||||
variant, err := oleutil.CallMethod(dispatch, method, params...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
object := variant.ToIDispatch()
|
||||
if object == nil {
|
||||
return nil, fmt.Errorf("%s returned no object", method)
|
||||
}
|
||||
|
||||
return object, nil
|
||||
}
|
||||
|
||||
// dispatchProperty reads a COM property that holds an object.
|
||||
func dispatchProperty(dispatch *ole.IDispatch, property string) (*ole.IDispatch, error) {
|
||||
variant, err := oleutil.GetProperty(dispatch, property)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
object := variant.ToIDispatch()
|
||||
if object == nil {
|
||||
return nil, fmt.Errorf("property %s holds no object", property)
|
||||
}
|
||||
|
||||
return object, nil
|
||||
}
|
||||
@@ -267,18 +267,38 @@ func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) {
|
||||
return SystemDNSSettings{}, fmt.Errorf("sending the command: %w", err)
|
||||
}
|
||||
|
||||
var dnsSettings SystemDNSSettings
|
||||
dnsSettings, serverAddresses, err := parseSystemDNSSettings(b)
|
||||
if err != nil {
|
||||
return dnsSettings, err
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.origNameservers = serverAddresses
|
||||
s.mu.Unlock()
|
||||
|
||||
return dnsSettings, nil
|
||||
}
|
||||
|
||||
// parseSystemDNSSettings parses the output of `scutil show State:/Network/Service/<id>/DNS`.
|
||||
// Lines that don't match the expected "index : value" shape are skipped: hosts with unusual
|
||||
// network services (e.g. orphaned hardware ports) can produce entries without a value.
|
||||
func parseSystemDNSSettings(out []byte) (SystemDNSSettings, []netip.Addr, error) {
|
||||
// port is not exposed by scutil, default to 53
|
||||
dnsSettings := SystemDNSSettings{ServerPort: DefaultPort}
|
||||
var serverAddresses []netip.Addr
|
||||
inSearchDomainsArray := false
|
||||
inServerAddressesArray := false
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(b))
|
||||
scanner := bufio.NewScanner(bytes.NewReader(out))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
switch {
|
||||
case strings.HasPrefix(line, "DomainName :"):
|
||||
domainName := strings.TrimSpace(strings.Split(line, ":")[1])
|
||||
dnsSettings.Domains = append(dnsSettings.Domains, domainName)
|
||||
domainName := strings.TrimSpace(strings.TrimPrefix(line, "DomainName :"))
|
||||
if domainName != "" {
|
||||
dnsSettings.Domains = append(dnsSettings.Domains, domainName)
|
||||
}
|
||||
continue
|
||||
case line == "SearchDomains : <array> {":
|
||||
inSearchDomainsArray = true
|
||||
continue
|
||||
@@ -288,36 +308,45 @@ func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) {
|
||||
case line == "}":
|
||||
inSearchDomainsArray = false
|
||||
inServerAddressesArray = false
|
||||
continue
|
||||
}
|
||||
|
||||
if !inSearchDomainsArray && !inServerAddressesArray {
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.SplitN(line, " : ", 2)
|
||||
if len(parts) != 2 {
|
||||
log.Debugf("skipping unexpected scutil DNS line %q", line)
|
||||
continue
|
||||
}
|
||||
value := strings.TrimSpace(parts[1])
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if inSearchDomainsArray {
|
||||
searchDomain := strings.Split(line, " : ")[1]
|
||||
dnsSettings.Domains = append(dnsSettings.Domains, searchDomain)
|
||||
} else if inServerAddressesArray {
|
||||
address := strings.Split(line, " : ")[1]
|
||||
if ip, err := netip.ParseAddr(address); err == nil && !ip.IsUnspecified() {
|
||||
ip = ip.Unmap()
|
||||
serverAddresses = append(serverAddresses, ip)
|
||||
// Prefer the first IPv4 server as ServerIP since our DNS listener is IPv4.
|
||||
if !dnsSettings.ServerIP.IsValid() && ip.Is4() {
|
||||
dnsSettings.ServerIP = ip
|
||||
}
|
||||
}
|
||||
dnsSettings.Domains = append(dnsSettings.Domains, value)
|
||||
continue
|
||||
}
|
||||
|
||||
ip, err := netip.ParseAddr(value)
|
||||
if err != nil || ip.IsUnspecified() {
|
||||
continue
|
||||
}
|
||||
ip = ip.Unmap()
|
||||
serverAddresses = append(serverAddresses, ip)
|
||||
// Prefer the first IPv4 server as ServerIP since our DNS listener is IPv4.
|
||||
if !dnsSettings.ServerIP.IsValid() && ip.Is4() {
|
||||
dnsSettings.ServerIP = ip
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return dnsSettings, err
|
||||
return dnsSettings, serverAddresses, err
|
||||
}
|
||||
|
||||
// default to 53 port
|
||||
dnsSettings.ServerPort = DefaultPort
|
||||
|
||||
s.mu.Lock()
|
||||
s.origNameservers = serverAddresses
|
||||
s.mu.Unlock()
|
||||
|
||||
return dnsSettings, nil
|
||||
return dnsSettings, serverAddresses, nil
|
||||
}
|
||||
|
||||
func (s *systemConfigurator) getOriginalNameservers() []netip.Addr {
|
||||
@@ -435,11 +464,15 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) {
|
||||
router := ""
|
||||
for scanner.Scan() {
|
||||
text := scanner.Text()
|
||||
parts := strings.SplitN(text, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(text, "PrimaryService") {
|
||||
primaryService = strings.TrimSpace(strings.Split(text, ":")[1])
|
||||
primaryService = strings.TrimSpace(parts[1])
|
||||
}
|
||||
if strings.Contains(text, "Router") {
|
||||
router = strings.TrimSpace(strings.Split(text, ":")[1])
|
||||
router = strings.TrimSpace(parts[1])
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil && err != io.EOF {
|
||||
|
||||
@@ -328,6 +328,120 @@ func removeTestDNSKey(key string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func TestParseSystemDNSSettings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
output string
|
||||
expectedDomains []string
|
||||
expectedServers []netip.Addr
|
||||
expectedIP netip.Addr
|
||||
}{
|
||||
{
|
||||
name: "well_formed",
|
||||
output: `<dictionary> {
|
||||
DomainName : example.com
|
||||
SearchDomains : <array> {
|
||||
0 : example.com
|
||||
1 : corp.example.com
|
||||
}
|
||||
ServerAddresses : <array> {
|
||||
0 : 192.168.1.1
|
||||
1 : fd00::53
|
||||
}
|
||||
}
|
||||
`,
|
||||
expectedDomains: []string{"example.com", "example.com", "corp.example.com"},
|
||||
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1"), netip.MustParseAddr("fd00::53")},
|
||||
expectedIP: netip.MustParseAddr("192.168.1.1"),
|
||||
},
|
||||
{
|
||||
// entries without a value after the separator used to panic with
|
||||
// "index out of range [1] with length 1"
|
||||
name: "malformed_array_entries_skipped",
|
||||
output: `<dictionary> {
|
||||
SearchDomains : <array> {
|
||||
0 :
|
||||
(null)
|
||||
|
||||
1 : corp.example.com
|
||||
}
|
||||
ServerAddresses : <array> {
|
||||
0 :
|
||||
1 : 192.168.1.1
|
||||
}
|
||||
}
|
||||
`,
|
||||
expectedDomains: []string{"corp.example.com"},
|
||||
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
|
||||
expectedIP: netip.MustParseAddr("192.168.1.1"),
|
||||
},
|
||||
{
|
||||
name: "domain_name_without_value_skipped",
|
||||
output: `<dictionary> {
|
||||
DomainName :
|
||||
ServerAddresses : <array> {
|
||||
0 : 192.168.1.1
|
||||
}
|
||||
}
|
||||
`,
|
||||
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
|
||||
expectedIP: netip.MustParseAddr("192.168.1.1"),
|
||||
},
|
||||
{
|
||||
name: "ipv6_first_prefers_ipv4_server_ip",
|
||||
output: `<dictionary> {
|
||||
ServerAddresses : <array> {
|
||||
0 : fd00::53
|
||||
1 : 192.168.1.1
|
||||
}
|
||||
}
|
||||
`,
|
||||
expectedServers: []netip.Addr{netip.MustParseAddr("fd00::53"), netip.MustParseAddr("192.168.1.1")},
|
||||
expectedIP: netip.MustParseAddr("192.168.1.1"),
|
||||
},
|
||||
{
|
||||
name: "invalid_and_unspecified_addresses_skipped",
|
||||
output: `<dictionary> {
|
||||
ServerAddresses : <array> {
|
||||
0 : (null)
|
||||
1 : 0.0.0.0
|
||||
2 : 192.168.1.1
|
||||
}
|
||||
}
|
||||
`,
|
||||
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
|
||||
expectedIP: netip.MustParseAddr("192.168.1.1"),
|
||||
},
|
||||
{
|
||||
name: "v4_mapped_address_unmapped",
|
||||
output: `<dictionary> {
|
||||
ServerAddresses : <array> {
|
||||
0 : ::ffff:192.168.1.1
|
||||
}
|
||||
}
|
||||
`,
|
||||
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
|
||||
expectedIP: netip.MustParseAddr("192.168.1.1"),
|
||||
},
|
||||
{
|
||||
name: "empty_output",
|
||||
output: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
settings, servers, err := parseSystemDNSSettings([]byte(tc.output))
|
||||
require.NoError(t, err, "parsing should not fail")
|
||||
|
||||
assert.Equal(t, tc.expectedDomains, settings.Domains, "domains should match")
|
||||
assert.Equal(t, tc.expectedServers, servers, "server addresses should match")
|
||||
assert.Equal(t, tc.expectedIP, settings.ServerIP, "server IP should match")
|
||||
assert.Equal(t, DefaultPort, settings.ServerPort, "server port should default to 53")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOriginalNameservers(t *testing.T) {
|
||||
configurator := &systemConfigurator{
|
||||
createdKeys: make(map[string]struct{}),
|
||||
|
||||
@@ -31,10 +31,28 @@ var (
|
||||
dnsFlushResolverCacheFn = dnsapi.NewProc("DnsFlushResolverCache")
|
||||
)
|
||||
|
||||
// Registry locations of the host DNS configuration this package programs,
|
||||
// exported so a diagnostic reader reports the same locations that are written.
|
||||
const (
|
||||
dnsPolicyConfigMatchPath = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig\NetBird-Match`
|
||||
gpoDnsPolicyRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig`
|
||||
gpoDnsPolicyConfigMatchPath = gpoDnsPolicyRoot + `\NetBird-Match`
|
||||
// NRPTKeyPrefix starts the name of every NRPT rule key this client creates.
|
||||
NRPTKeyPrefix = "NetBird-Match"
|
||||
|
||||
// DNSPolicyConfigRoot holds the NRPT rules of the local policy store.
|
||||
DNSPolicyConfigRoot = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig`
|
||||
|
||||
// GPODNSPolicyConfigRoot holds the NRPT rules of the group policy store,
|
||||
// which takes precedence over the local one when it is present.
|
||||
GPODNSPolicyConfigRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig`
|
||||
|
||||
// InterfaceConfigPath and InterfaceConfigPathV6 hold the per-interface DNS
|
||||
// settings, keyed by interface GUID, in separate hives per address family.
|
||||
InterfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`
|
||||
InterfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`
|
||||
)
|
||||
|
||||
const (
|
||||
dnsPolicyConfigMatchPath = DNSPolicyConfigRoot + `\` + NRPTKeyPrefix
|
||||
gpoDnsPolicyConfigMatchPath = GPODNSPolicyConfigRoot + `\` + NRPTKeyPrefix
|
||||
|
||||
dnsPolicyConfigVersionKey = "Version"
|
||||
dnsPolicyConfigVersionValue = 2
|
||||
@@ -45,8 +63,6 @@ const (
|
||||
|
||||
nrptMaxDomainsPerRule = 50
|
||||
|
||||
interfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`
|
||||
interfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`
|
||||
interfaceConfigNameServerKey = "NameServer"
|
||||
interfaceConfigDhcpNameSrvKey = "DhcpNameServer"
|
||||
interfaceConfigSearchListKey = "SearchList"
|
||||
@@ -84,7 +100,7 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) {
|
||||
}
|
||||
|
||||
var useGPO bool
|
||||
k, err := registry.OpenKey(registry.LOCAL_MACHINE, gpoDnsPolicyRoot, registry.QUERY_VALUE)
|
||||
k, err := registry.OpenKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.QUERY_VALUE)
|
||||
if err != nil {
|
||||
log.Debugf("failed to open GPO DNS policy root: %v", err)
|
||||
} else {
|
||||
@@ -123,7 +139,7 @@ func (r *registryConfigurator) captureOriginalNameservers() ([]netip.Addr, error
|
||||
seen := make(map[netip.Addr]struct{})
|
||||
var out []netip.Addr
|
||||
var merr *multierror.Error
|
||||
for _, root := range []string{interfaceConfigPath, interfaceConfigPathV6} {
|
||||
for _, root := range []string{InterfaceConfigPath, InterfaceConfigPathV6} {
|
||||
addrs, err := r.captureFromTcpipRoot(root)
|
||||
if err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("%s: %w", root, err))
|
||||
@@ -496,7 +512,7 @@ func (r *registryConfigurator) deleteInterfaceRegistryKeyProperty(propertyKey st
|
||||
}
|
||||
|
||||
func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) {
|
||||
regKeyPath := interfaceConfigPath + "\\" + r.guid
|
||||
regKeyPath := InterfaceConfigPath + "\\" + r.guid
|
||||
regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.SET_VALUE)
|
||||
if err != nil {
|
||||
return regKey, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", regKeyPath, err)
|
||||
|
||||
@@ -87,9 +87,10 @@ func (pm *ProfileManager) SetActiveProfileState(state *ProfileState) error {
|
||||
|
||||
// RemoveProfileState deletes the per-profile state file (which holds the
|
||||
// account email used for the SSO login hint and the UI display). Called after
|
||||
// a successful logout so a logged-out profile no longer shows a stale account
|
||||
// email. The state file only stores the email, so deleting it is equivalent to
|
||||
// clearing it; the next SSO login recreates it. A missing file is not an error.
|
||||
// profile removal; logout keeps the file so the next login can pass the email
|
||||
// as the login_hint. The state file only stores the email, so deleting it is
|
||||
// equivalent to clearing it; the next SSO login recreates it. A missing file
|
||||
// is not an error.
|
||||
func (pm *ProfileManager) RemoveProfileState(profileName string) error {
|
||||
configDir, err := getConfigDir()
|
||||
if err != nil {
|
||||
|
||||
@@ -2,54 +2,183 @@ package ipfwdstate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
|
||||
)
|
||||
|
||||
// IPForwardingState is a struct that keeps track of the IP forwarding state.
|
||||
// todo: read initial state of the IP forwarding from the system and reset the state based on it.
|
||||
// todo: separate v4/v6 forwarding state, since the sysctls are independent
|
||||
// (net.ipv4.ip_forward vs net.ipv6.conf.all.forwarding). Currently the nftables
|
||||
// manager shares one instance between both routers, which works only because
|
||||
// EnableIPForwarding enables both sysctls in a single call.
|
||||
// IPForwardingState tracks v4 and v6 IP-forwarding sysctl enables with
|
||||
// independent refcounts so a v4-only routing setup doesn't flip v6 sysctls.
|
||||
type IPForwardingState struct {
|
||||
enabledCounter int
|
||||
mu sync.Mutex
|
||||
|
||||
v4Count int
|
||||
v6Count int
|
||||
|
||||
// routingV4/routingV6 track whether the routing path currently holds a
|
||||
// reference, so repeated EnableRouting calls (one per network-map update)
|
||||
// hold at most one reference per family and an unpaired DisableRouting
|
||||
// can't release references held by DNAT rules.
|
||||
routingV4 bool
|
||||
routingV6 bool
|
||||
|
||||
wgIfaceName string
|
||||
v6Saved map[string]int
|
||||
}
|
||||
|
||||
func NewIPForwardingState() *IPForwardingState {
|
||||
return &IPForwardingState{}
|
||||
// NewIPForwardingState returns a state tracker for the IP-forwarding sysctls.
|
||||
// wgIfaceName is excluded from the per-interface accept_ra handling.
|
||||
func NewIPForwardingState(wgIfaceName string) *IPForwardingState {
|
||||
return &IPForwardingState{wgIfaceName: wgIfaceName}
|
||||
}
|
||||
|
||||
func (f *IPForwardingState) RequestForwarding() error {
|
||||
if f.enabledCounter != 0 {
|
||||
f.enabledCounter++
|
||||
// Counts returns the current v4 and v6 refcounts. Intended for diagnostics
|
||||
// and tests.
|
||||
func (f *IPForwardingState) Counts() (v4, v6 int) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.v4Count, f.v6Count
|
||||
}
|
||||
|
||||
// RequestRouting takes the forwarding references for the routing path. It is
|
||||
// idempotent: while routing already holds a reference, further calls don't
|
||||
// increment the refcounts, and a v4-only request releases a previously held v6
|
||||
// reference. A v6 sysctl failure is logged and not returned so it can't take
|
||||
// down v4 routing (the sysctl may be unwritable, e.g. read-only /proc/sys or
|
||||
// IPv6 disabled on the kernel command line); v6 is retried on the next call.
|
||||
func (f *IPForwardingState) RequestRouting(v6 bool) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if !f.routingV4 {
|
||||
if err := f.requestV4(); err != nil {
|
||||
return err
|
||||
}
|
||||
f.routingV4 = true
|
||||
}
|
||||
|
||||
if !v6 {
|
||||
if !f.routingV6 {
|
||||
return nil
|
||||
}
|
||||
f.routingV6 = false
|
||||
return f.releaseV6()
|
||||
}
|
||||
|
||||
if f.routingV6 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := systemops.EnableIPForwarding(); err != nil {
|
||||
return fmt.Errorf("failed to enable IP forwarding with sysctl: %w", err)
|
||||
if err := f.requestV6(); err != nil {
|
||||
log.Warnf("enable IPv6 forwarding for routing: %v", err)
|
||||
return nil
|
||||
}
|
||||
f.enabledCounter = 1
|
||||
log.Info("IP forwarding enabled")
|
||||
|
||||
f.routingV6 = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *IPForwardingState) ReleaseForwarding() error {
|
||||
if f.enabledCounter == 0 {
|
||||
return nil
|
||||
// ReleaseRouting releases the references RequestRouting holds. Calls without a
|
||||
// held reference are no-ops.
|
||||
func (f *IPForwardingState) ReleaseRouting() error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if f.routingV4 {
|
||||
f.routingV4 = false
|
||||
f.releaseV4()
|
||||
}
|
||||
|
||||
if f.enabledCounter > 1 {
|
||||
f.enabledCounter--
|
||||
return nil
|
||||
if f.routingV6 {
|
||||
f.routingV6 = false
|
||||
return f.releaseV6()
|
||||
}
|
||||
|
||||
// if failed to disable IP forwarding we anyway decrement the counter
|
||||
f.enabledCounter = 0
|
||||
|
||||
// todo call systemops.DisableIPForwarding()
|
||||
return nil
|
||||
}
|
||||
|
||||
// RequestForwarding enables the family's forwarding sysctl on first request.
|
||||
func (f *IPForwardingState) RequestForwarding(v6 bool) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if v6 {
|
||||
return f.requestV6()
|
||||
}
|
||||
return f.requestV4()
|
||||
}
|
||||
|
||||
// ReleaseForwarding decrements the family counter. The last v6 release restores
|
||||
// what enable captured. v4 stays on: net.ipv4.ip_forward is co-owned by other
|
||||
// tooling (docker, k8s, libvirt).
|
||||
func (f *IPForwardingState) ReleaseForwarding(v6 bool) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if v6 {
|
||||
return f.releaseV6()
|
||||
}
|
||||
f.releaseV4()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *IPForwardingState) requestV4() error {
|
||||
if f.v4Count == 0 {
|
||||
if err := systemops.EnableV4IPForwarding(); err != nil {
|
||||
return fmt.Errorf("enable IPv4 forwarding: %w", err)
|
||||
}
|
||||
log.Info("IPv4 forwarding enabled")
|
||||
}
|
||||
f.v4Count++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *IPForwardingState) releaseV4() {
|
||||
if f.v4Count > 0 {
|
||||
f.v4Count--
|
||||
}
|
||||
}
|
||||
|
||||
func (f *IPForwardingState) requestV6() error {
|
||||
if f.v6Count == 0 {
|
||||
saved, err := systemops.EnableV6IPForwarding(f.wgIfaceName)
|
||||
if err != nil {
|
||||
if rerr := systemops.DisableV6IPForwarding(saved); rerr != nil {
|
||||
log.Warnf("rollback partial v6 sysctls: %v", rerr)
|
||||
}
|
||||
return fmt.Errorf("enable IPv6 forwarding: %w", err)
|
||||
}
|
||||
// A failed restore on a previous release keeps its saved values; those
|
||||
// are the true originals, so keep them over what this enable captured.
|
||||
if f.v6Saved == nil {
|
||||
f.v6Saved = saved
|
||||
} else {
|
||||
for k, v := range saved {
|
||||
if _, ok := f.v6Saved[k]; !ok {
|
||||
f.v6Saved[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Info("IPv6 forwarding enabled")
|
||||
}
|
||||
f.v6Count++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *IPForwardingState) releaseV6() error {
|
||||
if f.v6Count == 0 {
|
||||
return nil
|
||||
}
|
||||
f.v6Count--
|
||||
if f.v6Count > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keep the saved values on failure so a later release or enable/release
|
||||
// cycle can still restore them; re-restoring an already-restored key is a
|
||||
// no-op since the sysctl already holds the desired value.
|
||||
if err := systemops.DisableV6IPForwarding(f.v6Saved); err != nil {
|
||||
return fmt.Errorf("disable IPv6 forwarding: %w", err)
|
||||
}
|
||||
f.v6Saved = nil
|
||||
log.Info("IPv6 forwarding disabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
//go:build privileged
|
||||
|
||||
package ipfwdstate
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestRequestRoutingV6ToV4Transition verifies that a v4-only routing request
|
||||
// releases a previously held routing-owned v6 reference without touching
|
||||
// references held by DNAT rules.
|
||||
func TestRequestRoutingV6ToV4Transition(t *testing.T) {
|
||||
f := NewIPForwardingState("wt-fwd-test")
|
||||
|
||||
require.NoError(t, f.RequestRouting(true), "request routing with v6")
|
||||
v4, v6 := f.Counts()
|
||||
assert.Equal(t, 1, v4, "v4 reference held")
|
||||
assert.Equal(t, 1, v6, "v6 reference held")
|
||||
|
||||
require.NoError(t, f.RequestRouting(false), "request routing v4-only")
|
||||
v4, v6 = f.Counts()
|
||||
assert.Equal(t, 1, v4, "v4 reference kept")
|
||||
assert.Equal(t, 0, v6, "routing-owned v6 reference released")
|
||||
|
||||
// A DNAT-held reference survives a v4-only routing request.
|
||||
require.NoError(t, f.RequestForwarding(true), "dnat v6 reference")
|
||||
require.NoError(t, f.RequestRouting(false), "repeat v4-only request")
|
||||
_, v6 = f.Counts()
|
||||
assert.Equal(t, 1, v6, "dnat-held v6 reference survives")
|
||||
require.NoError(t, f.ReleaseForwarding(true), "release dnat v6 reference")
|
||||
|
||||
require.NoError(t, f.ReleaseRouting(), "release routing")
|
||||
v4, v6 = f.Counts()
|
||||
assert.Equal(t, 0, v4, "all v4 references released")
|
||||
assert.Equal(t, 0, v6, "all v6 references released")
|
||||
}
|
||||
@@ -58,11 +58,7 @@ func Setup(wgIface iface) (map[string]int, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Escape '%' and '.' so they survive the dot-to-slash conversion in Set()
|
||||
safeName := strings.ReplaceAll(intf.Name, "%", percentEscape)
|
||||
safeName = strings.ReplaceAll(safeName, ".", dotEscape)
|
||||
|
||||
i := fmt.Sprintf(rpFilterInterfacePath, safeName)
|
||||
i := fmt.Sprintf(rpFilterInterfacePath, EscapeInterfaceName(intf.Name))
|
||||
oldVal, err := Set(i, 2, true)
|
||||
if err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
@@ -74,6 +70,13 @@ func Setup(wgIface iface) (map[string]int, error) {
|
||||
return keys, nberrors.FormatErrorOrNil(result)
|
||||
}
|
||||
|
||||
// EscapeInterfaceName escapes '%' and '.' in an interface name (e.g. VLANs
|
||||
// like eth0.100) so the name survives the dot-to-slash conversion in Set.
|
||||
func EscapeInterfaceName(name string) string {
|
||||
safe := strings.ReplaceAll(name, "%", percentEscape)
|
||||
return strings.ReplaceAll(safe, ".", dotEscape)
|
||||
}
|
||||
|
||||
// Set sets a sysctl configuration, if onlyIfOne is true it will only set the new value if it's set to 1
|
||||
func Set(key string, desiredValue int, onlyIfOne bool) (int, error) {
|
||||
path := strings.ReplaceAll(key, ".", "/")
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build windows
|
||||
|
||||
package systemops
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSortRouteCandidates(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
candidates []candidateRoute
|
||||
wantOrder []uint32
|
||||
}{
|
||||
{
|
||||
name: "longest prefix wins over metrics",
|
||||
candidates: []candidateRoute{
|
||||
{interfaceIndex: 1, prefixLength: 0, routeMetric: 0, interfaceMetric: 5},
|
||||
{interfaceIndex: 2, prefixLength: 24, routeMetric: 100, interfaceMetric: 50},
|
||||
},
|
||||
wantOrder: []uint32{2, 1},
|
||||
},
|
||||
{
|
||||
// Windows ranks equal-length prefixes by route metric + interface metric,
|
||||
// so a higher route metric on a low metric interface can still win.
|
||||
name: "combined metric beats route metric alone",
|
||||
candidates: []candidateRoute{
|
||||
{interfaceIndex: 8, prefixLength: 0, routeMetric: 0, interfaceMetric: 100},
|
||||
{interfaceIndex: 5, prefixLength: 0, routeMetric: 10, interfaceMetric: 5},
|
||||
},
|
||||
wantOrder: []uint32{5, 8},
|
||||
},
|
||||
{
|
||||
name: "lower combined metric wins",
|
||||
candidates: []candidateRoute{
|
||||
{interfaceIndex: 5, prefixLength: 0, routeMetric: 300, interfaceMetric: 5},
|
||||
{interfaceIndex: 8, prefixLength: 0, routeMetric: 0, interfaceMetric: 100},
|
||||
},
|
||||
wantOrder: []uint32{8, 5},
|
||||
},
|
||||
{
|
||||
name: "equal combined metric falls back to route metric",
|
||||
candidates: []candidateRoute{
|
||||
{interfaceIndex: 1, prefixLength: 0, routeMetric: 20, interfaceMetric: 10},
|
||||
{interfaceIndex: 2, prefixLength: 0, routeMetric: 5, interfaceMetric: 25},
|
||||
},
|
||||
wantOrder: []uint32{2, 1},
|
||||
},
|
||||
{
|
||||
// The metrics are uint32 on the Windows side, so the sum must not wrap.
|
||||
name: "combined metric beyond the uint32 range",
|
||||
candidates: []candidateRoute{
|
||||
{interfaceIndex: 1, prefixLength: 0, routeMetric: math.MaxUint32, interfaceMetric: 5},
|
||||
{interfaceIndex: 2, prefixLength: 0, routeMetric: math.MaxUint32 - 10, interfaceMetric: 5},
|
||||
},
|
||||
wantOrder: []uint32{2, 1},
|
||||
},
|
||||
{
|
||||
name: "unknown interface metric ranks on route metric only",
|
||||
candidates: []candidateRoute{
|
||||
{interfaceIndex: 1, prefixLength: 0, routeMetric: 30, interfaceMetric: -1},
|
||||
{interfaceIndex: 2, prefixLength: 0, routeMetric: 5, interfaceMetric: 10},
|
||||
},
|
||||
wantOrder: []uint32{2, 1},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sortRouteCandidates(tt.candidates)
|
||||
|
||||
got := make([]uint32, 0, len(tt.candidates))
|
||||
for _, c := range tt.candidates {
|
||||
got = append(got, c.interfaceIndex)
|
||||
}
|
||||
assert.Equal(t, tt.wantOrder, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -32,8 +32,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnableIPForwarding() error {
|
||||
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
|
||||
func EnableV4IPForwarding() error {
|
||||
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnableV6IPForwarding(string) (map[string]int, error) {
|
||||
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
|
||||
return map[string]int{}, nil
|
||||
}
|
||||
|
||||
func DisableV6IPForwarding(map[string]int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -58,8 +58,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnableIPForwarding() error {
|
||||
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
|
||||
func EnableV4IPForwarding() error {
|
||||
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnableV6IPForwarding(string) (map[string]int, error) {
|
||||
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
|
||||
return map[string]int{}, nil
|
||||
}
|
||||
|
||||
func DisableV6IPForwarding(map[string]int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -763,13 +763,10 @@ func flushRoutes(tableID, family int) error {
|
||||
return nberrors.FormatErrorOrNil(result)
|
||||
}
|
||||
|
||||
func EnableIPForwarding() error {
|
||||
func EnableV4IPForwarding() error {
|
||||
if _, err := sysctl.Set(ipv4ForwardingPath, 1, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sysctl.Set(ipv6ForwardingPath, 1, false); err != nil {
|
||||
log.Warnf("failed to enable IPv6 forwarding: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -43,8 +43,17 @@ func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error
|
||||
return r.genericRemoveVPNRoute(prefix, intf)
|
||||
}
|
||||
|
||||
func EnableIPForwarding() error {
|
||||
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
|
||||
func EnableV4IPForwarding() error {
|
||||
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnableV6IPForwarding(string) (map[string]int, error) {
|
||||
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
|
||||
return map[string]int{}, nil
|
||||
}
|
||||
|
||||
func DisableV6IPForwarding(map[string]int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -882,26 +882,40 @@ func getInterfaceMetric(interfaceIndex uint32, family int16) int {
|
||||
return int(ipInterfaceRow.Metric)
|
||||
}
|
||||
|
||||
// sortRouteCandidates sorts route candidates by priority: prefix length -> route metric -> interface metric
|
||||
// sortRouteCandidates sorts route candidates by priority: prefix length -> combined metric -> route metric.
|
||||
// Windows prefers the longest matching prefix and, among prefixes of the same length, the lowest metric, see
|
||||
// https://learn.microsoft.com/en-us/windows-hardware/customize/desktop/unattend/microsoft-windows-tcpip-interfaces-interface-routes-route-metric
|
||||
func sortRouteCandidates(candidates []candidateRoute) {
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if candidates[i].prefixLength != candidates[j].prefixLength {
|
||||
return candidates[i].prefixLength > candidates[j].prefixLength
|
||||
}
|
||||
if candidates[i].routeMetric != candidates[j].routeMetric {
|
||||
return candidates[i].routeMetric < candidates[j].routeMetric
|
||||
mi, mj := combinedMetric(candidates[i]), combinedMetric(candidates[j])
|
||||
if mi != mj {
|
||||
return mi < mj
|
||||
}
|
||||
return candidates[i].interfaceMetric < candidates[j].interfaceMetric
|
||||
return candidates[i].routeMetric < candidates[j].routeMetric
|
||||
})
|
||||
}
|
||||
|
||||
// combinedMetric returns the effective metric Windows uses to rank routes with an equal prefix length:
|
||||
// the sum of the route metric and the metric of the interface the route is on, see
|
||||
// https://learn.microsoft.com/en-us/windows-server/networking/technologies/network-subsystem/net-sub-interface-metric
|
||||
// An unknown interface metric contributes nothing.
|
||||
func combinedMetric(candidate candidateRoute) uint64 {
|
||||
if candidate.interfaceMetric < 0 {
|
||||
return uint64(candidate.routeMetric)
|
||||
}
|
||||
return uint64(candidate.routeMetric) + uint64(candidate.interfaceMetric)
|
||||
}
|
||||
|
||||
// GetBestInterface finds the best interface for reaching a destination,
|
||||
// excluding the VPN interface to avoid routing loops.
|
||||
//
|
||||
// Route selection priority:
|
||||
// 1. Longest prefix match (most specific route)
|
||||
// 2. Lowest route metric
|
||||
// 3. Lowest interface metric
|
||||
// 2. Lowest combined metric (route metric + interface metric)
|
||||
// 3. Lowest route metric.
|
||||
func GetBestInterface(dest netip.Addr, vpnIntf string) (*net.Interface, error) {
|
||||
var skipInterfaceIndex int
|
||||
if vpnIntf != "" {
|
||||
@@ -925,7 +939,6 @@ func GetBestInterface(dest netip.Addr, vpnIntf string) (*net.Interface, error) {
|
||||
return nil, fmt.Errorf("no route to %s", dest)
|
||||
}
|
||||
|
||||
// Sort routes: prefix length -> route metric -> interface metric
|
||||
sortRouteCandidates(candidates)
|
||||
|
||||
for _, candidate := range candidates {
|
||||
|
||||
92
client/internal/routemanager/systemops/v6forwarding_linux.go
Normal file
92
client/internal/routemanager/systemops/v6forwarding_linux.go
Normal file
@@ -0,0 +1,92 @@
|
||||
//go:build !android
|
||||
|
||||
package systemops
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
nberrors "github.com/netbirdio/netbird/client/errors"
|
||||
"github.com/netbirdio/netbird/client/internal/routemanager/sysctl"
|
||||
)
|
||||
|
||||
const (
|
||||
// 1 (default) accepts RAs only while forwarding is off; 2 keeps RA
|
||||
// acceptance on regardless, so RA-installed host defaults survive our
|
||||
// v6 forwarding flip.
|
||||
acceptRAInterfacePath = "net.ipv6.conf.%s.accept_ra"
|
||||
acceptRADefaultPath = "net.ipv6.conf.default.accept_ra"
|
||||
acceptRAProcPathFormat = "/proc/sys/net/ipv6/conf/%s/accept_ra"
|
||||
)
|
||||
|
||||
// EnableV6IPForwarding bumps accept_ra=2 on host v6 interfaces before flipping
|
||||
// forwarding=1, so RA-installed host defaults survive. Returns the prior values
|
||||
// of sysctls we actually changed; entries already at the target are omitted.
|
||||
func EnableV6IPForwarding(wgIfaceName string) (map[string]int, error) {
|
||||
saved := map[string]int{}
|
||||
bumpAcceptRA(saved, wgIfaceName)
|
||||
|
||||
oldVal, err := sysctl.Set(ipv6ForwardingPath, 1, false)
|
||||
if err != nil {
|
||||
return saved, err
|
||||
}
|
||||
if oldVal != 1 {
|
||||
saved[ipv6ForwardingPath] = oldVal
|
||||
}
|
||||
return saved, nil
|
||||
}
|
||||
|
||||
// DisableV6IPForwarding restores what EnableV6IPForwarding captured.
|
||||
func DisableV6IPForwarding(saved map[string]int) error {
|
||||
var result *multierror.Error
|
||||
for key, value := range saved {
|
||||
if _, err := sysctl.Set(key, value, false); err != nil {
|
||||
result = multierror.Append(result, fmt.Errorf("restore %s: %w", key, err))
|
||||
}
|
||||
}
|
||||
return nberrors.FormatErrorOrNil(result)
|
||||
}
|
||||
|
||||
func bumpAcceptRA(saved map[string]int, wgIfaceName string) {
|
||||
// Also bump conf.default so interfaces created while forwarding is on
|
||||
// (hotplug, new Wi-Fi/dock) inherit accept_ra=2 and keep accepting RAs.
|
||||
bumpAcceptRAKey(saved, acceptRADefaultPath)
|
||||
|
||||
interfaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
log.Warnf("list interfaces for accept_ra: %v", err)
|
||||
return
|
||||
}
|
||||
for _, intf := range interfaces {
|
||||
if intf.Name == "lo" || intf.Name == wgIfaceName {
|
||||
continue
|
||||
}
|
||||
bumpAcceptRAForInterface(saved, intf.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func bumpAcceptRAForInterface(saved map[string]int, name string) {
|
||||
// Build procfs path from name, not the dotted key: VLAN names like eth0.100.
|
||||
if _, err := os.Stat(fmt.Sprintf(acceptRAProcPathFormat, name)); err != nil {
|
||||
return
|
||||
}
|
||||
bumpAcceptRAKey(saved, fmt.Sprintf(acceptRAInterfacePath, sysctl.EscapeInterfaceName(name)))
|
||||
}
|
||||
|
||||
func bumpAcceptRAKey(saved map[string]int, key string) {
|
||||
// onlyIfOne=true: leave admin overrides (0, 2) alone.
|
||||
oldVal, err := sysctl.Set(key, 2, true)
|
||||
if err != nil {
|
||||
log.Warnf("bump %s: %v", key, err)
|
||||
return
|
||||
}
|
||||
// With onlyIfOne, a write only happened when the old value was 1; values
|
||||
// left untouched (0, 2) must not be recorded for restore.
|
||||
if oldVal == 1 {
|
||||
saved[key] = oldVal
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package systemops
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"net/netip"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
@@ -29,6 +30,7 @@ func ensureIPv6DefaultRoute(t *testing.T) {
|
||||
}
|
||||
if err := netlink.RouteAdd(route); err != nil {
|
||||
if errors.Is(err, syscall.EEXIST) {
|
||||
requireUsableIPv6Nexthop(t)
|
||||
return
|
||||
}
|
||||
t.Skipf("install IPv6 fallback default route: %v", err)
|
||||
@@ -38,4 +40,36 @@ func ensureIPv6DefaultRoute(t *testing.T) {
|
||||
t.Logf("delete IPv6 fallback default route: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
requireUsableIPv6Nexthop(t)
|
||||
}
|
||||
|
||||
// requireUsableIPv6Nexthop skips the test unless the resolved IPv6 default
|
||||
// nexthop can actually carry a route. Installing the default route succeeding
|
||||
// does not imply the kernel accepts it as a nexthop for a concrete prefix.
|
||||
func requireUsableIPv6Nexthop(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
nexthop, err := GetNextHop(netip.IPv6Unspecified())
|
||||
if err != nil {
|
||||
t.Skipf("resolve IPv6 default nexthop: %v", err)
|
||||
}
|
||||
|
||||
probe := &netlink.Route{
|
||||
Scope: netlink.SCOPE_UNIVERSE,
|
||||
Table: syscall.RT_TABLE_MAIN,
|
||||
Family: netlink.FAMILY_V6,
|
||||
Dst: &net.IPNet{IP: net.ParseIP("100::64"), Mask: net.CIDRMask(128, 128)},
|
||||
}
|
||||
require.NoError(t, addNextHop(nexthop, probe), "build IPv6 probe route")
|
||||
|
||||
switch err := netlink.RouteAdd(probe); {
|
||||
case err == nil:
|
||||
if err := netlink.RouteDel(probe); err != nil && !errors.Is(err, syscall.ESRCH) {
|
||||
t.Logf("delete IPv6 probe route: %v", err)
|
||||
}
|
||||
case errors.Is(err, syscall.EEXIST):
|
||||
default:
|
||||
t.Skipf("IPv6 nexthop %s unusable for route installation: %v", nexthop, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5628,9 +5628,13 @@ func (x *GetPeerSSHHostKeyResponse) GetFound() bool {
|
||||
type RequestJWTAuthRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// hint for OIDC login_hint parameter (typically email address)
|
||||
Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"`
|
||||
// hasGraphicalSession tells the daemon that the caller has a graphical session,
|
||||
// which decides whether PKCE or the device code flow is preferred. The daemon
|
||||
// cannot detect this itself: it does not inherit the session environment.
|
||||
HasGraphicalSession bool `protobuf:"varint,2,opt,name=hasGraphicalSession,proto3" json:"hasGraphicalSession,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *RequestJWTAuthRequest) Reset() {
|
||||
@@ -5670,6 +5674,13 @@ func (x *RequestJWTAuthRequest) GetHint() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RequestJWTAuthRequest) GetHasGraphicalSession() bool {
|
||||
if x != nil {
|
||||
return x.HasGraphicalSession
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RequestJWTAuthResponse contains authentication flow information
|
||||
type RequestJWTAuthResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -5894,9 +5905,13 @@ type RequestExtendAuthSessionRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Optional OIDC login_hint (typically the user's email) to pre-fill the
|
||||
// IdP login form.
|
||||
Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"`
|
||||
// hasGraphicalSession tells the daemon that the caller has a graphical session,
|
||||
// which decides whether PKCE or the device code flow is preferred. The daemon
|
||||
// cannot detect this itself: it does not inherit the session environment.
|
||||
HasGraphicalSession bool `protobuf:"varint,2,opt,name=hasGraphicalSession,proto3" json:"hasGraphicalSession,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *RequestExtendAuthSessionRequest) Reset() {
|
||||
@@ -5936,6 +5951,13 @@ func (x *RequestExtendAuthSessionRequest) GetHint() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RequestExtendAuthSessionRequest) GetHasGraphicalSession() bool {
|
||||
if x != nil {
|
||||
return x.HasGraphicalSession
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RequestExtendAuthSessionResponse carries the verification URI the UI
|
||||
// should open in a browser. The daemon retains the flow state and resolves
|
||||
// it via WaitExtendAuthSession.
|
||||
@@ -7503,9 +7525,10 @@ const file_daemon_proto_rawDesc = "" +
|
||||
"sshHostKey\x12\x16\n" +
|
||||
"\x06peerIP\x18\x02 \x01(\tR\x06peerIP\x12\x1a\n" +
|
||||
"\bpeerFQDN\x18\x03 \x01(\tR\bpeerFQDN\x12\x14\n" +
|
||||
"\x05found\x18\x04 \x01(\bR\x05found\"9\n" +
|
||||
"\x05found\x18\x04 \x01(\bR\x05found\"k\n" +
|
||||
"\x15RequestJWTAuthRequest\x12\x17\n" +
|
||||
"\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01B\a\n" +
|
||||
"\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01\x120\n" +
|
||||
"\x13hasGraphicalSession\x18\x02 \x01(\bR\x13hasGraphicalSessionB\a\n" +
|
||||
"\x05_hint\"\x9a\x02\n" +
|
||||
"\x16RequestJWTAuthResponse\x12(\n" +
|
||||
"\x0fverificationURI\x18\x01 \x01(\tR\x0fverificationURI\x128\n" +
|
||||
@@ -7525,9 +7548,10 @@ const file_daemon_proto_rawDesc = "" +
|
||||
"\x14WaitJWTTokenResponse\x12\x14\n" +
|
||||
"\x05token\x18\x01 \x01(\tR\x05token\x12\x1c\n" +
|
||||
"\ttokenType\x18\x02 \x01(\tR\ttokenType\x12\x1c\n" +
|
||||
"\texpiresIn\x18\x03 \x01(\x03R\texpiresIn\"C\n" +
|
||||
"\texpiresIn\x18\x03 \x01(\x03R\texpiresIn\"u\n" +
|
||||
"\x1fRequestExtendAuthSessionRequest\x12\x17\n" +
|
||||
"\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01B\a\n" +
|
||||
"\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01\x120\n" +
|
||||
"\x13hasGraphicalSession\x18\x02 \x01(\bR\x13hasGraphicalSessionB\a\n" +
|
||||
"\x05_hint\"\xe0\x01\n" +
|
||||
" RequestExtendAuthSessionResponse\x12(\n" +
|
||||
"\x0fverificationURI\x18\x01 \x01(\tR\x0fverificationURI\x128\n" +
|
||||
|
||||
@@ -894,6 +894,10 @@ message GetPeerSSHHostKeyResponse {
|
||||
message RequestJWTAuthRequest {
|
||||
// hint for OIDC login_hint parameter (typically email address)
|
||||
optional string hint = 1;
|
||||
// hasGraphicalSession tells the daemon that the caller has a graphical session,
|
||||
// which decides whether PKCE or the device code flow is preferred. The daemon
|
||||
// cannot detect this itself: it does not inherit the session environment.
|
||||
bool hasGraphicalSession = 2;
|
||||
}
|
||||
|
||||
// RequestJWTAuthResponse contains authentication flow information
|
||||
@@ -937,6 +941,10 @@ message RequestExtendAuthSessionRequest {
|
||||
// Optional OIDC login_hint (typically the user's email) to pre-fill the
|
||||
// IdP login form.
|
||||
optional string hint = 1;
|
||||
// hasGraphicalSession tells the daemon that the caller has a graphical session,
|
||||
// which decides whether PKCE or the device code flow is preferred. The daemon
|
||||
// cannot detect this itself: it does not inherit the session environment.
|
||||
bool hasGraphicalSession = 2;
|
||||
}
|
||||
|
||||
// RequestExtendAuthSessionResponse carries the verification URI the UI
|
||||
|
||||
@@ -1723,8 +1723,8 @@ func (s *Server) RequestJWTAuth(
|
||||
hint = profilemanager.GetLoginHint()
|
||||
}
|
||||
|
||||
isDesktop := isUnixRunningDesktop()
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isDesktop, false, hint)
|
||||
// the daemon has no graphical session of its own, only the caller can answer this
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, config, msg.GetHasGraphicalSession(), false, hint)
|
||||
if err != nil {
|
||||
return nil, gstatus.Errorf(codes.Internal, "failed to create OAuth flow: %v", err)
|
||||
}
|
||||
@@ -1827,8 +1827,8 @@ func (s *Server) RequestExtendAuthSession(
|
||||
hint = profilemanager.GetLoginHint()
|
||||
}
|
||||
|
||||
isDesktop := isUnixRunningDesktop()
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isDesktop, false, hint)
|
||||
// the daemon has no graphical session of its own, only the caller can answer this
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, config, msg.GetHasGraphicalSession(), false, hint)
|
||||
if err != nil {
|
||||
return nil, gstatus.Errorf(codes.Internal, "failed to create OAuth flow: %v", err)
|
||||
}
|
||||
@@ -2000,13 +2000,6 @@ func (s *Server) ExposeService(req *proto.ExposeServiceRequest, srv proto.Daemon
|
||||
return nil
|
||||
}
|
||||
|
||||
func isUnixRunningDesktop() bool {
|
||||
if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" {
|
||||
return false
|
||||
}
|
||||
return os.Getenv("DESKTOP_SESSION") != "" || os.Getenv("XDG_CURRENT_DESKTOP") != ""
|
||||
}
|
||||
|
||||
func (s *Server) runProbes(ctx context.Context, waitForProbeResult bool) {
|
||||
if s.connectClient == nil {
|
||||
return
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
"github.com/netbirdio/netbird/util"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -92,7 +93,8 @@ func printAuthInstructions(stderr io.Writer, authResponse *proto.RequestJWTAuthR
|
||||
|
||||
// RequestJWTToken requests or retrieves a JWT token for SSH authentication
|
||||
func RequestJWTToken(ctx context.Context, client proto.DaemonServiceClient, stdout, stderr io.Writer, useCache bool, hint string, openBrowser func(string) error) (string, error) {
|
||||
req := &proto.RequestJWTAuthRequest{}
|
||||
// the ssh client runs in the user's session, the daemon does not: tell it what we can see
|
||||
req := &proto.RequestJWTAuthRequest{HasGraphicalSession: util.HasGraphicalSession()}
|
||||
if hint != "" {
|
||||
req.Hint = &hint
|
||||
}
|
||||
@@ -193,4 +195,3 @@ func buildAddressList(hostname string, remote net.Addr) []string {
|
||||
}
|
||||
return addresses
|
||||
}
|
||||
|
||||
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/grpc/codes"
|
||||
gstatus "google.golang.org/grpc/status"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
)
|
||||
|
||||
@@ -58,10 +60,21 @@ func (s *Session) RequestExtend(ctx context.Context, p ExtendStartParams) (Exten
|
||||
return ExtendStartResult{}, err
|
||||
}
|
||||
|
||||
req := &proto.RequestExtendAuthSessionRequest{}
|
||||
if p.Hint != "" {
|
||||
h := p.Hint
|
||||
req.Hint = &h
|
||||
// a request from the UI implies a graphical session, which the daemon cannot detect itself
|
||||
req := &proto.RequestExtendAuthSessionRequest{HasGraphicalSession: true}
|
||||
hint := p.Hint
|
||||
if hint == "" {
|
||||
pm := profilemanager.NewProfileManager()
|
||||
if active, perr := pm.GetActiveProfile(); perr != nil {
|
||||
log.Debugf("failed to get active profile for login hint: %v", perr)
|
||||
} else if state, serr := pm.GetProfileState(active.ID); serr != nil {
|
||||
log.Debugf("failed to get profile state for login hint: %v", serr)
|
||||
} else {
|
||||
hint = state.Email
|
||||
}
|
||||
}
|
||||
if hint != "" {
|
||||
req.Hint = &hint
|
||||
}
|
||||
|
||||
resp, err := cli.RequestExtendAuthSession(ctx, req)
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
"lint": "eslint \"src/**/*.{ts,tsx}\"",
|
||||
"lint:fix": "eslint \"src/**/*.{ts,tsx}\" --fix",
|
||||
"check": "pnpm lint && pnpm typecheck && pnpm format:check",
|
||||
"check:fix": "pnpm lint:fix && pnpm format && pnpm typecheck"
|
||||
"check:fix": "pnpm lint:fix && pnpm format && pnpm typecheck",
|
||||
"i18n:check": "node ../i18n/check-translations.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
|
||||
104
client/ui/i18n/check-translations.mjs
Normal file
104
client/ui/i18n/check-translations.mjs
Normal file
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env node
|
||||
// Validates that every shipped translation bundle carries exactly the same set
|
||||
// of keys as the English source of truth. English (en) defines the keys; every
|
||||
// other locale declared in _index.json must match it 1:1:
|
||||
//
|
||||
// - no missing keys — a missing key silently falls back to English at runtime
|
||||
// (see i18n bundle fallback), so the gap never surfaces to users or CI
|
||||
// without this check;
|
||||
// - no orphaned keys — keys left behind after an English key is renamed or
|
||||
// removed are dead weight and a sign the locale is drifting.
|
||||
//
|
||||
// Pure Node, no dependencies, so it runs without installing the frontend
|
||||
// toolchain.
|
||||
//
|
||||
// Local: node client/ui/i18n/check-translations.mjs (or: pnpm i18n:check)
|
||||
// CI: .github/workflows/ui-translations.yml
|
||||
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const SOURCE = "en";
|
||||
const localesDir = join(dirname(fileURLToPath(import.meta.url)), "locales");
|
||||
const isCI = Boolean(process.env.GITHUB_ACTIONS);
|
||||
|
||||
function readJSON(path) {
|
||||
return JSON.parse(readFileSync(path, "utf8"));
|
||||
}
|
||||
|
||||
function keysOf(langCode) {
|
||||
return Object.keys(readJSON(join(localesDir, langCode, "common.json")));
|
||||
}
|
||||
|
||||
// Emit a GitHub Actions annotation so failures render inline on the PR diff.
|
||||
function annotate(file, message) {
|
||||
if (isCI) console.log(`::error file=${file}::${message}`);
|
||||
}
|
||||
|
||||
const index = readJSON(join(localesDir, "_index.json"));
|
||||
const declared = index.languages.map((l) => l.code);
|
||||
|
||||
if (!declared.includes(SOURCE)) {
|
||||
console.error(`FATAL: source language "${SOURCE}" is not declared in _index.json`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const sourceKeys = keysOf(SOURCE);
|
||||
const sourceSet = new Set(sourceKeys);
|
||||
console.log(`Source of truth: ${SOURCE}/common.json — ${sourceKeys.length} keys\n`);
|
||||
|
||||
let failed = false;
|
||||
|
||||
for (const code of declared) {
|
||||
if (code === SOURCE) continue;
|
||||
const file = `client/ui/i18n/locales/${code}/common.json`;
|
||||
|
||||
let keys;
|
||||
try {
|
||||
keys = keysOf(code);
|
||||
} catch (e) {
|
||||
failed = true;
|
||||
const msg = `bundle is declared in _index.json but common.json is missing or invalid (${e.message})`;
|
||||
console.error(`✗ ${code}: ${msg}`);
|
||||
annotate("client/ui/i18n/locales/_index.json", `${code}: ${msg}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const set = new Set(keys);
|
||||
const missing = sourceKeys.filter((k) => !set.has(k));
|
||||
const extra = keys.filter((k) => !sourceSet.has(k));
|
||||
|
||||
if (missing.length === 0 && extra.length === 0) {
|
||||
console.log(`✓ ${code}: ${keys.length} keys`);
|
||||
continue;
|
||||
}
|
||||
|
||||
failed = true;
|
||||
console.error(`✗ ${code}: ${keys.length} keys (expected ${sourceKeys.length})`);
|
||||
if (missing.length) {
|
||||
console.error(` missing ${missing.length}: ${missing.join(", ")}`);
|
||||
annotate(file, `Missing ${missing.length} key(s) present in ${SOURCE}: ${missing.join(", ")}`);
|
||||
}
|
||||
if (extra.length) {
|
||||
console.error(` extra ${extra.length}: ${extra.join(", ")}`);
|
||||
annotate(file, `Has ${extra.length} key(s) not present in ${SOURCE}: ${extra.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Locale directories present on disk but not declared in _index.json are never
|
||||
// loaded by the app — surface them so dead translation files don't rot silently.
|
||||
const onDisk = readdirSync(localesDir, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory())
|
||||
.map((e) => e.name);
|
||||
const undeclared = onDisk.filter((d) => !declared.includes(d));
|
||||
if (undeclared.length) {
|
||||
console.warn(`\n⚠ locale directories not declared in _index.json (not shipped): ${undeclared.join(", ")}`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
if (failed) {
|
||||
console.error("Translation check FAILED — every locale must match the English key set.");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("Translation check passed — all locales match the English key set.");
|
||||
@@ -1312,6 +1312,9 @@
|
||||
"daemon.outdated.description": {
|
||||
"message": "このアプリを使用するには NetBird サービスを更新してください。"
|
||||
},
|
||||
"daemon.outdated.download": {
|
||||
"message": "最新版をダウンロード"
|
||||
},
|
||||
"error.jwt_clock_skew": {
|
||||
"message": "サインインに失敗しました: このデバイスの時計がサーバーと同期していません。システムの時計を同期してからもう一度お試しください。"
|
||||
},
|
||||
|
||||
@@ -108,10 +108,11 @@ func (s *Connection) Login(ctx context.Context, p LoginParams) (LoginResult, err
|
||||
}
|
||||
|
||||
req := &proto.LoginRequest{
|
||||
ManagementUrl: p.ManagementURL,
|
||||
SetupKey: p.SetupKey,
|
||||
Hostname: p.Hostname,
|
||||
IsUnixDesktopClient: runtime.GOOS == "linux",
|
||||
ManagementUrl: p.ManagementURL,
|
||||
SetupKey: p.SetupKey,
|
||||
Hostname: p.Hostname,
|
||||
// a login driven by the UI always has a graphical session available
|
||||
IsUnixDesktopClient: true,
|
||||
}
|
||||
if profileName != "" {
|
||||
req.ProfileName = ptrStr(profileName)
|
||||
@@ -122,8 +123,16 @@ func (s *Connection) Login(ctx context.Context, p LoginParams) (LoginResult, err
|
||||
if p.PreSharedKey != "" {
|
||||
req.OptionalPreSharedKey = ptrStr(p.PreSharedKey)
|
||||
}
|
||||
if p.Hint != "" {
|
||||
req.Hint = ptrStr(p.Hint)
|
||||
hint := p.Hint
|
||||
if hint == "" && profileID != "" {
|
||||
if state, serr := profilemanager.NewProfileManager().GetProfileState(profilemanager.ID(profileID)); serr == nil {
|
||||
hint = state.Email
|
||||
} else {
|
||||
log.Debugf("failed to get profile state for login hint: %v", serr)
|
||||
}
|
||||
}
|
||||
if hint != "" {
|
||||
req.Hint = ptrStr(hint)
|
||||
}
|
||||
|
||||
resp, err := cli.Login(ctx, req)
|
||||
@@ -227,16 +236,6 @@ func (s *Connection) Logout(ctx context.Context, p LogoutParams) error {
|
||||
return s.classifyDaemonError(err)
|
||||
}
|
||||
|
||||
// The daemon runs as root and can't reach the user-owned per-profile state
|
||||
// file holding the account email (see Profiles.List), so clear the stale
|
||||
// email here; the next SSO login recreates it.
|
||||
if p.ProfileName != "" {
|
||||
if err := profilemanager.NewProfileManager().RemoveProfileState(p.ProfileName); err != nil {
|
||||
// Non-fatal: the logout itself succeeded.
|
||||
log.Warnf("failed to remove profile state for %s: %v", p.ProfileName, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -260,7 +259,7 @@ func (s *Connection) waitSSOLogin(ctx context.Context, p WaitSSOParams) (string,
|
||||
|
||||
// Persist the account email the same way the CLI does after its own
|
||||
// WaitSSOLogin: the daemon returns it but cannot store it, since it runs as
|
||||
// root and the per-profile state file is user-owned (see Logout below).
|
||||
// root and the per-profile state file is user-owned (see Profiles.List).
|
||||
// Without this the profile has no email, so Profiles.List shows no account
|
||||
// and later logins and session extends go out without a login_hint —
|
||||
// leaving the IdP to guess which account was meant.
|
||||
|
||||
@@ -162,8 +162,9 @@ func (s *Profiles) Remove(ctx context.Context, p ProfileRef) error {
|
||||
}
|
||||
|
||||
// The daemon deletes what it owns but runs as root, so it leaves the
|
||||
// user-owned state file holding the account email behind (same split as
|
||||
// Connection.Logout). Legacy profiles are keyed by name rather than by a
|
||||
// user-owned state file holding the account email behind. Logout keeps the
|
||||
// email on purpose so later logins can pass it as the login_hint; profile
|
||||
// removal is what deletes it. Legacy profiles are keyed by name rather than by a
|
||||
// generated ID, so a recreated profile of the same name would inherit the
|
||||
// deleted one's email and offer it as the login_hint.
|
||||
//
|
||||
|
||||
@@ -23,9 +23,10 @@ import (
|
||||
// model the client asks for. The proxy prices off the REQUEST model, not the
|
||||
// upstream response model, so a made-up model id billed at operator rates lets
|
||||
// these tests assert exact costs without a real vendor key.
|
||||
// Sourced from the harness so the counts can't drift from the mock's config.
|
||||
const (
|
||||
vllmPromptTokens = 11
|
||||
vllmCompletionTokens = 2
|
||||
vllmPromptTokens = harness.VLLMChatInputTokens
|
||||
vllmCompletionTokens = harness.VLLMChatOutputTokens
|
||||
)
|
||||
|
||||
// pricedEnv is a connected single-provider agent-network deployment pointed at
|
||||
@@ -169,23 +170,48 @@ func chatOnce(t *testing.T, ctx context.Context, env pricedEnv, model, sessionID
|
||||
return body
|
||||
}
|
||||
|
||||
// findAccessLogBySession polls the access-log page for the row carrying sessionID.
|
||||
func findAccessLogBySession(t *testing.T, ctx context.Context, sessionID string) api.AgentNetworkAccessLog {
|
||||
t.Helper()
|
||||
var row api.AgentNetworkAccessLog
|
||||
require.Eventually(t, func() bool {
|
||||
logs, lerr := srv.ListAccessLogs(ctx)
|
||||
if lerr != nil {
|
||||
return false
|
||||
}
|
||||
for _, r := range logs.Data {
|
||||
if r.SessionId != nil && *r.SessionId == sessionID {
|
||||
row = r
|
||||
return true
|
||||
// accessLogIngestWindow is how long a single request's access-log row is given
|
||||
// to appear before the caller gives up on it.
|
||||
// accessLogIngestWindow bounds how long a row may take to appear after its
|
||||
// request returned. The proxy streams each entry to management with a 10s send
|
||||
// timeout of its own, so a request whose send hits one full timeout and is
|
||||
// retried has not yet missed anything real — 30s left barely three send
|
||||
// attempts of headroom and lost the race on a loaded runner.
|
||||
const accessLogIngestWindow = 60 * time.Second
|
||||
|
||||
// lookupAccessLogBySession polls the access-log page for the row carrying
|
||||
// sessionID and reports whether it arrived within the window. It never fails
|
||||
// the test: callers that can recover — by firing a fresh request under a new
|
||||
// session — need to see the miss rather than die on it.
|
||||
func lookupAccessLogBySession(ctx context.Context, sessionID string, within time.Duration) (api.AgentNetworkAccessLog, bool) {
|
||||
deadline := time.Now().Add(within)
|
||||
for {
|
||||
if logs, lerr := srv.ListAccessLogs(ctx); lerr == nil {
|
||||
for _, r := range logs.Data {
|
||||
if r.SessionId != nil && *r.SessionId == sessionID {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}, 30*time.Second, 2*time.Second, "session id %q must be recorded in an access-log row", sessionID)
|
||||
if time.Now().After(deadline) {
|
||||
return api.AgentNetworkAccessLog{}, false
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return api.AgentNetworkAccessLog{}, false
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// findAccessLogBySession polls the access-log page for the row carrying
|
||||
// sessionID, failing the test if it never lands. Use it for a request whose row
|
||||
// must exist; where a missing row is a recoverable race, use
|
||||
// lookupAccessLogBySession and retry.
|
||||
func findAccessLogBySession(t *testing.T, ctx context.Context, sessionID string) api.AgentNetworkAccessLog {
|
||||
t.Helper()
|
||||
row, ok := lookupAccessLogBySession(ctx, sessionID, accessLogIngestWindow)
|
||||
require.True(t, ok, "session id %q must be recorded in an access-log row", sessionID)
|
||||
return row
|
||||
}
|
||||
|
||||
@@ -319,6 +345,11 @@ func TestPriceChangeUpdatesRecordedCost(t *testing.T) {
|
||||
outRateA = 0.020
|
||||
inRateB = 0.050 // 5x / 4x the original, so a repriced row is unmistakable
|
||||
outRateB = 0.080
|
||||
// Per-attempt ingest wait, shorter than the default so a request that
|
||||
// produces no row costs one retry rather than most of the budget, and an
|
||||
// overall deadline long enough to hold several attempts.
|
||||
repriceIngestWindow = 20 * time.Second
|
||||
repriceDeadline = 180 * time.Second
|
||||
)
|
||||
|
||||
env := provisionPricedProvider(t, ctx, "reprice", []api.AgentNetworkProviderModel{
|
||||
@@ -353,10 +384,15 @@ func TestPriceChangeUpdatesRecordedCost(t *testing.T) {
|
||||
// reading its cost, so an un-ingested row is never mistaken for "still rate A".
|
||||
// The expected new input cost is unmistakably higher than rate A, so a
|
||||
// lingering old-rate row can't satisfy the check.
|
||||
//
|
||||
// Every way an iteration can come up short — the request failing, its row not
|
||||
// landing, or the row still carrying rate A — is a symptom of the same
|
||||
// in-flight rebuild, so each one retries under a fresh session rather than
|
||||
// ending the test. Only the outer deadline is fatal.
|
||||
wantInputB := float64(vllmPromptTokens) / 1000 * inRateB
|
||||
var repriced api.AgentNetworkAccessLog
|
||||
var lastSession string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
deadline := time.Now().Add(repriceDeadline)
|
||||
for time.Now().Before(deadline) {
|
||||
lastSession = fmt.Sprintf("e2e-session-reprice-b-%d", time.Now().UnixNano())
|
||||
code, _, cerr := env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireChat, customModel, "Reply with exactly: pong", lastSession)
|
||||
@@ -364,7 +400,15 @@ func TestPriceChangeUpdatesRecordedCost(t *testing.T) {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
row := findAccessLogBySession(t, ctx, lastSession)
|
||||
row, ok := lookupAccessLogBySession(ctx, lastSession, repriceIngestWindow)
|
||||
if !ok {
|
||||
// No row for this request. The provider update rebuilds the proxy's
|
||||
// middleware chain, and a request served mid-rebuild can complete
|
||||
// without a resolved provider — 200 to the caller, nothing to
|
||||
// attribute, so no row is ever written for it. Fire another one.
|
||||
t.Logf("no access-log row for session %q within %s; retrying under a fresh session", lastSession, repriceIngestWindow)
|
||||
continue
|
||||
}
|
||||
if inDelta(row.InputCostUsd, wantInputB, 1e-6) {
|
||||
repriced = row
|
||||
break
|
||||
@@ -630,3 +674,47 @@ func inDelta(a, b, tol float64) bool {
|
||||
}
|
||||
return d <= tol
|
||||
}
|
||||
|
||||
// TestCustomDatedModelKeepsItsOwnPrice covers the review fix that anchored the
|
||||
// release-date fallback to Claude ids. Pricing looks every model up through
|
||||
// that helper, so while it matched a bare trailing date any operator id ending
|
||||
// in eight digits inherited the rate of its undated sibling — a silent
|
||||
// mis-bill on models NetBird knows nothing about.
|
||||
func TestCustomDatedModelKeepsItsOwnPrice(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
baseModel = "internal-llm"
|
||||
datedModel = "internal-llm-20250101"
|
||||
baseIn = 0.010
|
||||
baseOut = 0.020
|
||||
// An order of magnitude apart, so a row billed at the wrong entry is
|
||||
// unmistakable rather than a rounding argument.
|
||||
datedIn = 0.100
|
||||
datedOut = 0.200
|
||||
)
|
||||
|
||||
env := provisionPricedProvider(t, ctx, "customdated", []api.AgentNetworkProviderModel{
|
||||
{Id: baseModel, InputPer1k: baseIn, OutputPer1k: baseOut},
|
||||
{Id: datedModel, InputPer1k: datedIn, OutputPer1k: datedOut},
|
||||
})
|
||||
|
||||
t.Run("the undated id bills at its own rate", func(t *testing.T) {
|
||||
session := fmt.Sprintf("e2e-session-customdated-base-%d", time.Now().UnixNano())
|
||||
chatOnce(t, ctx, env, baseModel, session)
|
||||
assertOpenAICostAtRates(t, findAccessLogBySession(t, ctx, session), baseIn, baseOut)
|
||||
})
|
||||
|
||||
t.Run("the dated id keeps its own rate", func(t *testing.T) {
|
||||
session := fmt.Sprintf("e2e-session-customdated-dated-%d", time.Now().UnixNano())
|
||||
chatOnce(t, ctx, env, datedModel, session)
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
assertOpenAICostAtRates(t, row, datedIn, datedOut)
|
||||
|
||||
// Spelled out because it is the regression: inheriting the sibling's
|
||||
// rate would bill this request at a tenth of its price.
|
||||
assert.Greater(t, row.InputCostUsd, float64(vllmPromptTokens)/1000*baseIn*2,
|
||||
"a custom dated id must not inherit the undated entry's rate")
|
||||
})
|
||||
}
|
||||
|
||||
400
e2e/agentnetwork/discovery_live_test.go
Normal file
400
e2e/agentnetwork/discovery_live_test.go
Normal file
@@ -0,0 +1,400 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
sharedllm "github.com/netbirdio/netbird/shared/llm"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// TestLiveModelDiscovery drives model discovery against the REAL vendor
|
||||
// endpoints — OpenAI, Anthropic, Bedrock and Vertex — rather than the mock.
|
||||
//
|
||||
// The mock upstream proves the filter's mechanics: it advertises ids we chose,
|
||||
// so a listing narrowing to the ones we authorised is arithmetic we already
|
||||
// controlled both sides of. What it cannot prove is that the filter survives
|
||||
// contact with a real catalogue — ids we never enumerated, dated builds whose
|
||||
// suffix the vendor picks, surfaces that answer a listing request with
|
||||
// something other than a listing. That is what this covers, and it is the part
|
||||
// a QA engineer would otherwise have to walk through by hand.
|
||||
//
|
||||
// One proxy serves every case. Each provider gets its own group, policy and
|
||||
// client, because a model-less request matches exactly ONE route
|
||||
// (matchModelless): with two providers authorised for the same caller, the
|
||||
// listing would go to whichever won the tiebreak and the other would go
|
||||
// untested. Group-scoping the caller makes each provider the only candidate
|
||||
// for its own client.
|
||||
func TestLiveModelDiscovery(t *testing.T) {
|
||||
cases := liveDiscoveryCases()
|
||||
if len(cases) == 0 {
|
||||
t.Skip("no provider keys set; source ~/.llm-keys to run live model discovery")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
t.Logf("[discovery] live matrix: %s", strings.Join(caseNames(cases), ", "))
|
||||
|
||||
// Provision every provider, group and policy before the proxy starts: the
|
||||
// proxy takes a configuration snapshot at connect time and does not
|
||||
// reconcile provider changes made afterwards.
|
||||
keys := make(map[string]string, len(cases))
|
||||
for i := range cases {
|
||||
keys[cases[i].name] = provisionLiveDiscovery(t, ctx, &cases[i])
|
||||
}
|
||||
|
||||
endpoint, firstIP, firstClient, px := connectClient(t, ctx, "disc-live", keys[cases[0].name])
|
||||
clients := map[string]*harness.Client{cases[0].name: firstClient}
|
||||
ips := map[string]string{cases[0].name: firstIP}
|
||||
for _, tc := range cases[1:] {
|
||||
cl := joinClient(t, ctx, px, endpoint, keys[tc.name])
|
||||
ip, err := cl.ResolveProxyIP(ctx, endpoint)
|
||||
require.NoError(t, err, "resolve endpoint from the %s client", tc.name)
|
||||
clients[tc.name] = cl
|
||||
ips[tc.name] = ip
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runLiveDiscoveryCase(t, ctx, tc, clients[tc.name], endpoint, ips[tc.name])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// discoveryOutcome is what a discovery request must produce end to end. The
|
||||
// three are genuinely different contracts, not degrees of success: only the
|
||||
// first puts a bounded listing in front of the caller.
|
||||
type discoveryOutcome int
|
||||
|
||||
const (
|
||||
// outcomeFiltered: the proxy routes the request and bounds the response to
|
||||
// what the caller may use.
|
||||
outcomeFiltered discoveryOutcome = iota
|
||||
// outcomeDenied: no provider of this shape can serve the surface, so the
|
||||
// proxy refuses rather than rewriting the request onto an upstream that
|
||||
// would 404 it. The caller gets a NetBird error, not a vendor one.
|
||||
outcomeDenied
|
||||
// outcomeUpstreamNoListing: the proxy routes the request to the configured
|
||||
// upstream, and the vendor does not implement the endpoint there. Proxy
|
||||
// side correct, product side a dead end — see the Bedrock case.
|
||||
outcomeUpstreamNoListing
|
||||
)
|
||||
|
||||
// liveDiscoveryCase is one provider's discovery surface and what the proxy
|
||||
// must make of it.
|
||||
type liveDiscoveryCase struct {
|
||||
name string
|
||||
catalogID string
|
||||
upstream string
|
||||
apiKey string
|
||||
|
||||
// path is the discovery endpoint the client calls. Not every surface uses
|
||||
// /v1/models: Bedrock lists inference profiles instead.
|
||||
path string
|
||||
// headers the vendor requires on a bare GET (Anthropic versions its API
|
||||
// through a header, and rejects a request without one).
|
||||
headers []string
|
||||
|
||||
// models the provider record enumerates. Empty models a gateway record,
|
||||
// which enumerates nothing and claims everything.
|
||||
models []string
|
||||
// allowlist, when non-empty, is a guardrail narrowing the policy below the
|
||||
// provider's own enumeration — the second of the two bounds discovery
|
||||
// applies, and the only one a provider record alone cannot demonstrate.
|
||||
allowlist []string
|
||||
|
||||
// outcome is what this surface must produce end to end.
|
||||
outcome discoveryOutcome
|
||||
|
||||
// permitted is every id allowed to survive filtering, in the form the
|
||||
// provider record registers it. A surviving id counts as permitted when it
|
||||
// matches one of these outright or after Anthropic date-normalisation.
|
||||
permitted []string
|
||||
// wantHidden are ids the upstream is known to advertise and the bound must
|
||||
// remove. Only set where we enumerate the model ourselves, so the
|
||||
// expectation cannot rot when a vendor changes its catalogue.
|
||||
wantHidden []string
|
||||
}
|
||||
|
||||
// liveDiscoveryCases builds the matrix from whichever provider credentials are
|
||||
// present, mirroring availableProviders' env-var gating so a partial key set
|
||||
// still yields partial coverage.
|
||||
func liveDiscoveryCases() []liveDiscoveryCase {
|
||||
var cases []liveDiscoveryCase
|
||||
|
||||
// OpenAI enumerates TWO real models and the policy permits one. That is
|
||||
// the only case here where both bounds are observable at once: the
|
||||
// upstream advertises dozens of ids, the provider record cuts them to two,
|
||||
// and the guardrail cuts those to one.
|
||||
if k := os.Getenv("OPENAI_TOKEN"); k != "" {
|
||||
cases = append(cases, liveDiscoveryCase{
|
||||
name: "openai", catalogID: "openai_api", upstream: "https://api.openai.com", apiKey: k,
|
||||
path: "/v1/models",
|
||||
models: []string{"gpt-4o-mini", "gpt-4o"},
|
||||
allowlist: []string{"gpt-4o-mini"},
|
||||
outcome: outcomeFiltered,
|
||||
permitted: []string{"gpt-4o-mini"},
|
||||
wantHidden: []string{"gpt-4o"},
|
||||
})
|
||||
}
|
||||
|
||||
// Anthropic is the surface Claude Code actually calls. Its listing returns
|
||||
// DATED build ids (claude-haiku-4-5-20251001) while the provider record
|
||||
// registers the undated id, so this is the case that proves the filter's
|
||||
// date-normalisation against ids the vendor chose rather than ids we wrote.
|
||||
if k := os.Getenv("ANTHROPIC_TOKEN"); k != "" {
|
||||
cases = append(cases, liveDiscoveryCase{
|
||||
name: "anthropic", catalogID: "anthropic_api", upstream: "https://api.anthropic.com", apiKey: k,
|
||||
path: "/v1/models",
|
||||
headers: []string{"anthropic-version: 2023-06-01"},
|
||||
models: []string{"claude-haiku-4-5"},
|
||||
outcome: outcomeFiltered,
|
||||
permitted: []string{"claude-haiku-4-5"},
|
||||
})
|
||||
}
|
||||
|
||||
// Bedrock lists inference profiles, not models: matchModelless routes
|
||||
// /inference-profiles to a Bedrock route and refuses /v1/models for one.
|
||||
//
|
||||
// The request reaches AWS and AWS refuses it — bedrock-runtime answers
|
||||
// <UnknownOperationException/>, because ListInferenceProfiles is a CONTROL
|
||||
// PLANE operation served by bedrock.<region>.amazonaws.com, not the runtime
|
||||
// host. A provider record carries one upstream and it has to be the runtime
|
||||
// host for InvokeModel to work, so no Bedrock record can serve a listing as
|
||||
// the model stands today.
|
||||
//
|
||||
// The mock upstream hides this entirely: it answers /inference-profiles on
|
||||
// the same listener as everything else, so the routing test passes there
|
||||
// while the real endpoint 404s. That is the whole reason this file exists,
|
||||
// so the case is kept, asserting what actually happens.
|
||||
if k := os.Getenv("AWS_BEARER_TOKEN_BEDROCK"); k != "" {
|
||||
region := os.Getenv("AWS_REGION")
|
||||
if region == "" {
|
||||
region = "eu-central-1"
|
||||
}
|
||||
model := os.Getenv("AWS_BEDROCK_MODEL")
|
||||
if model == "" {
|
||||
model = "global.anthropic.claude-sonnet-4-6"
|
||||
}
|
||||
cases = append(cases, liveDiscoveryCase{
|
||||
name: "bedrock", catalogID: "bedrock_api",
|
||||
upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k,
|
||||
path: "/inference-profiles",
|
||||
models: []string{sharedllm.NormalizeAnthropicModel(strings.TrimPrefix(model, "global."))},
|
||||
outcome: outcomeUpstreamNoListing,
|
||||
})
|
||||
}
|
||||
|
||||
// Vertex carries the model in the rawPredict path and serves no listing
|
||||
// endpoint at all, so the proxy must refuse discovery rather than rewrite
|
||||
// it onto an upstream that would 404.
|
||||
if sa := os.Getenv("GOOGLE_VERTEX_SA_BASE64"); sa != "" {
|
||||
if project := os.Getenv("GOOGLE_VERTEX_PROJECT"); project != "" {
|
||||
region := os.Getenv("GOOGLE_VERTEX_REGION")
|
||||
if region == "" {
|
||||
region = "global"
|
||||
}
|
||||
host := "aiplatform.googleapis.com"
|
||||
if region != "global" {
|
||||
host = region + "-aiplatform.googleapis.com"
|
||||
}
|
||||
cases = append(cases, liveDiscoveryCase{
|
||||
name: "vertex", catalogID: "vertex_ai_api", upstream: "https://" + host,
|
||||
apiKey: "keyfile::" + sa,
|
||||
path: "/v1/models",
|
||||
outcome: outcomeDenied,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return cases
|
||||
}
|
||||
|
||||
// provisionLiveDiscovery creates the group, provider, optional guardrail and
|
||||
// policy for one case, and returns the setup key a client joins that group
|
||||
// with. Scoping each provider to its own group is what keeps it the only
|
||||
// candidate for its own client's model-less request.
|
||||
func provisionLiveDiscovery(t *testing.T, ctx context.Context, tc *liveDiscoveryCase) string {
|
||||
t.Helper()
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-disc-live-" + tc.name})
|
||||
require.NoError(t, err, "create group for %s", tc.name)
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-disc-live-" + tc.name,
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key for %s", tc.name)
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext for %s", tc.name)
|
||||
|
||||
req := api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-disc-live-" + tc.name,
|
||||
ProviderId: tc.catalogID,
|
||||
UpstreamUrl: tc.upstream,
|
||||
ApiKey: &tc.apiKey,
|
||||
Enabled: ptr(true),
|
||||
}
|
||||
if len(tc.models) > 0 {
|
||||
models := make([]api.AgentNetworkProviderModel, 0, len(tc.models))
|
||||
for _, id := range tc.models {
|
||||
models = append(models, api.AgentNetworkProviderModel{Id: id, InputPer1k: 0.001, OutputPer1k: 0.002})
|
||||
}
|
||||
req.Models = &models
|
||||
}
|
||||
prov, err := srv.CreateProvider(ctx, req)
|
||||
require.NoError(t, err, "create provider %s", tc.name)
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
polReq := api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-disc-live-" + tc.name,
|
||||
Enabled: ptr(true),
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
}
|
||||
if len(tc.allowlist) > 0 {
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-disc-live-" + tc.name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = tc.allowlist
|
||||
g, gerr := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, gerr, "create guardrail for %s", tc.name)
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), g.Id) })
|
||||
polReq.GuardrailIds = &[]string{g.Id}
|
||||
}
|
||||
pol, err := srv.CreatePolicy(ctx, polReq)
|
||||
require.NoError(t, err, "create policy for %s", tc.name)
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
return sk.Key
|
||||
}
|
||||
|
||||
// runLiveDiscoveryCase issues the discovery request and reports everything the
|
||||
// vendor said before asserting on any of it. The log is the point on the first
|
||||
// run: a live catalogue is the one input we do not control, so a failure has to
|
||||
// arrive with the response that caused it rather than just a count.
|
||||
func runLiveDiscoveryCase(t *testing.T, ctx context.Context, tc liveDiscoveryCase, cl *harness.Client, endpoint, proxyIP string) {
|
||||
t.Helper()
|
||||
|
||||
// A single request is enough for the two non-listing outcomes, and retrying
|
||||
// them would burn the retry window waiting for a status that is never
|
||||
// coming.
|
||||
if tc.outcome != outcomeFiltered {
|
||||
code, body, err := cl.Get(ctx, endpoint, proxyIP, tc.path, tc.headers)
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
t.Logf("[discovery] %s GET %s -> %d; body: %s", tc.name, tc.path, code, truncate(body, 2000))
|
||||
assert.NotEqual(t, 200, code,
|
||||
"%s serves no bounded listing, so a 200 here would mean the caller was handed a picker nothing narrows; body: %s",
|
||||
tc.name, truncate(body, 2000))
|
||||
|
||||
// Which side refused is the whole distinction between these two
|
||||
// outcomes, and a NetBird error is the thing that tells them apart: the
|
||||
// middleware chain stamps its own name on anything it generates.
|
||||
if tc.outcome == outcomeDenied {
|
||||
assert.True(t, isProxyError(body),
|
||||
"%s serves no listing endpoint at all, so the proxy must refuse the request itself rather than forward it to an upstream that would answer for us; body: %s",
|
||||
tc.name, truncate(body, 2000))
|
||||
return
|
||||
}
|
||||
assert.False(t, isProxyError(body),
|
||||
"%s discovery must be routed to the configured upstream and refused by the vendor, not blocked by the proxy; body: %s",
|
||||
tc.name, truncate(body, 2000))
|
||||
return
|
||||
}
|
||||
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Get(ctx, endpoint, proxyIP, tc.path, tc.headers)
|
||||
}, 200)
|
||||
t.Logf("[discovery] %s GET %s -> %d; body: %s", tc.name, tc.path, code, truncate(body, 4000))
|
||||
require.Equal(t, 200, code, "%s discovery must be served; body: %s", tc.name, truncate(body, 2000))
|
||||
|
||||
ids, ok := listingIDs(body)
|
||||
require.Truef(t, ok,
|
||||
"%s answered discovery with something other than a {\"data\":[{\"id\":…}]} listing, which the filter forwards untouched — the caller would get an unbounded picker; body: %s",
|
||||
tc.name, truncate(body, 2000))
|
||||
sort.Strings(ids)
|
||||
t.Logf("[discovery] %s: %d ids after filtering: %s", tc.name, len(ids), strings.Join(ids, ", "))
|
||||
|
||||
require.NotEmpty(t, ids, "%s filtered the listing down to nothing; the caller would see an empty picker", tc.name)
|
||||
|
||||
permitted := make(map[string]struct{}, len(tc.permitted)*2)
|
||||
for _, id := range tc.permitted {
|
||||
permitted[id] = struct{}{}
|
||||
permitted[sharedllm.NormalizeAnthropicModel(id)] = struct{}{}
|
||||
}
|
||||
for _, id := range ids {
|
||||
_, direct := permitted[id]
|
||||
_, normalised := permitted[sharedllm.NormalizeAnthropicModel(id)]
|
||||
assert.Truef(t, direct || normalised,
|
||||
"%s offered %q, which no policy on this route permits — every entry the picker shows must be a request the guardrail would allow", tc.name, id)
|
||||
}
|
||||
for _, hidden := range tc.wantHidden {
|
||||
assert.NotContainsf(t, ids, hidden,
|
||||
"%s offered %q, which the provider enumerates but the policy does not permit", tc.name, hidden)
|
||||
}
|
||||
}
|
||||
|
||||
// isProxyError reports whether a response body was generated by the middleware
|
||||
// chain rather than forwarded from a vendor. Every chain-generated error names
|
||||
// the middleware that raised it, which no upstream's error body does — so this
|
||||
// separates "the proxy refused" from "the proxy routed it and the vendor
|
||||
// refused", the two failures that otherwise look alike from the client side.
|
||||
func isProxyError(body string) bool {
|
||||
return strings.Contains(body, `"middleware":`)
|
||||
}
|
||||
|
||||
// listingIDs pulls the model ids out of a listing response. ok is false when
|
||||
// the body is not the {"data":[{"id":…}]} shape the filter recognises.
|
||||
func listingIDs(body string) ([]string, bool) {
|
||||
var doc struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
if doc.Data == nil {
|
||||
return nil, false
|
||||
}
|
||||
ids := make([]string, 0, len(doc.Data))
|
||||
for _, entry := range doc.Data {
|
||||
ids = append(ids, entry.ID)
|
||||
}
|
||||
return ids, true
|
||||
}
|
||||
|
||||
func caseNames(cases []liveDiscoveryCase) []string {
|
||||
names := make([]string, 0, len(cases))
|
||||
for _, c := range cases {
|
||||
names = append(names, c.name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// truncate bounds a logged response body. A live catalogue can run to tens of
|
||||
// kilobytes, and the useful part is the front.
|
||||
func truncate(s string, limit int) string {
|
||||
if len(s) <= limit {
|
||||
return s
|
||||
}
|
||||
return s[:limit] + "… (" + strconv.Itoa(len(s)-limit) + " more bytes)"
|
||||
}
|
||||
168
e2e/agentnetwork/discovery_multipolicy_test.go
Normal file
168
e2e/agentnetwork/discovery_multipolicy_test.go
Normal file
@@ -0,0 +1,168 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// TestDiscoveryBoundToCallersPolicies covers a model listing on a provider two
|
||||
// teams reach under different allowlists.
|
||||
//
|
||||
// Bounding the listing by the provider's enumerated models alone is not enough
|
||||
// once more than one policy is in play: the caller would be offered every model
|
||||
// any team may use, and each one outside their own policy is a request the
|
||||
// guardrail refuses a moment later — the empty-or-wrong picker this endpoint
|
||||
// exists to avoid, just moved one level up.
|
||||
//
|
||||
// The client joins the main group only. Both models are enumerated by the same
|
||||
// provider and both are advertised by the upstream, so a listing that leaked
|
||||
// the other team's model would visibly contain it.
|
||||
func TestDiscoveryBoundToCallersPolicies(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grpMain, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-disc-mp-main"})
|
||||
require.NoError(t, err, "create main group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpMain.Id) })
|
||||
|
||||
grpOther, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-disc-mp-other"})
|
||||
require.NoError(t, err, "create other group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpOther.Id) })
|
||||
|
||||
ephemeral := false
|
||||
mkKey := func(name, groupID string) string {
|
||||
sk, kerr := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: name,
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{groupID},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, kerr, "mint setup key %s", name)
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
return sk.Key
|
||||
}
|
||||
// One client per group. The second is what makes the first assertion mean
|
||||
// something: without a client that DOES see the other team's model, its
|
||||
// absence from the main client's listing could equally be a policy that
|
||||
// never propagated.
|
||||
keyMain := mkKey("e2e-disc-mp-main-client", grpMain.Id)
|
||||
keyOther := mkKey("e2e-disc-mp-other-client", grpOther.Id)
|
||||
|
||||
// One provider enumerating both models the upstream advertises, so the
|
||||
// listing is narrowed by policy rather than by what the provider serves.
|
||||
staticKey := "static-e2e-token"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-disc-mp",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{
|
||||
{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.001},
|
||||
{Id: harness.VLLMUnlistedModel, InputPer1k: 0.001, OutputPer1k: 0.001},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
mkGuardrail := func(name, model string) api.AgentNetworkGuardrail {
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{model}
|
||||
g, gerr := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, gerr, "create guardrail %s", name)
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), g.Id) })
|
||||
return g
|
||||
}
|
||||
gMain := mkGuardrail("e2e-disc-mp-main", harness.VLLMModel)
|
||||
gOther := mkGuardrail("e2e-disc-mp-other", harness.VLLMUnlistedModel)
|
||||
|
||||
enabled := true
|
||||
polMain, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-disc-mp-main",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpMain.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{gMain.Id},
|
||||
})
|
||||
require.NoError(t, err, "create main policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polMain.Id) })
|
||||
|
||||
// The other team's policy, on the same provider, permitting the model the
|
||||
// client must never be offered.
|
||||
polOther, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-disc-mp-other",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpOther.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{gOther.Id},
|
||||
})
|
||||
require.NoError(t, err, "create other policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polOther.Id) })
|
||||
|
||||
endpoint, proxyIP, clMain, px := connectClient(t, ctx, "disc-mp", keyMain)
|
||||
clOther := joinClient(t, ctx, px, endpoint, keyOther)
|
||||
|
||||
listing := func(t *testing.T, cl *harness.Client, ip string) string {
|
||||
t.Helper()
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Get(ctx, endpoint, ip, "/v1/models?limit=1000", nil)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "discovery must be served; body: %s", body)
|
||||
return body
|
||||
}
|
||||
|
||||
otherIP, err := clOther.ResolveProxyIP(ctx, endpoint)
|
||||
require.NoError(t, err, "resolve endpoint from the other client")
|
||||
|
||||
// The other team's client first: seeing its own model proves polOther is
|
||||
// live, so the main client's listing is narrowed by policy scoping rather
|
||||
// than by the other policy having failed to apply at all.
|
||||
otherBody := listing(t, clOther, otherIP)
|
||||
assert.Contains(t, otherBody, harness.VLLMUnlistedModel,
|
||||
"the other group's policy must be in force, or this test proves nothing")
|
||||
assert.NotContains(t, otherBody, harness.VLLMModel,
|
||||
"and it must not be offered the main group's model either — isolation runs both ways")
|
||||
|
||||
mainBody := listing(t, clMain, proxyIP)
|
||||
assert.Contains(t, mainBody, harness.VLLMModel,
|
||||
"the model the caller's own policy permits must reach the picker")
|
||||
assert.NotContains(t, mainBody, harness.VLLMUnlistedModel,
|
||||
"a model only another group's policy permits must not be offered to this caller")
|
||||
}
|
||||
|
||||
// joinClient starts a second tunnel client against an already-running proxy, so
|
||||
// a test can drive the same endpoint as two different group memberships without
|
||||
// paying for a second proxy.
|
||||
func joinClient(t *testing.T, ctx context.Context, px *harness.Proxy, endpoint, setupKey string) *harness.Client {
|
||||
t.Helper()
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, setupKey)
|
||||
require.NoError(t, err, "start second client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "second client must connect to management")
|
||||
if _, err := cl.ResolveProxyIP(ctx, endpoint); err != nil {
|
||||
t.Fatalf("second client could not resolve the endpoint: %v", err)
|
||||
}
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("second client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
return cl
|
||||
}
|
||||
455
e2e/agentnetwork/gateway_protocol_test.go
Normal file
455
e2e/agentnetwork/gateway_protocol_test.go
Normal file
@@ -0,0 +1,455 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// Models each catalog surface is registered with in the matrix below. They
|
||||
// differ per provider so the router's choice is unambiguous: a request that
|
||||
// lands on the wrong provider record fails the surface assertion instead of
|
||||
// passing by coincidence.
|
||||
const (
|
||||
matrixAnthropicModel = "claude-sonnet-5"
|
||||
matrixBedrockModel = "anthropic.claude-sonnet-5"
|
||||
// matrixBedrockPathModel is what a Bedrock SDK client puts in the URL: a
|
||||
// cross-region inference profile with a release date and version suffix.
|
||||
// The proxy must normalise it back to matrixBedrockModel to route and price.
|
||||
matrixBedrockPathModel = "us.anthropic.claude-sonnet-5-20250101-v1:0"
|
||||
// matrixVertexModel differs from the Anthropic record's model on purpose:
|
||||
// a shared id would leave two routes claiming it and make which one serves
|
||||
// /v1/messages depend on declaration order.
|
||||
matrixVertexModel = "claude-haiku-4-5"
|
||||
matrixVertexProject = "e2e-project"
|
||||
matrixVertexRegion = "us-east5"
|
||||
)
|
||||
|
||||
// gatewayEnv is a connected client plus a set of provider records, all pointed
|
||||
// at one mock upstream, so several wire shapes can be driven over a single
|
||||
// tunnel.
|
||||
type gatewayEnv struct {
|
||||
endpoint string
|
||||
proxyIP string
|
||||
client *harness.Client
|
||||
proxy *harness.Proxy
|
||||
vllm *harness.VLLM
|
||||
// providerIDs maps the catalog id to the created provider record id.
|
||||
providerIDs map[string]string
|
||||
}
|
||||
|
||||
// provisionGatewayMatrix brings up one mock upstream and one provider record
|
||||
// per catalog surface, all authorised for the same group by a single policy.
|
||||
// Sharing one proxy and client keeps the wire-shape cases to one tunnel setup;
|
||||
// each case still creates its own session id so its access-log row is findable.
|
||||
func provisionGatewayMatrix(t *testing.T, ctx context.Context) gatewayEnv {
|
||||
t.Helper()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gw-matrix"})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gw-matrix-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
// The mock ignores auth, so a dummy credential satisfies each catalog
|
||||
// entry's auth template. Vertex is the exception: its api_key is a GCP
|
||||
// service-account keyfile the proxy mints an OAuth token from, and a dummy
|
||||
// one cannot mint. That is deliberate — the Vertex case below asserts on
|
||||
// routing, which happens before the token mint.
|
||||
dummyKey := "sk-gw-e2e"
|
||||
dummyKeyfile := "keyfile::" + "e2e-not-a-real-service-account-key"
|
||||
|
||||
specs := []struct {
|
||||
name string
|
||||
catalogID string
|
||||
apiKey string
|
||||
models []api.AgentNetworkProviderModel
|
||||
}{
|
||||
{
|
||||
name: "openai", catalogID: "openai_api", apiKey: dummyKey,
|
||||
models: []api.AgentNetworkProviderModel{{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002}},
|
||||
},
|
||||
{
|
||||
name: "anthropic", catalogID: "anthropic_api", apiKey: dummyKey,
|
||||
models: []api.AgentNetworkProviderModel{{Id: matrixAnthropicModel, InputPer1k: 0.003, OutputPer1k: 0.015}},
|
||||
},
|
||||
{
|
||||
name: "bedrock", catalogID: "bedrock_api", apiKey: dummyKey,
|
||||
models: []api.AgentNetworkProviderModel{{Id: matrixBedrockModel, InputPer1k: 0.003, OutputPer1k: 0.015}},
|
||||
},
|
||||
{
|
||||
name: "vertex", catalogID: "vertex_ai_api", apiKey: dummyKeyfile,
|
||||
models: []api.AgentNetworkProviderModel{{Id: matrixVertexModel, InputPer1k: 0.001, OutputPer1k: 0.005}},
|
||||
},
|
||||
}
|
||||
|
||||
providerIDs := make(map[string]string, len(specs))
|
||||
ids := make([]string, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
key := spec.apiKey
|
||||
models := spec.models
|
||||
prov, perr := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-gw-" + spec.name,
|
||||
ProviderId: spec.catalogID,
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &key,
|
||||
Enabled: ptr(true),
|
||||
Models: &models,
|
||||
})
|
||||
require.NoError(t, perr, "create %s provider", spec.name)
|
||||
id := prov.Id
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) })
|
||||
providerIDs[spec.catalogID] = id
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
// Uncapped token limit: never blocks the handful of tokens driven here, but
|
||||
// switches on usage metering so consumption and cost land in the row.
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gw-matrix",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: ids,
|
||||
Limits: &api.AgentNetworkPolicyLimits{
|
||||
TokenLimit: api.AgentNetworkPolicyTokenLimit{
|
||||
Enabled: true,
|
||||
GroupCap: 10_000_000,
|
||||
UserCap: 10_000_000,
|
||||
WindowSeconds: 60,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, "gw-matrix", sk.Key)
|
||||
return gatewayEnv{
|
||||
endpoint: endpoint,
|
||||
proxyIP: proxyIP,
|
||||
client: cl,
|
||||
proxy: px,
|
||||
vllm: vllm,
|
||||
providerIDs: providerIDs,
|
||||
}
|
||||
}
|
||||
|
||||
// connectClient starts a proxy and a tunnel client for the shared account and
|
||||
// waits until the client can reach the proxy peer, returning the endpoint and
|
||||
// the proxy's tunnel IP to pin requests to.
|
||||
func connectClient(t *testing.T, ctx context.Context, name, setupKey string) (string, string, *harness.Client, *harness.Proxy) {
|
||||
t.Helper()
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-"+name+"-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, setupKey)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
// The probe resolves the endpoint and its first packet wakes the lazy proxy
|
||||
// peer, so WaitProxyPeer then observes it connected.
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
return settings.Endpoint, proxyIP, cl, px
|
||||
}
|
||||
|
||||
// callUntil retries an HTTP call through the tunnel until it returns one of the
|
||||
// wanted statuses or the deadline passes, absorbing the DNS and tunnel jitter
|
||||
// the first call through a fresh tunnel can hit. The last status and body are
|
||||
// returned either way so the caller can assert with real detail.
|
||||
func callUntil(t *testing.T, call func() (int, string, error), want ...int) (int, string) {
|
||||
t.Helper()
|
||||
wanted := make(map[int]struct{}, len(want))
|
||||
for _, w := range want {
|
||||
wanted[w] = struct{}{}
|
||||
}
|
||||
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
c, b, err := call()
|
||||
if err == nil {
|
||||
code, body = c, b
|
||||
if _, ok := wanted[code]; ok {
|
||||
return code, body
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
|
||||
// TestGatewayProtocolProviderMatrix drives one request per wire shape over a
|
||||
// single tunnel, with a provider record per catalog surface behind it. It is
|
||||
// the regression net for the routing and parser-selection changes: each case
|
||||
// asserts the surface the request was metered under and the token counts that
|
||||
// surface's own usage block carries, so a request parsed by the wrong provider's
|
||||
// parser meters zero and fails rather than passing on a coincidence.
|
||||
func TestGatewayProtocolProviderMatrix(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionGatewayMatrix(t, ctx)
|
||||
diag := func() string {
|
||||
return fmt.Sprintf("\n=== upstream logs ===\n%s\n=== proxy logs ===\n%s",
|
||||
env.vllm.Logs(context.Background()), env.proxy.Logs(context.Background()))
|
||||
}
|
||||
|
||||
t.Run("openai chat completions", func(t *testing.T) {
|
||||
session := "e2e-gw-openai"
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireChat, harness.VLLMModel, "ping", session)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "openai chat must succeed; body: %s%s", body, diag())
|
||||
require.Contains(t, body, "chat.completion", "body must be an OpenAI completion; got: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
require.NotNil(t, row.Provider)
|
||||
assert.Equal(t, "openai", *row.Provider, "the OpenAI chat path must meter under the openai surface")
|
||||
assert.Equal(t, int64(harness.VLLMChatInputTokens), row.InputTokens, "OpenAI usage block must be read")
|
||||
assert.Equal(t, int64(harness.VLLMChatOutputTokens), row.OutputTokens)
|
||||
})
|
||||
|
||||
t.Run("anthropic messages", func(t *testing.T) {
|
||||
session := "e2e-gw-anthropic"
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, matrixAnthropicModel, "ping", session)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "anthropic messages must succeed; body: %s%s", body, diag())
|
||||
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
require.NotNil(t, row.Provider)
|
||||
assert.Equal(t, "anthropic", *row.Provider, "the /v1/messages path must meter under the anthropic surface")
|
||||
// These counts only appear if the Anthropic parser read the response:
|
||||
// its usage fields are named differently from the OpenAI block.
|
||||
assert.Equal(t, int64(harness.VLLMMessagesInputTokens), row.InputTokens,
|
||||
"Anthropic input_tokens must be read; zero here means the wrong parser ran")
|
||||
assert.Equal(t, int64(harness.VLLMMessagesOutputTokens), row.OutputTokens)
|
||||
assert.Positive(t, row.CachedInputTokens, "the Anthropic cache-read bucket must be recorded")
|
||||
assert.Positive(t, row.CostUsd, "a metered request must carry a cost")
|
||||
require.NotNil(t, row.ResolvedProviderId)
|
||||
assert.Equal(t, env.providerIDs["anthropic_api"], *row.ResolvedProviderId,
|
||||
"a vendor-tagged request must not cross to another provider's record")
|
||||
})
|
||||
|
||||
t.Run("bedrock invoke normalises the path model", func(t *testing.T) {
|
||||
session := "e2e-gw-bedrock"
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Bedrock(ctx, env.endpoint, env.proxyIP, matrixBedrockPathModel, "ping", session)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "bedrock invoke must succeed; body: %s%s", body, diag())
|
||||
|
||||
row := findAccessLogBySession(t, ctx, session)
|
||||
require.NotNil(t, row.Provider)
|
||||
assert.Equal(t, "bedrock", *row.Provider, "a native Bedrock path must meter under the bedrock surface")
|
||||
require.NotNil(t, row.Model)
|
||||
assert.Equal(t, matrixBedrockModel, *row.Model,
|
||||
"the inference-profile prefix, release date and version suffix must be normalised away")
|
||||
assert.Equal(t, int64(harness.VLLMMessagesInputTokens), row.InputTokens)
|
||||
})
|
||||
|
||||
t.Run("anthropic token counting", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.PostJSON(ctx, env.endpoint, env.proxyIP, "/v1/messages/count_tokens",
|
||||
fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"ping"}]}`, matrixAnthropicModel),
|
||||
[]string{"anthropic-version: 2023-06-01"})
|
||||
}, 200)
|
||||
assert.Equal(t, 200, code, "token counting must route rather than deny; body: %s%s", body, diag())
|
||||
})
|
||||
|
||||
t.Run("bedrock token counting", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.PostJSON(ctx, env.endpoint, env.proxyIP,
|
||||
"/model/"+matrixBedrockPathModel+"/count-tokens",
|
||||
`{"input":{"converse":{"messages":[{"role":"user","content":[{"text":"ping"}]}]}}}`, nil)
|
||||
}, 200)
|
||||
assert.Equal(t, 200, code,
|
||||
"the Bedrock count-tokens action must route; denying it pushes counting onto the billable inference path; body: %s%s",
|
||||
body, diag())
|
||||
})
|
||||
|
||||
t.Run("vertex token counting reaches its provider", func(t *testing.T) {
|
||||
// The dummy service-account key cannot mint an OAuth token, so the
|
||||
// request stops at the upstream credential. Both outcomes render as
|
||||
// 403, so the deny code is what distinguishes them: upstream_auth_failed
|
||||
// means the path resolved to the Vertex route and only the credential
|
||||
// failed, while model_not_routable would mean the method segment was
|
||||
// swallowed into the model id and no route ever claimed it.
|
||||
path := fmt.Sprintf("/v1/projects/%s/locations/%s/publishers/anthropic/models/%s/count-tokens:rawPredict",
|
||||
matrixVertexProject, matrixVertexRegion, matrixVertexModel)
|
||||
_, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.PostJSON(ctx, env.endpoint, env.proxyIP, path,
|
||||
`{"anthropic_version":"vertex-2023-10-16","messages":[{"role":"user","content":"ping"}]}`, nil)
|
||||
}, 403)
|
||||
assert.NotContains(t, body, "model_not_routable",
|
||||
"the count-tokens method segment must not be parsed as part of the model id; body: %s%s", body, diag())
|
||||
assert.Contains(t, body, "llm_policy.upstream_auth_failed",
|
||||
"the request must reach the Vertex route and fail only at the credential; body: %s%s", body, diag())
|
||||
})
|
||||
|
||||
t.Run("connection warming probe", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Get(ctx, env.endpoint, env.proxyIP, "/api/hello", nil)
|
||||
}, 200)
|
||||
assert.NotEqual(t, 403, code,
|
||||
"the warm-up probe carries no model and must not be refused as unroutable; body: %s%s", body, diag())
|
||||
})
|
||||
|
||||
t.Run("unknown model denies in the caller's error shape", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages,
|
||||
"claude-not-a-real-model-9", "ping", "e2e-gw-unknown")
|
||||
}, 403)
|
||||
require.Equal(t, 403, code, "a model no provider claims must still be refused; body: %s%s", body, diag())
|
||||
|
||||
// The NetBird fields stay where they were for existing consumers.
|
||||
assert.Contains(t, body, "llm_policy.model_not_routable", "the deny code must be preserved")
|
||||
// And the vendor's own envelope rides alongside, so the client can show
|
||||
// the reason instead of an unexplained API error.
|
||||
assert.Contains(t, body, `"type":"error"`, "an Anthropic caller must get the Anthropic error envelope")
|
||||
assert.Contains(t, body, "permission_error", "403 must map to the vendor's permission error type")
|
||||
})
|
||||
}
|
||||
|
||||
// TestModelDiscoveryWithModelAllowlist covers gateway model discovery on an
|
||||
// account that restricts models, which is the configuration that broke: the
|
||||
// listing carries no model, and the per-model allowlist fails closed on an
|
||||
// undetermined one, so discovery denied for exactly the accounts using the
|
||||
// feature. It also asserts the allowlist still refuses a model outside it, so
|
||||
// the exemption cannot be read as a way around the gate.
|
||||
func TestModelDiscoveryWithModelAllowlist(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gw-discovery"})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gw-discovery-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
// One provider enumerating a single model, while the upstream's own listing
|
||||
// advertises two. The proxy must serve the shorter list.
|
||||
dummyKey := "sk-discovery-e2e"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-gw-discovery",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &dummyKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{
|
||||
{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
// The model allowlist is what makes this a regression test: without a
|
||||
// guardrail enabled, discovery was never gated in the first place.
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-gw-discovery-allowlist"
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{harness.VLLMModel}
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gw-discovery",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, "gw-discovery", sk.Key)
|
||||
diag := func() string {
|
||||
return fmt.Sprintf("\n=== upstream logs ===\n%s\n=== proxy logs ===\n%s",
|
||||
vllm.Logs(context.Background()), px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
t.Run("listing is served and bounded by policy", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Get(ctx, endpoint, proxyIP, "/v1/models?limit=1000", nil)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code,
|
||||
"discovery must not be refused because the request carries no model; body: %s%s", body, diag())
|
||||
|
||||
assert.Contains(t, body, harness.VLLMModel, "the authorised model must reach the picker")
|
||||
assert.NotContains(t, body, harness.VLLMUnlistedModel,
|
||||
"a model the policy does not authorise must not be offered; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("allowlist still refuses a model outside it", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Chat(ctx, endpoint, proxyIP, harness.WireChat,
|
||||
harness.VLLMUnlistedModel, "ping", "e2e-gw-discovery-blocked")
|
||||
}, 403)
|
||||
require.Equal(t, 403, code,
|
||||
"exempting model-less endpoints must not exempt inference; body: %s%s", body, diag())
|
||||
assert.True(t,
|
||||
strings.Contains(body, "llm_policy.model_blocked") || strings.Contains(body, "llm_policy.model_not_routable"),
|
||||
"the refusal must name a model policy code; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("allowlisted model still routes", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return cl.Chat(ctx, endpoint, proxyIP, harness.WireChat,
|
||||
harness.VLLMModel, "ping", "e2e-gw-discovery-allowed")
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "the allowlisted model must still be served; body: %s%s", body, diag())
|
||||
})
|
||||
}
|
||||
242
e2e/agentnetwork/gateway_review_test.go
Normal file
242
e2e/agentnetwork/gateway_review_test.go
Normal file
@@ -0,0 +1,242 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// The cases in this file cover behaviour that arrived from code review, after
|
||||
// the gateway-protocol end-to-end tests were written. Each had unit coverage
|
||||
// only; none needed a new harness capability, which is why they belong here
|
||||
// rather than on a manual checklist.
|
||||
|
||||
// TestNonInferenceEndpointsAreAuthorised covers the two review findings on the
|
||||
// endpoints that carry no body: the per-model lookup must be authorised
|
||||
// against the same allowlist that bounds the listing beside it, and only a read
|
||||
// method may claim the non-inference exemption that skips the token pre-flight.
|
||||
func TestNonInferenceEndpointsAreAuthorised(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionDiscoveryProvider(t, ctx)
|
||||
|
||||
t.Run("lookup of an authorised model succeeds", func(t *testing.T) {
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Get(ctx, env.endpoint, env.proxyIP, "/v1/models/"+harness.VLLMModel, nil)
|
||||
}, 200)
|
||||
assert.Equal(t, 200, code, "an allowlisted model must remain reachable; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("lookup of an unauthorised model is refused", func(t *testing.T) {
|
||||
code, body, err := env.client.Get(ctx, env.endpoint, env.proxyIP, "/v1/models/"+harness.VLLMUnlistedModel, nil)
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
assert.Equal(t, 403, code,
|
||||
"a model the policy does not authorise must not be confirmed by the detail lookup; body: %s", body)
|
||||
})
|
||||
|
||||
// A write must not claim the exemption that lets the listing skip the token
|
||||
// pre-flight. The body names no model on purpose: that is what a request
|
||||
// probing for the exemption looks like, and it is the case the method gate
|
||||
// exists to refuse. (A POST that does name a model is a different thing —
|
||||
// it routes and meters as the inference request it is.)
|
||||
for _, path := range []string{"/v1/models", "/v1/models/" + harness.VLLMModel, "/api/hello"} {
|
||||
t.Run("write to "+path+" is refused", func(t *testing.T) {
|
||||
code, body, err := env.client.PostJSON(ctx, env.endpoint, env.proxyIP, path,
|
||||
`{"messages":[{"role":"user","content":"hi"}]}`, nil)
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
assert.NotEqual(t, 200, code,
|
||||
"a write to a non-inference path must not be served unmetered; body: %s", body)
|
||||
})
|
||||
}
|
||||
|
||||
// A request carrying the sub-agent attribution headers must still be served
|
||||
// and metered normally. Asserting the ids themselves is not possible yet:
|
||||
// the parser lifts them onto the request's metadata, but nothing persists
|
||||
// them, so they have no queryable surface to check against.
|
||||
t.Run("sub-agent headers do not disturb the request", func(t *testing.T) {
|
||||
sessionID := fmt.Sprintf("e2e-session-agentid-%d", time.Now().UnixNano())
|
||||
code, body, err := env.client.PostJSON(ctx, env.endpoint, env.proxyIP, "/v1/chat/completions",
|
||||
fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}`, harness.VLLMModel),
|
||||
[]string{
|
||||
"x-session-id: " + sessionID,
|
||||
"x-claude-code-agent-id: agent-child-7",
|
||||
"x-claude-code-parent-agent-id: agent-root-1",
|
||||
})
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
require.Equal(t, 200, code, "the request must succeed; body: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
assert.Positive(t, row.InputTokens, "the request must still be metered normally")
|
||||
})
|
||||
}
|
||||
|
||||
// TestDatedModelIdRouting covers both halves of the dated-id rule that review
|
||||
// tightened: a dated id still reaches an undated registration, but a route
|
||||
// pinned to one dated build must never serve a different one.
|
||||
func TestDatedModelIdRouting(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
undated = "claude-sonnet-9"
|
||||
datedA = "claude-sonnet-9-20250101"
|
||||
datedB = "claude-sonnet-9-20250202"
|
||||
)
|
||||
|
||||
t.Run("a dated id reaches its undated registration", func(t *testing.T) {
|
||||
env := provisionModelProvider(t, ctx, "dated-undated", "anthropic_api", undated)
|
||||
|
||||
sessionID := fmt.Sprintf("e2e-session-dated-%d", time.Now().UnixNano())
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, datedA, "Reply with exactly: pong", sessionID)
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "a pinned release of a registered family must route; body: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
assert.Positive(t, row.InputTokens, "the dated request must price at the registered rate, not zero")
|
||||
})
|
||||
|
||||
t.Run("a route pinned to one dated build refuses another", func(t *testing.T) {
|
||||
env := provisionModelProvider(t, ctx, "dated-pinned", "anthropic_api", datedA)
|
||||
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, datedA, "Reply with exactly: pong", "")
|
||||
}, 200)
|
||||
require.Equal(t, 200, code, "the exact dated id must still route; body: %s", body)
|
||||
|
||||
code, body, err := env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireMessages, datedB, "Reply with exactly: pong", "")
|
||||
require.NoError(t, err, "request must reach the proxy")
|
||||
assert.Equal(t, 403, code,
|
||||
"a provider pinned to one dated build must not serve another; body: %s", body)
|
||||
})
|
||||
}
|
||||
|
||||
// TestBedrockInferenceProfilesReachTheUpstream covers the startup lookup a
|
||||
// Bedrock client makes. The proxy forwards it to the configured upstream rather
|
||||
// than denying it, so what comes back is the upstream's answer — never a
|
||||
// NetBird policy rejection.
|
||||
func TestBedrockInferenceProfilesReachTheUpstream(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionModelProvider(t, ctx, "infprofiles", "bedrock_api", "anthropic.claude-sonnet-5")
|
||||
|
||||
code, body := callUntil(t, func() (int, string, error) {
|
||||
return env.client.Get(ctx, env.endpoint, env.proxyIP, "/inference-profiles", nil)
|
||||
}, 200)
|
||||
|
||||
assert.Equal(t, 200, code, "the lookup must reach the upstream; body: %s", body)
|
||||
assert.NotContains(t, body, "llm_policy.",
|
||||
"the proxy must not answer a control-plane lookup with a policy denial")
|
||||
assert.Contains(t, body, "inferenceProfileSummaries",
|
||||
"the upstream's own answer must come back untouched")
|
||||
}
|
||||
|
||||
// provisionDiscoveryProvider brings up one mock-backed provider enumerating a
|
||||
// single model, with an allowlist guardrail in effect, plus a connected client.
|
||||
func provisionDiscoveryProvider(t *testing.T, ctx context.Context) pricedEnv {
|
||||
t.Helper()
|
||||
env := provisionModelProvider(t, ctx, "noninference", "openai_api", harness.VLLMModel)
|
||||
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-noninference-allowlist-" + fmt.Sprint(time.Now().UnixNano())
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{harness.VLLMModel}
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
_, err = srv.UpdatePolicy(ctx, env.policyID, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-noninference",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{env.groupID},
|
||||
DestinationProviderIds: []string{env.providerID},
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "attach guardrail to policy")
|
||||
return env
|
||||
}
|
||||
|
||||
// provisionModelProvider brings up the mock, one provider under the given
|
||||
// catalog id enumerating exactly one model, an authorising policy, and a
|
||||
// connected proxy + client.
|
||||
func provisionModelProvider(t *testing.T, ctx context.Context, name, catalogID, model string) pricedEnv {
|
||||
t.Helper()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
suffix := strings.ToLower(name)
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gwr-" + suffix})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gwr-" + suffix + "-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
dummyKey := "sk-gwr-e2e"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-gwr-" + suffix,
|
||||
ProviderId: catalogID,
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &dummyKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{
|
||||
{Id: model, InputPer1k: 0.001, OutputPer1k: 0.002},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gwr-" + suffix,
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
Limits: &api.AgentNetworkPolicyLimits{
|
||||
TokenLimit: api.AgentNetworkPolicyTokenLimit{
|
||||
Enabled: true,
|
||||
GroupCap: 10_000_000,
|
||||
UserCap: 10_000_000,
|
||||
WindowSeconds: 60,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, "gwr-"+suffix, sk.Key)
|
||||
return pricedEnv{
|
||||
providerID: prov.Id,
|
||||
groupID: grp.Id,
|
||||
policyID: pol.Id,
|
||||
upstream: vllm.URL,
|
||||
endpoint: endpoint,
|
||||
proxyIP: proxyIP,
|
||||
client: cl,
|
||||
proxy: px,
|
||||
}
|
||||
}
|
||||
199
e2e/agentnetwork/streaming_test.go
Normal file
199
e2e/agentnetwork/streaming_test.go
Normal file
@@ -0,0 +1,199 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// streamedModel is priced high enough that a mis-metered request is obvious in
|
||||
// the recorded cost, and named so it cannot collide with another test's route.
|
||||
const streamedModel = "e2e-streamed-model"
|
||||
|
||||
const (
|
||||
streamInRate = 0.010
|
||||
streamOutRate = 0.020
|
||||
// The cache-read bucket is priced separately from input, so a run that
|
||||
// folded the two together fails the per-bucket assertions below.
|
||||
streamCacheReadRate = 0.001
|
||||
)
|
||||
|
||||
// TestStreamingResponseMetersInputTokens is the end-to-end guard for the
|
||||
// metering bug this endpoint's gateway-protocol work fixed.
|
||||
//
|
||||
// On a streamed answer the input-token count exists only in the opening
|
||||
// message_start event; every later frame reports output. A response read with
|
||||
// the wrong vendor's parser — the shape a gateway record produces when it names
|
||||
// one API surface and serves another — never looks at that event, so input
|
||||
// metered as zero and the bulk of the bill silently vanished. Nothing in the
|
||||
// suite sent stream: true before this test, so the whole branch went unrun.
|
||||
//
|
||||
// The provider points at the mock's streaming listener, which answers every
|
||||
// request as SSE with token counts that differ from the buffered surface. That
|
||||
// difference is the point: passing these assertions is only possible if the
|
||||
// stream accumulator ran.
|
||||
func TestStreamingResponseMetersInputTokens(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionStreamingProvider(t, ctx, "anthropic_api")
|
||||
|
||||
sessionID := fmt.Sprintf("e2e-session-stream-%d", time.Now().UnixNano())
|
||||
code, body := chatStreamUntil(t, ctx, env, harness.WireMessages, streamedModel, sessionID)
|
||||
require.Equal(t, 200, code, "streamed chat must succeed; body: %s", body)
|
||||
assert.Contains(t, body, "message_start",
|
||||
"the client must receive the event stream itself, not a buffered rewrite of it")
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
|
||||
assert.Equal(t, harness.VLLMStreamInputTokens, int(row.InputTokens),
|
||||
"input tokens live in message_start; zero here is the bug this test exists for")
|
||||
assert.Equal(t, harness.VLLMStreamOutputTokens, int(row.OutputTokens),
|
||||
"output tokens ride message_delta and supersede the message_start seed")
|
||||
assert.Equal(t, harness.VLLMStreamCacheReadTokens, int(row.CachedInputTokens),
|
||||
"the Anthropic cache bucket rides message_start too, and only its own parser reads it")
|
||||
|
||||
// The Anthropic surface bills cache reads additively, so the input bucket
|
||||
// prices the full input count rather than a remainder.
|
||||
wantInput := float64(harness.VLLMStreamInputTokens) / 1000 * streamInRate
|
||||
wantOutput := float64(harness.VLLMStreamOutputTokens) / 1000 * streamOutRate
|
||||
assert.InDelta(t, wantInput, row.InputCostUsd, 1e-6, "input cost must price the streamed input tokens")
|
||||
assert.InDelta(t, wantOutput, row.OutputCostUsd, 1e-6, "output cost must price the streamed output tokens")
|
||||
assert.Greater(t, row.CostUsd, 0.0, "a streamed request must never record as free")
|
||||
}
|
||||
|
||||
// TestStreamingOnGatewayTypedProvider drives the same streamed Anthropic call
|
||||
// through a provider record whose catalog id names the OpenAI surface — the
|
||||
// exact misconfiguration that hid the bug, since gateway records commonly pin
|
||||
// one parser while the upstream serves another shape entirely.
|
||||
//
|
||||
// The router must choose the parser from the request path rather than the
|
||||
// record's provider id, or the Anthropic usage block goes unread and input
|
||||
// meters at zero all over again.
|
||||
func TestStreamingOnGatewayTypedProvider(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env := provisionStreamingProvider(t, ctx, "openai_api")
|
||||
|
||||
sessionID := fmt.Sprintf("e2e-session-stream-gw-%d", time.Now().UnixNano())
|
||||
code, body := chatStreamUntil(t, ctx, env, harness.WireMessages, streamedModel, sessionID)
|
||||
require.Equal(t, 200, code, "streamed chat through a gateway record must succeed; body: %s", body)
|
||||
|
||||
row := findAccessLogBySession(t, ctx, sessionID)
|
||||
|
||||
assert.Equal(t, harness.VLLMStreamInputTokens, int(row.InputTokens),
|
||||
"a record typed openai_api must still read the Anthropic usage block it is actually serving")
|
||||
assert.Equal(t, harness.VLLMStreamOutputTokens, int(row.OutputTokens),
|
||||
"output tokens must survive the surface mismatch too")
|
||||
assert.InDelta(t, float64(harness.VLLMStreamInputTokens)/1000*streamInRate, row.InputCostUsd, 1e-6,
|
||||
"the request must be priced on the surface it spoke, not the one the record names")
|
||||
}
|
||||
|
||||
// provisionStreamingProvider brings up the mock, one provider pointed at its
|
||||
// streaming listener under the given catalog id, a policy authorising it, and a
|
||||
// connected proxy + client.
|
||||
func provisionStreamingProvider(t *testing.T, ctx context.Context, catalogID string) pricedEnv {
|
||||
t.Helper()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
name := "stream-" + catalogID
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-" + name})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-" + name + "-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
dummyKey := "sk-stream-e2e"
|
||||
cacheRead := streamCacheReadRate
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: name,
|
||||
ProviderId: catalogID,
|
||||
UpstreamUrl: vllm.StreamURL,
|
||||
ApiKey: &dummyKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{{
|
||||
Id: streamedModel,
|
||||
InputPer1k: streamInRate,
|
||||
OutputPer1k: streamOutRate,
|
||||
CacheReadPer1k: &cacheRead,
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-" + name,
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
Limits: &api.AgentNetworkPolicyLimits{
|
||||
TokenLimit: api.AgentNetworkPolicyTokenLimit{
|
||||
Enabled: true,
|
||||
GroupCap: 10_000_000,
|
||||
UserCap: 10_000_000,
|
||||
WindowSeconds: 60,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
endpoint, proxyIP, cl, px := connectClient(t, ctx, name, sk.Key)
|
||||
return pricedEnv{
|
||||
providerID: prov.Id,
|
||||
groupID: grp.Id,
|
||||
policyID: pol.Id,
|
||||
upstream: vllm.StreamURL,
|
||||
endpoint: endpoint,
|
||||
proxyIP: proxyIP,
|
||||
client: cl,
|
||||
proxy: px,
|
||||
}
|
||||
}
|
||||
|
||||
// chatStreamUntil drives one streamed chat, retrying to absorb the tunnel and
|
||||
// DNS jitter a first call through a fresh peer can hit.
|
||||
func chatStreamUntil(t *testing.T, ctx context.Context, env pricedEnv, kind, model, sessionID string) (int, string) {
|
||||
t.Helper()
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
c, b, cerr := env.client.ChatStream(ctx, env.endpoint, env.proxyIP, kind, model, "Reply with exactly: pong", sessionID)
|
||||
if cerr == nil {
|
||||
code, body = c, b
|
||||
if code == 200 {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
if code != 200 {
|
||||
t.Logf("=== proxy logs ===\n%s", env.proxy.Logs(context.Background()))
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -199,12 +200,18 @@ func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want st
|
||||
const (
|
||||
// curlExitCouldNotResolve is curl's exit code for a DNS resolution failure, distinct from connection-level failures.
|
||||
curlExitCouldNotResolve = 6
|
||||
// dnsProbeRetryWindow bounds DNS-failure retries: the synthesized zone lands a beat after management connects, so early NXDOMAIN is propagation; a zone still absent after this window is a real failure.
|
||||
dnsProbeRetryWindow = 30 * time.Second
|
||||
dnsProbeRetryInterval = 2 * time.Second
|
||||
// curlExitCouldNotConnect is curl's exit code for a connection that never
|
||||
// established. The probe exists to WAKE the lazy proxy peer, so the first
|
||||
// attempt legitimately arrives before WireGuard has brought the tunnel up
|
||||
// and fails here — which is propagation, exactly like an early NXDOMAIN,
|
||||
// and belongs inside the retry window rather than failing the test outright.
|
||||
curlExitCouldNotConnect = 7
|
||||
// endpointProbeRetryWindow bounds retries of the transient failures above: the synthesized zone and the tunnel both land a beat after management connects. Still failing after this window is a real failure.
|
||||
endpointProbeRetryWindow = 30 * time.Second
|
||||
endpointProbeRetryInterval = 2 * time.Second
|
||||
)
|
||||
|
||||
// ResolveProxyIP GETs https://<endpoint>/ from the client's netns: any HTTP status proves DNS + tunnel and wakes the lazy proxy peer; only DNS failures retry, within dnsProbeRetryWindow. Returns the connected IP for --resolve pinning.
|
||||
// ResolveProxyIP GETs https://<endpoint>/ from the client's netns: any HTTP status proves DNS + tunnel and wakes the lazy proxy peer; DNS and connect failures retry, within endpointProbeRetryWindow. Returns the connected IP for --resolve pinning.
|
||||
func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) {
|
||||
args := []string{
|
||||
"run", "--rm",
|
||||
@@ -215,7 +222,7 @@ func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string,
|
||||
"-w", "%{remote_ip}",
|
||||
"https://" + endpoint + "/",
|
||||
}
|
||||
deadline := time.Now().Add(dnsProbeRetryWindow)
|
||||
deadline := time.Now().Add(endpointProbeRetryWindow)
|
||||
for {
|
||||
cmd := exec.CommandContext(ctx, "docker", args...)
|
||||
var stdout, stderr strings.Builder
|
||||
@@ -231,21 +238,29 @@ func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string,
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) || exitErr.ExitCode() != curlExitCouldNotResolve {
|
||||
if !errors.As(err, &exitErr) || !isTransientProbeExit(exitErr.ExitCode()) {
|
||||
return "", fmt.Errorf("no HTTP response from %s: %w (%s)", endpoint, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
dnsErr := fmt.Errorf("DNS resolution failed for %s: %s", endpoint, strings.TrimSpace(stderr.String()))
|
||||
if time.Until(deadline) < dnsProbeRetryInterval {
|
||||
return "", dnsErr
|
||||
probeErr := fmt.Errorf("endpoint %s not reachable yet: %s", endpoint, strings.TrimSpace(stderr.String()))
|
||||
if time.Until(deadline) < endpointProbeRetryInterval {
|
||||
return "", probeErr
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("%w (%w)", dnsErr, ctx.Err())
|
||||
case <-time.After(dnsProbeRetryInterval):
|
||||
return "", fmt.Errorf("%w (%w)", probeErr, ctx.Err())
|
||||
case <-time.After(endpointProbeRetryInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// isTransientProbeExit reports whether a curl exit code describes a state the
|
||||
// endpoint is expected to pass THROUGH on its way up, rather than a settled
|
||||
// failure. Anything else — TLS refusal, a protocol error, a bad argument —
|
||||
// would still be failing after the retry window, so it fails immediately.
|
||||
func isTransientProbeExit(code int) bool {
|
||||
return code == curlExitCouldNotResolve || code == curlExitCouldNotConnect
|
||||
}
|
||||
|
||||
// Wire shapes for Chat.
|
||||
const (
|
||||
// WireChat is the OpenAI-compatible /v1/chat/completions shape.
|
||||
@@ -292,6 +307,27 @@ func (cl *Client) ChatPrefixed(ctx context.Context, endpoint, proxyIP, pathPrefi
|
||||
return cl.post(ctx, endpoint, proxyIP, pathPrefix+path, body, withSessionID(headers, sessionID))
|
||||
}
|
||||
|
||||
// ChatStream is Chat with "stream": true in the request body, so the proxy's
|
||||
// request parser marks the call as streaming and its response parser takes the
|
||||
// SSE accumulator rather than the buffered-body path. Pair it with a provider
|
||||
// pointed at VLLM.StreamURL, which answers every request as an event stream.
|
||||
func (cl *Client) ChatStream(ctx context.Context, endpoint, proxyIP, kind, model, prompt, sessionID string) (int, string, error) {
|
||||
var path, body string
|
||||
var headers []string
|
||||
switch kind {
|
||||
case WireMessages:
|
||||
path = "/v1/messages"
|
||||
headers = []string{"anthropic-version: 2023-06-01"}
|
||||
body = fmt.Sprintf(`{"model":%q,"max_tokens":2048,"stream":true,"messages":[{"role":"user","content":%q}]}`, model, prompt)
|
||||
default:
|
||||
path = "/v1/chat/completions"
|
||||
// include_usage is what makes a real OpenAI stream emit its final usage
|
||||
// frame; without it the last chunk carries no tokens at all.
|
||||
body = fmt.Sprintf(`{"model":%q,"stream":true,"stream_options":{"include_usage":true},"messages":[{"role":"user","content":%q}]}`, model, prompt)
|
||||
}
|
||||
return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(headers, sessionID))
|
||||
}
|
||||
|
||||
// Vertex issues an Anthropic-on-Vertex rawPredict POST over the tunnel. Unlike
|
||||
// Chat, the model is carried in the request path (project/region/model), so the
|
||||
// proxy routes by path and mints the service-account OAuth token; the body uses
|
||||
@@ -322,10 +358,29 @@ func withSessionID(headers []string, sessionID string) []string {
|
||||
return append(headers, "x-session-id: "+sessionID)
|
||||
}
|
||||
|
||||
// post runs curl in a throwaway container sharing the client's network
|
||||
// namespace so the request traverses the WireGuard tunnel, pinning the endpoint
|
||||
// to the proxy IP. It returns the HTTP status and response body.
|
||||
// Get issues a GET to the agent-network endpoint over the client's tunnel.
|
||||
// Model discovery and the connection-warming probe are read-only endpoints
|
||||
// that carry no body, so they can't go through the chat helpers.
|
||||
func (cl *Client) Get(ctx context.Context, endpoint, proxyIP, path string, extraHeaders []string) (int, string, error) {
|
||||
return cl.do(ctx, http.MethodGet, endpoint, proxyIP, path, "", extraHeaders)
|
||||
}
|
||||
|
||||
// PostJSON issues an arbitrary JSON POST over the client's tunnel, for wire
|
||||
// shapes the typed helpers don't cover (token counting, say).
|
||||
func (cl *Client) PostJSON(ctx context.Context, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) {
|
||||
return cl.do(ctx, http.MethodPost, endpoint, proxyIP, path, body, extraHeaders)
|
||||
}
|
||||
|
||||
// post issues a JSON POST. Retained as the shorthand the chat helpers use.
|
||||
func (cl *Client) post(ctx context.Context, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) {
|
||||
return cl.do(ctx, http.MethodPost, endpoint, proxyIP, path, body, extraHeaders)
|
||||
}
|
||||
|
||||
// do runs curl in a throwaway container sharing the client's network
|
||||
// namespace so the request traverses the WireGuard tunnel, pinning the endpoint
|
||||
// to the proxy IP. It returns the HTTP status and response body. An empty body
|
||||
// sends no payload, which is what a GET needs.
|
||||
func (cl *Client) do(ctx context.Context, method, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) {
|
||||
url := "https://" + endpoint + path
|
||||
args := []string{
|
||||
"run", "--rm",
|
||||
@@ -334,13 +389,15 @@ func (cl *Client) post(ctx context.Context, endpoint, proxyIP, path, body string
|
||||
"-sk", "--connect-timeout", "5", "--max-time", "90",
|
||||
"--resolve", endpoint + ":443:" + proxyIP,
|
||||
"-o", "/dev/stderr", "-w", "%{http_code}",
|
||||
"-X", "POST", url,
|
||||
"-X", method, url,
|
||||
"-H", "Content-Type: application/json",
|
||||
}
|
||||
for _, h := range extraHeaders {
|
||||
args = append(args, "-H", h)
|
||||
}
|
||||
args = append(args, "--data", body)
|
||||
if body != "" {
|
||||
args = append(args, "--data", body)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "docker", args...)
|
||||
// -w writes the status code to stdout; -o /dev/stderr writes the body to
|
||||
// stderr so we can capture both separately.
|
||||
|
||||
@@ -18,18 +18,63 @@ const (
|
||||
vllmImage = "nginx:alpine"
|
||||
vllmAlias = "vllm"
|
||||
vllmPort = "8000/tcp"
|
||||
// vllmStreamPort serves the same wire shapes as an SSE stream. See the
|
||||
// nginx config for why streaming lives on its own listener.
|
||||
vllmStreamPort = "8001/tcp"
|
||||
// VLLMModel is the served model id the mock advertises and echoes back. It
|
||||
// matches a real small model commonly served by vLLM so the provider's
|
||||
// enumerated model and the client's request line up.
|
||||
VLLMModel = "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
// VLLMUnlistedModel is a second id the mock's model listing advertises but
|
||||
// no test provider enumerates, so a filtered listing is observably shorter
|
||||
// than the upstream's own.
|
||||
VLLMUnlistedModel = "Qwen/Qwen2.5-7B-Instruct"
|
||||
)
|
||||
|
||||
// Token counts the mock reports per wire shape. Tests assert on these rather
|
||||
// than on "> 0" so a response parsed with the wrong provider's parser (which
|
||||
// would read a different field, or none) fails loudly instead of passing on
|
||||
// a coincidental non-zero.
|
||||
const (
|
||||
// VLLMChatInputTokens / VLLMChatOutputTokens ride the OpenAI usage block.
|
||||
VLLMChatInputTokens = 11
|
||||
VLLMChatOutputTokens = 2
|
||||
// VLLMMessagesInputTokens / VLLMMessagesOutputTokens ride the Anthropic
|
||||
// usage block, whose field names the OpenAI parser cannot read.
|
||||
VLLMMessagesInputTokens = 17
|
||||
VLLMMessagesOutputTokens = 3
|
||||
)
|
||||
|
||||
// Token counts the streaming surface reports. They differ from the
|
||||
// non-streaming ones on purpose: a test that asserts these numbers proves the
|
||||
// SSE accumulator ran, rather than a buffered JSON body having been parsed.
|
||||
//
|
||||
// Input and cache-read arrive on message_start; output arrives on
|
||||
// message_delta and supersedes the seed value message_start carries. Any
|
||||
// parser that cannot read message_start reports zero input tokens — which is
|
||||
// exactly the bug these counts exist to catch.
|
||||
const (
|
||||
VLLMStreamInputTokens = 29
|
||||
VLLMStreamOutputTokens = 5
|
||||
VLLMStreamCacheReadTokens = 7
|
||||
)
|
||||
|
||||
// vllmNginxConf emulates a vLLM OpenAI-compatible server over plain HTTP (vLLM's
|
||||
// default: no TLS, port 8000). It answers /v1/models with a one-model list and
|
||||
// any chat/completions path with a canned OpenAI-shaped chat completion carrying
|
||||
// a non-zero usage block, so the proxy's OpenAI parser records real token
|
||||
// consumption. Running actual vLLM in CI is infeasible (GPU + multi-GB model
|
||||
// default: no TLS, port 8000), and additionally answers the wire shapes the
|
||||
// other catalog surfaces speak so one mock can stand in for every provider the
|
||||
// proxy routes to. Running actual vLLM in CI is infeasible (GPU + multi-GB model
|
||||
// download), so this stands in for the wire contract the proxy depends on.
|
||||
//
|
||||
// Each shape answers with its own vendor's usage block, so a response parsed
|
||||
// under the wrong surface meters zero rather than passing by accident:
|
||||
//
|
||||
// - /v1/chat/completions (and any unmatched path): OpenAI chat completion.
|
||||
// - /v1/messages: Anthropic Messages, snake_case usage plus a cache bucket.
|
||||
// - /model/{id}/invoke: Bedrock InvokeModel, which carries the Anthropic body.
|
||||
// - the token-counting endpoints: a count, with no usage block at all.
|
||||
//
|
||||
// The model listing advertises two models so a policy that authorises one
|
||||
// produces an observably shorter list than the upstream's own.
|
||||
const vllmNginxConf = `pid /tmp/nginx.pid;
|
||||
events {}
|
||||
http {
|
||||
@@ -37,13 +82,75 @@ http {
|
||||
listen 8000;
|
||||
location = /v1/models {
|
||||
default_type application/json;
|
||||
return 200 '{"object":"list","data":[{"id":"Qwen/Qwen2.5-0.5B-Instruct","object":"model","owned_by":"vllm"}]}';
|
||||
return 200 '{"object":"list","data":[{"id":"Qwen/Qwen2.5-0.5B-Instruct","object":"model","owned_by":"vllm"},{"id":"Qwen/Qwen2.5-7B-Instruct","object":"model","owned_by":"vllm"}]}';
|
||||
}
|
||||
location = /v1/messages {
|
||||
default_type application/json;
|
||||
return 200 '{"id":"msg_e2e","type":"message","role":"assistant","model":"claude-sonnet-5","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":17,"output_tokens":3,"cache_read_input_tokens":5}}';
|
||||
}
|
||||
location = /v1/messages/count_tokens {
|
||||
default_type application/json;
|
||||
return 200 '{"input_tokens":7}';
|
||||
}
|
||||
location ~ ^/model/.+/invoke$ {
|
||||
default_type application/json;
|
||||
return 200 '{"id":"msg_e2e_bedrock","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":17,"output_tokens":3,"cache_read_input_tokens":5}}';
|
||||
}
|
||||
location ~ ^/model/.+/count-tokens$ {
|
||||
default_type application/json;
|
||||
return 200 '{"inputTokens":9}';
|
||||
}
|
||||
location = /api/hello {
|
||||
return 200;
|
||||
}
|
||||
location = /inference-profiles {
|
||||
default_type application/json;
|
||||
return 200 '{"inferenceProfileSummaries":[{"inferenceProfileId":"us.anthropic.claude-sonnet-5","status":"ACTIVE"}]}';
|
||||
}
|
||||
location / {
|
||||
default_type application/json;
|
||||
return 200 '{"id":"chatcmpl-e2e-vllm","object":"chat.completion","created":1700000000,"model":"Qwen/Qwen2.5-0.5B-Instruct","choices":[{"index":0,"message":{"role":"assistant","content":"pong"},"finish_reason":"stop"}],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}';
|
||||
}
|
||||
}
|
||||
|
||||
# The streaming surface, on its own port so the response content type is a
|
||||
# property of the listener rather than of a per-request branch: nginx sets
|
||||
# Content-Type from default_type, which cannot be varied inside an "if", and
|
||||
# a second Content-Type via add_header would leave the proxy reading the
|
||||
# wrong one. A provider record pointed at this port streams every answer.
|
||||
#
|
||||
# Input and cache-read tokens ride message_start, output rides message_delta
|
||||
# — the split that makes a stream different from a buffered body, and the
|
||||
# reason a parser that ignores message_start meters input as zero.
|
||||
server {
|
||||
listen 8001;
|
||||
location = /v1/messages {
|
||||
default_type text/event-stream;
|
||||
return 200 'event: message_start
|
||||
data: {"type":"message_start","message":{"id":"msg_e2e_stream","type":"message","role":"assistant","model":"claude-sonnet-5","content":[],"usage":{"input_tokens":29,"output_tokens":1,"cache_read_input_tokens":7}}}
|
||||
|
||||
event: content_block_delta
|
||||
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
|
||||
|
||||
event: message_delta
|
||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}
|
||||
|
||||
event: message_stop
|
||||
data: {"type":"message_stop"}
|
||||
|
||||
';
|
||||
}
|
||||
location / {
|
||||
default_type text/event-stream;
|
||||
return 200 'data: {"choices":[{"delta":{"content":"pong"}}]}
|
||||
|
||||
data: {"choices":[],"usage":{"prompt_tokens":29,"completion_tokens":5,"total_tokens":34}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
';
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
@@ -55,6 +162,10 @@ type VLLM struct {
|
||||
workDir string
|
||||
// URL is the upstream URL the vllm provider points at (http://<alias>:8000).
|
||||
URL string
|
||||
// StreamURL is the same mock's streaming listener. A provider pointed here
|
||||
// answers every request as SSE, so the proxy's streaming accumulator runs
|
||||
// instead of its buffered-body parser.
|
||||
StreamURL string
|
||||
}
|
||||
|
||||
// StartVLLM runs the mock vLLM server on the shared network over plain HTTP.
|
||||
@@ -73,14 +184,17 @@ func StartVLLM(ctx context.Context, c *Combined) (*VLLM, error) {
|
||||
|
||||
req := testcontainers.ContainerRequest{
|
||||
Image: vllmImage,
|
||||
ExposedPorts: []string{vllmPort},
|
||||
ExposedPorts: []string{vllmPort, vllmStreamPort},
|
||||
Networks: []string{c.network.Name},
|
||||
NetworkAliases: map[string][]string{c.network.Name: {vllmAlias}},
|
||||
Cmd: []string{"nginx", "-c", "/conf/nginx.conf", "-g", "daemon off;"},
|
||||
HostConfigModifier: func(hc *container.HostConfig) {
|
||||
hc.Binds = append(hc.Binds, workDir+":/conf:ro")
|
||||
},
|
||||
WaitingFor: wait.ForListeningPort(vllmPort).WithStartupTimeout(60 * time.Second),
|
||||
WaitingFor: wait.ForAll(
|
||||
wait.ForListeningPort(vllmPort),
|
||||
wait.ForListeningPort(vllmStreamPort),
|
||||
).WithStartupTimeout(60 * time.Second),
|
||||
}
|
||||
|
||||
ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
@@ -92,7 +206,12 @@ func StartVLLM(ctx context.Context, c *Combined) (*VLLM, error) {
|
||||
return nil, fmt.Errorf("start vllm container: %w", err)
|
||||
}
|
||||
|
||||
return &VLLM{container: ctr, workDir: workDir, URL: "http://" + vllmAlias + ":8000"}, nil
|
||||
return &VLLM{
|
||||
container: ctr,
|
||||
workDir: workDir,
|
||||
URL: "http://" + vllmAlias + ":8000",
|
||||
StreamURL: "http://" + vllmAlias + ":8001",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Logs returns the vLLM container logs, for diagnostics on failure.
|
||||
|
||||
2
go.mod
2
go.mod
@@ -57,6 +57,7 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.9.0
|
||||
github.com/gliderlabs/ssh v0.3.8
|
||||
github.com/go-jose/go-jose/v4 v4.1.4
|
||||
github.com/go-ole/go-ole v1.3.0
|
||||
github.com/gobwas/ws v1.4.0
|
||||
github.com/goccy/go-yaml v1.18.0
|
||||
github.com/godbus/dbus/v5 v5.2.2
|
||||
@@ -199,7 +200,6 @@ require (
|
||||
github.com/go-ldap/ldap/v3 v3.4.13 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/analysis v0.23.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.1 // indirect
|
||||
|
||||
@@ -111,6 +111,59 @@ check_nb_domain() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Non-interactive configuration
|
||||
# ------------------------------
|
||||
# Every prompt below can be pre-answered with an environment variable, so the
|
||||
# script runs unattended (cloud-init, CI, Terraform, curl | bash). resolve()
|
||||
# is the single place that decides env var vs prompt vs default; the read_*
|
||||
# helpers stay pure prompts.
|
||||
#
|
||||
# Supported env vars:
|
||||
# NETBIRD_DOMAIN domain/FQDN (required)
|
||||
# NETBIRD_LETSENCRYPT_EMAIL ACME email (required for built-in Traefik)
|
||||
# NETBIRD_AGENT_NETWORK true enables the agent-network preset
|
||||
# NETBIRD_REVERSE_PROXY_TYPE 0-5 (default 0 = built-in Traefik)
|
||||
# NETBIRD_ENABLE_PROXY true/false (default false)
|
||||
# NETBIRD_ENABLE_CROWDSEC true/false (default false)
|
||||
# NETBIRD_TRAEFIK_EXTERNAL_NETWORK external-Traefik network (type 1)
|
||||
# NETBIRD_TRAEFIK_ENTRYPOINT external-Traefik entrypoint (type 1, default websecure)
|
||||
# NETBIRD_TRAEFIK_CERTRESOLVER external-Traefik cert resolver (type 1)
|
||||
# NETBIRD_BIND_LOCALHOST_ONLY true/false (default true, types 2-5)
|
||||
# NETBIRD_EXTERNAL_PROXY_NETWORK docker network to join (types 2-4)
|
||||
# NETBIRD_NON_INTERACTIVE true forces unattended mode even with a TTY
|
||||
|
||||
# tty_available succeeds only when we may prompt: never when the operator has
|
||||
# set NETBIRD_NON_INTERACTIVE=true, otherwise only when /dev/tty can actually
|
||||
# be opened. A PTY can be attached in automation (CI runners, some
|
||||
# provisioners), so the env override is the authoritative signal and the
|
||||
# /dev/tty probe is the fallback. /dev/tty is a world-rw device node even with
|
||||
# no terminal, so a permission test ([ -r ]) is not enough - we must open it.
|
||||
tty_available() {
|
||||
[[ "${NETBIRD_NON_INTERACTIVE:-}" == "true" ]] && return 1
|
||||
{ true < /dev/tty; } 2>/dev/null
|
||||
}
|
||||
|
||||
# resolve ENV_VAR_NAME DEFAULT PROMPT_FN [prompt args...]
|
||||
# env var set and non-empty -> its value
|
||||
# interactive -> PROMPT_FN "$@" (prompt behavior unchanged)
|
||||
# otherwise -> DEFAULT, or abort when DEFAULT is "required"
|
||||
resolve() {
|
||||
local env_name="$1" default="$2" prompt_fn="$3"
|
||||
shift 3
|
||||
local env_value="${!env_name:-}"
|
||||
if [[ -n "$env_value" ]]; then
|
||||
echo "$env_value"
|
||||
elif tty_available; then
|
||||
"$prompt_fn" "$@"
|
||||
elif [[ "$default" == "required" ]]; then
|
||||
echo "$env_name is required for a non-interactive install." > /dev/stderr
|
||||
exit 1
|
||||
else
|
||||
echo "$default"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
read_nb_domain() {
|
||||
READ_NETBIRD_DOMAIN=""
|
||||
echo -n "Enter the domain you want to use for NetBird (e.g. netbird.my-domain.com): " > /dev/stderr
|
||||
@@ -383,7 +436,14 @@ initialize_default_values() {
|
||||
}
|
||||
|
||||
configure_domain() {
|
||||
# Domain is validated (not a free-form value), so it keeps its own guard
|
||||
# rather than going through resolve(): a valid NETBIRD_DOMAIN is used as-is,
|
||||
# otherwise we prompt, or abort when there is no terminal to prompt on.
|
||||
if ! check_nb_domain "$NETBIRD_DOMAIN"; then
|
||||
if ! tty_available; then
|
||||
echo "NETBIRD_DOMAIN is required for a non-interactive install." > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
NETBIRD_DOMAIN=$(read_nb_domain)
|
||||
fi
|
||||
|
||||
@@ -411,11 +471,7 @@ apply_agent_network_preset() {
|
||||
ENABLE_PROXY="true"
|
||||
ENABLE_CROWDSEC="false"
|
||||
|
||||
if [[ -n "${NETBIRD_LETSENCRYPT_EMAIL}" ]]; then
|
||||
TRAEFIK_ACME_EMAIL="${NETBIRD_LETSENCRYPT_EMAIL}"
|
||||
else
|
||||
TRAEFIK_ACME_EMAIL=$(read_traefik_acme_email)
|
||||
fi
|
||||
TRAEFIK_ACME_EMAIL=$(resolve NETBIRD_LETSENCRYPT_EMAIL required read_traefik_acme_email)
|
||||
|
||||
echo "" > /dev/stderr
|
||||
echo "Agent-network preset enabled (NETBIRD_AGENT_NETWORK=true):" > /dev/stderr
|
||||
@@ -437,35 +493,35 @@ configure_reverse_proxy() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Prompt for reverse proxy type
|
||||
REVERSE_PROXY_TYPE=$(read_reverse_proxy_type)
|
||||
# Reverse proxy type (env NETBIRD_REVERSE_PROXY_TYPE, else prompt, else 0)
|
||||
REVERSE_PROXY_TYPE=$(resolve NETBIRD_REVERSE_PROXY_TYPE 0 read_reverse_proxy_type)
|
||||
|
||||
# Handle built-in Traefik prompts (option 0)
|
||||
if [[ "$REVERSE_PROXY_TYPE" == "0" ]]; then
|
||||
TRAEFIK_ACME_EMAIL=$(read_traefik_acme_email)
|
||||
ENABLE_PROXY=$(read_enable_proxy)
|
||||
TRAEFIK_ACME_EMAIL=$(resolve NETBIRD_LETSENCRYPT_EMAIL required read_traefik_acme_email)
|
||||
ENABLE_PROXY=$(resolve NETBIRD_ENABLE_PROXY false read_enable_proxy)
|
||||
if [[ "$ENABLE_PROXY" == "true" ]]; then
|
||||
ENABLE_CROWDSEC=$(read_enable_crowdsec)
|
||||
ENABLE_CROWDSEC=$(resolve NETBIRD_ENABLE_CROWDSEC false read_enable_crowdsec)
|
||||
fi
|
||||
fi
|
||||
|
||||
# Handle external Traefik-specific prompts (option 1)
|
||||
if [[ "$REVERSE_PROXY_TYPE" == "1" ]]; then
|
||||
TRAEFIK_EXTERNAL_NETWORK=$(read_traefik_network)
|
||||
TRAEFIK_ENTRYPOINT=$(read_traefik_entrypoint)
|
||||
TRAEFIK_CERTRESOLVER=$(read_traefik_certresolver)
|
||||
TRAEFIK_EXTERNAL_NETWORK=$(resolve NETBIRD_TRAEFIK_EXTERNAL_NETWORK "" read_traefik_network)
|
||||
TRAEFIK_ENTRYPOINT=$(resolve NETBIRD_TRAEFIK_ENTRYPOINT websecure read_traefik_entrypoint)
|
||||
TRAEFIK_CERTRESOLVER=$(resolve NETBIRD_TRAEFIK_CERTRESOLVER "" read_traefik_certresolver)
|
||||
fi
|
||||
|
||||
# Handle port binding for external proxy options (2-5)
|
||||
if [[ "$REVERSE_PROXY_TYPE" -ge 2 ]]; then
|
||||
BIND_LOCALHOST_ONLY=$(read_port_binding_preference)
|
||||
BIND_LOCALHOST_ONLY=$(resolve NETBIRD_BIND_LOCALHOST_ONLY true read_port_binding_preference)
|
||||
fi
|
||||
|
||||
# Handle Docker network prompts for external proxies (options 2-4)
|
||||
case "$REVERSE_PROXY_TYPE" in
|
||||
2) EXTERNAL_PROXY_NETWORK=$(read_proxy_docker_network "Nginx") ;;
|
||||
3) EXTERNAL_PROXY_NETWORK=$(read_proxy_docker_network "Nginx Proxy Manager") ;;
|
||||
4) EXTERNAL_PROXY_NETWORK=$(read_proxy_docker_network "Caddy") ;;
|
||||
2) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Nginx") ;;
|
||||
3) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Nginx Proxy Manager") ;;
|
||||
4) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Caddy") ;;
|
||||
*) ;; # No network prompt for other options
|
||||
esac
|
||||
return 0
|
||||
@@ -643,8 +699,13 @@ start_services_and_show_instructions() {
|
||||
print_post_setup_instructions
|
||||
|
||||
echo ""
|
||||
echo -n "Press Enter when your reverse proxy is configured (or Ctrl+C to exit)... "
|
||||
read -r < /dev/tty
|
||||
if tty_available; then
|
||||
echo -n "Press Enter when your reverse proxy is configured (or Ctrl+C to exit)... "
|
||||
read -r < /dev/tty
|
||||
else
|
||||
echo "Non-interactive mode: starting NetBird containers now. Finish configuring"
|
||||
echo "your reverse proxy using the instructions above so it can reach them."
|
||||
fi
|
||||
|
||||
echo -e "$MSG_STARTING_SERVICES"
|
||||
$DOCKER_COMPOSE_COMMAND up -d
|
||||
|
||||
@@ -113,8 +113,61 @@ type Provider struct {
|
||||
// upstream provider + credentials on Portkey's hosted side).
|
||||
ExtraHeaders []ExtraHeader
|
||||
Models []Model
|
||||
// Discovery, when non-nil, describes how to ask this vendor which
|
||||
// models the operator's own credential can actually reach, so the
|
||||
// provider form can offer a live list instead of only the hand-curated
|
||||
// Models above. Nil for entries with no listing endpoint (gateways
|
||||
// vary too much) — those keep free-text entry.
|
||||
Discovery *Discovery
|
||||
}
|
||||
|
||||
// ListingShape names the response envelope a vendor returns its model
|
||||
// listing in. Every vendor invented its own, and none of them can be
|
||||
// guessed from the request, so the catalog states it.
|
||||
type ListingShape string
|
||||
|
||||
const (
|
||||
// ShapeOpenAIData is {"data":[{"id":…}]} — OpenAI, and Anthropic, which
|
||||
// adopted the same envelope.
|
||||
ShapeOpenAIData ListingShape = "openai_data"
|
||||
// ShapeBedrockInferenceProfiles is
|
||||
// {"inferenceProfileSummaries":[{"inferenceProfileId":…}]}. The ids carry
|
||||
// the region prefix that makes them invocable, which is exactly what an
|
||||
// operator cannot reconstruct by hand.
|
||||
ShapeBedrockInferenceProfiles ListingShape = "bedrock_inference_profiles"
|
||||
// ShapeVertexPublisherModels is {"publisherModels":[{"name":…}]}, where
|
||||
// name is a resource path and the invocable id is its last segment joined
|
||||
// to a separate versionId field.
|
||||
ShapeVertexPublisherModels ListingShape = "vertex_publisher_models"
|
||||
)
|
||||
|
||||
// Discovery describes one vendor's model-listing endpoint.
|
||||
//
|
||||
// Host is deliberately separate from the provider record's upstream URL:
|
||||
// Bedrock serves listings from the control plane (bedrock.<region>) while
|
||||
// inference must go to the runtime host (bedrock-runtime.<region>), so the
|
||||
// two cannot be the same value. Empty Host means "use the record's own
|
||||
// upstream", which is right for every vendor that serves both from one host.
|
||||
//
|
||||
// The regionPlaceholder in Host is substituted from the provider record's
|
||||
// region. Deriving the discovery host from the catalog rather than accepting
|
||||
// one from the caller is also what keeps this from being an open proxy: the
|
||||
// only hosts management will dial are the ones written here.
|
||||
type Discovery struct {
|
||||
Host string
|
||||
Path string
|
||||
Query string
|
||||
Shape ListingShape
|
||||
// Headers are static headers the vendor requires beyond the credential
|
||||
// (Anthropic versions its API through one and rejects a request without
|
||||
// it). The auth header itself comes from AuthHeaderName/Template.
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
// RegionPlaceholder is replaced in Discovery.Host by the provider record's
|
||||
// configured region.
|
||||
const RegionPlaceholder = "<region>"
|
||||
|
||||
// ExtraHeader names a single optional per-provider routing/config
|
||||
// header. Catalog declares N of these per provider type; the operator
|
||||
// fills any subset on the provider record (see Provider.ExtraValues).
|
||||
@@ -245,8 +298,12 @@ var providers = []Provider{
|
||||
AuthHeaderTemplate: "Bearer ${API_KEY}",
|
||||
DefaultContentType: "application/json",
|
||||
BrandColor: "#10A37F",
|
||||
ParserID: "openai",
|
||||
PricingSurfaces: []string{"openai"},
|
||||
Discovery: &Discovery{
|
||||
Path: "/v1/models",
|
||||
Shape: ShapeOpenAIData,
|
||||
},
|
||||
ParserID: "openai",
|
||||
PricingSurfaces: []string{"openai"},
|
||||
// Pricing + context windows cross-checked against LiteLLM's
|
||||
// model_prices_and_context_window.json. Notable corrections from
|
||||
// earlier values: o4-mini repriced from $4/$16 to $1.10/$4.40
|
||||
@@ -284,8 +341,18 @@ var providers = []Provider{
|
||||
AuthHeaderTemplate: "${API_KEY}",
|
||||
DefaultContentType: "application/json",
|
||||
BrandColor: "#D97757",
|
||||
ParserID: "anthropic",
|
||||
PricingSurfaces: []string{"anthropic"},
|
||||
Discovery: &Discovery{
|
||||
Path: "/v1/models",
|
||||
// The default page is short and a picker wants the whole
|
||||
// catalogue in one call.
|
||||
Query: "limit=1000",
|
||||
Shape: ShapeOpenAIData,
|
||||
// Anthropic versions its API through a header and refuses a
|
||||
// request that omits it, listing included.
|
||||
Headers: map[string]string{"anthropic-version": "2023-06-01"},
|
||||
},
|
||||
ParserID: "anthropic",
|
||||
PricingSurfaces: []string{"anthropic"},
|
||||
// Per Anthropic's current model lineup. Pricing in USD per 1k
|
||||
// tokens. Context windows: 4.6+ family is 1M; Haiku 4.5 stays at
|
||||
// 200K. claude-3-7-sonnet and claude-3-5-haiku retired
|
||||
@@ -296,6 +363,8 @@ var providers = []Provider{
|
||||
// account to be on >= 30-day data retention or all requests
|
||||
// 400.
|
||||
Models: []Model{
|
||||
{ID: "claude-opus-5", Label: "Claude Opus 5", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "claude-sonnet-5", Label: "Claude Sonnet 5", InputPer1k: 0.003, OutputPer1k: 0.015, CacheReadPer1k: 0.0003, CacheCreationPer1k: 0.00375, ContextWindow: 1000000},
|
||||
{ID: "claude-fable-5", Label: "Claude Fable 5", InputPer1k: 0.010, OutputPer1k: 0.050, CacheReadPer1k: 0.001, CacheCreationPer1k: 0.0125, ContextWindow: 1000000},
|
||||
{ID: "claude-opus-4-8", Label: "Claude Opus 4.8", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "claude-opus-4-7", Label: "Claude Opus 4.7", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
@@ -343,6 +412,22 @@ var providers = []Provider{
|
||||
AuthHeaderTemplate: "Bearer ${API_KEY}",
|
||||
DefaultContentType: "application/json",
|
||||
BrandColor: "#FF9900",
|
||||
// Listings come from the CONTROL PLANE, not the runtime host in
|
||||
// DefaultHost above: ListInferenceProfiles is not an operation
|
||||
// bedrock-runtime implements, and answers <UnknownOperationException/>
|
||||
// there. Inference has to go to the runtime host, so the two hosts
|
||||
// genuinely differ and Discovery.Host carries the difference.
|
||||
//
|
||||
// Inference profiles rather than foundation models because the profile
|
||||
// id is the invocable one: it carries the region prefix (eu., us.,
|
||||
// global.) that AWS requires and that cannot be derived from the
|
||||
// configured region — an eu-central-1 account legitimately holds
|
||||
// global.* profiles.
|
||||
Discovery: &Discovery{
|
||||
Host: "bedrock." + RegionPlaceholder + ".amazonaws.com",
|
||||
Path: "/inference-profiles",
|
||||
Shape: ShapeBedrockInferenceProfiles,
|
||||
},
|
||||
// ParserID stays empty (path-style dispatch via IsBedrockPathStyle);
|
||||
// the request parser meters these under the "bedrock" surface.
|
||||
PricingSurfaces: []string{"bedrock"},
|
||||
@@ -355,6 +440,8 @@ var providers = []Provider{
|
||||
// Llama 3.3 70B entry kept unchanged — LiteLLM tracks only
|
||||
// per-region Llama 3 entries; standalone 3.3 not yet listed.
|
||||
Models: []Model{
|
||||
{ID: "anthropic.claude-opus-5", Label: "Claude Opus 5 (Bedrock)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "anthropic.claude-sonnet-5", Label: "Claude Sonnet 5 (Bedrock)", InputPer1k: 0.003, OutputPer1k: 0.015, CacheReadPer1k: 0.0003, CacheCreationPer1k: 0.00375, ContextWindow: 1000000},
|
||||
{ID: "anthropic.claude-opus-4-8", Label: "Claude Opus 4.8 (Bedrock)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "anthropic.claude-opus-4-7", Label: "Claude Opus 4.7 (Bedrock)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "anthropic.claude-opus-4-6", Label: "Claude Opus 4.6 (Bedrock)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
@@ -391,6 +478,15 @@ var providers = []Provider{
|
||||
AuthHeaderTemplate: "Bearer ${API_KEY}",
|
||||
DefaultContentType: "application/json",
|
||||
BrandColor: "#4285F4",
|
||||
// Only the v1beta1 publisher listing answers: the v1 form and the
|
||||
// project-scoped form under BOTH versions return 404. That means the
|
||||
// list is publisher-global — it cannot say which models this project
|
||||
// has enabled — so it is offered as a suggestion beside the catalog
|
||||
// rather than replacing it. See the discovery e2e for the probes.
|
||||
Discovery: &Discovery{
|
||||
Path: "/v1beta1/publishers/anthropic/models",
|
||||
Shape: ShapeVertexPublisherModels,
|
||||
},
|
||||
// ParserID stays empty (path-style dispatch via IsVertexPathStyle);
|
||||
// Anthropic-on-Vertex requests are metered under the "anthropic"
|
||||
// surface with the bare, unversioned model id.
|
||||
@@ -406,6 +502,8 @@ var providers = []Provider{
|
||||
// exists — the router denies unmeterable publishers rather than forward
|
||||
// them uncounted.
|
||||
Models: []Model{
|
||||
{ID: "claude-opus-5", Label: "Claude Opus 5 (Vertex)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "claude-sonnet-5", Label: "Claude Sonnet 5 (Vertex)", InputPer1k: 0.003, OutputPer1k: 0.015, CacheReadPer1k: 0.0003, CacheCreationPer1k: 0.00375, ContextWindow: 1000000},
|
||||
{ID: "claude-fable-5", Label: "Claude Fable 5 (Vertex)", InputPer1k: 0.010, OutputPer1k: 0.050, CacheReadPer1k: 0.001, CacheCreationPer1k: 0.0125, ContextWindow: 1000000},
|
||||
{ID: "claude-opus-4-8", Label: "Claude Opus 4.8 (Vertex)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
{ID: "claude-opus-4-7", Label: "Claude Opus 4.7 (Vertex)", InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625, ContextWindow: 1000000},
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestClaudeLineupSelectable pins the models Claude Code resolves to by
|
||||
// default. A model absent from the lineup can't be ticked on a provider
|
||||
// record, so llm_router denies it as not-routable and the operator has no
|
||||
// way to authorise the client's own default.
|
||||
func TestClaudeLineupSelectable(t *testing.T) {
|
||||
for providerID, wanted := range map[string][]string{
|
||||
"anthropic_api": {"claude-opus-5", "claude-sonnet-5", "claude-haiku-4-5"},
|
||||
"bedrock_api": {"anthropic.claude-opus-5", "anthropic.claude-sonnet-5", "anthropic.claude-haiku-4-5"},
|
||||
"vertex_ai_api": {"claude-opus-5", "claude-sonnet-5", "claude-haiku-4-5"},
|
||||
} {
|
||||
provider, ok := Lookup(providerID)
|
||||
require.True(t, ok, "catalog must define %s", providerID)
|
||||
|
||||
selectable := make(map[string]Model, len(provider.Models))
|
||||
for _, m := range provider.Models {
|
||||
selectable[m.ID] = m
|
||||
}
|
||||
for _, id := range wanted {
|
||||
model, found := selectable[id]
|
||||
require.True(t, found, "%s must offer %s", providerID, id)
|
||||
assert.NotEmpty(t, model.Label, "%s/%s needs a label for the picker", providerID, id)
|
||||
assert.Positive(t, model.InputPer1k, "%s/%s needs an input rate", providerID, id)
|
||||
assert.Positive(t, model.OutputPer1k, "%s/%s needs an output rate", providerID, id)
|
||||
assert.Positive(t, model.ContextWindow, "%s/%s needs a context window", providerID, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/modeldiscovery"
|
||||
nbcontext "github.com/netbirdio/netbird/management/server/context"
|
||||
"github.com/netbirdio/netbird/shared/auth"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// discoveryManagerStub records what the handler asked for and returns a canned
|
||||
// answer. The Manager interface is embedded rather than implemented: only the
|
||||
// one method is reachable from this handler, and a call to any other should
|
||||
// fail loudly rather than silently return a zero value.
|
||||
type discoveryManagerStub struct {
|
||||
agentnetwork.Manager
|
||||
|
||||
gotReq modeldiscovery.Request
|
||||
gotRecordID string
|
||||
models []modeldiscovery.Model
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *discoveryManagerStub) DiscoverProviderModels(
|
||||
_ context.Context, _, _ string, req modeldiscovery.Request, recordID string,
|
||||
) ([]modeldiscovery.Model, error) {
|
||||
s.gotReq = req
|
||||
s.gotRecordID = recordID
|
||||
return s.models, s.err
|
||||
}
|
||||
|
||||
// postDiscovery drives the handler with an authenticated request.
|
||||
func postDiscovery(t *testing.T, stub *discoveryManagerStub, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
h := &handler{manager: stub}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/agent-network/catalog/providers/models", strings.NewReader(body))
|
||||
req = req.WithContext(nbcontext.SetUserAuthInContext(req.Context(), auth.UserAuth{
|
||||
AccountId: "acc-1",
|
||||
UserId: "user-1",
|
||||
}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.discoverProviderModels(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestDiscoverModelsReturnsTheVendorList(t *testing.T) {
|
||||
stub := &discoveryManagerStub{models: []modeldiscovery.Model{
|
||||
{ID: "eu.anthropic.claude-haiku-4-5-20251001-v1:0", Label: "EU Claude Haiku 4.5", PricingKnown: true},
|
||||
{ID: "global.cohere.embed-v4:0", Label: "Global Cohere Embed v4"},
|
||||
}}
|
||||
|
||||
rec := postDiscovery(t, stub, `{
|
||||
"catalog_provider_id":"bedrock_api",
|
||||
"upstream_url":"https://bedrock-runtime.eu-central-1.amazonaws.com",
|
||||
"api_key":"aws-bearer"
|
||||
}`)
|
||||
require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
|
||||
|
||||
var out api.AgentNetworkModelDiscoveryResponse
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
|
||||
require.Len(t, out.Models, 2)
|
||||
|
||||
assert.Equal(t, "eu.anthropic.claude-haiku-4-5-20251001-v1:0", out.Models[0].Id)
|
||||
assert.True(t, out.Models[0].PricingKnown)
|
||||
// An unpriced model must say so rather than arriving indistinguishable
|
||||
// from a priced one: registering it silently would meter at zero.
|
||||
assert.False(t, out.Models[1].PricingKnown)
|
||||
|
||||
assert.Equal(t, "bedrock_api", stub.gotReq.CatalogID)
|
||||
assert.Equal(t, "aws-bearer", stub.gotReq.APIKey)
|
||||
assert.Empty(t, stub.gotRecordID)
|
||||
}
|
||||
|
||||
func TestDiscoverModelsUsesAStoredRecordWithoutAKey(t *testing.T) {
|
||||
stub := &discoveryManagerStub{}
|
||||
|
||||
rec := postDiscovery(t, stub, `{"catalog_provider_id":"openai_api","provider_id":"prov-42"}`)
|
||||
require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String())
|
||||
|
||||
// The dashboard refreshes a saved provider's list without ever holding
|
||||
// the credential, so the record id has to reach the manager.
|
||||
assert.Equal(t, "prov-42", stub.gotRecordID)
|
||||
assert.Empty(t, stub.gotReq.APIKey)
|
||||
}
|
||||
|
||||
// TestDiscoverModelsRefusesMixedCredentials covers the case where a caller
|
||||
// names a saved provider AND supplies a key. Accepting it would run an
|
||||
// arbitrary credential under the identity of a record the caller may only be
|
||||
// permitted to read.
|
||||
func TestDiscoverModelsRefusesMixedCredentials(t *testing.T) {
|
||||
stub := &discoveryManagerStub{}
|
||||
|
||||
rec := postDiscovery(t, stub, `{
|
||||
"catalog_provider_id":"openai_api",
|
||||
"provider_id":"prov-42",
|
||||
"api_key":"sk-attacker"
|
||||
}`)
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
assert.Empty(t, stub.gotRecordID, "the request must be refused before it reaches the manager")
|
||||
}
|
||||
|
||||
// TestDiscoverModelsReportsNoDiscoveryDistinctly matters because the caller
|
||||
// falls back to the catalog's own model list on this outcome. Collapsing it
|
||||
// into a generic 500 would turn "this provider has no listing endpoint" into
|
||||
// "something went wrong", and the form would show an error instead of a list.
|
||||
func TestDiscoverModelsReportsNoDiscoveryDistinctly(t *testing.T) {
|
||||
stub := &discoveryManagerStub{err: modeldiscovery.ErrNoDiscovery}
|
||||
|
||||
rec := postDiscovery(t, stub, `{"catalog_provider_id":"litellm_proxy","upstream_url":"https://gw.example.com","api_key":"sk"}`)
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, rec.Code)
|
||||
}
|
||||
|
||||
func TestDiscoverModelsRejectsMalformedRequests(t *testing.T) {
|
||||
for name, body := range map[string]string{
|
||||
"not json": `{`,
|
||||
"no catalog provider": `{"api_key":"sk"}`,
|
||||
"blank catalog provide": `{"catalog_provider_id":" ","api_key":"sk"}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
stub := &discoveryManagerStub{}
|
||||
rec := postDiscovery(t, stub, body)
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/modeldiscovery"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/pricing"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
nbcontext "github.com/netbirdio/netbird/management/server/context"
|
||||
@@ -32,6 +34,7 @@ type handler struct {
|
||||
func RegisterEndpoints(manager agentnetwork.Manager, router *mux.Router) {
|
||||
h := &handler{manager: manager}
|
||||
router.HandleFunc("/agent-network/catalog/providers", h.getCatalogProviders).Methods("GET", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/catalog/providers/models", h.discoverProviderModels).Methods("POST", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/providers", h.getAllProviders).Methods("GET", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/providers", h.createProvider).Methods("POST", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/providers/{providerId}", h.getProvider).Methods("GET", "OPTIONS")
|
||||
@@ -61,6 +64,73 @@ func (h *handler) getCatalogProviders(w http.ResponseWriter, r *http.Request) {
|
||||
util.WriteJSONObject(r.Context(), w, out)
|
||||
}
|
||||
|
||||
// discoverProviderModels asks the vendor which models the operator's own
|
||||
// credential can reach, so the provider form can offer a live list rather than
|
||||
// only the static catalog.
|
||||
func (h *handler) discoverProviderModels(w http.ResponseWriter, r *http.Request) {
|
||||
userAuth, err := nbcontext.GetUserAuthFromContext(r.Context())
|
||||
if err != nil {
|
||||
util.WriteError(r.Context(), err, w)
|
||||
return
|
||||
}
|
||||
|
||||
var body api.AgentNetworkModelDiscoveryRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
util.WriteErrorResponse("invalid json", http.StatusBadRequest, w)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(body.CatalogProviderId) == "" {
|
||||
util.WriteErrorResponse("catalog_provider_id is required", http.StatusBadRequest, w)
|
||||
return
|
||||
}
|
||||
|
||||
recordID := strValue(body.ProviderId)
|
||||
req := modeldiscovery.Request{
|
||||
CatalogID: body.CatalogProviderId,
|
||||
UpstreamURL: strValue(body.UpstreamUrl),
|
||||
APIKey: strValue(body.ApiKey),
|
||||
}
|
||||
// One source of credential or the other, never a mix: taking a key from
|
||||
// the request while addressing a saved record would let a caller run an
|
||||
// arbitrary credential against a provider they can only read.
|
||||
if recordID != "" && req.APIKey != "" {
|
||||
util.WriteErrorResponse("provide either provider_id or api_key, not both", http.StatusBadRequest, w)
|
||||
return
|
||||
}
|
||||
|
||||
models, err := h.manager.DiscoverProviderModels(r.Context(), userAuth.AccountId, userAuth.UserId, req, recordID)
|
||||
if err != nil {
|
||||
// A provider with no listing endpoint is a fact about the catalog
|
||||
// entry, not a failure: the caller falls back to the catalog's own
|
||||
// models, so it must be able to tell the two apart.
|
||||
if errors.Is(err, modeldiscovery.ErrNoDiscovery) {
|
||||
util.WriteErrorResponse(err.Error(), http.StatusUnprocessableEntity, w)
|
||||
return
|
||||
}
|
||||
util.WriteError(r.Context(), err, w)
|
||||
return
|
||||
}
|
||||
|
||||
out := api.AgentNetworkModelDiscoveryResponse{Models: make([]api.AgentNetworkDiscoveredModel, 0, len(models))}
|
||||
for _, m := range models {
|
||||
entry := api.AgentNetworkDiscoveredModel{Id: m.ID, PricingKnown: m.PricingKnown}
|
||||
if m.Label != "" {
|
||||
label := m.Label
|
||||
entry.Label = &label
|
||||
}
|
||||
out.Models = append(out.Models, entry)
|
||||
}
|
||||
util.WriteJSONObject(r.Context(), w, out)
|
||||
}
|
||||
|
||||
// strValue reads an optional string field, treating absent as empty.
|
||||
func strValue(v *string) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(*v)
|
||||
}
|
||||
|
||||
// applyDefaultPricing overwrites the catalog response's model rates with
|
||||
// the LIVE default pricing table, which may differ from the compiled-in
|
||||
// catalog rates when the operator provides a defaults_llm_pricing.yaml.
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/labelgen"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/modeldiscovery"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/sessionkey"
|
||||
@@ -50,6 +51,7 @@ type Manager interface {
|
||||
CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error)
|
||||
UpdateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error)
|
||||
DeleteProvider(ctx context.Context, accountID, userID, providerID string) error
|
||||
DiscoverProviderModels(ctx context.Context, accountID, userID string, req modeldiscovery.Request, recordID string) ([]modeldiscovery.Model, error)
|
||||
|
||||
GetAllPolicies(ctx context.Context, accountID, userID string) ([]*types.Policy, error)
|
||||
GetPolicy(ctx context.Context, accountID, userID, policyID string) (*types.Policy, error)
|
||||
@@ -123,6 +125,11 @@ type managerImpl struct {
|
||||
permissionsManager permissions.Manager
|
||||
proxyController proxy.Controller
|
||||
|
||||
// modelDiscovery queries vendors for the models a credential can reach.
|
||||
// A field rather than a package call so tests can drive it without
|
||||
// reaching the network.
|
||||
modelDiscovery *modeldiscovery.Client
|
||||
|
||||
// reconcileCache holds the last set of synthesised proxy mappings
|
||||
// per account, each paired with the proxy that served it, so a change
|
||||
// of serving proxy can be diffed without re-deriving it.
|
||||
@@ -151,6 +158,7 @@ func NewManager(
|
||||
accountManager: accountManager,
|
||||
permissionsManager: permissionsManager,
|
||||
proxyController: proxyController,
|
||||
modelDiscovery: &modeldiscovery.Client{},
|
||||
reconcileCache: make(map[string]map[string]syntheticMapping),
|
||||
labelRng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||
}
|
||||
@@ -170,6 +178,37 @@ func (m *managerImpl) GetProvider(ctx context.Context, accountID, userID, provid
|
||||
return m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID)
|
||||
}
|
||||
|
||||
// DiscoverProviderModels asks the vendor which models a credential can reach.
|
||||
//
|
||||
// recordID, when set, names an existing provider whose stored credential and
|
||||
// upstream are used instead of the ones in req — so the dashboard can refresh
|
||||
// the list without ever holding the key. Reading a stored credential is a read
|
||||
// of that provider, and is permission-checked as one.
|
||||
//
|
||||
// Gated on Create rather than Read: this spends the operator's credential
|
||||
// against a third party, which is not something a read-only role should be
|
||||
// able to make the server do.
|
||||
func (m *managerImpl) DiscoverProviderModels(ctx context.Context, accountID, userID string, req modeldiscovery.Request, recordID string) ([]modeldiscovery.Model, error) {
|
||||
if err := m.requirePermission(ctx, accountID, userID, modules.AgentNetworkProviders, operations.Create); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if recordID != "" {
|
||||
record, err := m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, recordID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The catalog id comes from the stored record too: letting the caller
|
||||
// name a different one would run a provider's credential against
|
||||
// whichever vendor endpoint they picked.
|
||||
req.CatalogID = record.ProviderID
|
||||
req.UpstreamURL = record.UpstreamURL
|
||||
req.APIKey = record.APIKey
|
||||
}
|
||||
|
||||
return m.modelDiscovery.Fetch(ctx, req)
|
||||
}
|
||||
|
||||
// CreateProvider persists a new provider for the account. Providers have no
|
||||
// settings side effects: the account's endpoint is bootstrapped separately and
|
||||
// explicitly via CreateSettings, and every provider in the account routes
|
||||
@@ -1017,6 +1056,10 @@ func (*mockManager) GetAllProviders(_ context.Context, _, _ string) ([]*types.Pr
|
||||
return []*types.Provider{}, nil
|
||||
}
|
||||
|
||||
func (*mockManager) DiscoverProviderModels(_ context.Context, _, _ string, _ modeldiscovery.Request, _ string) ([]modeldiscovery.Model, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (*mockManager) GetProvider(_ context.Context, _, _, _ string) (*types.Provider, error) {
|
||||
return &types.Provider{}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
// Package modeldiscovery asks a vendor which models an operator's own
|
||||
// credential can reach, so the provider form can offer a live list instead of
|
||||
// only the catalog's hand-curated one.
|
||||
//
|
||||
// The catalog cannot know two things that matter. It goes stale — its entries
|
||||
// carry comments tracking which models a vendor retired on which date — and it
|
||||
// cannot see an account: which OpenAI models an org is entitled to, which
|
||||
// Bedrock inference profiles a given account and region hold, which Vertex
|
||||
// models a project has enabled. Those are exactly the facts an operator needs
|
||||
// when filling in a provider record, and only the vendor has them.
|
||||
//
|
||||
// The vendor is authoritative for the model ID. The catalog remains
|
||||
// authoritative for pricing, and a discovered model the catalog cannot price
|
||||
// is reported as such rather than silently registered at a rate of zero.
|
||||
package modeldiscovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/oauth2/google"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
)
|
||||
|
||||
const (
|
||||
// fetchTimeout bounds one vendor call end to end. A listing is a single
|
||||
// small GET; anything slower is a vendor problem and the operator is
|
||||
// waiting on a form.
|
||||
fetchTimeout = 8 * time.Second
|
||||
// maxListingBytes bounds the response we will buffer. The largest real
|
||||
// listing observed is Bedrock's foundation-model catalogue at ~70KB, so
|
||||
// this is a wide margin over anything legitimate.
|
||||
maxListingBytes = 2 << 20
|
||||
// gcpScope matches the scope llm_router mints Vertex tokens under, so a
|
||||
// credential that works for discovery works for inference too.
|
||||
gcpScope = "https://www.googleapis.com/auth/cloud-platform"
|
||||
// vertexKeyfilePrefix marks an api_key that is a base64 service-account
|
||||
// JSON key rather than a bearer token.
|
||||
vertexKeyfilePrefix = "keyfile::"
|
||||
)
|
||||
|
||||
// ErrNoDiscovery is returned for a catalog entry that declares no listing
|
||||
// endpoint. Gateways vary too much to have one, and the caller should fall
|
||||
// back to the catalog list plus free-text entry rather than treating this as
|
||||
// a failure.
|
||||
var ErrNoDiscovery = errors.New("provider has no model-discovery endpoint")
|
||||
|
||||
// Model is one discovered model.
|
||||
type Model struct {
|
||||
// ID is the identifier to register on the provider record, in the form the
|
||||
// vendor issues it. For Bedrock that is the region-prefixed inference
|
||||
// profile id, which is the only form AWS accepts at invoke time.
|
||||
ID string
|
||||
// Label is the vendor's display name where it supplies one.
|
||||
Label string
|
||||
// PricingKnown reports whether the shipped pricing table can price this
|
||||
// model. False means the operator must set rates, or the request would
|
||||
// meter at zero.
|
||||
PricingKnown bool
|
||||
}
|
||||
|
||||
// Request identifies which vendor to ask and with what credential.
|
||||
type Request struct {
|
||||
// CatalogID selects the catalog entry, which supplies the endpoint, the
|
||||
// auth header and the response shape. The caller never supplies those.
|
||||
CatalogID string
|
||||
// UpstreamURL is the provider record's configured upstream. It is used
|
||||
// only when the catalog entry declares no discovery host of its own.
|
||||
UpstreamURL string
|
||||
// Region substitutes the catalog host's <region> placeholder.
|
||||
Region string
|
||||
// APIKey is the operator's credential, exactly as stored on the record.
|
||||
APIKey string
|
||||
}
|
||||
|
||||
// Client fetches model listings. The zero value is usable; Resolver and
|
||||
// HTTPClient exist so tests can drive it against a local server.
|
||||
type Client struct {
|
||||
HTTPClient *http.Client
|
||||
// Resolver looks up the host for the SSRF check. Nil uses the default.
|
||||
Resolver *net.Resolver
|
||||
// AllowPrivateHosts disables the private-address guard. Only tests set it:
|
||||
// their server is on loopback, which is precisely what the guard blocks.
|
||||
AllowPrivateHosts bool
|
||||
}
|
||||
|
||||
// Fetch returns the models the credential can reach.
|
||||
func (c *Client) Fetch(ctx context.Context, req Request) ([]Model, error) {
|
||||
entry, ok := catalog.Lookup(req.CatalogID)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown catalog provider %q", req.CatalogID)
|
||||
}
|
||||
if entry.Discovery == nil {
|
||||
return nil, ErrNoDiscovery
|
||||
}
|
||||
|
||||
endpoint, err := c.discoveryURL(entry, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build discovery request: %w", err)
|
||||
}
|
||||
if err := applyAuth(httpReq, entry, req.APIKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for name, value := range entry.Discovery.Headers {
|
||||
httpReq.Header.Set(name, value)
|
||||
}
|
||||
httpReq.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := c.httpClient().Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reach %s: %w", entry.Name, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxListingBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s listing: %w", entry.Name, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
// Surface the vendor's own status. An operator whose key lacks a scope
|
||||
// needs to see 403 rather than a generic failure.
|
||||
return nil, fmt.Errorf("%s returned %d for its model listing", entry.Name, resp.StatusCode)
|
||||
}
|
||||
|
||||
ids, err := parseListing(entry.Discovery.Shape, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decorate(entry, ids), nil
|
||||
}
|
||||
|
||||
// discoveryURL builds the listing URL and refuses one that does not point at a
|
||||
// public host.
|
||||
//
|
||||
// The path, query and (for Bedrock) the host all come from the catalog rather
|
||||
// than from the caller, so the only operator-controlled part is the host of an
|
||||
// entry whose listing lives on its own upstream. That still has to be checked:
|
||||
// management holds credentials for every provider, and an upstream pointed at
|
||||
// an internal address would turn this endpoint into a probe of the management
|
||||
// server's own network.
|
||||
func (c *Client) discoveryURL(entry catalog.Provider, req Request) (string, error) {
|
||||
host := entry.Discovery.Host
|
||||
if host == "" {
|
||||
parsed, err := url.Parse(strings.TrimSpace(req.UpstreamURL))
|
||||
if err != nil || parsed.Host == "" {
|
||||
return "", fmt.Errorf("provider upstream %q is not a usable URL", req.UpstreamURL)
|
||||
}
|
||||
host = parsed.Host
|
||||
}
|
||||
if strings.Contains(host, catalog.RegionPlaceholder) {
|
||||
region := strings.TrimSpace(req.Region)
|
||||
if region == "" {
|
||||
// A provider record carries no region field: the region lives
|
||||
// inside the upstream host the operator already configured, so
|
||||
// read it back out rather than asking them for it twice.
|
||||
region = regionFromUpstream(entry, req.UpstreamURL)
|
||||
}
|
||||
if region == "" {
|
||||
return "", fmt.Errorf("%s discovery needs a region, and none could be read from the provider upstream", entry.Name)
|
||||
}
|
||||
host = strings.ReplaceAll(host, catalog.RegionPlaceholder, region)
|
||||
}
|
||||
|
||||
target := &url.URL{Scheme: "https", Host: host, Path: entry.Discovery.Path, RawQuery: entry.Discovery.Query}
|
||||
if err := c.checkPublicHost(target.Hostname()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return target.String(), nil
|
||||
}
|
||||
|
||||
// regionFromUpstream recovers the region an operator embedded in the provider
|
||||
// upstream, by matching it against the catalog's own host template. Bedrock's
|
||||
// template is "bedrock-runtime.<region>.amazonaws.com" and Vertex's is
|
||||
// "<region>-aiplatform.googleapis.com", so the region is whatever sits between
|
||||
// the fixed halves. Returns empty when the upstream does not match the
|
||||
// template, which is the case for a custom or proxied endpoint.
|
||||
func regionFromUpstream(entry catalog.Provider, upstreamURL string) string {
|
||||
prefix, suffix, found := strings.Cut(entry.DefaultHost, catalog.RegionPlaceholder)
|
||||
if !found {
|
||||
return ""
|
||||
}
|
||||
parsed, err := url.Parse(strings.TrimSpace(upstreamURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
host := parsed.Hostname()
|
||||
if host == "" {
|
||||
// A bare host with no scheme parses as a path, not a host.
|
||||
host = strings.TrimSpace(upstreamURL)
|
||||
}
|
||||
if !strings.HasPrefix(host, prefix) || !strings.HasSuffix(host, suffix) {
|
||||
return ""
|
||||
}
|
||||
region := host[len(prefix) : len(host)-len(suffix)]
|
||||
if region == "" || strings.Contains(region, ".") {
|
||||
return ""
|
||||
}
|
||||
return region
|
||||
}
|
||||
|
||||
// checkPublicHost refuses hosts that resolve to an address the management
|
||||
// server should never be asked to reach on an operator's behalf.
|
||||
func (c *Client) checkPublicHost(host string) error {
|
||||
if c.AllowPrivateHosts {
|
||||
return nil
|
||||
}
|
||||
if host == "" {
|
||||
return errors.New("discovery host is empty")
|
||||
}
|
||||
resolver := c.Resolver
|
||||
if resolver == nil {
|
||||
resolver = net.DefaultResolver
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
addrs, err := resolver.LookupNetIP(ctx, "ip", host)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve discovery host %q: %w", host, err)
|
||||
}
|
||||
// Every address must be public: a name that resolves to one public and one
|
||||
// loopback address is still a way to reach loopback.
|
||||
for _, addr := range addrs {
|
||||
if !isPublic(addr) {
|
||||
return fmt.Errorf("discovery host %q resolves to a non-public address", host)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPublic reports whether an address is one we are willing to dial.
|
||||
func isPublic(addr netip.Addr) bool {
|
||||
addr = addr.Unmap()
|
||||
switch {
|
||||
case !addr.IsValid(),
|
||||
addr.IsLoopback(),
|
||||
addr.IsPrivate(),
|
||||
addr.IsLinkLocalUnicast(),
|
||||
addr.IsLinkLocalMulticast(),
|
||||
addr.IsInterfaceLocalMulticast(),
|
||||
addr.IsMulticast(),
|
||||
addr.IsUnspecified():
|
||||
return false
|
||||
}
|
||||
// 100.64.0.0/10 (carrier NAT) is where NetBird's own overlay addresses
|
||||
// live, so it is emphatically not somewhere to send a provider credential.
|
||||
if addr.Is4() {
|
||||
b := addr.As4()
|
||||
if b[0] == 100 && b[1] >= 64 && b[1] <= 127 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// applyAuth sets the credential header the catalog entry declares. A Vertex
|
||||
// service-account key is exchanged for an OAuth token first, the same way the
|
||||
// proxy does at request time.
|
||||
func applyAuth(req *http.Request, entry catalog.Provider, apiKey string) error {
|
||||
key := strings.TrimSpace(apiKey)
|
||||
if key == "" {
|
||||
return fmt.Errorf("%s discovery needs an API key", entry.Name)
|
||||
}
|
||||
if rest, ok := strings.CutPrefix(key, vertexKeyfilePrefix); ok {
|
||||
token, err := mintGCPToken(req.Context(), rest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key = token
|
||||
}
|
||||
name := entry.AuthHeaderName
|
||||
if name == "" {
|
||||
name = "Authorization"
|
||||
}
|
||||
template := entry.AuthHeaderTemplate
|
||||
if template == "" {
|
||||
template = "${API_KEY}"
|
||||
}
|
||||
req.Header.Set(name, strings.ReplaceAll(template, "${API_KEY}", key))
|
||||
return nil
|
||||
}
|
||||
|
||||
// mintGCPToken exchanges a base64 service-account key for an access token.
|
||||
func mintGCPToken(ctx context.Context, saKeyB64 string) (string, error) {
|
||||
jsonKey, err := base64.StdEncoding.DecodeString(strings.TrimSpace(saKeyB64))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decode service-account key: %w", err)
|
||||
}
|
||||
conf, err := google.JWTConfigFromJSON(jsonKey, gcpScope)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse service-account key: %w", err)
|
||||
}
|
||||
tok, err := conf.TokenSource(ctx).Token()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("mint gcp token: %w", err)
|
||||
}
|
||||
return tok.AccessToken, nil
|
||||
}
|
||||
|
||||
// decorate turns raw vendor ids into the models the caller renders, marking
|
||||
// each with whether the shipped pricing table can price it.
|
||||
func decorate(entry catalog.Provider, ids []listedModel) []Model {
|
||||
priced := make(map[string]struct{}, len(entry.Models))
|
||||
for _, m := range entry.Models {
|
||||
priced[m.ID] = struct{}{}
|
||||
}
|
||||
|
||||
out := make([]Model, 0, len(ids))
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, listed := range ids {
|
||||
if listed.id == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[listed.id]; dup {
|
||||
continue
|
||||
}
|
||||
seen[listed.id] = struct{}{}
|
||||
|
||||
// The catalog keys pricing by the normalised id while the vendor
|
||||
// issues the wire form, so normalise before asking whether we can
|
||||
// price it — otherwise every Bedrock profile would report unpriced.
|
||||
_, known := priced[normalizeForPricing(entry.ID, listed.id)]
|
||||
out = append(out, Model{ID: listed.id, Label: listed.label, PricingKnown: known})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Client) httpClient() *http.Client {
|
||||
if c.HTTPClient != nil {
|
||||
return c.HTTPClient
|
||||
}
|
||||
return &http.Client{
|
||||
Timeout: fetchTimeout,
|
||||
// A redirect is a way to move the request to a host the guard above
|
||||
// never checked, so none are followed.
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
package modeldiscovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
)
|
||||
|
||||
// stubTransport answers every request with one canned response and records the
|
||||
// request it was given, so a test can assert on the URL and headers the client
|
||||
// built without a network round trip.
|
||||
type stubTransport struct {
|
||||
status int
|
||||
body string
|
||||
got *http.Request
|
||||
}
|
||||
|
||||
func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
s.got = req
|
||||
status := s.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Body: io.NopCloser(strings.NewReader(s.body)),
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newStubClient returns a client that never leaves the process. The host guard
|
||||
// is disabled because it would otherwise resolve the vendor's real name, which
|
||||
// would make these tests depend on DNS.
|
||||
func newStubClient(status int, body string) (*Client, *stubTransport) {
|
||||
tr := &stubTransport{status: status, body: body}
|
||||
return &Client{
|
||||
HTTPClient: &http.Client{Transport: tr},
|
||||
AllowPrivateHosts: true,
|
||||
}, tr
|
||||
}
|
||||
|
||||
// The payloads below are trimmed from what the vendors actually returned in
|
||||
// the discovery e2e, rather than invented, so a parser that only works against
|
||||
// an idealised shape fails here.
|
||||
|
||||
const openAIListing = `{"object":"list","data":[
|
||||
{"id":"gpt-4o-mini","object":"model","created":1721172741,"owned_by":"system"},
|
||||
{"id":"gpt-4o","object":"model","created":1715367049,"owned_by":"system"}
|
||||
]}`
|
||||
|
||||
const anthropicListing = `{"data":[
|
||||
{"type":"model","id":"claude-haiku-4-5-20251001","display_name":"Claude Haiku 4.5"},
|
||||
{"type":"model","id":"claude-sonnet-4-6","display_name":"Claude Sonnet 4.6"}
|
||||
],"has_more":false}`
|
||||
|
||||
const bedrockListing = `{"inferenceProfileSummaries":[
|
||||
{"inferenceProfileId":"eu.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"inferenceProfileName":"EU Anthropic Claude Haiku 4.5","status":"ACTIVE","type":"SYSTEM_DEFINED"},
|
||||
{"inferenceProfileId":"global.cohere.embed-v4:0",
|
||||
"inferenceProfileName":"Global Cohere Embed v4","status":"ACTIVE","type":"SYSTEM_DEFINED"},
|
||||
{"inferenceProfileId":"eu.meta.llama3-2-1b-instruct-v1:0",
|
||||
"inferenceProfileName":"EU Meta Llama 3.2 1B","status":"INACTIVE","type":"SYSTEM_DEFINED"}
|
||||
]}`
|
||||
|
||||
const vertexListing = `{"publisherModels":[
|
||||
{"name":"publishers/anthropic/models/claude-3-opus","versionId":"20240229","launchStage":"GA"},
|
||||
{"name":"publishers/anthropic/models/claude-sonnet-4-5","versionId":"20250929","launchStage":"GA"}
|
||||
]}`
|
||||
|
||||
func TestFetchOpenAIListing(t *testing.T) {
|
||||
cl, tr := newStubClient(http.StatusOK, openAIListing)
|
||||
|
||||
models, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "openai_api",
|
||||
UpstreamURL: "https://api.openai.com",
|
||||
APIKey: "sk-test",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "https://api.openai.com/v1/models", tr.got.URL.String())
|
||||
assert.Equal(t, "Bearer sk-test", tr.got.Header.Get("Authorization"),
|
||||
"the credential must be injected through the catalog's auth template")
|
||||
assert.Equal(t, []string{"gpt-4o-mini", "gpt-4o"}, ids(models))
|
||||
for _, m := range models {
|
||||
assert.True(t, m.PricingKnown, "both models are in the shipped catalog: %s", m.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAnthropicSendsTheVersionHeader(t *testing.T) {
|
||||
cl, tr := newStubClient(http.StatusOK, anthropicListing)
|
||||
|
||||
models, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "anthropic_api",
|
||||
UpstreamURL: "https://api.anthropic.com",
|
||||
APIKey: "sk-ant-test",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Anthropic rejects a request without the version header, so a listing
|
||||
// that reached us at all proves it was sent — but assert it, because the
|
||||
// failure mode otherwise only shows up against the live API.
|
||||
assert.Equal(t, "2023-06-01", tr.got.Header.Get("anthropic-version"))
|
||||
assert.Equal(t, "sk-ant-test", tr.got.Header.Get("x-api-key"),
|
||||
"Anthropic takes a bare key under its own header, not a Bearer token")
|
||||
assert.Equal(t, "limit=1000", tr.got.URL.RawQuery)
|
||||
|
||||
assert.Equal(t, []string{"claude-haiku-4-5-20251001", "claude-sonnet-4-6"}, ids(models))
|
||||
assert.Equal(t, "Claude Haiku 4.5", models[0].Label)
|
||||
}
|
||||
|
||||
func TestFetchBedrockUsesTheControlPlaneAndKeepsWireIDs(t *testing.T) {
|
||||
cl, tr := newStubClient(http.StatusOK, bedrockListing)
|
||||
|
||||
models, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "bedrock_api",
|
||||
// The record's upstream is the RUNTIME host, which does not serve
|
||||
// listings. The catalog's own discovery host must win over it.
|
||||
UpstreamURL: "https://bedrock-runtime.eu-central-1.amazonaws.com",
|
||||
Region: "eu-central-1",
|
||||
APIKey: "aws-bearer",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "https://bedrock.eu-central-1.amazonaws.com/inference-profiles",
|
||||
tr.got.URL.String(), "listings come from the control plane, not the runtime host")
|
||||
|
||||
// Region-prefixed ids verbatim: the prefix is what makes them invocable
|
||||
// and it cannot be reconstructed — global.* alongside eu.* is exactly the
|
||||
// case that defeats deriving it from the configured region.
|
||||
assert.Equal(t, []string{
|
||||
"eu.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"global.cohere.embed-v4:0",
|
||||
}, ids(models), "an INACTIVE profile must not be offered")
|
||||
|
||||
assert.True(t, models[0].PricingKnown,
|
||||
"the catalog prices anthropic.claude-haiku-4-5, which this id normalises to")
|
||||
assert.False(t, models[1].PricingKnown,
|
||||
"cohere embed is not in the shipped Bedrock catalog, so the operator must price it")
|
||||
}
|
||||
|
||||
func TestFetchVertexJoinsNameAndVersion(t *testing.T) {
|
||||
cl, _ := newStubClient(http.StatusOK, vertexListing)
|
||||
|
||||
models, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "vertex_ai_api",
|
||||
UpstreamURL: "https://us-east5-aiplatform.googleapis.com",
|
||||
Region: "us-east5",
|
||||
APIKey: "ya29.test-token",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Vertex addresses a model as "<id>@<version>" on rawPredict, and splits
|
||||
// those across two fields in the listing.
|
||||
assert.Equal(t, []string{"claude-3-opus@20240229", "claude-sonnet-4-5@20250929"}, ids(models))
|
||||
assert.Equal(t, "claude-3-opus", models[0].Label)
|
||||
}
|
||||
|
||||
func TestFetchSurfacesTheVendorStatus(t *testing.T) {
|
||||
cl, _ := newStubClient(http.StatusForbidden, `{"error":{"message":"no access"}}`)
|
||||
|
||||
_, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "openai_api",
|
||||
UpstreamURL: "https://api.openai.com",
|
||||
APIKey: "sk-test",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "403",
|
||||
"an operator whose key lacks access needs to see which status the vendor returned")
|
||||
}
|
||||
|
||||
func TestFetchRejectsAProviderWithoutDiscovery(t *testing.T) {
|
||||
cl, _ := newStubClient(http.StatusOK, openAIListing)
|
||||
|
||||
_, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "litellm_proxy",
|
||||
UpstreamURL: "https://gateway.example.com",
|
||||
APIKey: "sk-test",
|
||||
})
|
||||
assert.ErrorIs(t, err, ErrNoDiscovery,
|
||||
"a gateway with no listing endpoint must be distinguishable from a failure, so the caller can fall back")
|
||||
}
|
||||
|
||||
func TestFetchRequiresACredential(t *testing.T) {
|
||||
cl, _ := newStubClient(http.StatusOK, openAIListing)
|
||||
|
||||
_, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "openai_api",
|
||||
UpstreamURL: "https://api.openai.com",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "API key")
|
||||
}
|
||||
|
||||
func TestDiscoveryURLNeedsARegionWhenTheHostTemplatesOne(t *testing.T) {
|
||||
cl, _ := newStubClient(http.StatusOK, bedrockListing)
|
||||
|
||||
// An upstream that matches no catalog template — a proxy in front of
|
||||
// Bedrock, say — leaves nothing to read the region from. Refusing beats
|
||||
// guessing: an unsubstituted placeholder would dial a host that does not
|
||||
// exist, and a guessed region would dial the wrong account's endpoint.
|
||||
_, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "bedrock_api",
|
||||
UpstreamURL: "https://bedrock.internal-proxy.example.com",
|
||||
APIKey: "aws-bearer",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "region")
|
||||
}
|
||||
|
||||
// TestHostGuardRejectsNonPublicAddresses is the SSRF guard. Management holds a
|
||||
// credential for every provider, so an upstream pointed at an internal address
|
||||
// would turn discovery into a way to probe — and hand a token to — the
|
||||
// management server's own network.
|
||||
func TestHostGuardRejectsNonPublicAddresses(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
addr string
|
||||
want bool
|
||||
}{
|
||||
{"loopback v4", "127.0.0.1", false},
|
||||
{"loopback v6", "::1", false},
|
||||
{"private 10/8", "10.0.0.5", false},
|
||||
{"private 172.16/12", "172.16.4.1", false},
|
||||
{"private 192.168/16", "192.168.1.1", false},
|
||||
{"link-local", "169.254.169.254", false}, // cloud metadata
|
||||
{"unspecified", "0.0.0.0", false},
|
||||
{"multicast", "224.0.0.1", false},
|
||||
{"netbird overlay 100.64/10", "100.90.1.2", false},
|
||||
{"v4-mapped loopback", "::ffff:127.0.0.1", false},
|
||||
{"public v4", "1.1.1.1", true},
|
||||
{"public v6", "2606:4700:4700::1111", true},
|
||||
{"just outside CGNAT", "100.128.0.1", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
addr, err := netip.ParseAddr(tc.addr)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want, isPublic(addr))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostGuardResolvesAndRejectsLocalhost(t *testing.T) {
|
||||
cl := &Client{}
|
||||
err := cl.checkPublicHost("localhost")
|
||||
require.Error(t, err, "a name resolving to loopback must be refused, not just a literal address")
|
||||
assert.Contains(t, err.Error(), "non-public")
|
||||
}
|
||||
|
||||
// TestEveryDiscoveryEntryHasAParser keeps the catalog and the parser table from
|
||||
// drifting: adding a Discovery block with a shape nothing parses would fail
|
||||
// only at runtime, in front of an operator.
|
||||
func TestEveryDiscoveryEntryHasAParser(t *testing.T) {
|
||||
for _, entry := range catalog.All() {
|
||||
if entry.Discovery == nil {
|
||||
continue
|
||||
}
|
||||
t.Run(entry.ID, func(t *testing.T) {
|
||||
assert.NotEmpty(t, entry.Discovery.Path, "a discovery entry needs a path")
|
||||
_, err := parseListing(entry.Discovery.Shape, []byte(`{}`))
|
||||
assert.NoError(t, err, "shape %q has no parser", entry.Discovery.Shape)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func ids(models []Model) []string {
|
||||
out := make([]string, 0, len(models))
|
||||
for _, m := range models {
|
||||
out = append(out, m.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestRegionIsReadBackFromTheUpstream covers the reason the API takes no
|
||||
// region field: a provider record has none, and the operator already encoded
|
||||
// it in the upstream host when they configured inference.
|
||||
func TestRegionIsReadBackFromTheUpstream(t *testing.T) {
|
||||
cl, tr := newStubClient(http.StatusOK, bedrockListing)
|
||||
|
||||
_, err := cl.Fetch(context.Background(), Request{
|
||||
CatalogID: "bedrock_api",
|
||||
UpstreamURL: "https://bedrock-runtime.us-west-2.amazonaws.com",
|
||||
APIKey: "aws-bearer",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "bedrock.us-west-2.amazonaws.com", tr.got.URL.Host)
|
||||
}
|
||||
|
||||
func TestRegionFromUpstream(t *testing.T) {
|
||||
bedrock, ok := catalog.Lookup("bedrock_api")
|
||||
require.True(t, ok)
|
||||
vertex, ok := catalog.Lookup("vertex_ai_api")
|
||||
require.True(t, ok)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
entry catalog.Provider
|
||||
upstream string
|
||||
want string
|
||||
}{
|
||||
{"bedrock runtime host", bedrock, "https://bedrock-runtime.eu-central-1.amazonaws.com", "eu-central-1"},
|
||||
{"bedrock without scheme", bedrock, "bedrock-runtime.ap-south-1.amazonaws.com", "ap-south-1"},
|
||||
{"vertex regional host", vertex, "https://us-east5-aiplatform.googleapis.com", "us-east5"},
|
||||
// A proxied or self-hosted upstream matches no template, and guessing
|
||||
// a region from it would build a URL pointing somewhere arbitrary.
|
||||
{"unrelated upstream", bedrock, "https://llm.internal.example.com", ""},
|
||||
{"vertex global host has no region segment", vertex, "https://aiplatform.googleapis.com", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, regionFromUpstream(tc.entry, tc.upstream))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package modeldiscovery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
sharedllm "github.com/netbirdio/netbird/shared/llm"
|
||||
)
|
||||
|
||||
// listedModel is one entry lifted out of a vendor listing before the catalog
|
||||
// is consulted about it.
|
||||
type listedModel struct {
|
||||
id string
|
||||
label string
|
||||
}
|
||||
|
||||
// parseListing extracts model ids from a vendor listing. Each vendor invented
|
||||
// its own envelope, and the shape is declared by the catalog rather than
|
||||
// sniffed, so a vendor that changes shape fails loudly instead of silently
|
||||
// returning nothing.
|
||||
func parseListing(shape catalog.ListingShape, body []byte) ([]listedModel, error) {
|
||||
switch shape {
|
||||
case catalog.ShapeOpenAIData:
|
||||
return parseOpenAIData(body)
|
||||
case catalog.ShapeBedrockInferenceProfiles:
|
||||
return parseBedrockInferenceProfiles(body)
|
||||
case catalog.ShapeVertexPublisherModels:
|
||||
return parseVertexPublisherModels(body)
|
||||
default:
|
||||
return nil, fmt.Errorf("no parser for listing shape %q", shape)
|
||||
}
|
||||
}
|
||||
|
||||
// parseOpenAIData reads {"data":[{"id":…}]}, which OpenAI defined and
|
||||
// Anthropic adopted. Anthropic additionally supplies display_name.
|
||||
func parseOpenAIData(body []byte) ([]listedModel, error) {
|
||||
var doc struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
return nil, fmt.Errorf("decode model listing: %w", err)
|
||||
}
|
||||
out := make([]listedModel, 0, len(doc.Data))
|
||||
for _, entry := range doc.Data {
|
||||
out = append(out, listedModel{id: entry.ID, label: entry.DisplayName})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// parseBedrockInferenceProfiles reads
|
||||
// {"inferenceProfileSummaries":[{"inferenceProfileId":…}]}.
|
||||
//
|
||||
// The profile id is taken verbatim because its region prefix (eu., us.,
|
||||
// global.) is what makes it invocable, and it is not derivable from the
|
||||
// configured region — an account in one region legitimately holds global.*
|
||||
// profiles alongside its regional ones.
|
||||
//
|
||||
// Only ACTIVE profiles are offered: AWS reports others, and registering one
|
||||
// would produce a model that routes inside NetBird and fails at AWS.
|
||||
func parseBedrockInferenceProfiles(body []byte) ([]listedModel, error) {
|
||||
var doc struct {
|
||||
Summaries []struct {
|
||||
ID string `json:"inferenceProfileId"`
|
||||
Name string `json:"inferenceProfileName"`
|
||||
Status string `json:"status"`
|
||||
} `json:"inferenceProfileSummaries"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
return nil, fmt.Errorf("decode inference-profile listing: %w", err)
|
||||
}
|
||||
out := make([]listedModel, 0, len(doc.Summaries))
|
||||
for _, entry := range doc.Summaries {
|
||||
if entry.Status != "" && !strings.EqualFold(entry.Status, "ACTIVE") {
|
||||
continue
|
||||
}
|
||||
out = append(out, listedModel{id: entry.ID, label: entry.Name})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// parseVertexPublisherModels reads {"publisherModels":[{"name":…}]}, where
|
||||
// name is a resource path ("publishers/anthropic/models/claude-3-opus") and
|
||||
// the version lives in a separate field.
|
||||
//
|
||||
// Vertex addresses a model as "<id>@<version>" on the rawPredict path, so the
|
||||
// two are joined here: reporting the bare name would hand the operator an id
|
||||
// that looks usable and is not.
|
||||
func parseVertexPublisherModels(body []byte) ([]listedModel, error) {
|
||||
var doc struct {
|
||||
Models []struct {
|
||||
Name string `json:"name"`
|
||||
VersionID string `json:"versionId"`
|
||||
} `json:"publisherModels"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
return nil, fmt.Errorf("decode publisher-model listing: %w", err)
|
||||
}
|
||||
out := make([]listedModel, 0, len(doc.Models))
|
||||
for _, entry := range doc.Models {
|
||||
id := entry.Name
|
||||
if slash := strings.LastIndex(id, "/"); slash >= 0 {
|
||||
id = id[slash+1:]
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
label := id
|
||||
if entry.VersionID != "" {
|
||||
id += "@" + entry.VersionID
|
||||
}
|
||||
out = append(out, listedModel{id: id, label: label})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// normalizeForPricing maps a vendor's wire id onto the key the catalog prices
|
||||
// it under. It mirrors the synthesiser's normalizePricingModelID: the two must
|
||||
// agree, or a model reported here as priced would meter at the default rate
|
||||
// instead of the operator's.
|
||||
func normalizeForPricing(catalogProviderID, modelID string) string {
|
||||
switch {
|
||||
case catalog.IsBedrockPathStyle(catalogProviderID):
|
||||
return sharedllm.NormalizeBedrockModel(modelID)
|
||||
case catalog.IsVertexPathStyle(catalogProviderID):
|
||||
return sharedllm.NormalizeVertexModel(modelID)
|
||||
default:
|
||||
return modelID
|
||||
}
|
||||
}
|
||||
@@ -47,17 +47,11 @@ var supplementalDefaults = map[string]map[string]Entry{
|
||||
"gpt-5-nano": {InputPer1k: 0.00005, OutputPer1k: 0.0004, CachedInputPer1k: 0.000005},
|
||||
},
|
||||
"anthropic": {
|
||||
// claude-opus-5 is not yet in the catalog lineup but gateway /
|
||||
// grandfathered traffic uses it; priced so it isn't skipped.
|
||||
"claude-opus-5": {InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625},
|
||||
// "kimi-k3[1m]" is the 1M-context alias some Claude Code guides
|
||||
// configure against Moonshot's Anthropic-compatible endpoint;
|
||||
// priced identically to kimi-k3 so those requests aren't skipped.
|
||||
"kimi-k3[1m]": {InputPer1k: 0.003, OutputPer1k: 0.015, CacheReadPer1k: 0.0003},
|
||||
},
|
||||
"bedrock": {
|
||||
"anthropic.claude-opus-5": {InputPer1k: 0.005, OutputPer1k: 0.025, CacheReadPer1k: 0.0005, CacheCreationPer1k: 0.00625},
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -82,6 +82,11 @@ anthropic:
|
||||
output_per_1k: 0.015
|
||||
cache_read_per_1k: 0.0003
|
||||
cache_creation_per_1k: 0.00375
|
||||
claude-sonnet-5:
|
||||
input_per_1k: 0.003
|
||||
output_per_1k: 0.015
|
||||
cache_read_per_1k: 0.0003
|
||||
cache_creation_per_1k: 0.00375
|
||||
kimi-k3:
|
||||
input_per_1k: 0.003
|
||||
output_per_1k: 0.015
|
||||
@@ -145,6 +150,11 @@ bedrock:
|
||||
output_per_1k: 0.015
|
||||
cache_read_per_1k: 0.0003
|
||||
cache_creation_per_1k: 0.00375
|
||||
anthropic.claude-sonnet-5:
|
||||
input_per_1k: 0.003
|
||||
output_per_1k: 0.015
|
||||
cache_read_per_1k: 0.0003
|
||||
cache_creation_per_1k: 0.00375
|
||||
meta.llama3-3-70b-instruct:
|
||||
input_per_1k: 0.00072
|
||||
output_per_1k: 0.00072
|
||||
|
||||
@@ -116,11 +116,13 @@ func TestDefaultTable_PinnedRates(t *testing.T) {
|
||||
assert.InDelta(t, 0.010, fable.InputPer1k, 1e-9, "claude-fable-5 input")
|
||||
assert.InDelta(t, 0.0125, fable.CacheCreationPer1k, 1e-9, "claude-fable-5 cache creation")
|
||||
|
||||
// Supplementals present on their surfaces.
|
||||
// Every id below must stay priced whichever source provides it: the
|
||||
// catalog lineup for the current Claude 5 family, supplementalDefaults
|
||||
// for the ids the dashboard deliberately doesn't offer.
|
||||
for surface, ids := range map[string][]string{
|
||||
"openai": {"gpt-5", "gpt-5-mini", "gpt-5-nano"},
|
||||
"anthropic": {"claude-opus-5", "kimi-k3[1m]", "kimi-k3"},
|
||||
"bedrock": {"anthropic.claude-opus-5"},
|
||||
"anthropic": {"claude-opus-5", "claude-sonnet-5", "kimi-k3[1m]", "kimi-k3"},
|
||||
"bedrock": {"anthropic.claude-opus-5", "anthropic.claude-sonnet-5"},
|
||||
} {
|
||||
for _, id := range ids {
|
||||
_, ok := table[surface][id]
|
||||
|
||||
@@ -211,7 +211,19 @@ func SynthesizeServices(ctx context.Context, s store.Store, accountID string) ([
|
||||
|
||||
groupIndex := indexProviderGroups(enabledPolicies)
|
||||
|
||||
routerCfgJSON, err := buildRouterConfigJSON(enabledProviders, groupIndex)
|
||||
// The proxy guardrail is a per-provider fail-closed backstop; the
|
||||
// authoritative per-policy/group decision is management's
|
||||
// SelectPolicyForRequest. A provider lands in that map only when every
|
||||
// authorising policy restricts models.
|
||||
providerAllowlists := buildProviderAllowlists(enabledPolicies, guardrailsByID)
|
||||
|
||||
// Discovery gets the finer view: per policy rather than flattened per
|
||||
// provider, so a listing can be bounded to what the calling groups may
|
||||
// actually use instead of the union across everyone who reaches the
|
||||
// provider.
|
||||
modelPolicies := buildModelPolicies(enabledPolicies, guardrailsByID)
|
||||
|
||||
routerCfgJSON, err := buildRouterConfigJSON(enabledProviders, groupIndex, modelPolicies)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -228,11 +240,6 @@ func SynthesizeServices(ctx context.Context, s store.Store, accountID string) ([
|
||||
|
||||
mergedGuardrails := mergeGuardrails(enabledPolicies, guardrailsByID)
|
||||
applyAccountCollectionControls(&mergedGuardrails, settings)
|
||||
// The proxy guardrail is a per-provider fail-closed backstop; the
|
||||
// authoritative per-policy/group decision is management's
|
||||
// SelectPolicyForRequest. A provider lands in this map only when every
|
||||
// authorising policy restricts models.
|
||||
providerAllowlists := buildProviderAllowlists(enabledPolicies, guardrailsByID)
|
||||
guardrailJSON, err := marshalGuardrailConfig(providerAllowlists, mergedGuardrails.PromptCapture)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -351,6 +358,11 @@ type routerProviderRoute struct {
|
||||
AuthHeaderName string `json:"auth_header_name"`
|
||||
AuthHeaderValue string `json:"auth_header_value"`
|
||||
AllowedGroupIDs []string `json:"allowed_group_ids,omitempty"`
|
||||
// ModelPolicies is one entry per enabled policy authorising this provider,
|
||||
// carrying that policy's source groups and the models it permits. The
|
||||
// router bounds a model listing with it, so a provider two groups reach
|
||||
// under different allowlists offers each only its own.
|
||||
ModelPolicies []routerModelPolicy `json:"model_policies,omitempty"`
|
||||
// Vertex marks a Google Vertex AI provider, whose requests carry the
|
||||
// model in the URL path. The router selects it by path, bypassing the
|
||||
// model/vendor table.
|
||||
@@ -422,7 +434,7 @@ func indexProviderGroups(policies []*types.Policy) map[string][]string {
|
||||
// path-prefix tiebreak. Providers no enabled policy authorises
|
||||
// (orphans) are intentionally OMITTED so the router never observes a
|
||||
// route with an empty ACL.
|
||||
func buildRouterConfigJSON(providers []*types.Provider, groupIndex map[string][]string) ([]byte, error) {
|
||||
func buildRouterConfigJSON(providers []*types.Provider, groupIndex map[string][]string, modelPolicies map[string][]routerModelPolicy) ([]byte, error) {
|
||||
cfg := routerConfig{Providers: make([]routerProviderRoute, 0, len(providers))}
|
||||
for _, p := range providers {
|
||||
groups, hasPolicy := groupIndex[p.ID]
|
||||
@@ -449,6 +461,7 @@ func buildRouterConfigJSON(providers []*types.Provider, groupIndex map[string][]
|
||||
AuthHeaderName: headerName,
|
||||
AuthHeaderValue: headerValue,
|
||||
AllowedGroupIDs: groups,
|
||||
ModelPolicies: modelPolicies[p.ID],
|
||||
Vertex: catalog.IsVertexPathStyle(p.ProviderID),
|
||||
Bedrock: catalog.IsBedrockPathStyle(p.ProviderID),
|
||||
GCPServiceAccountKeyB64: gcpSAKeyB64,
|
||||
@@ -1098,3 +1111,46 @@ func mergeGuardrail(g *types.Guardrail, merged *MergedGuardrails) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// routerModelPolicy mirrors the router's ModelPolicyRule: one authorising
|
||||
// policy's source groups plus the models it permits. Models is nil for a
|
||||
// policy that sets no model allowlist, which lifts the restriction for the
|
||||
// groups it binds — so nil and empty must survive the round trip distinctly.
|
||||
type routerModelPolicy struct {
|
||||
GroupIDs []string `json:"group_ids"`
|
||||
Models []string `json:"models"`
|
||||
}
|
||||
|
||||
// buildModelPolicies indexes, per provider, one rule for each enabled policy
|
||||
// authorising it: the policy's source groups and the models its guardrail
|
||||
// permits.
|
||||
//
|
||||
// This is deliberately finer than buildProviderAllowlists, which flattens the
|
||||
// same inputs into one list per provider for the proxy's fail-closed guardrail.
|
||||
// A flattened list cannot answer "what may THIS caller see", so a provider two
|
||||
// teams reach under different allowlists would offer each team the other's
|
||||
// models — a picker full of entries the next request refuses. Keeping the
|
||||
// source groups alongside the models lets the router answer it at request time,
|
||||
// where it knows the caller's groups.
|
||||
func buildModelPolicies(policies []*types.Policy, byID map[string]*types.Guardrail) map[string][]routerModelPolicy {
|
||||
out := make(map[string][]routerModelPolicy)
|
||||
for _, p := range policies {
|
||||
if p == nil || len(p.SourceGroups) == 0 {
|
||||
continue
|
||||
}
|
||||
restricted, models := policyModelAllowlist(p, byID)
|
||||
rule := routerModelPolicy{GroupIDs: append([]string(nil), p.SourceGroups...)}
|
||||
if restricted {
|
||||
// Never nil when restricted: an allowlist permitting nothing must
|
||||
// stay distinguishable from no allowlist at all.
|
||||
rule.Models = append([]string{}, models...)
|
||||
}
|
||||
for _, providerID := range p.DestinationProviderIDs {
|
||||
if providerID == "" {
|
||||
continue
|
||||
}
|
||||
out[providerID] = append(out[providerID], rule)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
)
|
||||
@@ -93,3 +94,75 @@ func TestBuildProviderAllowlists(t *testing.T) {
|
||||
"an enabled-but-empty allowlist is restricted with an empty set, not unrestricted")
|
||||
})
|
||||
}
|
||||
|
||||
// policyForGroups builds an enabled policy binding the given source groups to
|
||||
// the given providers under an optional guardrail.
|
||||
func policyForGroups(id string, groups []string, guardrailIDs []string, providerIDs ...string) *types.Policy {
|
||||
return &types.Policy{
|
||||
ID: id,
|
||||
Enabled: true,
|
||||
SourceGroups: groups,
|
||||
DestinationProviderIDs: providerIDs,
|
||||
GuardrailIDs: guardrailIDs,
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildModelPolicies covers the finer index discovery needs. Where
|
||||
// buildProviderAllowlists flattens every authorising policy into one list per
|
||||
// provider — enough for a fail-closed backstop, but blind to who is asking —
|
||||
// this keeps each policy's source groups beside its models so the router can
|
||||
// bound a listing to the calling groups.
|
||||
func TestBuildModelPolicies(t *testing.T) {
|
||||
byID := map[string]*types.Guardrail{
|
||||
"g-4o": allowlistGuardrail("g-4o", "acc-1", "gpt-4o"),
|
||||
"g-opus": allowlistGuardrail("g-opus", "acc-1", "claude-opus-4"),
|
||||
"g-disabled": {ID: "g-disabled", Checks: types.GuardrailChecks{ModelAllowlist: types.GuardrailModelAllowlist{Enabled: false, Models: []string{"gpt-4o"}}}},
|
||||
}
|
||||
|
||||
t.Run("each policy keeps its own groups and models", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForGroups("p1", []string{"grp-eng"}, []string{"g-4o"}, "prov-x"),
|
||||
policyForGroups("p2", []string{"grp-sales"}, []string{"g-opus"}, "prov-x"),
|
||||
}
|
||||
got := buildModelPolicies(policies, byID)
|
||||
assert.Equal(t, []routerModelPolicy{
|
||||
{GroupIDs: []string{"grp-eng"}, Models: []string{"gpt-4o"}},
|
||||
{GroupIDs: []string{"grp-sales"}, Models: []string{"claude-opus-4"}},
|
||||
}, got["prov-x"],
|
||||
"the two policies must stay separable so neither group is offered the other's models")
|
||||
})
|
||||
|
||||
t.Run("an unrestricted policy carries nil models", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForGroups("p1", []string{"grp-eng"}, []string{"g-4o"}, "prov-x"),
|
||||
policyForGroups("p2", []string{"grp-admin"}, nil, "prov-x"),
|
||||
}
|
||||
got := buildModelPolicies(policies, byID)
|
||||
assert.Nil(t, got["prov-x"][1].Models,
|
||||
"no allowlist must reach the router as nil, which lifts the restriction for its groups")
|
||||
})
|
||||
|
||||
t.Run("a disabled allowlist is not a restriction", func(t *testing.T) {
|
||||
policies := []*types.Policy{policyForGroups("p1", []string{"grp-eng"}, []string{"g-disabled"}, "prov-x")}
|
||||
got := buildModelPolicies(policies, byID)
|
||||
assert.Nil(t, got["prov-x"][0].Models,
|
||||
"a guardrail with the allowlist check off restricts nothing")
|
||||
})
|
||||
|
||||
t.Run("an enabled allowlist with no models permits nothing", func(t *testing.T) {
|
||||
byIDEmpty := map[string]*types.Guardrail{
|
||||
"g-empty": {ID: "g-empty", Checks: types.GuardrailChecks{ModelAllowlist: types.GuardrailModelAllowlist{Enabled: true}}},
|
||||
}
|
||||
policies := []*types.Policy{policyForGroups("p1", []string{"grp-eng"}, []string{"g-empty"}, "prov-x")}
|
||||
got := buildModelPolicies(policies, byIDEmpty)
|
||||
require.NotNil(t, got["prov-x"][0].Models,
|
||||
"an empty allowlist must not arrive as nil — that would read as unrestricted")
|
||||
assert.Empty(t, got["prov-x"][0].Models)
|
||||
})
|
||||
|
||||
t.Run("a policy binding no groups is skipped", func(t *testing.T) {
|
||||
policies := []*types.Policy{policyForGroups("p1", nil, []string{"g-4o"}, "prov-x")}
|
||||
assert.Empty(t, buildModelPolicies(policies, byID),
|
||||
"a policy with no source groups authorises nobody, so it bounds nobody's listing")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,6 +13,14 @@ func NormalizeBedrockModel(modelID string) string {
|
||||
return sharedllm.NormalizeBedrockModel(modelID)
|
||||
}
|
||||
|
||||
// NormalizeAnthropicModel strips the trailing "-YYYYMMDD" release-date suffix
|
||||
// from an Anthropic model id so a dated id a client pins matches the undated
|
||||
// one the operator registered. Thin delegate to shared/llm for the same
|
||||
// contract reason as the two below.
|
||||
func NormalizeAnthropicModel(modelID string) string {
|
||||
return sharedllm.NormalizeAnthropicModel(modelID)
|
||||
}
|
||||
|
||||
// NormalizeVertexModel strips the "@version" suffix from a Vertex AI model id
|
||||
// so it matches the catalog/pricing key. Thin delegate to shared/llm, kept
|
||||
// beside NormalizeBedrockModel for the same contract reason.
|
||||
|
||||
@@ -10,6 +10,8 @@ package pricing
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
sharedllm "github.com/netbirdio/netbird/shared/llm"
|
||||
)
|
||||
|
||||
// Entry is a single model's input and output pricing, expressed in USD per
|
||||
@@ -92,7 +94,10 @@ func NewTable(raw map[string]map[string]EntryJSON) (*Table, error) {
|
||||
return &Table{entries: entries}, nil
|
||||
}
|
||||
|
||||
// Lookup returns the entry for the given provider surface and model.
|
||||
// Lookup returns the entry for the given provider surface and model. A
|
||||
// dated Anthropic id falls back to its undated form, so a client pinning
|
||||
// "claude-sonnet-4-5-20250929" bills at the registered "claude-sonnet-4-5"
|
||||
// rate instead of recording no cost at all.
|
||||
func (t *Table) Lookup(provider, model string) (Entry, bool) {
|
||||
if t == nil {
|
||||
return Entry{}, false
|
||||
@@ -101,7 +106,14 @@ func (t *Table) Lookup(provider, model string) (Entry, bool) {
|
||||
if !ok {
|
||||
return Entry{}, false
|
||||
}
|
||||
e, ok := byModel[model]
|
||||
if e, found := byModel[model]; found {
|
||||
return e, true
|
||||
}
|
||||
undated := sharedllm.NormalizeAnthropicModel(model)
|
||||
if undated == model {
|
||||
return Entry{}, false
|
||||
}
|
||||
e, ok := byModel[undated]
|
||||
return e, ok
|
||||
}
|
||||
|
||||
|
||||
@@ -175,3 +175,22 @@ func TestNewTable_NilAndEmpty(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries, "nil in, empty (never-matching) map out for the per-record map")
|
||||
}
|
||||
|
||||
// TestLookup_DatedAnthropicIDFallsBackToUndated covers a client pinning a
|
||||
// release date on a model priced under its undated id. Without the
|
||||
// fallback the request records no cost at all.
|
||||
func TestLookup_DatedAnthropicIDFallsBackToUndated(t *testing.T) {
|
||||
table, err := NewTable(map[string]map[string]EntryJSON{
|
||||
"anthropic": {
|
||||
"claude-sonnet-4-5": {InputPer1K: 0.003, OutputPer1K: 0.015},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "table must build from a valid defaults map")
|
||||
|
||||
entry, ok := table.Lookup("anthropic", "claude-sonnet-4-5-20250929")
|
||||
require.True(t, ok, "a dated id must resolve to the undated entry")
|
||||
assert.InDelta(t, 0.003, entry.InputPer1K, 1e-9, "dated id must bill at the registered rate")
|
||||
|
||||
_, ok = table.Lookup("anthropic", "claude-sonnet-9-9-20250929")
|
||||
assert.False(t, ok, "an unknown family must stay unpriced")
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/llm"
|
||||
"github.com/netbirdio/netbird/proxy/internal/llm/pricing"
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware"
|
||||
)
|
||||
@@ -175,13 +176,28 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
|
||||
// Anthropic route still bills its cache buckets additively.
|
||||
func (m *Middleware) lookupCosts(md []middleware.KV, surface, model string, inTokens, outTokens, cachedTokens, cacheCreationTokens int64) (pricing.Costs, bool) {
|
||||
if recordID := lookupKV(md, middleware.KeyLLMResolvedProviderID); recordID != "" {
|
||||
if entry, ok := m.perRecord[recordID][model]; ok {
|
||||
if entry, ok := perRecordEntry(m.perRecord[recordID], model); ok {
|
||||
return pricing.EntryCosts(entry, surface, inTokens, outTokens, cachedTokens, cacheCreationTokens), true
|
||||
}
|
||||
}
|
||||
return m.defaults.Costs(surface, model, inTokens, outTokens, cachedTokens, cacheCreationTokens)
|
||||
}
|
||||
|
||||
// perRecordEntry resolves the operator's stored price for a model on one
|
||||
// provider record, falling back to the undated form of a dated Anthropic id
|
||||
// so a client that pins a release date still bills at the registered rate.
|
||||
func perRecordEntry(byModel map[string]pricing.Entry, model string) (pricing.Entry, bool) {
|
||||
if entry, ok := byModel[model]; ok {
|
||||
return entry, true
|
||||
}
|
||||
undated := llm.NormalizeAnthropicModel(model)
|
||||
if undated == model {
|
||||
return pricing.Entry{}, false
|
||||
}
|
||||
entry, ok := byModel[undated]
|
||||
return entry, ok
|
||||
}
|
||||
|
||||
// usd renders a cost as the fixed-precision string every cost.usd_* key
|
||||
// carries, so the per-bucket values and the aggregates round identically.
|
||||
//
|
||||
|
||||
@@ -84,8 +84,10 @@ func (m *Middleware) MutationsSupported() bool { return false }
|
||||
func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) {
|
||||
model, modelPresent := lookupMetadata(in.Metadata, middleware.KeyLLMModel)
|
||||
providerID, _ := lookupMetadata(in.Metadata, middleware.KeyLLMResolvedProviderID)
|
||||
surface, _ := lookupMetadata(in.Metadata, middleware.KeyLLMProvider)
|
||||
nonInference, _ := lookupMetadata(in.Metadata, middleware.KeyLLMNonInference)
|
||||
|
||||
if denial := m.evaluateAllowlist(providerID, model, modelPresent); denial != nil {
|
||||
if denial := m.evaluateAllowlist(providerID, surface, model, modelPresent, nonInference == "true"); denial != nil {
|
||||
return denial, nil
|
||||
}
|
||||
|
||||
@@ -114,7 +116,7 @@ func (m *Middleware) Close() error { return nil }
|
||||
// evaluateAllowlist denies when the resolved provider's allowlist rejects the
|
||||
// model; nil means proceed. Scoped to the provider llm_router resolved, so an
|
||||
// unrestricted provider (absent from config) is never caught by another's list.
|
||||
func (m *Middleware) evaluateAllowlist(providerID, model string, modelPresent bool) *middleware.Output {
|
||||
func (m *Middleware) evaluateAllowlist(providerID, surface, model string, modelPresent, nonInference bool) *middleware.Output {
|
||||
if len(m.cfg.ProviderAllowlists) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -122,7 +124,7 @@ func (m *Middleware) evaluateAllowlist(providerID, model string, modelPresent bo
|
||||
// if this request targets a restricted provider — fail closed. llm_router
|
||||
// normally stamps the provider first, so this is a defensive guard.
|
||||
if providerID == "" {
|
||||
return denyModel("", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
return denyModel(surface, "", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
}
|
||||
allowlist, restricted := m.cfg.ProviderAllowlists[providerID]
|
||||
if !restricted {
|
||||
@@ -133,18 +135,29 @@ func (m *Middleware) evaluateAllowlist(providerID, model string, modelPresent bo
|
||||
// Fail closed: with an allowlist in effect for this provider, a request whose
|
||||
// model the parser couldn't extract (absent/empty) is denied. This enforces
|
||||
// the allowlist for path-routed providers (Bedrock, Vertex) with no body model.
|
||||
//
|
||||
// The exception is a non-inference endpoint the router already authorised.
|
||||
// The model listing and the connection-warming probe name no model
|
||||
// anywhere — not in a body, not in the path — so failing closed here
|
||||
// rejected model discovery for exactly the accounts that configured an
|
||||
// allowlist, which is the outage this endpoint is meant to avoid. The
|
||||
// per-model lookup does name one (the router stamps it from the path), so
|
||||
// it still falls through to the allowlist check below.
|
||||
if !modelPresent || normaliseModel(model) == "" {
|
||||
return denyModel("", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
if nonInference {
|
||||
return nil
|
||||
}
|
||||
return denyModel(surface, "", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
}
|
||||
if modelInAllowlist(allowlist, model) {
|
||||
return nil
|
||||
}
|
||||
return denyModel(model, denyCodeModel, denyMessageModel, denyReasonModel)
|
||||
return denyModel(surface, model, denyCodeModel, denyMessageModel, denyReasonModel)
|
||||
}
|
||||
|
||||
// denyModel builds a 403 deny Output for a model-allowlist rejection. model is
|
||||
// included in the details only when non-empty.
|
||||
func denyModel(model, code, message, reason string) *middleware.Output {
|
||||
func denyModel(surface, model, code, message, reason string) *middleware.Output {
|
||||
details := map[string]string{}
|
||||
if model != "" {
|
||||
details["model"] = model
|
||||
@@ -156,6 +169,7 @@ func denyModel(model, code, message, reason string) *middleware.Output {
|
||||
Code: code,
|
||||
Message: message,
|
||||
Details: details,
|
||||
Surface: surface,
|
||||
},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMPolicyDecision, Value: "deny"},
|
||||
|
||||
@@ -343,3 +343,52 @@ func TestFactoryNormalisesAllowlist(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out2.Decision, "trimmed entry must still match")
|
||||
}
|
||||
|
||||
// TestAllowlistSkipsNonInferenceWithoutModel covers the reported regression:
|
||||
// GET /v1/models carries no model anywhere, so the fail-closed rule above
|
||||
// denied model discovery for exactly the accounts that configured a provider
|
||||
// allowlist — the clients that read a 403 here render an empty model picker.
|
||||
// The router authorises those endpoints by path before the guardrail sees
|
||||
// them, so an absent model there is expected rather than undeterminable.
|
||||
func TestAllowlistSkipsNonInferenceWithoutModel(t *testing.T) {
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision,
|
||||
"model discovery must not be refused because it names no model")
|
||||
}
|
||||
|
||||
// TestAllowlistStillAppliesToNonInferenceWithModel pins that the exemption is
|
||||
// scoped to requests that genuinely name nothing. The per-model lookup
|
||||
// (GET /v1/models/{id}) is non-inference too, but the router stamps the model
|
||||
// from its path, so the allowlist must still decide it — otherwise the
|
||||
// exemption becomes a way to confirm a model the policy blocks.
|
||||
func TestAllowlistStillAppliesToNonInferenceWithModel(t *testing.T) {
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
|
||||
t.Run("model in the allowlist", func(t *testing.T) {
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"},
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision,
|
||||
"an allowlisted model must stay reachable")
|
||||
})
|
||||
|
||||
t.Run("model outside the allowlist", func(t *testing.T) {
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMNonInference, Value: "true"},
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-5"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision,
|
||||
"non-inference must not become a way past the allowlist")
|
||||
require.NotNil(t, out.DenyReason)
|
||||
assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code,
|
||||
"a named but blocked model is blocked, not unknown")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -217,6 +217,32 @@ func applyHeaderPair(rule *HeaderPairRule, in *middleware.Input) *middleware.Mut
|
||||
return mutations
|
||||
}
|
||||
|
||||
// bodyInjectableSurfaces are the request-body dialects that accept the
|
||||
// OpenAI-standard identity fields this middleware writes. A surface
|
||||
// outside this set gets header-only stamping: "user" and "metadata.tags"
|
||||
// are not part of the Anthropic Messages schema, which rejects unknown
|
||||
// top-level fields and permits only "user_id" under metadata, so writing
|
||||
// them into an Anthropic-shaped body turns a working request into a 400.
|
||||
// Claude Code speaks that shape through gateway records pinned to the
|
||||
// OpenAI parser, so the check keys on the detected surface rather than
|
||||
// on the provider record.
|
||||
var bodyInjectableSurfaces = map[string]struct{}{
|
||||
"openai": {},
|
||||
// An empty surface means no parser claimed the path (a custom gateway
|
||||
// base). Those upstreams are OpenAI-compatible by convention, so keep
|
||||
// the long-standing behaviour rather than silently dropping identity.
|
||||
"": {},
|
||||
}
|
||||
|
||||
// bodyAcceptsOpenAIIdentity reports whether the request body may carry the
|
||||
// OpenAI-standard identity fields, read from the surface llm_request_parser
|
||||
// resolved from the request path.
|
||||
func bodyAcceptsOpenAIIdentity(in *middleware.Input) bool {
|
||||
surface, _ := lookupMetadata(in.Metadata, middleware.KeyLLMProvider)
|
||||
_, ok := bodyInjectableSurfaces[surface]
|
||||
return ok
|
||||
}
|
||||
|
||||
// injectIntoBody parses the request body and writes the supplied
|
||||
// identity dimensions into it. Tags land at metadata.tags (creating
|
||||
// the metadata object when absent); the user identity lands at the
|
||||
@@ -225,6 +251,8 @@ func applyHeaderPair(rule *HeaderPairRule, in *middleware.Input) *middleware.Mut
|
||||
// was written. Returns ok=false (no mutation) when:
|
||||
//
|
||||
// - both inputs are empty (nothing to write);
|
||||
// - the body speaks a dialect without these fields (see
|
||||
// bodyInjectableSurfaces);
|
||||
// - the body is empty or truncated (we don't have the full document
|
||||
// to safely round-trip);
|
||||
// - the body isn't a JSON object (skip silently — this middleware
|
||||
@@ -245,6 +273,9 @@ func injectIntoBody(in *middleware.Input, tags []string, userID string) ([]byte,
|
||||
if in == nil || len(in.Body) == 0 || in.BodyTruncated {
|
||||
return nil, false
|
||||
}
|
||||
if !bodyAcceptsOpenAIIdentity(in) {
|
||||
return nil, false
|
||||
}
|
||||
var doc map[string]any
|
||||
if err := json.Unmarshal(in.Body, &doc); err != nil {
|
||||
return nil, false
|
||||
|
||||
@@ -704,3 +704,57 @@ func TestInject_ExtraHeaders_EmptyValueSkipped(t *testing.T) {
|
||||
"empty extra value must not be stamped")
|
||||
}
|
||||
}
|
||||
|
||||
// TestInject_AnthropicBodyIsNotRewritten pins the shape gate. Claude Code
|
||||
// reaches a LiteLLM record on /v1/messages, where "user" is not a
|
||||
// permitted top-level field and metadata accepts only "user_id", so
|
||||
// writing the OpenAI-standard fields would turn a working request into a
|
||||
// 400 naming a field the client never sent. Header stamping still runs, so
|
||||
// spend tracking and per-end-user budgets keep working.
|
||||
func TestInject_AnthropicBodyIsNotRewritten(t *testing.T) {
|
||||
rule := liteLLMRuleWithBody()
|
||||
rule.HeaderPair.EndUserIDInBody = true
|
||||
mw := New(Config{Providers: []ProviderInjection{rule}})
|
||||
|
||||
in := newInput(litellmProvider, "alice", []string{"grp-eng"})
|
||||
in.UserEmail = "alice@example.com"
|
||||
in.Metadata = append(in.Metadata, middleware.KV{Key: middleware.KeyLLMProvider, Value: "anthropic"})
|
||||
in.Body = []byte(`{"model":"claude-sonnet-5","messages":[]}`)
|
||||
|
||||
out, err := mw.Invoke(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.Mutations)
|
||||
assert.Empty(t, out.Mutations.BodyReplace,
|
||||
"an Anthropic-shaped body must reach the upstream unmodified")
|
||||
|
||||
var endUser string
|
||||
for _, kv := range out.Mutations.HeadersAdd {
|
||||
if kv.Key == "x-litellm-end-user-id" {
|
||||
endUser = kv.Value
|
||||
}
|
||||
}
|
||||
assert.Equal(t, "alice@example.com", endUser,
|
||||
"header stamping must still carry identity when body inject is skipped")
|
||||
}
|
||||
|
||||
// TestInject_OpenAIBodyStillRewritten guards the gate against
|
||||
// over-reaching: the OpenAI surface must keep its body-level identity,
|
||||
// which is the only path LiteLLM's tag-budget check reads.
|
||||
func TestInject_OpenAIBodyStillRewritten(t *testing.T) {
|
||||
mw := New(Config{Providers: []ProviderInjection{liteLLMRuleWithBody()}})
|
||||
|
||||
in := newInput(litellmProvider, "alice", []string{"grp-eng"})
|
||||
in.Metadata = append(in.Metadata, middleware.KV{Key: middleware.KeyLLMProvider, Value: "openai"})
|
||||
in.Body = []byte(`{"model":"gpt-4o-mini","messages":[]}`)
|
||||
|
||||
out, err := mw.Invoke(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.Mutations)
|
||||
require.NotEmpty(t, out.Mutations.BodyReplace, "the OpenAI surface still gets body tags")
|
||||
|
||||
var doc map[string]any
|
||||
require.NoError(t, json.Unmarshal(out.Mutations.BodyReplace, &doc))
|
||||
meta, ok := doc["metadata"].(map[string]any)
|
||||
require.True(t, ok, "metadata must be an object")
|
||||
assert.NotEmpty(t, meta["tags"], "metadata.tags must still be written")
|
||||
}
|
||||
|
||||
@@ -84,6 +84,15 @@ func (m *Middleware) Invoke(ctx context.Context, in *middleware.Input) (*middlew
|
||||
return allowNoAttribution(), nil
|
||||
}
|
||||
|
||||
// Model-listing and other non-inference endpoints carry no model, and
|
||||
// management's per-model allowlist fails closed on an empty one. The
|
||||
// router has already authorised the route against the caller's groups
|
||||
// and the request consumes no tokens, so gating it on a model that
|
||||
// cannot exist would only break gateway model discovery.
|
||||
if lookupKV(in.Metadata, middleware.KeyLLMNonInference) == "true" {
|
||||
return allowNoAttribution(), nil
|
||||
}
|
||||
|
||||
providerID := lookupKV(in.Metadata, middleware.KeyLLMResolvedProviderID)
|
||||
if providerID == "" {
|
||||
// llm_router didn't emit a resolved provider id — usually
|
||||
@@ -117,7 +126,7 @@ func (m *Middleware) Invoke(ctx context.Context, in *middleware.Input) (*middlew
|
||||
}
|
||||
|
||||
if resp.GetDecision() == "deny" {
|
||||
return denyFromManagement(resp), nil
|
||||
return denyFromManagement(resp, lookupKV(in.Metadata, middleware.KeyLLMProvider)), nil
|
||||
}
|
||||
return allowFromManagement(resp), nil
|
||||
}
|
||||
@@ -161,7 +170,7 @@ func allowFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.O
|
||||
// envelope. The deny code surfaces verbatim through the framework's
|
||||
// fixed JSON template; arbitrary middleware bytes can't reach the
|
||||
// wire.
|
||||
func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.Output {
|
||||
func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse, surface string) *middleware.Output {
|
||||
code := resp.GetDenyCode()
|
||||
if code == "" {
|
||||
code = "llm_policy.cap_exceeded"
|
||||
@@ -176,6 +185,7 @@ func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.Ou
|
||||
DenyReason: &middleware.DenyReason{
|
||||
Code: code,
|
||||
Message: denyMessageForCode(code),
|
||||
Surface: surface,
|
||||
},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMPolicyDecision, Value: "deny"},
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user