sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-10-13 08:59:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

[client] Add nblink, an unprivileged forwarder into the network (#7955)

Browse Source 
Reaching an internal service meant installing the agent, which needs root
for a TUN device. nblink runs the embedded client in netstack mode and
forwards local HTTP listeners to upstreams inside the tunnel, so it works on
managed machines and in rootless containers. Upstream names resolve
in-process.

Forwards bind loopback unless --allow-public-bind is given. The listener
refuses rebound and cross-site browser requests and bounds stalled request
bodies. It ships as release archives and a distroless image that runs under
an arbitrary UID.

An embedded client given no StatePath now keeps its state in memory rather
than falling back to the installed agent's state file, which every embedded
client in a proxy process also shared. The state manager gets an explicit
in-memory store for it.
This commit is contained in:
Maycon Santos
2026-10-12 13:54:15 +02:00
committed by GitHub
parent fe7841721a
commit 66a28fd666
28 changed files with 4891 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/agent-network-e2e.yml
+1 -1
View File
@@ -52,7 +52,7 @@ jobs:
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client') }}
key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client', 'e2e/harness/Dockerfile.nblink') }}
restore-keys: |
${{ runner.os }}-anet-e2e-buildx-
.goreleaser.yaml
+54
View File
@@ -216,6 +216,29 @@ builds:
- -s -w -X main.Version={{.Version}} -X main.Commit={{.Commit}} -X main.BuildDate={{.CommitDate}}
mod_timestamp: "{{ .CommitTimestamp }}"
- id: nblink
dir: client/link/cmd/nblink
env: [CGO_ENABLED=0]
binary: nblink
goos:
- linux
- darwin
- windows
goarch:
- amd64
- arm64
- arm
goarm:
- 7
ignore:
- goos: windows
goarch: arm
- goos: darwin
goarch: arm
ldflags:
- -s -w -X github.com/netbirdio/netbird/version.version={{.Version}}
mod_timestamp: "{{ .CommitTimestamp }}"
- id: netbird-idp-migrate
dir: tools/idp-migrate
env:
@@ -280,6 +303,15 @@ archives:
builds:
- netbird-mysql-migrate
name_template: "netbird-mysql-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
- id: nblink
builds:
- nblink
name_template: "nblink_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
# goreleaser archives as tar.gz everywhere unless told otherwise, and
# nblink is downloaded and extracted by hand on Windows too.
format_overrides:
- goos: windows
format: zip
nfpms:
- maintainer: Netbird <dev@netbird.io>
@@ -619,6 +651,28 @@ dockers_v2:
"org.opencontainers.image.revision": "{{.FullCommit}}"
"org.opencontainers.image.source": "{{.GitURL}}"
"maintainer": "dev@netbird.io"
- id: nblink
disable: "{{ .Env.SKIP_DOCKER_PUSH }}"
ids:
- nblink
images:
- netbirdio/nblink
- ghcr.io/netbirdio/nblink
tags:
- "{{ .Version }}"
- "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}"
dockerfile: client/Dockerfile-nblink
platforms:
- linux/amd64
- linux/arm64
- linux/arm
annotations:
"org.opencontainers.image.created": "{{.Date}}"
"org.opencontainers.image.title": "nblink"
"org.opencontainers.image.version": "{{.Version}}"
"org.opencontainers.image.revision": "{{.FullCommit}}"
"org.opencontainers.image.source": "{{.GitURL}}"
"maintainer": "dev@netbird.io"
- id: proxy-ubi
disable: "{{ .Env.SKIP_DOCKER_PUSH }}"
ids:
client/Dockerfile-nblink
+34
View File
@@ -0,0 +1,34 @@
# nblink runs unprivileged: no TUN device, no capabilities, no host networking.
#
# The image is built for platforms that assign an arbitrary UID at run time, so
# it must not depend on a passwd entry and its state directory has to be
# writable by the root group rather than by one named user.
FROM alpine:3.22 AS build
# Group 0 owns the state directory so a container started as any UID in that
# group can still persist its identity. Mode 770 rather than g=u, because the
# directory holds the peer's keys and nothing outside owner and group should
# read them, including when the path is bind-mounted from a host.
RUN mkdir -p /tmp/state/nblink \
&& chgrp -R 0 /tmp/state/nblink \
&& chmod -R 770 /tmp/state/nblink
FROM gcr.io/distroless/static-debian13:nonroot
ARG TARGETPLATFORM
COPY ${TARGETPLATFORM}/nblink /usr/local/bin/nblink
# COPY of a directory copies its contents and creates the target with mode
# 755, which group 0 cannot write. Copying the parent makes nblink/ itself the
# copied entry, so its mode survives.
COPY --from=build --chown=65532:0 /tmp/state/ /var/lib/
# Numeric so the runtime can substitute its own UID without a passwd lookup.
USER 65532:0
# No NB_STATE_DIR by default: state stays in memory, so the image runs on a
# read-only root filesystem without a volume. Set NB_STATE_DIR=/var/lib/nblink
# and mount a volume there to keep the peer identity; a volume mounted over
# the directory starts out with its group-0 ownership.
ENV NB_LOG_LEVEL=info
ENTRYPOINT ["/usr/local/bin/nblink"]
client/embed/embed.go
+13 -3
View File
@@ -54,6 +54,9 @@ type Client struct {
jwtToken string
connect *internal.ConnectClient
recorder *peer.Status
// inMemoryState is set when no StatePath was given in netstack mode,
// where the client changes nothing on the host that needs restoring.
inMemoryState bool
}
// Options configures a new Client.
@@ -78,7 +81,8 @@ type Options struct {
NoUserspace bool
// ConfigPath is the path to the netbird config file. If empty, the config will be stored in memory and not persisted.
ConfigPath string
// StatePath is the path to the netbird state file
// StatePath is the path to the netbird state file. When empty in netstack
// mode, state is kept in memory; with NoUserspace the default path is used.
StatePath string
// DisableClientRoutes disables the client routes
DisableClientRoutes bool
@@ -193,7 +197,6 @@ func New(opts Options) (*Client, error) {
}
if opts.StatePath != "" {
// TODO: Disable state if path not provided
if err := os.Setenv("NB_DNS_STATE_FILE", opts.StatePath); err != nil {
return nil, fmt.Errorf("setenv: %w", err)
}
@@ -261,6 +264,9 @@ func New(opts Options) (*Client, error) {
jwtToken: opts.JWTToken,
config: config,
recorder: peer.NewRecorder(config.ManagementURL.String()),
// Without a path the default would be the installed agent's state
// file, which this client must neither read nor overwrite.
inMemoryState: opts.StatePath == "" && !opts.NoUserspace,
}, nil
}
@@ -292,7 +298,11 @@ func (c *Client) Start(startCtx context.Context) error {
if err, _ := authClient.Login(ctx, c.setupKey, c.jwtToken); err != nil {
return fmt.Errorf("login: %w", err)
}
client := internal.NewConnectClient(ctx, c.config, c.recorder)
var connectOpts []internal.ConnectClientOption
if c.inMemoryState {
connectOpts = append(connectOpts, internal.WithInMemoryState())
}
client := internal.NewConnectClient(ctx, c.config, c.recorder, connectOpts...)
client.SetSyncResponsePersistence(true)
// either startup error (permanent backoff err) or nil err (successful engine up)
client/grpc/dialer_generic.go
+8 -16
View File
@@ -6,12 +6,9 @@ import (
"context"
"fmt"
"net"
"os/user"
"os"
"runtime"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
@@ -45,18 +42,13 @@ func WithSweeper(sweeper Sweeper) grpc.DialOption {
}
func dialContext(ctx context.Context, addr string) (net.Conn, error) {
if runtime.GOOS == "linux" {
currentUser, err := user.Current()
if err != nil {
return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err)
}
// the custom dialer requires root permissions which are not required for use cases run as non-root
if currentUser.Uid != "0" {
log.Debug("Not running as root, using standard dialer")
dialer := &net.Dialer{}
return dialer.DialContext(ctx, "tcp", addr)
}
// The custom dialer requires root permissions which are not required for
// use cases run as non-root. The effective UID is read directly: a passwd
// lookup fails without cgo for a UID that has no entry, as on OpenShift.
if runtime.GOOS == "linux" && os.Geteuid() != 0 {
log.Debug("Not running as root, using standard dialer")
dialer := &net.Dialer{}
return dialer.DialContext(ctx, "tcp", addr)
}
conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr)
client/internal/connect.go
+20 -3
View File
@@ -72,6 +72,9 @@ type ConnectClient struct {
updateManager *updater.Manager
persistSyncResponse bool
// inMemoryState keeps the state manager off disk, for clients that change
// nothing on the host that a later run would have to restore.
inMemoryState bool
// netMgr gates every reconnection loop on OS-reported network
// availability and sweeps connections on network change.
@@ -84,6 +87,12 @@ type ConnectClient struct {
// ConnectClientOption configures optional ConnectClient behavior.
type ConnectClientOption func(*ConnectClient)
// WithInMemoryState keeps client state in memory instead of the default state
// file, so the client neither reads nor writes another installation's state.
func WithInMemoryState() ConnectClientOption {
return func(c *ConnectClient) { c.inMemoryState = true }
}
// WithNetEvents injects the OS network event handling.
func WithNetEvents(events *netevents.Manager) ConnectClientOption {
return func(c *ConnectClient) { c.netMgr = events }
@@ -272,8 +281,15 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
return err
}
// path stays alongside the manager because the engine derives StateDir
// from it; an in-memory client leaves it empty.
var path string
if runtime.GOOS == "ios" || runtime.GOOS == "android" {
var stateManager *statemanager.Manager
switch {
case c.inMemoryState:
log.Debugf("client state is kept in memory")
stateManager = statemanager.NewInMemory()
case runtime.GOOS == "ios" || runtime.GOOS == "android":
// On mobile, use the provided state file path directly
if !fileExists(mobileDependency.StateFilePath) {
if err := createFile(mobileDependency.StateFilePath); err != nil {
@@ -282,11 +298,12 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
}
}
path = mobileDependency.StateFilePath
} else {
stateManager = statemanager.New(path)
default:
sm := profilemanager.NewServiceManager("")
path = sm.GetStatePath()
stateManager = statemanager.New(path)
}
stateManager := statemanager.New(path)
stateManager.RegisterState(&sshconfig.ShutdownState{})
if c.updateManager != nil {
client/internal/statemanager/manager.go
+16 -47
View File
@@ -1,13 +1,9 @@
package statemanager
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"reflect"
"sync"
"time"
@@ -17,7 +13,6 @@ import (
"golang.org/x/exp/maps"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/util"
)
const (
@@ -56,7 +51,7 @@ type Manager struct {
cancel context.CancelFunc
done chan struct{}
filePath string
store store
// holds the states that are registered with the manager and that are to be persisted
states map[string]State
// holds the state names that have been updated and need to be persisted with the next save
@@ -65,10 +60,20 @@ type Manager struct {
stateTypes map[string]reflect.Type
}
// New creates a new Manager instance
// New creates a Manager that persists its states to filePath.
func New(filePath string) *Manager {
return newManager(&fileStore{path: filePath})
}
// NewInMemory creates a Manager that persists nothing, for a client that
// changes nothing on the host and so has no state a later run must restore.
func NewInMemory() *Manager {
return newManager(memoryStore{})
}
func newManager(s store) *Manager {
return &Manager{
filePath: filePath,
store: s,
states: make(map[string]State),
dirty: make(map[string]struct{}),
stateTypes: make(map[string]reflect.Type),
@@ -274,7 +279,7 @@ func (m *Manager) PersistState(ctx context.Context) error {
done := make(chan error, 1)
start := time.Now()
go func() {
done <- util.WriteBytesWithRestrictedPermission(ctx, m.filePath, bs)
done <- m.store.save(ctx, bs)
}()
select {
@@ -293,45 +298,9 @@ func (m *Manager) PersistState(ctx context.Context) error {
return nil
}
// loadStateFile reads and unmarshals the state file into a map of raw JSON messages
// loadStateFile returns the states an earlier run persisted.
func (m *Manager) loadStateFile(deleteCorrupt bool) (map[string]json.RawMessage, error) {
data, err := os.ReadFile(m.filePath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
log.Debugf("state file %s does not exist", m.filePath)
return nil, nil // nolint:nilnil
}
return nil, fmt.Errorf("read state file: %w", err)
}
var rawStates map[string]json.RawMessage
if err := json.Unmarshal(data, &rawStates); err != nil {
if len(bytes.TrimSpace(data)) == 0 {
log.Warnf("state file %s is empty (%d bytes)", m.filePath, len(data))
} else {
log.Warnf("state file %s has malformed content (%d bytes)", m.filePath, len(data))
}
m.handleCorruptedState(deleteCorrupt)
return nil, fmt.Errorf("unmarshal states: %w", err)
}
return rawStates, nil
}
// handleCorruptedState creates a backup of a corrupted state file by moving it
func (m *Manager) handleCorruptedState(deleteCorrupt bool) {
if !deleteCorrupt {
return
}
log.Warn("State file appears to be corrupted, attempting to back it up")
backupPath := fmt.Sprintf("%s.corrupted.%d", m.filePath, time.Now().UnixNano())
if err := os.Rename(m.filePath, backupPath); err != nil {
log.Errorf("Failed to backup corrupted state file: %v", err)
return
}
log.Infof("Created backup of corrupted state file at: %s", backupPath)
return m.store.load(deleteCorrupt)
}
// loadSingleRawState unmarshals a raw state into a concrete state object
client/internal/statemanager/manager_test.go
+65
View File
@@ -0,0 +1,65 @@
package statemanager
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type testState struct {
Value string `json:"value"`
}
func (s *testState) Name() string { return "test_state" }
func TestInMemoryManagerNeverTouchesDisk(t *testing.T) {
// Run from an empty directory, so a write to a relative or empty path
// would leave a file behind here.
dir := t.TempDir()
t.Chdir(dir)
m := NewInMemory()
m.RegisterState(&testState{})
require.NoError(t, m.UpdateState(&testState{Value: "kept"}))
require.NoError(t, m.PersistState(context.Background()), "persisting in memory must succeed")
entries, err := os.ReadDir(dir)
require.NoError(t, err)
assert.Empty(t, entries, "an in-memory manager must not write a state file")
assert.Equal(t, &testState{Value: "kept"}, m.GetState(&testState{}), "the state must still be held in memory")
assert.NoError(t, m.LoadState(&testState{}), "loading with nothing on disk must succeed")
names, err := m.GetSavedStateNames()
require.NoError(t, err)
assert.Empty(t, names, "an in-memory manager has no saved states")
}
// An empty path is a misconfiguration rather than a request for memory: the
// in-memory manager has its own constructor, so a path that resolved to
// nothing must surface instead of silently discarding every write.
func TestManagerWithoutAPathFailsToPersist(t *testing.T) {
m := New("")
m.RegisterState(&testState{})
require.NoError(t, m.UpdateState(&testState{Value: "kept"}))
assert.Error(t, m.PersistState(context.Background()),
"an empty path must report that it cannot persist, not quietly drop the state")
}
func TestManagerPersistsToItsFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "state.json")
m := New(path)
m.RegisterState(&testState{})
require.NoError(t, m.UpdateState(&testState{Value: "kept"}))
require.NoError(t, m.PersistState(context.Background()))
names, err := New(path).GetSavedStateNames()
require.NoError(t, err)
assert.Equal(t, []string{"test_state"}, names, "a manager with a path must write its state there")
}
client/internal/statemanager/store.go
+91
View File
@@ -0,0 +1,91 @@
package statemanager
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"time"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/util"
)
// store is where a Manager keeps its states between runs.
type store interface {
// load returns the states persisted by an earlier run, or an empty map
// when there are none. deleteCorrupt asks the store to move a state it
// cannot parse out of the way.
load(deleteCorrupt bool) (map[string]json.RawMessage, error)
// save replaces the persisted states with data.
save(ctx context.Context, data []byte) error
}
// fileStore persists states to a file, which is what an installed agent needs:
// it changes routing, DNS and firewall rules on the host, and a later run has
// to know what to put back.
type fileStore struct {
path string
}
// memoryStore keeps nothing. An embedded client in netstack mode changes
// nothing on the host, so there is no state a later run has to restore, and
// writing one would mean reading and overwriting another installation's file.
type memoryStore struct{}
func (f *fileStore) load(deleteCorrupt bool) (map[string]json.RawMessage, error) {
data, err := os.ReadFile(f.path)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
log.Debugf("state file %s does not exist", f.path)
return nil, nil //nolint:nilnil // no file is not a failure, there is simply nothing to load
}
return nil, fmt.Errorf("read state file: %w", err)
}
var rawStates map[string]json.RawMessage
if err := json.Unmarshal(data, &rawStates); err != nil {
if len(bytes.TrimSpace(data)) == 0 {
log.Warnf("state file %s is empty (%d bytes)", f.path, len(data))
} else {
log.Warnf("state file %s has malformed content (%d bytes)", f.path, len(data))
}
f.handleCorrupted(deleteCorrupt)
return nil, fmt.Errorf("unmarshal states: %w", err)
}
return rawStates, nil
}
func (f *fileStore) save(ctx context.Context, data []byte) error {
return util.WriteBytesWithRestrictedPermission(ctx, f.path, data)
}
// handleCorrupted moves a state file that cannot be parsed aside, so the next
// run starts from nothing rather than failing on the same content forever.
func (f *fileStore) handleCorrupted(deleteCorrupt bool) {
if !deleteCorrupt {
return
}
log.Warn("State file appears to be corrupted, attempting to back it up")
backupPath := fmt.Sprintf("%s.corrupted.%d", f.path, time.Now().UnixNano())
if err := os.Rename(f.path, backupPath); err != nil {
log.Errorf("Failed to backup corrupted state file: %v", err)
return
}
log.Infof("Created backup of corrupted state file at: %s", backupPath)
}
func (memoryStore) load(bool) (map[string]json.RawMessage, error) {
return map[string]json.RawMessage{}, nil
}
func (memoryStore) save(context.Context, []byte) error {
return nil
}
client/link/README.md
+225
View File
@@ -0,0 +1,225 @@
# nblink
`nblink` forwards local ports into a NetBird network from an unprivileged
process. It runs the client in userspace mode, so there is no TUN device to
create and no elevated privileges to grant. That lets it run on managed
machines and in rootless containers where the normal agent cannot.
Overlay names on the upstream side are resolved inside the tunnel, so
`/etc/resolv.conf` is never touched.
## Install
Download the archive for your platform, extract it, and run the binary. There
is no installer and no service to register.
## Use
A forward is one self-describing argument, and the flag repeats:
```sh
nblink --forward 'http://8080=https://grafana.internal'
```
The scheme on the left selects the listener type. The right side is the
upstream reached over the network. Giving only a port binds loopback, so the
example above listens on `127.0.0.1:8080`.
Without a setup key, `nblink` opens a browser to log in and prints the URL so
it also works over SSH.
```sh
# several forwards from one session
nblink --forward 'http://8080=https://grafana.internal' \
--forward 'http://9090=https://prometheus.internal'
# non-interactive
nblink --forward 'http://8080=https://grafana.internal' --setup-key "$NB_SETUP_KEY"
# check the configuration without touching the network
nblink --forward 'http://8080=https://grafana.internal' --check
```
Port `0` asks the operating system for a free port. The address actually bound
is printed at startup.
## Container
Every flag has a matching `NB_`-prefixed environment variable, which is how the
image is configured. `NB_FORWARD` takes a comma separated list.
```sh
docker run --rm -p 127.0.0.1:8080:8080 \
--mount type=bind,src="$(pwd)/nb_setup_key",dst=/run/secrets/nb_setup_key,readonly \
-e NB_SETUP_KEY=file:/run/secrets/nb_setup_key \
-e NB_FORWARD='http://0.0.0.0:8080=https://grafana.internal' \
-e NB_ALLOW_PUBLIC_BIND=true \
netbirdio/nblink
```
The key file has to exist before the run and be readable by the container
user, which is UID 65532 in group 0. A bind mount keeps the host file's owner
and mode, so a root-owned key written with the usual `chmod 600` is not
readable and the run fails on startup with `permission denied`. Either give the
file to that UID (`chown 65532`) or keep it mode `640` in group 0
(`chgrp 0`). If the source path does not exist, `--mount` refuses to start the
container; the older `-v` form creates a directory there instead, which the read
then reports as a directory.
The image needs no `--privileged`, no `--cap-add NET_ADMIN`, no
`--device /dev/net/tun` and no host networking.
Inside a container the listener has to bind `0.0.0.0`, because a published
port arrives on the container's own interface rather than its loopback. That
still requires `NB_ALLOW_PUBLIC_BIND`, so exposing a forward is always
something the operator wrote down. Publish it on the host's loopback as above:
a bare `-p 8080:8080` publishes on every host interface, which hands the
peer's identity to anyone on the network who can reach the port.
A public listener accepts requests addressed to `localhost` or to an IP
address. Any name other machines use to reach it, such as a DNS name or a
Kubernetes Service name, has to be listed with `--allowed-host`
(`NB_ALLOWED_HOST`). See [Limits](#limits) for why.
The image keeps its state in memory, so it runs on a read-only root filesystem
and each start registers a new peer. To keep one identity across restarts, set
`NB_STATE_DIR=/var/lib/nblink` and mount a volume there:
```sh
docker run --rm --read-only -p 127.0.0.1:8080:8080 \
-v nblink-state:/var/lib/nblink -e NB_STATE_DIR=/var/lib/nblink \
...
```
Without a state directory, use an ephemeral setup key so the peers left behind
by earlier starts are removed once they go offline, rather than piling up.
A kept identity is not yet a kept login. Without a setup key every start runs
the browser flow, even when the peer's session is still valid, because the
token is resolved before the stored profile is opened. Give a restart that has
to be unattended a setup key.
## Kubernetes and OpenShift
The image fits OpenShift's `restricted-v2` SCC and the Kubernetes `restricted`
Pod Security Standard as it is:
- It runs as any UID. OpenShift assigns one from the namespace range and adds
group 0, and everything the process writes is owned by group 0.
- It needs no capabilities, no privilege escalation, no TUN device and no host
networking, so `capabilities: {drop: [ALL]}` and
`allowPrivilegeEscalation: false` hold.
- With state in memory it needs no writable path, so
`readOnlyRootFilesystem: true` holds.
The simplest shape is a sidecar. The application in the same pod reaches the
forward on `localhost`, so the listener stays on loopback and needs neither
`NB_ALLOW_PUBLIC_BIND` nor a Service:
```yaml
containers:
- name: nblink
image: netbirdio/nblink
env:
- name: NB_FORWARD
value: http://8080=https://grafana.internal
- name: NB_SETUP_KEY
value: file:/run/secrets/nblink/setup-key
volumeMounts:
- name: nblink-key
mountPath: /run/secrets/nblink
readOnly: true
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
capabilities: {drop: [ALL]}
seccompProfile: {type: RuntimeDefault}
volumes:
- name: nblink-key
secret:
secretName: nblink-setup-key
defaultMode: 0440
```
A Secret volume is owned by the pod's `fsGroup`, which OpenShift sets for
every pod, so mode `0440` is readable by the container and by nothing else.
To keep the identity across pod restarts, mount a persistent volume at
`/var/lib/nblink` and set `NB_STATE_DIR` to it. `fsGroup` makes the volume
writable by the arbitrary UID. A `Deployment` with more than one replica
cannot share one state directory: every replica is its own peer, so give
each one its own volume through a `StatefulSet`, or keep state in memory with
an ephemeral setup key.
Serving a forward to other pods through a Service needs `NB_ALLOW_PUBLIC_BIND`
and every name callers use in `NB_ALLOWED_HOST`, for example
`grafana-link,grafana-link.monitoring.svc,grafana-link.monitoring.svc.cluster.local`.
Each caller acts under the forwarder peer's identity, so restrict who can
reach the Service with a `NetworkPolicy`. Do not expose it through a Route or
an Ingress.
## Configuration
| Flag | Environment | Default | Purpose |
| --- | --- | --- | --- |
| `--forward` | `NB_FORWARD` | | Forward spec, repeatable. Comma separated in the environment. |
| `--setup-key` | `NB_SETUP_KEY` | | Non-interactive login. Accepts a `file:` prefix. |
| `--management-url` | `NB_MANAGEMENT_URL` | NetBird cloud | Management server. |
| `--hostname` | `NB_HOSTNAME` | host name | Peer name in the network. |
| `--state-dir` | `NB_STATE_DIR` | memory | Where identity and state persist. |
| `--log-level` | `NB_LOG_LEVEL` | `info` | Log level. |
| `--allow-public-bind` | `NB_ALLOW_PUBLIC_BIND` | `false` | Required before binding a non-loopback address. |
| `--allowed-host` | `NB_ALLOWED_HOST` | | Extra name a listener accepts in `Host`, repeatable. Comma separated in the environment. |
| `--no-browser` | `NB_NO_BROWSER` | `false` | Print the login URL instead of opening a browser. |
| `--check` | `NB_CHECK` | `false` | Validate, print the effective forwards, exit. |
A secret given as `file:/path` is read from that file, so a container can mount
it instead of exposing it in the process environment.
## Limits
`nblink` only dials out. It refuses inbound connections and does not reach the
host's own network, so the peer cannot become a route into the machine it runs
on.
A loopback forward answers only requests whose `Host` names the loopback
interface or a name listed with `--allowed-host`, and refuses any request a
browser marks as belonging to another page, through either a cross-origin
`Origin` or a cross-site `Sec-Fetch-Site`. A page the user visits can point its
own hostname at `127.0.0.1` or embed the address directly, and without those
checks the browser could reach the upstream through this listener under the
peer's identity. Ordinary callers send neither header and are unaffected.
A public forward applies the same checks, and also accepts any IP address in
`Host`, since a page cannot rebind an address. The `Host` check matters there
too: a container port published on the host's loopback is reached by a
rebound page with a matching `Origin`, and its unlisted name is the only thing
that gives it away.
A name listed with `--allowed-host` is trusted rather than merely permitted,
and the list applies to every forward in the process, loopback ones included.
`Host` is the one signal a rebound page cannot forge, and listing a name gives
that up for it: a page served on that name can resolve it to the listener and
send a matching `Host` and `Origin`, which the browser then marks same-origin.
List only names you control, and prefer a fully qualified one, since a
single-label name can be answered by anything on the local network.
A forward on `0.0.0.0` listens on IPv4 only, and one on `::` on IPv6 only.
`nblink` does not ask the local router for a port mapping through UPnP,
NAT-PMP or PCP, since an unprivileged forwarder should not reconfigure the
network it runs on. Connections are still established through hole punching
or the relay. Set `NB_DISABLE_NAT_MAPPER=false` to allow it.
A browser too old to send `Sec-Fetch-Site`, which browsers have sent since
2020, can still reach a loopback forward with an embedded no-cors GET.
Userspace mode carries TCP, UDP and ping only. It cannot act as an exit node or
a routing peer and does not take over system DNS. Ports below 1024 need
privileges the process usually does not have, though some container runtimes
lower that boundary.
This build forwards HTTP. The `tcp`, `udp` and `socks5` schemes are reserved by
the grammar and rejected with a message saying so, so adding them later needs
no change to how a forward is written.
client/link/auth.go
+101
View File
@@ -0,0 +1,101 @@
package link
import (
"context"
"fmt"
"io"
"os"
"strings"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/auth"
"github.com/netbirdio/netbird/client/internal/profilemanager"
"github.com/netbirdio/netbird/util"
)
// credentials carries exactly one proof of identity to the embedded client.
type credentials struct {
setupKey string
jwtToken string
}
// resolveCredentials returns a setup key when one was supplied, and otherwise
// drives the interactive login flow.
//
// The embedded client accepts a setup key or an already-issued token, so the
// browser flow is run here and only its result is handed over.
func resolveCredentials(ctx context.Context, cfg *Config) (credentials, error) {
if cfg.SetupKey != "" {
log.Info("authenticating with the configured setup key")
return credentials{setupKey: cfg.SetupKey}, nil
}
log.Info("no setup key configured, starting interactive login")
token, err := interactiveLogin(ctx, cfg)
if err != nil {
return credentials{}, err
}
return credentials{jwtToken: token}, nil
}
// interactiveLogin runs the standard browser login and returns the token to
// authenticate the peer with.
func interactiveLogin(ctx context.Context, cfg *Config) (string, error) {
// The flow only needs somewhere to read the management URL from and a key
// to talk to it with. The peer identity the client ends up using is
// created separately, so this config is deliberately not persisted.
flowConfig, err := profilemanager.CreateInMemoryConfig(profilemanager.ConfigInput{
ManagementURL: cfg.ManagementURL,
})
if err != nil {
return "", fmt.Errorf("prepare login: %w", err)
}
flow, err := auth.NewOAuthFlow(ctx, flowConfig, util.HasGraphicalSession(), false, "", false)
if err != nil {
return "", fmt.Errorf("start login: %w", err)
}
info, err := flow.RequestAuthInfo(ctx)
if err != nil {
return "", fmt.Errorf("request login: %w", err)
}
promptLogin(os.Stderr, util.OpenBrowser, info, cfg.NoBrowser)
token, err := flow.WaitToken(ctx, info)
if err != nil {
return "", fmt.Errorf("wait for login: %w", err)
}
return token.GetTokenToUse(), nil
}
// promptLogin writes the verification URL to out, which the caller points at
// stderr so stdout stays clean for piping, and opens a browser unless asked not
// to. The URL is always printed so the flow still works over SSH where no
// browser can open.
func promptLogin(out io.Writer, openBrowser func(string) error, info auth.AuthFlowInfo, noBrowser bool) {
uri := info.VerificationURIComplete
if uri == "" {
uri = info.VerificationURI
}
var b strings.Builder
fmt.Fprintf(&b, "\nOpen this URL to log in:\n\n %s\n", uri)
if info.UserCode != "" && !strings.Contains(uri, info.UserCode) {
fmt.Fprintf(&b, "\n and enter the code %s\n", info.UserCode)
}
b.WriteString("\n")
if _, err := io.WriteString(out, b.String()); err != nil {
log.Warnf("print login URL: %v", err)
}
if noBrowser {
return
}
if err := openBrowser(uri); err != nil {
log.Debugf("could not open a browser: %v", err)
}
}
client/link/auth_test.go
+70
View File
@@ -0,0 +1,70 @@
package link
import (
"context"
"errors"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/client/internal/auth"
)
// browserSpy records the URLs a login prompt tried to open.
type browserSpy struct {
opened []string
err error
}
func (b *browserSpy) open(uri string) error {
b.opened = append(b.opened, uri)
return b.err
}
func TestPromptLoginPrintsURLAndOpensBrowser(t *testing.T) {
var out strings.Builder
browser := &browserSpy{}
info := auth.AuthFlowInfo{VerificationURIComplete: "https://login.example/device?code=ABCD-EFGH", UserCode: "ABCD-EFGH"}
promptLogin(&out, browser.open, info, false)
assert.Contains(t, out.String(), info.VerificationURIComplete, "the URL must be printed")
assert.NotContains(t, out.String(), "enter the code", "a code already in the URL must not be repeated")
assert.Equal(t, []string{info.VerificationURIComplete}, browser.opened, "the browser must be opened on the same URL")
}
func TestPromptLoginNoBrowserStillPrintsURL(t *testing.T) {
var out strings.Builder
browser := &browserSpy{}
info := auth.AuthFlowInfo{VerificationURI: "https://login.example/device", UserCode: "WXYZ-1234"}
promptLogin(&out, browser.open, info, true)
assert.Contains(t, out.String(), "https://login.example/device", "the URL must be printed even without a browser")
assert.Contains(t, out.String(), "enter the code WXYZ-1234", "a code missing from the URL must be printed")
assert.Empty(t, browser.opened, "--no-browser must not open a browser")
}
func TestPromptLoginSurvivesBrowserFailure(t *testing.T) {
var out strings.Builder
browser := &browserSpy{err: errors.New("no display")}
info := auth.AuthFlowInfo{VerificationURIComplete: "https://login.example/authorize"}
promptLogin(&out, browser.open, info, false)
assert.Contains(t, out.String(), "https://login.example/authorize",
"the URL must be printed so the login still works where no browser opens")
}
func TestResolveCredentialsUsesSetupKeyWithoutLogin(t *testing.T) {
// An unroutable management URL: a login attempt would fail, so success
// proves the setup key path never starts one.
cfg := &Config{SetupKey: "11111111-2222-3333-4444-555555555555", ManagementURL: "https://management.invalid"}
creds, err := resolveCredentials(context.Background(), cfg)
require.NoError(t, err)
assert.Equal(t, cfg.SetupKey, creds.setupKey, "the configured key must be used")
assert.Empty(t, creds.jwtToken, "exactly one proof of identity must be handed over")
}
client/link/cmd/nblink/main.go
+69
View File
@@ -0,0 +1,69 @@
// Command nblink forwards local listeners into a NetBird network from an
// unprivileged process.
package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/spf13/cobra"
"github.com/netbirdio/netbird/client/link"
"github.com/netbirdio/netbird/util"
"github.com/netbirdio/netbird/version"
)
func main() {
if err := rootCmd().Execute(); err != nil {
fmt.Fprintf(os.Stderr, "nblink: %v\n", err)
os.Exit(1)
}
}
func rootCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "nblink",
Short: "Forward local ports into a NetBird network",
Long: "nblink runs a NetBird peer in userspace and forwards local listeners\n" +
"into the network. It needs no TUN device and no elevated privileges,\n" +
"so it runs on managed machines and in rootless containers.\n\n" +
"Every flag can also be set through an NB_-prefixed environment variable\n" +
"of the same name, which is how the container image is configured.",
Example: " nblink --forward 'http://8080=https://grafana.internal'\n" +
" nblink --forward 'http://127.0.0.1:8080=https://grafana.internal' --setup-key file:/run/secrets/key",
Version: version.NetbirdVersion(),
// A stray positional argument is a mistake, most often a forward spec
// that lost its --forward, and must not start the forwarder anyway.
Args: cobra.NoArgs,
// main prints the error, so cobra must not print it a second time.
SilenceUsage: true,
SilenceErrors: true,
}
raw := link.BindFlags(cmd)
cmd.RunE = func(cmd *cobra.Command, _ []string) error {
if err := link.SetFlagsFromEnvVars(cmd); err != nil {
return err
}
cfg, err := raw.Resolve()
if err != nil {
return err
}
if err := util.InitLog(cfg.LogLevel, util.LogConsole); err != nil {
return fmt.Errorf("init log: %w", err)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
return link.Run(ctx, cfg)
}
return cmd
}
client/link/cmd/nblink/main_test.go
+80
View File
@@ -0,0 +1,80 @@
package main
import (
"fmt"
"os"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/version"
)
// TestMain clears the ambient NB_ environment before any test runs.
//
// Every flag reads an NB_-prefixed variable, so a developer who has nblink
// configured in their own shell would otherwise hand these commands a
// configuration the test never asked for: an inherited NB_FORWARD alone makes
// the missing-forward case resolve successfully and the test fail. Clearing
// here rather than in execute keeps t.Setenv working for the tests that do
// want a variable set.
func TestMain(m *testing.M) {
for _, entry := range os.Environ() {
name, _, _ := strings.Cut(entry, "=")
if strings.HasPrefix(name, "NB_") {
if err := os.Unsetenv(name); err != nil {
fmt.Fprintf(os.Stderr, "unset %s: %v\n", name, err)
os.Exit(1)
}
}
}
os.Exit(m.Run())
}
func execute(t *testing.T, args ...string) (string, error) {
t.Helper()
cmd := rootCmd()
var out strings.Builder
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetArgs(args)
err := cmd.Execute()
return out.String(), err
}
// A forward spec that lost its --forward is the usual stray argument, and it
// must not start a forwarder with whatever else was configured.
func TestRejectsPositionalArguments(t *testing.T) {
_, err := execute(t, "--check", "--forward", "http://8080=https://grafana.internal", "http://9090=https://prometheus.internal")
require.Error(t, err, "a positional argument must be refused")
assert.Contains(t, err.Error(), "unknown command", "the error must point at the stray argument")
}
func TestRejectsInvalidEnvironmentValue(t *testing.T) {
t.Setenv("NB_ALLOW_PUBLIC_BIND", "maybe")
_, err := execute(t, "--check", "--forward", "http://8080=https://grafana.internal")
require.Error(t, err, "an unparseable environment value must stop the start")
assert.Contains(t, err.Error(), "NB_ALLOW_PUBLIC_BIND", "the error must name the variable")
}
func TestEnvironmentConfiguresAFullCheck(t *testing.T) {
t.Setenv("NB_FORWARD", "http://8080=https://grafana.internal,http://0.0.0.0:9090=https://prometheus.internal")
t.Setenv("NB_ALLOW_PUBLIC_BIND", "true")
t.Setenv("NB_CHECK", "true")
_, err := execute(t)
assert.NoError(t, err, "a complete environment configuration must pass the check without flags")
}
func TestRequiresAForward(t *testing.T) {
_, err := execute(t, "--check")
require.Error(t, err, "a run with no forward must be refused")
assert.Contains(t, err.Error(), "NB_FORWARD", "the error must name the environment variable too")
}
func TestPrintsVersion(t *testing.T) {
out, err := execute(t, "--version")
require.NoError(t, err)
assert.Contains(t, out, version.NetbirdVersion(), "--version must print the build version")
}
client/link/config.go
+245
View File
@@ -0,0 +1,245 @@
package link
import (
"fmt"
"net/netip"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/netbirdio/netbird/client/internal/profilemanager"
)
// envPrefix matches the prefix the agent uses, so an operator who already sets
// NB_SETUP_KEY or NB_MANAGEMENT_URL for the agent can reuse them here.
const envPrefix = "NB_"
// fileScheme marks a flag value that holds a path to read the real value from,
// so a container can pass a secret as a mounted file instead of an environment
// variable visible to anything that can read the process environment.
const fileScheme = "file:"
// hostnamePattern matches a DNS name per RFC 1123 section 2.1: dot-separated
// labels of letters, digits and inner hyphens.
var hostnamePattern = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$`)
// Config is the resolved runtime configuration.
type Config struct {
Forwards []Forward
AllowedHosts []string
SetupKey string
ManagementURL string
Hostname string
StateDir string
LogLevel string
AllowPublicBind bool
NoBrowser bool
Check bool
}
// rawConfig holds flag values before parsing and validation.
type rawConfig struct {
forwards []string
allowedHosts []string
setupKey string
managementURL string
hostname string
stateDir string
logLevel string
allowPublicBind bool
noBrowser bool
check bool
}
// Resolve validates the raw flag values and produces a runtime configuration.
// It reports every problem it can rather than stopping at the first, so an
// operator fixing a container environment sees the whole list in one run.
func (r *rawConfig) Resolve() (*Config, error) {
cfg := &Config{
ManagementURL: r.managementURL,
Hostname: r.hostname,
StateDir: r.stateDir,
LogLevel: r.logLevel,
AllowPublicBind: r.allowPublicBind,
NoBrowser: r.noBrowser,
Check: r.check,
}
setupKey, err := resolveSecret(r.setupKey)
if err != nil {
return nil, fmt.Errorf("setup key: %w", err)
}
cfg.SetupKey = setupKey
if len(r.forwards) == 0 {
return nil, fmt.Errorf("at least one --forward is required (env %s)", FlagNameToEnvVar("forward"))
}
var problems []string
cfg.AllowedHosts, problems = parseAllowedHosts(r.allowedHosts)
seen := make(map[string]string, len(r.forwards))
for _, spec := range r.forwards {
fwd, err := ParseForward(spec)
if err != nil {
problems = append(problems, err.Error())
continue
}
// Port 0 means the OS assigns one, so several such forwards do not
// collide even though they share a spec.
if prev, dup := seen[fwd.Listen]; dup && !strings.HasSuffix(fwd.Listen, ":0") {
problems = append(problems, fmt.Sprintf("forward %q: %s is already bound by %q", spec, fwd.Listen, prev))
continue
}
if !cfg.AllowPublicBind && !isLoopback(fwd.Listen) {
problems = append(problems, fmt.Sprintf(
"forward %q: %s is not loopback, pass --allow-public-bind (env %s) to expose it",
spec, fwd.Listen, FlagNameToEnvVar("allow-public-bind")))
continue
}
seen[fwd.Listen] = spec
fwd.AllowedHosts = cfg.AllowedHosts
cfg.Forwards = append(cfg.Forwards, fwd)
}
if len(problems) > 0 {
return nil, fmt.Errorf("invalid configuration:\n %s", strings.Join(problems, "\n "))
}
return cfg, nil
}
// ConfigPath returns where the peer identity is persisted, or an empty string
// when the client should keep it in memory only.
func (c *Config) ConfigPath() string {
if c.StateDir == "" {
return ""
}
return filepath.Join(c.StateDir, "config.json")
}
// StatePath returns where runtime state is persisted, or an empty string when
// no state directory was configured.
func (c *Config) StatePath() string {
if c.StateDir == "" {
return ""
}
return filepath.Join(c.StateDir, "state.json")
}
// BindFlags registers nblink's flags on cmd and returns the backing values.
// Every flag gains an NB_-prefixed environment variable of the same name, so
// the container configuration cannot drift from the command line.
func BindFlags(cmd *cobra.Command) *rawConfig {
raw := &rawConfig{}
f := cmd.PersistentFlags()
f.StringSliceVar(&raw.forwards, "forward", nil,
"forward spec scheme://[host:]port=upstream, repeatable (env: comma separated)")
f.StringSliceVar(&raw.allowedHosts, "allowed-host", nil,
"extra name a listener may be reached under, repeatable (env: comma separated)")
f.StringVar(&raw.setupKey, "setup-key", "",
"setup key for non-interactive login, accepts a file: prefix")
f.StringVar(&raw.managementURL, "management-url", profilemanager.DefaultManagementURL,
"management server URL")
f.StringVar(&raw.hostname, "hostname", "", "peer name in the network")
f.StringVar(&raw.stateDir, "state-dir", "",
"directory for config and state, kept in memory when empty")
f.StringVar(&raw.logLevel, "log-level", "info", "log level")
f.BoolVar(&raw.allowPublicBind, "allow-public-bind", false,
"permit forwards that bind an address other than loopback")
f.BoolVar(&raw.noBrowser, "no-browser", false,
"print the login URL instead of opening a browser")
f.BoolVar(&raw.check, "check", false,
"validate configuration, print the effective forwards and exit")
return raw
}
// SetFlagsFromEnvVars fills unset flags from NB_-prefixed environment
// variables derived from each flag name, so --management-url reads
// NB_MANAGEMENT_URL. A flag given on the command line wins.
//
// A value that does not parse is an error rather than a warning, so a
// container cannot start with a configuration other than the one it was given.
func SetFlagsFromEnvVars(cmd *cobra.Command) error {
flags := cmd.PersistentFlags()
var err error
flags.VisitAll(func(f *pflag.Flag) {
if err != nil || f.Changed {
return
}
env := FlagNameToEnvVar(f.Name)
value, ok := os.LookupEnv(env)
if !ok {
return
}
if setErr := flags.Set(f.Name, value); setErr != nil {
err = fmt.Errorf("%s: %w", env, setErr)
}
})
return err
}
// FlagNameToEnvVar converts a flag name to its environment variable, so
// allow-public-bind becomes NB_ALLOW_PUBLIC_BIND.
func FlagNameToEnvVar(name string) string {
return envPrefix + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
}
// parseAllowedHosts normalizes the names given with --allowed-host and reports
// every entry that is not a plain hostname.
func parseAllowedHosts(raw []string) ([]string, []string) {
var hosts, problems []string
for _, entry := range raw {
name := normalizeHostname(entry)
if _, err := netip.ParseAddr(name); err == nil {
problems = append(problems, fmt.Sprintf(
"allowed host %q: addresses are already accepted on a public listener, list names only", entry))
continue
}
if !hostnamePattern.MatchString(name) {
problems = append(problems, fmt.Sprintf(
"allowed host %q: must be a hostname, without scheme, port or wildcard", entry))
continue
}
hosts = append(hosts, name)
}
return hosts, problems
}
// normalizeHostname lowercases a name and drops a trailing root dot, so the
// spellings a client may send compare equal.
func normalizeHostname(name string) string {
return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), ".")
}
// resolveSecret reads a value that may carry a file: prefix naming the file
// that holds it. Whitespace around the file contents is trimmed so a trailing
// newline from the usual tooling does not become part of the secret.
func resolveSecret(value string) (string, error) {
path, ok := strings.CutPrefix(value, fileScheme)
if !ok {
return value, nil
}
if path == "" {
return "", fmt.Errorf("%s prefix with no path", fileScheme)
}
content, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read %s: %w", path, err)
}
// An empty file would otherwise look like no secret at all, silently
// turning a non-interactive start into a login that waits for a browser.
secret := strings.TrimSpace(string(content))
if secret == "" {
return "", fmt.Errorf("%s is empty", path)
}
return secret, nil
}
client/link/config_test.go
+219
View File
@@ -0,0 +1,219 @@
package link
import (
"os"
"path/filepath"
"testing"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newTestCommand wires the flags the same way the binary does, so the tests
// exercise the real flag and environment plumbing.
func newTestCommand() (*cobra.Command, *rawConfig) {
cmd := &cobra.Command{Use: "nblink"}
return cmd, BindFlags(cmd)
}
func TestFlagNameToEnvVar(t *testing.T) {
assert.Equal(t, "NB_FORWARD", FlagNameToEnvVar("forward"))
assert.Equal(t, "NB_ALLOW_PUBLIC_BIND", FlagNameToEnvVar("allow-public-bind"))
assert.Equal(t, "NB_MANAGEMENT_URL", FlagNameToEnvVar("management-url"))
}
func TestSetFlagsFromEnvVars(t *testing.T) {
cmd, raw := newTestCommand()
t.Setenv("NB_FORWARD", "http://8080=https://a.internal,http://8081=https://b.internal")
t.Setenv("NB_MANAGEMENT_URL", "https://mgmt.example:443")
require.NoError(t, SetFlagsFromEnvVars(cmd))
assert.Equal(t, []string{"http://8080=https://a.internal", "http://8081=https://b.internal"},
raw.forwards, "NB_FORWARD should split on commas into repeated forwards")
assert.Equal(t, "https://mgmt.example:443", raw.managementURL)
}
// A value given on the command line must win over the environment, otherwise a
// container's baked-in defaults could not be overridden at run time.
func TestFlagBeatsEnvVar(t *testing.T) {
cmd, raw := newTestCommand()
require.NoError(t, cmd.PersistentFlags().Set("log-level", "debug"))
t.Setenv("NB_LOG_LEVEL", "error")
require.NoError(t, SetFlagsFromEnvVars(cmd))
assert.Equal(t, "debug", raw.logLevel, "an explicit flag should not be overwritten by the environment")
}
// A container must not start with a configuration other than the one it was
// given, so an unparseable value stops startup instead of leaving the default.
func TestSetFlagsFromEnvVarsRejectsInvalidValue(t *testing.T) {
cmd, raw := newTestCommand()
t.Setenv("NB_ALLOW_PUBLIC_BIND", "yes-please")
err := SetFlagsFromEnvVars(cmd)
require.Error(t, err)
assert.Contains(t, err.Error(), "NB_ALLOW_PUBLIC_BIND", "the error should name the variable at fault")
assert.False(t, raw.allowPublicBind, "the default must not be silently kept in effect")
}
func TestResolveRequiresForward(t *testing.T) {
_, raw := newTestCommand()
_, err := raw.Resolve()
require.Error(t, err)
assert.Contains(t, err.Error(), "at least one --forward")
}
func TestResolveGatesPublicBind(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{"http://0.0.0.0:8080=https://grafana.internal"}
_, err := raw.Resolve()
require.Error(t, err)
assert.Contains(t, err.Error(), "allow-public-bind",
"binding a public address should require the explicit opt-in")
}
func TestResolveAllowsPublicBindWhenOptedIn(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{"http://0.0.0.0:8080=https://grafana.internal"}
raw.allowPublicBind = true
cfg, err := raw.Resolve()
require.NoError(t, err)
require.Len(t, cfg.Forwards, 1)
assert.Equal(t, "0.0.0.0:8080", cfg.Forwards[0].Listen)
}
func TestResolveRejectsDuplicateListener(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{
"http://8080=https://a.internal",
"http://127.0.0.1:8080=https://b.internal",
}
_, err := raw.Resolve()
require.Error(t, err)
assert.Contains(t, err.Error(), "already bound",
"two forwards on one address should be caught before binding")
}
// Every bad spec should be reported together, so fixing a container
// environment does not take one deploy per mistake.
func TestResolveReportsEveryProblem(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{
"http://70000=https://a.internal",
"gopher://8081=https://b.internal",
}
_, err := raw.Resolve()
require.Error(t, err)
assert.Contains(t, err.Error(), "out of range")
assert.Contains(t, err.Error(), "unknown scheme")
}
func TestResolveAllowedHosts(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{"http://0.0.0.0:8080=https://a.internal", "http://8081=https://b.internal"}
raw.allowPublicBind = true
raw.allowedHosts = []string{"NBLink.LAN.", "svc.ns.svc.cluster.local"}
cfg, err := raw.Resolve()
require.NoError(t, err)
assert.Equal(t, []string{"nblink.lan", "svc.ns.svc.cluster.local"}, cfg.AllowedHosts,
"allowed hosts must be stored in the form they are compared in")
for _, f := range cfg.Forwards {
assert.Equal(t, cfg.AllowedHosts, f.AllowedHosts, "every forward must carry the allowed hosts")
}
}
func TestResolveRejectsMalformedAllowedHosts(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{"http://8080=https://a.internal"}
raw.allowedHosts = []string{"http://nblink.lan", "nblink.lan:8080", "*.lan", "10.0.0.5", ""}
_, err := raw.Resolve()
require.Error(t, err)
for _, entry := range []string{`"http://nblink.lan"`, `"nblink.lan:8080"`, `"*.lan"`, `""`} {
assert.Contains(t, err.Error(), entry, "every malformed entry must be reported")
}
assert.Contains(t, err.Error(), "addresses are already accepted", "an address must be explained, not just refused")
}
func TestResolveSetupKeyFromFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "key")
// The trailing newline mirrors what a mounted secret usually contains.
require.NoError(t, os.WriteFile(path, []byte("A1B2C3D4-SETUP-KEY\n"), 0o600))
_, raw := newTestCommand()
raw.forwards = []string{"http://8080=https://a.internal"}
raw.setupKey = "file:" + path
cfg, err := raw.Resolve()
require.NoError(t, err)
assert.Equal(t, "A1B2C3D4-SETUP-KEY", cfg.SetupKey, "a mounted secret should be trimmed")
}
// Each port-0 forward gets its own OS-assigned port, so sharing the spec is
// not a collision.
func TestResolveAllowsSeveralEphemeralPorts(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{
"http://0=https://a.internal",
"http://0=https://b.internal",
}
cfg, err := raw.Resolve()
require.NoError(t, err)
assert.Len(t, cfg.Forwards, 2, "two ephemeral forwards should both be kept")
}
func TestResolveSetupKeyFileEmpty(t *testing.T) {
path := filepath.Join(t.TempDir(), "key")
require.NoError(t, os.WriteFile(path, []byte(" \n"), 0o600))
_, raw := newTestCommand()
raw.forwards = []string{"http://8080=https://a.internal"}
raw.setupKey = "file:" + path
_, err := raw.Resolve()
require.Error(t, err)
assert.Contains(t, err.Error(), "is empty",
"an empty secret should fail rather than fall back to interactive login")
}
func TestResolveSetupKeyFileMissing(t *testing.T) {
_, raw := newTestCommand()
raw.forwards = []string{"http://8080=https://a.internal"}
raw.setupKey = "file:" + filepath.Join(t.TempDir(), "absent")
_, err := raw.Resolve()
require.Error(t, err)
assert.Contains(t, err.Error(), "setup key")
}
func TestStatePathsFollowStateDir(t *testing.T) {
cfg := &Config{}
assert.Empty(t, cfg.ConfigPath(), "an unset state dir should keep identity in memory")
assert.Empty(t, cfg.StatePath())
cfg.StateDir = "/var/lib/nblink"
assert.Equal(t, filepath.Join("/var/lib/nblink", "config.json"), cfg.ConfigPath())
assert.Equal(t, filepath.Join("/var/lib/nblink", "state.json"), cfg.StatePath())
}
client/link/forward.go
+217
View File
@@ -0,0 +1,217 @@
// Package link implements nblink, an unprivileged forwarder that exposes local
// listeners and carries their traffic into the NetBird overlay.
//
// The process runs the embedded client in netstack mode, so it needs no TUN
// device and no elevated privileges. Overlay names on the upstream side are
// resolved inside the tunnel rather than through the host resolver.
package link
import (
"errors"
"fmt"
"net"
"net/netip"
"net/url"
"regexp"
"strconv"
"strings"
)
// defaultBindHost is used when a forward spec gives a port but no bind address.
// Binding loopback by default keeps the overlay unreachable from the rest of
// the network until the operator opts in.
const defaultBindHost = "127.0.0.1"
// ProtoHTTP forwards HTTP requests to an upstream URL over the overlay.
const ProtoHTTP = "http"
// supportedProtos lists the forward schemes this build accepts. Specs naming a
// known but unimplemented scheme are rejected with a clearer message than an
// unknown one would produce.
var (
supportedProtos = map[string]bool{ProtoHTTP: true}
plannedProtos = map[string]bool{"tcp": true, "udp": true, "socks5": true}
// schemePrefix matches a URL scheme at the start of a string, per RFC 3986
// section 3.1.
schemePrefix = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.\-]*://`)
// urlPassword matches the password half of a URL's userinfo. Errors quote
// the spec the operator typed, which would otherwise echo a password back
// to the terminal and into whatever captured it.
//
// The password runs to the last '@' of the authority, which is where
// net/url ends the userinfo, so a password containing an '@' is covered
// whole. The username ends at the first ':', as net/url splits it.
urlPassword = regexp.MustCompile(`(//[^/:\s]*):[^/\s]*@`)
)
// Forward is one local listener and the overlay address it carries traffic to.
type Forward struct {
// Proto is the listener type, taken from the scheme on the left of the spec.
Proto string
// Listen is the local bind address as host:port.
Listen string
// Upstream is the target reached over the overlay.
Upstream *url.URL
// Spec is the original text, used in logs and errors so the operator sees
// what they typed rather than a normalized form.
Spec string
// AllowedHosts are names, beyond loopback names and addresses, that the
// listener accepts in the Host header.
AllowedHosts []string
}
// String returns the forward in the spec form the operator wrote.
func (f Forward) String() string {
return fmt.Sprintf("%s://%s=%s", f.Proto, f.Listen, f.Upstream)
}
// ParseForward parses a forward spec of the form scheme://[host:]port=upstream.
//
// The scheme on the left selects the listener type, the right side is the
// overlay target. A spec that gives only a port binds loopback.
func ParseForward(spec string) (Forward, error) {
spec = strings.TrimSpace(spec)
if spec == "" {
return Forward{}, fmt.Errorf("empty forward spec")
}
schemeEnd := strings.Index(spec, "://")
if schemeEnd < 0 {
return Forward{}, fmt.Errorf("forward %q: missing scheme, want scheme://[host:]port=upstream", redactSpec(spec))
}
proto := strings.ToLower(spec[:schemeEnd])
// The left side cannot contain '=', so the first one always separates the
// listener from the upstream even when the upstream carries a query string.
sep := strings.Index(spec, "=")
if sep < 0 || sep < schemeEnd {
return Forward{}, fmt.Errorf("forward %q: missing '=' between listener and upstream", redactSpec(spec))
}
if !supportedProtos[proto] {
if plannedProtos[proto] {
return Forward{}, fmt.Errorf("forward %q: %s forwarding is not supported yet, this build handles %s", redactSpec(spec), proto, ProtoHTTP)
}
return Forward{}, fmt.Errorf("forward %q: unknown scheme %q", redactSpec(spec), proto)
}
listen, err := parseListen(spec[schemeEnd+len("://") : sep])
if err != nil {
return Forward{}, fmt.Errorf("forward %q: %w", redactSpec(spec), err)
}
upstream, err := parseUpstream(spec[sep+1:])
if err != nil {
return Forward{}, fmt.Errorf("forward %q: %w", redactSpec(spec), err)
}
return Forward{Proto: proto, Listen: listen, Upstream: upstream, Spec: spec}, nil
}
// redactSpec replaces any password inside a forward spec, so an error may
// still name the spec it refers to.
func redactSpec(spec string) string {
return urlPassword.ReplaceAllString(spec, "$1:xxxxx@")
}
// parseListen normalizes the listener side of a spec to host:port, defaulting
// the host to loopback when only a port is given.
func parseListen(raw string) (string, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return "", fmt.Errorf("missing listen address")
}
host, port := defaultBindHost, raw
if strings.Contains(raw, ":") {
h, p, err := net.SplitHostPort(raw)
if err != nil {
return "", fmt.Errorf("parse listen address %q: %w", raw, err)
}
if h != "" {
host = h
}
port = p
}
// Only literal addresses are accepted. A name would be resolved once by
// the loopback check and again by the bind, and an answer that changed in
// between would put a listener on a public address without the opt-in. A
// zone suffix stays allowed: it is part of the literal, resolved the same
// way at both points, and link-local addresses cannot be bound without it.
if strings.EqualFold(host, "localhost") {
host = defaultBindHost
}
if _, err := netip.ParseAddr(host); err != nil {
return "", fmt.Errorf("listen host %q must be an IP address or localhost", host)
}
n, err := strconv.Atoi(port)
if err != nil {
return "", fmt.Errorf("listen port %q is not a number", port)
}
// Port 0 asks the OS for a free port. The bound address is printed at
// startup, so the operator still learns where to point a client.
if n < 0 || n > 65535 {
return "", fmt.Errorf("listen port %d out of range 0-65535", n)
}
return net.JoinHostPort(host, port), nil
}
// parseUpstream validates the overlay target of an HTTP forward. The host is
// resolved later, inside the tunnel, so only the shape is checked here.
func parseUpstream(raw string) (*url.URL, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil, fmt.Errorf("missing upstream")
}
// Only a leading scheme counts. A "://" inside a path or query, as in a
// redirect parameter, belongs to the target rather than to this URL.
if !schemePrefix.MatchString(raw) {
raw = "https://" + raw
}
u, err := url.Parse(raw)
if err != nil {
// url.Error carries the whole URL, password included, so only the
// reason inside it is reported alongside the redacted form.
var parseErr *url.Error
if errors.As(err, &parseErr) {
err = parseErr.Err
}
return nil, fmt.Errorf("parse upstream %q: %w", redactSpec(raw), err)
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("upstream scheme %q must be http or https", u.Scheme)
}
if u.Hostname() == "" {
return nil, fmt.Errorf("upstream %q has no host", redactSpec(raw))
}
// Credentials in the upstream would reach the logs and the --check output,
// and the forwarder does not use them to authenticate anything.
if u.User != nil {
return nil, fmt.Errorf("upstream %q must not carry credentials", u.Redacted())
}
return u, nil
}
// isLoopback reports whether addr binds only the loopback interface. The host
// is always a literal address, because parseListen rejects names, so this
// decides the same address the bind will use.
func isLoopback(addr string) bool {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false
}
ip, err := netip.ParseAddr(host)
if err != nil {
return false
}
// Unmap first, so a v4-mapped form such as ::ffff:127.0.0.1 is recognised
// as the loopback address it is.
return ip.Unmap().IsLoopback()
}
client/link/forward_http.go
+422
View File
@@ -0,0 +1,422 @@
package link
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httputil"
"net/netip"
"net/url"
"os"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
const (
// dialTimeout bounds connection establishment only. Request and response
// bodies stream without a deadline so long downloads are not cut off.
dialTimeout = 30 * time.Second
// readHeaderTimeout bounds how long a caller may take to send request
// headers.
readHeaderTimeout = 10 * time.Second
// fetchSiteHeader is the browser's own account of how a request was
// initiated. It is the only one of the three guard signals that a no-cors
// GET carries.
fetchSiteHeader = "Sec-Fetch-Site"
// defaultHTTPPort fills in the port an origin leaves implicit.
defaultHTTPPort = "80"
idleConnTimeout = 90 * time.Second
tlsHandshakeTimeout = 10 * time.Second
expectContinueTimeout = time.Second
maxIdleConns = 100
// maxHeaderBytes caps the request headers a caller may send.
maxHeaderBytes = 32 << 10
// maxLoggedPath caps how much of a caller-supplied path reaches the log.
maxLoggedPath = 128
)
// bodyIdleTimeout bounds how long a request body may stall without delivering
// more bytes. It is a variable so tests can shorten it.
var bodyIdleTimeout = 30 * time.Second
// foreignFetchSites are the Sec-Fetch-Site values a browser sends when a page
// other than this listener's own initiated the request. A page on another port
// is same-site here, because a bare address or localhost is its own site, yet
// it is a different application with no more claim on this listener than any
// other page has.
var foreignFetchSites = map[string]bool{"cross-site": true, "same-site": true}
// DialFunc establishes a connection inside the overlay. The embedded client
// supplies the real implementation; tests supply their own.
type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
// httpForwarder serves one local HTTP listener and proxies what it accepts to
// an upstream reached over the overlay.
type httpForwarder struct {
forward Forward
server *http.Server
listener net.Listener
}
// idleTimeoutBody applies a fresh read deadline before every read, so a
// transfer that keeps progressing runs as long as it needs while a stalled one
// is cut off.
type idleTimeoutBody struct {
io.ReadCloser
controller *http.ResponseController
idle time.Duration
}
// newHTTPForwarder binds the local listener and prepares the reverse proxy.
// Binding happens here rather than in Serve so a port clash is reported before
// any forward is announced as ready.
func newHTTPForwarder(fwd Forward, dial DialFunc) (*httpForwarder, error) {
listener, err := net.Listen(listenNetwork(fwd.Listen), fwd.Listen)
if err != nil {
return nil, fmt.Errorf("listen on %s: %w", fwd.Listen, bindHint(fwd.Listen, err))
}
upstream := fwd.Upstream
proxy := &httputil.ReverseProxy{
Rewrite: func(r *httputil.ProxyRequest) {
r.SetURL(upstream)
// The upstream decides routing and TLS verification from Host, and
// it must see its own name rather than the local bind address.
r.Out.Host = upstream.Host
},
Transport: &http.Transport{
DialContext: boundedDial(dial),
ForceAttemptHTTP2: true,
MaxIdleConns: maxIdleConns,
// Without this the per-host default of two applies and MaxIdleConns
// never binds, so concurrent requests re-dial the overlay instead
// of reusing a connection.
MaxIdleConnsPerHost: maxIdleConns,
IdleConnTimeout: idleConnTimeout,
TLSHandshakeTimeout: tlsHandshakeTimeout,
ExpectContinueTimeout: expectContinueTimeout,
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
if errors.Is(err, context.Canceled) {
return
}
log.Debugf("%s %s %s: %v", fwd.Listen, r.Method, truncatePath(r.URL.Path), err)
// No upstream answered, so the connection is closed rather than
// kept for a next request. What is left of the body is still
// drained afterwards, bounded by the deadline armed in
// withBodyIdleTimeout.
rejectRequest(w, "upstream unreachable over the overlay", http.StatusBadGateway)
},
}
return &httpForwarder{
forward: fwd,
listener: listener,
server: &http.Server{
Handler: withBodyIdleTimeout(guardRebinding(proxy, isLoopback(fwd.Listen), fwd.AllowedHosts)),
// Bound how long a caller may take to send headers, so a slow
// sender cannot hold a connection and its goroutine open
// indefinitely. Neither deadline limits body streaming, so large
// uploads and downloads still run as long as they need.
ReadHeaderTimeout: readHeaderTimeout,
IdleTimeout: idleConnTimeout,
// Go's default allows about a megabyte of headers per connection,
// which a caller on a published port can spend freely.
MaxHeaderBytes: maxHeaderBytes,
},
}, nil
}
// Addr returns the address actually bound, which differs from the spec when
// the operator asked for port 0.
func (f *httpForwarder) Addr() string {
return f.listener.Addr().String()
}
// Serve runs until the forwarder is closed.
func (f *httpForwarder) Serve() error {
if err := f.server.Serve(f.listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
return fmt.Errorf("serve %s: %w", f.forward.Listen, err)
}
return nil
}
// Close stops the listener and waits for in-flight requests to finish, up to
// the deadline carried by ctx.
//
// Shutdown only closes listeners the server has taken over in Serve, so the
// raw listener is closed here as well. Otherwise a forwarder that was built
// but never served, which happens when a later bind in the same set fails,
// would hold its socket until the process exits.
func (f *httpForwarder) Close(ctx context.Context) error {
shutdownErr := f.server.Shutdown(ctx)
if err := f.listener.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
return err
}
return shutdownErr
}
func (b *idleTimeoutBody) Read(p []byte) (int, error) {
if err := b.setDeadline(time.Now().Add(b.idle)); err != nil {
return 0, err
}
n, err := b.ReadCloser.Read(p)
if errors.Is(err, io.EOF) {
// The body arrived in full, so its deadline has to go with it. The
// server keeps reading the connection while the handler streams its
// reply, and a deadline left over from the body would cut that reply
// off partway through.
//
// Only EOF clears it. On any other error the caller stalled or the
// connection broke, and the deadline is what stops the server from
// blocking forever as it drains what was never sent.
b.clearDeadline()
}
return n, err
}
func (b *idleTimeoutBody) setDeadline(t time.Time) error {
// A connection that cannot carry a deadline, which the controller reports
// as unsupported, still reads normally rather than failing the request.
if err := b.controller.SetReadDeadline(t); err != nil && !errors.Is(err, http.ErrNotSupported) {
return err
}
return nil
}
// clearDeadline removes any read deadline. A failure here cannot be acted on
// and must not mask the error the caller is already returning.
func (b *idleTimeoutBody) clearDeadline() {
if err := b.setDeadline(time.Time{}); err != nil {
log.Debugf("clear read deadline: %v", err)
}
}
// listenNetwork pins the address family to the literal the operator wrote.
// Plain "tcp" turns 0.0.0.0 into a dual-stack wildcard that also accepts IPv6,
// which exposes more than the spec asked for. A "6" network binds IPv6 only.
func listenNetwork(addr string) string {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return "tcp"
}
ip, err := netip.ParseAddr(host)
if err != nil {
return "tcp"
}
if ip.Is4() {
return "tcp4"
}
return "tcp6"
}
// boundedDial applies dialTimeout to connection establishment without
// constraining the lifetime of the connection it returns.
func boundedDial(dial DialFunc) DialFunc {
return func(ctx context.Context, network, addr string) (net.Conn, error) {
ctx, cancel := context.WithTimeout(ctx, dialTimeout)
defer cancel()
start := time.Now()
conn, err := dial(ctx, network, addr)
if err != nil {
log.Debugf("dial %s over the overlay failed after %s: %v",
addr, time.Since(start).Truncate(time.Millisecond), err)
return nil, err
}
log.Debugf("dial %s over the overlay took %s, remote %s",
addr, time.Since(start).Truncate(time.Millisecond), conn.RemoteAddr())
return conn, nil
}
}
// bindHint adds the likely cause when a bind fails for a reason an operator
// can act on. The privileged-port boundary is detected from the error rather
// than assumed from the port number, because containers often lower it to zero.
func bindHint(addr string, err error) error {
if errors.Is(err, os.ErrPermission) {
return fmt.Errorf("%w (nblink runs unprivileged, so %s likely needs a port above 1023)", err, addr)
}
return err
}
// withBodyIdleTimeout cuts off a caller that sends request headers and then
// stops sending the body.
//
// ReadHeaderTimeout covers only the headers, and IdleTimeout applies between
// requests rather than during one, so without this a caller could hold a
// connection and its handler open indefinitely. A total ReadTimeout would
// close that gap but would also break legitimate long uploads, so the deadline
// is refreshed on every read that makes progress instead.
func withBodyIdleTimeout(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Body != nil && r.Body != http.NoBody {
b := &idleTimeoutBody{
ReadCloser: r.Body,
controller: http.NewResponseController(w),
idle: bodyIdleTimeout,
}
// Armed before the handler runs rather than on the first read,
// because it is the only bound on the drain the server performs
// after the handler returns. That drain reads the connection
// directly and no response header suppresses it, so a handler
// that never touches the body would otherwise leave the
// connection and its goroutine held until the caller goes away.
//
// Replacing r.Body also hides it from net/http's own type-based
// checks, which short-circuit that drain for a body it recognises,
// so this deadline stands in for those too.
if err := b.setDeadline(time.Now().Add(b.idle)); err != nil {
log.Debugf("arm request body deadline: %v", err)
}
r.Body = b
}
next.ServeHTTP(w, r)
})
}
// guardRebinding rejects requests that a browser issued on behalf of some
// other site, and requests that reached a loopback listener under a name that
// is not its own.
//
// A loopback forwarder is reachable from any page the user visits. A site can
// point its own hostname at 127.0.0.1, and it can also embed the loopback
// address directly in an img, script or iframe URL. Either way the browser
// sends the request, and this listener would otherwise proxy it into the
// network under the peer's identity.
//
// Three signals separate those from a local caller. Host carries the
// attacker's name in the rebinding variant. Origin is present on any
// cross-origin fetch. Sec-Fetch-Site covers what the first two miss: the
// browser states how the request was initiated, including on the no-cors GETs
// that carry no Origin at all. Non-browser callers send none of the latter
// two and are unaffected.
//
// A browser too old to send Sec-Fetch-Site can still reach a loopback forward
// with an embedded no-cors GET. Browsers have sent it since 2020, so this
// residual needs a deliberately outdated one.
//
// The Host check holds on a publicly bound listener too, which is how a
// container port published to the host is reached. There a rebound page sends
// its own name with a matching Origin, so Host is the only signal left. Other
// machines reach a public listener by address, which is accepted; any name
// they use instead has to be listed with --allowed-host.
func guardRebinding(next http.Handler, loopbackOnly bool, allowedHosts []string) http.Handler {
allowed := make(map[string]bool, len(allowedHosts))
for _, h := range allowedHosts {
allowed[normalizeHostname(h)] = true
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !isAllowedHost(hostnameOf(r.Host), loopbackOnly, allowed) {
rejectRequest(w, "unexpected Host for this listener, list it with --allowed-host", http.StatusMisdirectedRequest)
return
}
if !isLocalCaller(r) {
rejectRequest(w, "cross-origin request refused", http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
})
}
// rejectRequest answers a request the forwarder will not proxy and closes the
// connection.
//
// Closing stops the connection being reused; it does not stop the server
// draining what is left of the request body. That drain runs from
// finishRequest whatever the response said, and it reads the original body
// rather than the handler's wrapper, so the only thing bounding it is the
// deadline withBodyIdleTimeout arms on the connection before the handler
// chain runs. Without that deadline a caller that stopped sending holds the
// connection and its goroutine even though the answer already went out.
func rejectRequest(w http.ResponseWriter, msg string, code int) {
w.Header().Set("Connection", "close")
http.Error(w, msg, code)
}
// isLocalCaller reports whether a request may be proxied, from what the
// browser says about where it came from.
func isLocalCaller(r *http.Request) bool {
if foreignFetchSites[strings.ToLower(r.Header.Get(fetchSiteHeader))] {
return false
}
// Only a browser sets Origin, and a legitimate one for this listener is
// same-origin. Anything else is a cross-site caller.
origin := r.Header.Get("Origin")
return origin == "" || isSameOrigin(origin, r.Host)
}
// hostnameOf strips any port from a Host header value.
func hostnameOf(host string) string {
if h, _, err := net.SplitHostPort(host); err == nil {
return h
}
return host
}
// isAllowedHost reports whether a request may arrive under host. Loopback
// names always may and listed names always may. A public listener also
// accepts any address, since a page cannot rebind an address it does not
// control.
func isAllowedHost(host string, loopbackOnly bool, allowed map[string]bool) bool {
if isLocalName(host) || allowed[normalizeHostname(host)] {
return true
}
if loopbackOnly {
return false
}
_, err := netip.ParseAddr(strings.Trim(host, "[]"))
return err == nil
}
// isLocalName reports whether host names this machine's loopback interface.
func isLocalName(host string) bool {
if strings.EqualFold(host, "localhost") {
return true
}
ip, err := netip.ParseAddr(strings.Trim(host, "[]"))
return err == nil && ip.Unmap().IsLoopback()
}
// isSameOrigin reports whether an Origin header names this listener. Scheme,
// host and port all have to match: the forwarder serves plaintext HTTP, and a
// page on another loopback port is a different origin like any other.
func isSameOrigin(origin, host string) bool {
u, err := url.Parse(origin)
if err != nil || u.Scheme != "http" {
return false
}
return normalizeAuthority(u.Host) == normalizeAuthority(host)
}
// normalizeAuthority reduces an authority to a comparable host:port, so two
// spellings of one origin match.
func normalizeAuthority(authority string) string {
host, port, err := net.SplitHostPort(authority)
if err != nil {
host, port = authority, defaultHTTPPort
}
return strings.ToLower(strings.Trim(host, "[]")) + ":" + port
}
// truncatePath shortens a caller-supplied path so a log line cannot be made
// arbitrarily long by the request that produced it.
func truncatePath(path string) string {
if len(path) <= maxLoggedPath {
return path
}
return path[:maxLoggedPath] + "..."
}
client/link/forward_http_test.go
+563
View File
@@ -0,0 +1,563 @@
package link
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// errDialNotExpected marks a test that must never reach the overlay, so a dial
// that does happen fails the test rather than passing silently.
var errDialNotExpected = errors.New("dial not expected in this test")
func refuseDial(context.Context, string, string) (net.Conn, error) {
return nil, errDialNotExpected
}
// overlayDialer stands in for the embedded client. It records what the
// forwarder asked to reach and then connects to a local test server, which
// lets the whole proxy path be exercised without a live network.
type overlayDialer struct {
target string
calls atomic.Int64
asked atomic.Value
}
func (d *overlayDialer) dial(ctx context.Context, network, addr string) (net.Conn, error) {
d.calls.Add(1)
d.asked.Store(addr)
var dialer net.Dialer
return dialer.DialContext(ctx, network, d.target)
}
func (d *overlayDialer) lastAsked() string {
v, _ := d.asked.Load().(string)
return v
}
// startForwarder wires a forwarder in front of upstream and returns its local
// base URL.
func startForwarder(t *testing.T, upstream *httptest.Server) (string, *overlayDialer) {
t.Helper()
upstreamURL, err := url.Parse(upstream.URL)
require.NoError(t, err)
// The spec names a hostname that does not resolve on this machine, proving
// the forwarder hands resolution to the dialer instead of the host.
fwd, err := ParseForward("http://127.0.0.1:0=http://grafana.internal")
require.NoError(t, err)
dialer := &overlayDialer{target: upstreamURL.Host}
f, err := newHTTPForwarder(fwd, dialer.dial)
require.NoError(t, err)
go func() { _ = f.Serve() }()
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = f.Close(ctx)
})
return "http://" + f.Addr(), dialer
}
func TestForwarderProxiesThroughOverlayDialer(t *testing.T) {
var gotHost, gotPath string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotHost, gotPath = r.Host, r.URL.RequestURI()
_, _ = io.WriteString(w, "reached upstream")
}))
defer upstream.Close()
base, dialer := startForwarder(t, upstream)
resp, err := http.Get(base + "/dashboard?orgId=1")
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "reached upstream", string(body))
assert.Equal(t, "/dashboard?orgId=1", gotPath, "path and query should be forwarded unchanged")
assert.Equal(t, "grafana.internal", gotHost,
"upstream must see its own Host, not the local bind address")
assert.Equal(t, int64(1), dialer.calls.Load(), "the overlay dialer should carry the request")
assert.Equal(t, "grafana.internal:80", dialer.lastAsked(),
"the overlay dialer should be asked for the upstream name, so it resolves in the tunnel")
}
func TestForwarderForwardsRequestBodyAndMethod(t *testing.T) {
var gotMethod, gotBody string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotMethod = r.Method
b, _ := io.ReadAll(r.Body)
gotBody = string(b)
w.WriteHeader(http.StatusCreated)
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
resp, err := http.Post(base+"/api", "application/json", strings.NewReader(`{"k":"v"}`))
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusCreated, resp.StatusCode)
assert.Equal(t, http.MethodPost, gotMethod)
assert.Equal(t, `{"k":"v"}`, gotBody, "request bodies should reach the upstream intact")
}
// A forward whose upstream cannot be reached must answer 502 rather than hang
// or leak the dial error to the caller.
func TestForwarderReportsUnreachableUpstream(t *testing.T) {
fwd, err := ParseForward("http://127.0.0.1:0=http://grafana.internal")
require.NoError(t, err)
refuse := func(context.Context, string, string) (net.Conn, error) {
return nil, &net.OpError{Op: "dial", Err: net.UnknownNetworkError("no route in overlay")}
}
f, err := newHTTPForwarder(fwd, refuse)
require.NoError(t, err)
go func() { _ = f.Serve() }()
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = f.Close(ctx)
})
resp, err := http.Get("http://" + f.Addr())
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusBadGateway, resp.StatusCode)
}
// Binding must fail at construction so a port clash surfaces before any
// forward is announced as ready.
func TestNewHTTPForwarderReportsBindConflict(t *testing.T) {
busy, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer busy.Close()
fwd, err := ParseForward("http://" + busy.Addr().String() + "=https://grafana.internal")
require.NoError(t, err)
_, err = newHTTPForwarder(fwd, refuseDial)
require.Error(t, err)
assert.Contains(t, err.Error(), "listen on")
}
// A forwarder that was built but never served still owns its socket. The
// bind-all-then-serve path closes earlier forwarders when a later bind fails,
// so Close has to release the port even though Serve never ran.
func TestCloseReleasesAnUnservedListener(t *testing.T) {
fwd, err := ParseForward("http://127.0.0.1:0=https://grafana.internal")
require.NoError(t, err)
f, err := newHTTPForwarder(fwd, refuseDial)
require.NoError(t, err)
addr := f.Addr()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
require.NoError(t, f.Close(ctx))
// The port is free again only if the listener was actually closed.
reclaimed, err := net.Listen("tcp", addr)
require.NoError(t, err, "Close should have released the listener")
require.NoError(t, reclaimed.Close())
}
// A caller that sends headers and then stalls its body must be cut off.
// ReadHeaderTimeout does not cover the body and IdleTimeout applies only
// between requests, so without the idle read deadline this connection would be
// held open indefinitely.
func TestStalledRequestBodyIsCutOff(t *testing.T) {
previous := bodyIdleTimeout
bodyIdleTimeout = 150 * time.Millisecond
t.Cleanup(func() { bodyIdleTimeout = previous })
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
conn, err := net.Dial("tcp", strings.TrimPrefix(base, "http://"))
require.NoError(t, err)
defer conn.Close()
// Announce a body, send one byte of it, then go quiet.
_, err = conn.Write([]byte("POST / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 1000\r\n\r\nx"))
require.NoError(t, err)
require.NoError(t, conn.SetReadDeadline(time.Now().Add(10*time.Second)))
_, err = io.ReadAll(conn)
// The server closes the connection rather than waiting on the rest of the
// body, so the read ends well before the generous deadline above.
require.NoError(t, err, "the connection should be closed by the server, not time out in the test")
}
func TestRedactSpec(t *testing.T) {
assert.Equal(t, "http://8080=https://user:xxxxx@a.internal",
redactSpec("http://8080=https://user:hunter2@a.internal"),
"a password must not survive into an error message")
assert.Equal(t, "http://8080=https://a.internal",
redactSpec("http://8080=https://a.internal"),
"a spec without credentials should be unchanged")
}
// The error that rejects a credential-bearing upstream quotes the spec, which
// must not carry the password back to the terminal.
func TestParseForwardDoesNotEchoPassword(t *testing.T) {
_, err := ParseForward("http://8080=https://user:hunter2@a.internal")
require.Error(t, err)
assert.NotContains(t, err.Error(), "hunter2", "the password must not appear anywhere in the error")
assert.Contains(t, err.Error(), "must not carry credentials")
}
// A POST whose body is fully delivered must still receive its whole response,
// even when the upstream takes longer to answer than the body idle timeout.
// The deadline that guards the body must not outlive it.
func TestCompletedUploadGetsFullResponse(t *testing.T) {
previous := bodyIdleTimeout
bodyIdleTimeout = 200 * time.Millisecond
t.Cleanup(func() { bodyIdleTimeout = previous })
payload := strings.Repeat("y", 4096)
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
time.Sleep(600 * time.Millisecond)
_, _ = io.WriteString(w, payload)
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
resp, err := http.Post(base+"/upload", "text/plain", strings.NewReader(strings.Repeat("x", 2048)))
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err, "the response must not be cut off once the body was fully read")
assert.Equal(t, len(payload), len(body), "the whole response body should arrive")
}
// A page that points its own hostname at this loopback listener must not be
// able to reach the upstream through it.
func TestLoopbackForwarderRejectsForeignHost(t *testing.T) {
var reached bool
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reached = true
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
req, err := http.NewRequest(http.MethodGet, base, nil)
require.NoError(t, err)
req.Host = "attacker.example"
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusMisdirectedRequest, resp.StatusCode)
assert.False(t, reached, "the upstream must not be reached under a foreign Host")
}
func TestLoopbackForwarderRejectsCrossOrigin(t *testing.T) {
var reached bool
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reached = true
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
req, err := http.NewRequest(http.MethodGet, base, nil)
require.NoError(t, err)
req.Header.Set("Origin", "https://attacker.example")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
assert.False(t, reached, "the upstream must not be reached from a cross-site page")
}
// The ordinary local callers must keep working: no Origin at all, and a
// same-origin browser request.
func TestLoopbackForwarderAllowsLocalCallers(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, "ok")
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
for _, origin := range []string{"", "http://" + strings.TrimPrefix(base, "http://")} {
req, err := http.NewRequest(http.MethodGet, base, nil)
require.NoError(t, err)
if origin != "" {
req.Header.Set("Origin", origin)
}
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
_ = resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode, "origin %q should be allowed", origin)
assert.Equal(t, "ok", string(body))
}
}
// A malformed upstream fails in the URL parser rather than in the credential
// check, so both error layers have to be redacted.
func TestParseForwardRedactsMalformedUpstream(t *testing.T) {
for _, spec := range []string{
"http://8080=https://user:hunter2@a.internal:bad",
"http://8080=https://user:hunter2@a.internal/%zz",
} {
_, err := ParseForward(spec)
require.Error(t, err)
assert.NotContains(t, err.Error(), "hunter2",
"a password must not survive a parse failure either")
}
}
// An embedded <img> or <script> aimed straight at the loopback address carries
// a loopback Host and no Origin, so Sec-Fetch-Site is the only signal that the
// request came from a page.
func TestLoopbackForwarderRejectsCrossSiteFetch(t *testing.T) {
for _, site := range []string{"cross-site", "same-site", "Cross-Site"} {
var reached bool
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reached = true
}))
base, _ := startForwarder(t, upstream)
req, err := http.NewRequest(http.MethodGet, base, nil)
require.NoError(t, err)
req.Header.Set("Sec-Fetch-Site", site)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
_ = resp.Body.Close()
upstream.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode, "Sec-Fetch-Site %q should be refused", site)
assert.False(t, reached, "the upstream must not be reached from a %s page", site)
}
}
// A browser reaches the listener legitimately when the user navigated to it or
// when the page it serves calls back into it.
func TestLoopbackForwarderAllowsFirstPartyFetch(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, "ok")
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
for _, site := range []string{"none", "same-origin"} {
req, err := http.NewRequest(http.MethodGet, base, nil)
require.NoError(t, err)
req.Header.Set("Sec-Fetch-Site", site)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
_ = resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode, "Sec-Fetch-Site %q should be allowed", site)
}
}
// A page served from another loopback port is a different origin, however
// local it is.
func TestLoopbackForwarderRejectsAnotherLocalPort(t *testing.T) {
var reached bool
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reached = true
}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
req, err := http.NewRequest(http.MethodGet, base, nil)
require.NoError(t, err)
req.Header.Set("Origin", "http://localhost:3000")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusForbidden, resp.StatusCode)
assert.False(t, reached, "a page on another local port must not reach the upstream")
}
// A rejected request is never read, so the deadline that bounds an accepted
// upload never arms for it. Closing the connection is what stops the server
// from draining a body the caller stopped sending.
func TestRejectedRequestClosesTheConnection(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
defer upstream.Close()
base, _ := startForwarder(t, upstream)
body, writer := io.Pipe()
t.Cleanup(func() { _ = writer.Close() })
req, err := http.NewRequest(http.MethodPost, base, body)
require.NoError(t, err)
req.Host = "attacker.example"
// The request is answered on another goroutine, so a server that waits on
// the stalled body fails the test by timing out rather than hanging it.
type rejection struct {
status int
closesConn bool
}
done := make(chan rejection, 1)
go func() {
resp, err := http.DefaultClient.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
done <- rejection{status: resp.StatusCode, closesConn: resp.Close}
}()
select {
case got := <-done:
assert.Equal(t, http.StatusMisdirectedRequest, got.status)
assert.True(t, got.closesConn, "the server must close a connection it rejected rather than drain the body")
case <-time.After(5 * time.Second):
t.Fatal("the rejected request was not answered while its body stalled")
}
}
// net/url ends the userinfo at the last '@', so a password containing one must
// be redacted whole rather than up to its first '@'.
func TestRedactSpecCoversPasswordWithAtSign(t *testing.T) {
redacted := redactSpec("http://8080=https://user:pa@ss@a.internal/%zz")
assert.Equal(t, "http://8080=https://user:xxxxx@a.internal/%zz", redacted)
assert.NotContains(t, redacted, "ss@a.internal", "no part of the password may survive")
}
func TestParseForwardRedactsPasswordWithAtSign(t *testing.T) {
_, err := ParseForward("http://8080=https://user:pa@ss@a.internal/%zz")
require.Error(t, err)
assert.NotContains(t, err.Error(), "ss@", "no part of the password may reach the error")
}
// stalledBodyProbe sends a request whose declared body is only partly
// delivered and then stops, as a caller holding a connection open would. It
// returns the status line, or an empty string when nothing came back inside
// wait.
//
// It speaks raw HTTP rather than using the client, because the shape that
// matters is an explicit Content-Length the sender never satisfies: a body
// handed to http.Client is sent chunked, which exercises a different path in
// the server's post-handler drain.
func stalledBodyProbe(t *testing.T, addr, host string, wait time.Duration) (status string, released bool) {
t.Helper()
conn, err := net.Dial("tcp", addr)
require.NoError(t, err, "connect to the forwarder")
t.Cleanup(func() { _ = conn.Close() })
req := fmt.Sprintf("POST / HTTP/1.1\r\nHost: %s\r\nContent-Length: 100\r\n\r\n0123456789", host)
_, err = conn.Write([]byte(req))
require.NoError(t, err, "send the partial request")
require.NoError(t, conn.SetReadDeadline(time.Now().Add(wait)))
buf := make([]byte, 1024)
n, err := conn.Read(buf)
if err != nil {
return "", false
}
status = strings.SplitN(string(buf[:n]), "\r\n", 2)[0]
// Read on until the connection ends. The answer going out is not the same
// thing as the connection being let go: the server drains the rest of the
// body afterwards, and that drain is where a stalled caller actually holds
// a descriptor and a goroutine. A timeout here means it is still held.
for {
if _, err := conn.Read(buf); err != nil {
var timeout net.Error
return status, !(errors.As(err, &timeout) && timeout.Timeout())
}
}
}
// A caller that declares a body and then stops sending must still be answered,
// on every path that does not read it. The server drains an unread body before
// reusing the connection, and that drain has no deadline of its own: whatever
// the forwarder leaves on the connection is what bounds it.
//
// Without this, a few thousand such connections exhaust the forwarder's file
// descriptors while each one holds a goroutine parked in that drain.
func TestStalledRequestBodyIsAnsweredOnEveryPath(t *testing.T) {
// The dial target refuses, so a request that passes the guard reaches the
// proxy's error handler instead of an upstream.
// Shortened so the test observes the bound rather than waiting out the
// production one.
previous := bodyIdleTimeout
bodyIdleTimeout = time.Second
t.Cleanup(func() { bodyIdleTimeout = previous })
base := serveForward(t, "http://127.0.0.1:0=http://grafana.internal", "127.0.0.1:1")
addr := strings.TrimPrefix(base, "http://")
cases := []struct {
name string
host string
want string
}{
{
name: "the guard rejects it",
host: "attacker.example",
want: "HTTP/1.1 421 Misdirected Request",
},
{
name: "the overlay dial fails",
host: "127.0.0.1",
want: "HTTP/1.1 502 Bad Gateway",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, released := stalledBodyProbe(t, addr, tc.host, 8*time.Second)
assert.Equal(t, tc.want, got,
"the caller must be answered rather than left waiting out a drain with no deadline")
assert.True(t, released,
"the connection must be let go as well as answered, or a stalled caller still holds it")
})
}
}
client/link/forward_http_upstream_test.go
+286
View File
@@ -0,0 +1,286 @@
package link
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// serveForward starts a forwarder for spec in front of target and returns its
// local base URL.
func serveForward(t *testing.T, spec, target string) string {
t.Helper()
fwd, err := ParseForward(spec)
require.NoError(t, err)
dialer := &overlayDialer{target: target}
f, err := newHTTPForwarder(fwd, dialer.dial)
require.NoError(t, err)
go func() { _ = f.Serve() }()
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = f.Close(ctx)
})
return "http://" + f.Addr()
}
func TestForwarderVerifiesUpstreamCertificate(t *testing.T) {
var reached atomic.Bool
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
reached.Store(true)
_, _ = io.WriteString(w, "should not be served")
}))
defer upstream.Close()
// The test server's certificate is signed by a CA no trust store holds, so
// a forwarder that verifies its upstream must refuse it.
u, err := url.Parse(upstream.URL)
require.NoError(t, err)
base := serveForward(t, "http://127.0.0.1:0=https://grafana.internal", u.Host)
resp, err := http.Get(base + "/")
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, http.StatusBadGateway, resp.StatusCode, "an untrusted upstream certificate must fail the request")
assert.Contains(t, string(body), "upstream unreachable over the overlay", "the 502 must be the forwarder's own answer")
assert.False(t, reached.Load(), "no request may reach an upstream whose certificate was not verified")
}
func TestForwarderJoinsUpstreamBasePath(t *testing.T) {
var gotURI string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotURI = r.URL.RequestURI()
}))
defer upstream.Close()
u, err := url.Parse(upstream.URL)
require.NoError(t, err)
base := serveForward(t, "http://127.0.0.1:0=http://grafana.internal/api", u.Host)
resp, err := http.Get(base + "/v1/query?q=up")
require.NoError(t, err)
_ = resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "/api/v1/query?q=up", gotURI, "the upstream's path must prefix the request path")
}
// An LLM reply streamed as server-sent events must reach the caller event by
// event. A forwarder that buffered the body would hold the first event until
// the upstream finished, which here never happens until the test has read it.
func TestForwarderStreamsEventsAsTheyArrive(t *testing.T) {
release := make(chan struct{})
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: first\n\n")
w.(http.Flusher).Flush()
<-release
_, _ = io.WriteString(w, "data: second\n\n")
}))
defer upstream.Close()
defer close(release)
u, err := url.Parse(upstream.URL)
require.NoError(t, err)
base := serveForward(t, "http://127.0.0.1:0=http://llm.internal", u.Host)
resp, err := http.Get(base + "/v1/chat/completions")
require.NoError(t, err)
defer resp.Body.Close()
firstLine := make(chan string, 1)
go func(body io.Reader) {
line, _ := bufio.NewReader(body).ReadString('\n')
firstLine <- line
}(resp.Body)
select {
case line := <-firstLine:
assert.Equal(t, "data: first\n", line, "the first event must arrive before the stream ends")
case <-time.After(5 * time.Second):
t.Fatal("the first event was buffered instead of streamed")
}
}
func TestForwarderServesConcurrentRequests(t *testing.T) {
var served atomic.Int64
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
served.Add(1)
_, _ = io.WriteString(w, r.URL.Query().Get("n"))
}))
defer upstream.Close()
u, err := url.Parse(upstream.URL)
require.NoError(t, err)
base := serveForward(t, "http://127.0.0.1:0=http://grafana.internal", u.Host)
const callers = 32
var wg sync.WaitGroup
errs := make(chan error, callers)
for i := range callers {
wg.Add(1)
go func(n int) {
defer wg.Done()
resp, err := http.Get(fmt.Sprintf("%s/?n=%d", base, n))
if err != nil {
errs <- err
return
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
errs <- err
return
}
if got := string(body); got != fmt.Sprint(n) {
errs <- fmt.Errorf("caller %d got the answer for %q", n, got)
}
}(i)
}
wg.Wait()
close(errs)
for err := range errs {
assert.NoError(t, err, "every concurrent caller must get its own answer")
}
assert.Equal(t, int64(callers), served.Load(), "every request must reach the upstream once")
}
// A container port published to the host is a public listener, and a page
// that rebinds its own name to that port sends a matching Origin and
// Sec-Fetch-Site. Host is the only signal that tells it apart, so a public
// listener accepts addresses and listed names only.
func TestGuardOnPublicListener(t *testing.T) {
next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
allowed := []string{"nblink.lan"}
cases := []struct {
name string
loopbackOnly bool
host string
headers map[string]string
want int
}{
{name: "public listener under a listed name", host: "nblink.lan:8080", want: http.StatusOK},
{name: "public listener under a listed name, other spelling", host: "NBLINK.LAN.:8080", want: http.StatusOK},
{name: "public listener under its LAN address", host: "192.168.1.20:8080", want: http.StatusOK},
{name: "public listener under an IPv6 address", host: "[fd00::20]:8080", want: http.StatusOK},
{name: "public listener under localhost", host: "localhost:8080", want: http.StatusOK},
{name: "public listener under an unlisted name", host: "printer.lan:8080", want: http.StatusMisdirectedRequest},
{
name: "public listener, rebound page", host: "attacker.example:8080",
headers: map[string]string{"Origin": "http://attacker.example:8080", fetchSiteHeader: "same-origin"},
want: http.StatusMisdirectedRequest,
},
{name: "loopback listener under a listed name", loopbackOnly: true, host: "nblink.lan:8080", want: http.StatusOK},
{name: "loopback listener under a LAN address", loopbackOnly: true, host: "192.168.1.20:8080", want: http.StatusMisdirectedRequest},
{
name: "public listener, cross-origin fetch", host: "nblink.lan:8080",
headers: map[string]string{"Origin": "https://attacker.example"}, want: http.StatusForbidden,
},
{
name: "public listener, cross-site no-cors GET", host: "nblink.lan:8080",
headers: map[string]string{fetchSiteHeader: "cross-site"}, want: http.StatusForbidden,
},
{
name: "public listener, same-origin page", host: "nblink.lan:8080",
headers: map[string]string{"Origin": "http://nblink.lan:8080"}, want: http.StatusOK,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://"+tc.host+"/", nil)
for k, v := range tc.headers {
req.Header.Set(k, v)
}
rec := httptest.NewRecorder()
guardRebinding(next, tc.loopbackOnly, allowed).ServeHTTP(rec, req)
assert.Equal(t, tc.want, rec.Code, "%s must be answered %d", tc.name, tc.want)
})
}
}
func TestListenNetwork(t *testing.T) {
cases := map[string]string{
"0.0.0.0:8080": "tcp4",
"127.0.0.1:8080": "tcp4",
"[::]:8080": "tcp6",
"[::1]:8080": "tcp6",
"[fe80::1%en0]:808": "tcp6",
}
for addr, want := range cases {
assert.Equal(t, want, listenNetwork(addr), "%s must bind %s only", addr, want)
}
}
// 0.0.0.0 asks for every IPv4 interface. Go's plain "tcp" would also accept
// IPv6, which the operator did not write.
func TestIPv4WildcardDoesNotListenOnIPv6(t *testing.T) {
probe, err := net.Listen("tcp6", "[::1]:0")
if err != nil {
t.Skip("no IPv6 loopback on this host")
}
_ = probe.Close()
fwd, err := ParseForward("http://0.0.0.0:0=http://grafana.internal")
require.NoError(t, err)
f, err := newHTTPForwarder(fwd, refuseDial)
require.NoError(t, err)
t.Cleanup(func() { _ = f.Close(context.Background()) })
_, port, err := net.SplitHostPort(f.Addr())
require.NoError(t, err)
assert.Equal(t, "0.0.0.0:"+port, f.Addr(), "the listener must report the IPv4 wildcard it was asked for")
conn, err := net.DialTimeout("tcp6", net.JoinHostPort("::1", port), time.Second)
if err == nil {
_ = conn.Close()
}
assert.Error(t, err, "an IPv4 wildcard must not accept IPv6 connections")
}
func TestBindHint(t *testing.T) {
denied := &net.OpError{Op: "listen", Err: os.NewSyscallError("bind", syscall.EACCES)}
err := bindHint("0.0.0.0:80", denied)
assert.ErrorIs(t, err, os.ErrPermission, "the hint must keep the original error")
assert.Contains(t, err.Error(), "needs a port above 1023", "a permission error must suggest an unprivileged port")
inUse := errors.New("address already in use")
assert.Equal(t, inUse, bindHint("127.0.0.1:8080", inUse), "other errors must pass through unchanged")
}
func TestForwardStringRoundTrips(t *testing.T) {
fwd, err := ParseForward("http://8080=https://grafana.internal/d")
require.NoError(t, err)
assert.Equal(t, "http://127.0.0.1:8080=https://grafana.internal/d", fwd.String(),
"String must print the normalized spec")
again, err := ParseForward(fwd.String())
require.NoError(t, err)
assert.Equal(t, fwd.Listen, again.Listen, "the printed spec must parse back to the same listener")
assert.Equal(t, fwd.Upstream.String(), again.Upstream.String(), "and to the same upstream")
assert.False(t, strings.Contains(fwd.String(), " "), "the spec must not carry stray whitespace")
}
client/link/forward_test.go
+170
View File
@@ -0,0 +1,170 @@
package link
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseForward(t *testing.T) {
tests := []struct {
name string
spec string
wantListen string
wantUpstream string
}{
{
name: "full listen address",
spec: "http://127.0.0.1:8080=https://grafana.internal",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "bare port defaults to loopback",
spec: "http://8080=https://grafana.internal",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "empty host defaults to loopback",
spec: "http://:8080=https://grafana.internal",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "upstream without scheme assumes https",
spec: "http://8080=grafana.internal",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "plain http upstream is preserved",
spec: "http://8080=http://grafana.internal:3000",
wantListen: "127.0.0.1:8080",
wantUpstream: "http://grafana.internal:3000",
},
{
name: "upstream query string survives the separator split",
spec: "http://8080=https://grafana.internal/d?orgId=1",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal/d?orgId=1",
},
{
name: "explicit public bind is parsed, gating happens later",
spec: "http://0.0.0.0:8080=https://grafana.internal",
wantListen: "0.0.0.0:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "ipv6 listen address",
spec: "http://[::1]:8080=https://grafana.internal",
wantListen: "[::1]:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "port zero asks the OS for a free port",
spec: "http://0=https://grafana.internal",
wantListen: "127.0.0.1:0",
wantUpstream: "https://grafana.internal",
},
{
name: "localhost normalizes to a literal address",
spec: "http://localhost:8080=https://grafana.internal",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal",
},
{
// A zone is part of the literal and is resolved the same way at
// the check and at the bind, and link-local cannot be bound
// without it.
name: "zone scoped ipv6 listen address",
spec: "http://[fe80::1%eth0]:8080=https://grafana.internal",
wantListen: "[fe80::1%eth0]:8080",
wantUpstream: "https://grafana.internal",
},
{
name: "a scheme inside the query is not mistaken for the upstream scheme",
spec: "http://8080=grafana.internal/login?next=https://grafana.internal/d",
wantListen: "127.0.0.1:8080",
wantUpstream: "https://grafana.internal/login?next=https://grafana.internal/d",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := ParseForward(tc.spec)
require.NoError(t, err)
assert.Equal(t, ProtoHTTP, got.Proto, "scheme should select the HTTP forwarder")
assert.Equal(t, tc.wantListen, got.Listen, "listen address should be normalized")
assert.Equal(t, tc.wantUpstream, got.Upstream.String(), "upstream should round-trip")
})
}
}
func TestParseForwardErrors(t *testing.T) {
tests := []struct {
name string
spec string
wantMsg string
}{
{name: "empty", spec: "", wantMsg: "empty forward spec"},
{name: "no scheme", spec: "8080=grafana.internal", wantMsg: "missing scheme"},
{name: "no separator", spec: "http://8080", wantMsg: "missing '='"},
{name: "unknown scheme", spec: "gopher://8080=grafana.internal", wantMsg: "unknown scheme"},
{name: "port out of range", spec: "http://70000=grafana.internal", wantMsg: "out of range"},
{name: "port not a number", spec: "http://http=grafana.internal", wantMsg: "not a number"},
{name: "missing upstream", spec: "http://8080=", wantMsg: "missing upstream"},
{name: "bad upstream scheme", spec: "http://8080=ftp://grafana.internal", wantMsg: "must be http or https"},
{name: "listen host is a name", spec: "http://grafana.internal:8080=https://a.internal", wantMsg: "must be an IP address or localhost"},
{name: "upstream carries credentials", spec: "http://8080=https://user:pass@grafana.internal", wantMsg: "must not carry credentials"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := ParseForward(tc.spec)
require.Error(t, err)
assert.Contains(t, err.Error(), tc.wantMsg, "error should name the actual problem")
})
}
}
// Schemes the grammar reserves for later must fail with a message that says so,
// rather than looking like a typo.
func TestParseForwardPlannedSchemes(t *testing.T) {
for _, proto := range []string{"tcp", "udp", "socks5"} {
t.Run(proto, func(t *testing.T) {
_, err := ParseForward(proto + "://5432=db.internal:5432")
require.Error(t, err)
assert.Contains(t, err.Error(), "not supported yet",
"a reserved scheme should report that it is unimplemented")
})
}
}
func TestIsLoopback(t *testing.T) {
tests := []struct {
addr string
want bool
}{
{"127.0.0.1:8080", true},
{"127.0.0.53:8080", true},
{"[::1]:8080", true},
{"0.0.0.0:8080", false},
{"192.168.1.10:8080", false},
{"[::]:8080", false},
{"not-an-address", false},
// A v4-mapped form is still the loopback address.
{"[::ffff:127.0.0.1]:8080", true},
// Link-local is not loopback, so it stays behind the opt-in.
{"[fe80::1%eth0]:8080", false},
// Names never reach this check, because parseListen rejects them.
{"localhost:8080", false},
}
for _, tc := range tests {
t.Run(tc.addr, func(t *testing.T) {
assert.Equal(t, tc.want, isLoopback(tc.addr))
})
}
}
client/link/link.go
+253
View File
@@ -0,0 +1,253 @@
package link
import (
"context"
"errors"
"fmt"
"io"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
log "github.com/sirupsen/logrus"
embed "github.com/netbirdio/netbird/client/embed"
)
const (
// startTimeout bounds the wait for the overlay session to come up.
startTimeout = 90 * time.Second
// shutdownTimeout bounds draining in-flight requests and closing the session.
shutdownTimeout = 10 * time.Second
// natMapperEnv is the embedded client's switch for UPnP, NAT-PMP and PCP
// port mapping on the local router.
natMapperEnv = "NB_DISABLE_NAT_MAPPER"
)
// Run starts the overlay session, serves every configured forward, and blocks
// until the process is signalled or a forward fails.
func Run(ctx context.Context, cfg *Config) error {
if cfg.Check {
return printEffectiveConfig(os.Stdout, cfg)
}
creds, err := resolveCredentials(ctx, cfg)
if err != nil {
return interruptedOr(ctx, err)
}
if err := disableNATMapperUnlessSet(); err != nil {
return err
}
client, err := embed.New(embed.Options{
DeviceName: cfg.Hostname,
SetupKey: creds.setupKey,
JWTToken: creds.jwtToken,
ManagementURL: cfg.ManagementURL,
ConfigPath: cfg.ConfigPath(),
StatePath: cfg.StatePath(),
LogLevel: cfg.LogLevel,
// nblink only dials out. Refusing inbound connections and host network
// access keeps the peer from becoming a route into the machine it runs on.
BlockInbound: true,
BlockLANAccess: true,
})
if err != nil {
return fmt.Errorf("create client: %w", err)
}
log.Infof("connecting to %s", cfg.ManagementURL)
if err := startSession(ctx, client.Start, func() { stopClient(client) }); err != nil {
return interruptedOr(ctx, err)
}
logSession(client)
forwards, err := startForwards(cfg, client.DialContext)
if err != nil {
stopClient(client)
return err
}
err = waitForShutdown(ctx, forwards)
shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancelShutdown()
closeForwards(shutdownCtx, forwards)
stopClient(client)
return err
}
// disableNATMapperUnlessSet keeps the embedded client from asking the local
// router for a port mapping. An unprivileged forwarder should not change the
// network it runs on, and ICE and the relay connect without it. An operator
// who sets the variable keeps their choice.
func disableNATMapperUnlessSet() error {
// An operator keeps their choice only when the value is one the consumer
// can read. Empty and unparseable values reach the consumer as false,
// which would enable the mapper rather than leave the default in place.
if val := os.Getenv(natMapperEnv); val != "" {
if _, err := strconv.ParseBool(val); err == nil {
return nil
}
log.Warnf("failed to parse %s=%q, disabling the NAT mapper", natMapperEnv, val)
}
if err := os.Setenv(natMapperEnv, "true"); err != nil {
return fmt.Errorf("set %s: %w", natMapperEnv, err)
}
return nil
}
// startSession runs start under startTimeout and returns as soon as ctx ends,
// even if start has not.
//
// The management dial inside the embedded client does not observe
// cancellation, so waiting for start to return would hold a signal during
// startup until the dial times out. A start that completes after the caller
// gave up is stopped in the background instead of left running.
func startSession(ctx context.Context, start func(context.Context) error, stop func()) error {
startCtx, cancel := context.WithTimeout(ctx, startTimeout)
done := make(chan error, 1)
go func() {
defer cancel()
done <- start(startCtx)
}()
select {
case err := <-done:
if err != nil {
return fmt.Errorf("start client: %w", err)
}
return nil
case <-ctx.Done():
go func() {
if err := <-done; err == nil {
stop()
}
}()
return ctx.Err()
}
}
// interruptedOr reports a signal during startup or login as a clean shutdown
// rather than a failure, and passes any other error through.
func interruptedOr(ctx context.Context, err error) error {
select {
case <-ctx.Done():
log.Infof("interrupted before the session was up, shutting down")
return nil
default:
return err
}
}
// startForwards binds every forward before serving any of them, so a clash on
// the last one does not leave the earlier ones running.
func startForwards(cfg *Config, dial DialFunc) ([]*httpForwarder, error) {
forwards := make([]*httpForwarder, 0, len(cfg.Forwards))
for _, fwd := range cfg.Forwards {
f, err := newHTTPForwarder(fwd, dial)
if err != nil {
closeForwards(context.Background(), forwards)
return nil, err
}
forwards = append(forwards, f)
}
for _, f := range forwards {
log.Infof("forwarding http://%s to %s over the overlay", f.Addr(), f.forward.Upstream)
}
return forwards, nil
}
// waitForShutdown blocks until a signal arrives, the context ends, or a
// forward stops serving.
func waitForShutdown(ctx context.Context, forwards []*httpForwarder) error {
serveErr := make(chan error, len(forwards))
for _, f := range forwards {
go func(f *httpForwarder) {
if err := f.Serve(); err != nil {
serveErr <- err
}
}(f)
}
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(stop)
select {
case err := <-serveErr:
return err
case sig := <-stop:
log.Infof("received %s, shutting down", sig)
return nil
case <-ctx.Done():
// The caller's context also ends on a signal, and select picks either
// ready case, so this path reports the shutdown too.
log.Infof("shutting down")
return nil
}
}
func closeForwards(ctx context.Context, forwards []*httpForwarder) {
for _, f := range forwards {
if err := f.Close(ctx); err != nil && !errors.Is(err, context.Canceled) {
log.Warnf("close forward %s: %v", f.forward.Listen, err)
}
}
}
func stopClient(client *embed.Client) {
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
if err := client.Stop(ctx); err != nil && !errors.Is(err, embed.ErrClientNotStarted) {
log.Warnf("stop client: %v", err)
}
}
// logSession reports control-plane state once the session is up, so a failure
// to reach an upstream later can be told apart from a session that never
// connected. The peer name and address stay at debug level, where the rest of
// the client keeps identifying details.
func logSession(client *embed.Client) {
status, err := client.Status()
if err != nil {
log.Infof("connected (status unavailable: %v)", err)
return
}
log.Infof("connected, management %t, signal %t, %d peers",
status.ManagementState.Connected, status.SignalState.Connected, len(status.Peers))
log.Debugf("peer %s has address %s", status.LocalPeerState.FQDN, status.LocalPeerState.IP)
}
// printEffectiveConfig writes the parsed forwards and exits without touching
// the network, so a container configuration can be checked before deploying it.
func printEffectiveConfig(out io.Writer, cfg *Config) error {
var b strings.Builder
fmt.Fprintf(&b, "management-url: %s\n", cfg.ManagementURL)
fmt.Fprintf(&b, "state-dir: %s\n", orDefault(cfg.StateDir, "(memory)"))
fmt.Fprintf(&b, "hostname: %s\n", orDefault(cfg.Hostname, "(host default)"))
fmt.Fprintf(&b, "setup-key: %t\n", cfg.SetupKey != "")
fmt.Fprintf(&b, "allowed-hosts: %s\n", orDefault(strings.Join(cfg.AllowedHosts, ","), "(none)"))
fmt.Fprintf(&b, "forwards: %d\n", len(cfg.Forwards))
for _, f := range cfg.Forwards {
fmt.Fprintf(&b, " %s -> %s\n", f.Listen, f.Upstream)
}
if _, err := io.WriteString(out, b.String()); err != nil {
return fmt.Errorf("write configuration: %w", err)
}
return nil
}
func orDefault(value, fallback string) string {
if value == "" {
return fallback
}
return value
}
client/link/link_test.go
+440
View File
@@ -0,0 +1,440 @@
package link
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// freePort asks the OS for a port and releases it, so the caller can name that
// port in a spec and still be the one that binds it.
func freePort(t *testing.T) string {
t.Helper()
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
_, port, err := net.SplitHostPort(l.Addr().String())
require.NoError(t, err)
require.NoError(t, l.Close())
return port
}
// mustParse builds a Forward from a spec, failing the test if the spec is bad.
func mustParse(t *testing.T, spec string) Forward {
t.Helper()
fwd, err := ParseForward(spec)
require.NoError(t, err, "spec %q must parse", spec)
return fwd
}
// One session serves every forward it was given, which is the whole point of
// the flag repeating: a user logs in once and reaches several services.
func TestStartForwardsServesEveryForward(t *testing.T) {
first := mustParse(t, "http://127.0.0.1:0=https://a.internal")
second := mustParse(t, "http://127.0.0.1:0=https://b.internal")
forwards, err := startForwards(&Config{Forwards: []Forward{first, second}}, refuseDial)
require.NoError(t, err)
t.Cleanup(func() { closeForwards(context.Background(), forwards) })
require.Len(t, forwards, 2, "both forwards should be bound")
for _, f := range forwards {
go func() { _ = f.Serve() }()
}
// Each listener answers on its own address. The dialer refuses, so a 502
// is the proof the request reached that forwarder and was routed onward,
// rather than the port belonging to something else.
for _, f := range forwards {
resp, rerr := http.Get("http://" + f.Addr())
require.NoError(t, rerr, "forward on %s must accept a request", f.Addr())
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
assert.Equal(t, http.StatusBadGateway, resp.StatusCode,
"a forward whose dialer refuses should answer 502, not fail to serve")
}
}
// Ports are bound before any forward is announced, so a clash on the last spec
// must not leave the earlier ones holding their sockets. The observable is the
// port: it has to be free again once startForwards has returned the error.
func TestStartForwardsReleasesEarlierPortsOnFailure(t *testing.T) {
firstPort := freePort(t)
clashPort := freePort(t)
// Something else already holds the second forward's port.
blocker, err := net.Listen("tcp", "127.0.0.1:"+clashPort)
require.NoError(t, err)
defer func() { _ = blocker.Close() }()
forwards, err := startForwards(&Config{Forwards: []Forward{
mustParse(t, "http://127.0.0.1:"+firstPort+"=https://a.internal"),
mustParse(t, "http://127.0.0.1:"+clashPort+"=https://b.internal"),
}}, refuseDial)
require.Error(t, err, "a clash on the second forward must fail the whole set")
assert.Nil(t, forwards, "no forwarder should be returned alongside the error")
assert.Contains(t, err.Error(), clashPort, "the error should name the port that clashed")
// The first forward's port is free again, which it would not be if its
// listener had been left open.
reclaimed, err := net.Listen("tcp", "127.0.0.1:"+firstPort)
require.NoError(t, err, "the first forward's port must be released when a later bind fails")
_ = reclaimed.Close()
}
// closeForwards releases every listener it is given, so a shutdown does not
// depend on the process exiting.
func TestCloseForwardsReleasesEveryPort(t *testing.T) {
port := freePort(t)
forwards, err := startForwards(&Config{Forwards: []Forward{
mustParse(t, "http://127.0.0.1:"+port+"=https://a.internal"),
}}, refuseDial)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
closeForwards(ctx, forwards)
reclaimed, err := net.Listen("tcp", "127.0.0.1:"+port)
require.NoError(t, err, "the port must be free once the forward is closed")
_ = reclaimed.Close()
}
// captureStdout runs fn with os.Stdout redirected and returns what was written.
func captureStdout(t *testing.T, fn func()) string {
t.Helper()
r, w, err := os.Pipe()
require.NoError(t, err)
original := os.Stdout
os.Stdout = w
done := make(chan string, 1)
go func() {
var sb strings.Builder
_, _ = io.Copy(&sb, r)
done <- sb.String()
}()
fn()
os.Stdout = original
require.NoError(t, w.Close())
out := <-done
_ = r.Close()
return out
}
// --check is what an operator runs to validate a container's configuration
// before deploying it, so it has to report the forwards it would serve and
// return without touching the network.
func TestCheckReportsTheEffectiveForwards(t *testing.T) {
cfg := &Config{
Check: true,
ManagementURL: "https://mgmt.example:443",
Hostname: "nblink-1",
Forwards: []Forward{
mustParse(t, "http://8080=https://grafana.internal"),
mustParse(t, "http://127.0.0.1:9090=https://prometheus.internal"),
},
}
var err error
out := captureStdout(t, func() { err = Run(context.Background(), cfg) })
require.NoError(t, err, "--check must succeed on a valid configuration")
assert.Contains(t, out, "https://mgmt.example:443")
assert.Contains(t, out, "nblink-1")
assert.Contains(t, out, "forwards: 2")
assert.Contains(t, out, "127.0.0.1:8080 -> https://grafana.internal",
"the listen address must be reported as resolved, not as it was typed")
assert.Contains(t, out, "127.0.0.1:9090 -> https://prometheus.internal")
assert.Contains(t, out, "state-dir: (memory)", "an unset state dir must be named, not left blank")
}
// --check reports only whether a setup key is configured. Printing the key
// itself would put a credential on the terminal and into whatever captured it,
// which is the one thing this output must never do.
func TestCheckDoesNotPrintTheSetupKey(t *testing.T) {
cfg := &Config{
Check: true,
ManagementURL: "https://mgmt.example:443",
SetupKey: "EEE7B5B1-1F8C-4C9A-9E2D-0123456789AB",
Forwards: []Forward{mustParse(t, "http://8080=https://grafana.internal")},
}
var err error
out := captureStdout(t, func() { err = Run(context.Background(), cfg) })
require.NoError(t, err)
assert.NotContains(t, out, cfg.SetupKey, "the setup key must never be printed")
assert.Contains(t, out, "setup-key: true", "its presence should still be reported")
}
// namedDialer routes each overlay name to its own local server, so a test can
// tell which upstream a request actually reached.
type namedDialer struct {
mu sync.Mutex
targets map[string]string
asked []string
}
func (d *namedDialer) dial(ctx context.Context, network, addr string) (net.Conn, error) {
d.mu.Lock()
d.asked = append(d.asked, addr)
target, ok := d.targets[addr]
d.mu.Unlock()
if !ok {
return nil, fmt.Errorf("no route to %s", addr)
}
var dialer net.Dialer
return dialer.DialContext(ctx, network, target)
}
func namedUpstream(t *testing.T, name string) string {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.WriteString(w, name)
}))
t.Cleanup(srv.Close)
u, err := url.Parse(srv.URL)
require.NoError(t, err)
return u.Host
}
// Every forward shares the one overlay dialer and asks it for its own
// upstream, so two specs reach two different services rather than the same one.
func TestStartForwardsRouteEachToItsOwnUpstream(t *testing.T) {
dialer := &namedDialer{targets: map[string]string{
"grafana.internal:80": namedUpstream(t, "grafana"),
"prometheus.internal:80": namedUpstream(t, "prometheus"),
}}
cfg := &Config{}
for _, spec := range []string{
"http://127.0.0.1:0=http://grafana.internal",
"http://127.0.0.1:0=http://prometheus.internal",
} {
fwd, err := ParseForward(spec)
require.NoError(t, err)
cfg.Forwards = append(cfg.Forwards, fwd)
}
forwards, err := startForwards(cfg, dialer.dial)
require.NoError(t, err)
require.Len(t, forwards, 2, "every configured forward must be started")
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
closeForwards(ctx, forwards)
})
for _, f := range forwards {
go func(f *httpForwarder) { _ = f.Serve() }(f)
}
require.NotEqual(t, forwards[0].Addr(), forwards[1].Addr(), "each forward must bind its own port")
for i, want := range []string{"grafana", "prometheus"} {
resp, err := http.Get("http://" + forwards[i].Addr() + "/")
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode, "forward %d must be served", i)
assert.Equal(t, want, string(body), "forward %d must reach its own upstream", i)
}
dialer.mu.Lock()
defer dialer.mu.Unlock()
assert.ElementsMatch(t, []string{"grafana.internal:80", "prometheus.internal:80"}, dialer.asked,
"both forwards must share the one overlay dialer, each asking for its own name")
}
func TestPrintEffectiveConfig(t *testing.T) {
const key = "11111111-2222-3333-4444-555555555555"
cfg := &Config{ManagementURL: "https://management.invalid", SetupKey: key}
for _, spec := range []string{
"http://8080=https://grafana.internal",
"http://127.0.0.1:0=http://prometheus.internal:9090/api",
} {
fwd, err := ParseForward(spec)
require.NoError(t, err)
cfg.Forwards = append(cfg.Forwards, fwd)
}
var out strings.Builder
require.NoError(t, printEffectiveConfig(&out, cfg))
assert.Equal(t, "management-url: https://management.invalid\n"+
"state-dir: (memory)\n"+
"hostname: (host default)\n"+
"setup-key: true\n"+
"allowed-hosts: (none)\n"+
"forwards: 2\n"+
" 127.0.0.1:8080 -> https://grafana.internal\n"+
" 127.0.0.1:0 -> http://prometheus.internal:9090/api\n",
out.String(), "the check must print the effective configuration")
assert.NotContains(t, out.String(), key, "the check must report a key is set, never the key")
}
// The embedded client's management dial ignores cancellation, so a start that
// does the same must not hold a signal until it gives up on its own.
func TestStartSessionReturnsWhenInterrupted(t *testing.T) {
release := make(chan error)
start := func(context.Context) error { return <-release }
var stopped atomic.Bool
ctx, cancel := context.WithCancel(context.Background())
result := make(chan error, 1)
go func() { result <- startSession(ctx, start, func() { stopped.Store(true) }) }()
cancel()
select {
case err := <-result:
assert.ErrorIs(t, err, context.Canceled, "an interrupted start must report the cancellation")
case <-time.After(2 * time.Second):
t.Fatal("startSession waited for a start that ignores cancellation")
}
// The start completes after the caller gave up. Its session must not be
// left running.
release <- nil
assert.Eventually(t, stopped.Load, 2*time.Second, 10*time.Millisecond,
"a session that came up after the interrupt must be stopped")
}
func TestStartSessionReportsStartFailure(t *testing.T) {
failure := errors.New("login: permission denied")
err := startSession(context.Background(), func(context.Context) error { return failure }, func() {
t.Error("a session that never started must not be stopped")
})
assert.ErrorIs(t, err, failure, "the start error must be returned")
assert.Contains(t, err.Error(), "start client", "the error must say which step failed")
}
func TestStartSessionBoundsTheStart(t *testing.T) {
var deadline time.Time
err := startSession(context.Background(), func(ctx context.Context) error {
deadline, _ = ctx.Deadline()
return nil
}, func() {})
require.NoError(t, err)
assert.WithinDuration(t, time.Now().Add(startTimeout), deadline, 5*time.Second,
"the start must run under startTimeout")
}
func TestInterruptedOr(t *testing.T) {
failure := errors.New("wait for login: context canceled")
ctx, cancel := context.WithCancel(context.Background())
assert.Equal(t, failure, interruptedOr(ctx, failure), "an error while running must be returned")
cancel()
assert.NoError(t, interruptedOr(ctx, failure), "an error caused by a signal must be a clean exit")
}
func TestWaitForShutdownReturnsWhenAForwardStops(t *testing.T) {
fwd, err := ParseForward("http://127.0.0.1:0=http://grafana.internal")
require.NoError(t, err)
f, err := newHTTPForwarder(fwd, refuseDial)
require.NoError(t, err)
// Closing the raw listener under a serving forwarder is how a forward
// dies on its own, and the process must not keep running without it.
result := make(chan error, 1)
go func() { result <- waitForShutdown(context.Background(), []*httpForwarder{f}) }()
time.Sleep(100 * time.Millisecond)
require.NoError(t, f.listener.Close())
select {
case err := <-result:
assert.Error(t, err, "a forward that stopped serving must end the run with an error")
case <-time.After(5 * time.Second):
t.Fatal("waitForShutdown kept running after its only forward stopped")
}
_ = f.Close(context.Background())
}
func TestWaitForShutdownReturnsWhenContextEnds(t *testing.T) {
fwd, err := ParseForward("http://127.0.0.1:0=http://grafana.internal")
require.NoError(t, err)
f, err := newHTTPForwarder(fwd, refuseDial)
require.NoError(t, err)
t.Cleanup(func() { _ = f.Close(context.Background()) })
ctx, cancel := context.WithCancel(context.Background())
result := make(chan error, 1)
go func() { result <- waitForShutdown(ctx, []*httpForwarder{f}) }()
cancel()
select {
case err := <-result:
assert.NoError(t, err, "a cancelled context is a clean shutdown")
case <-time.After(5 * time.Second):
t.Fatal("waitForShutdown ignored the cancelled context")
}
}
func TestDisableNATMapperUnlessSet(t *testing.T) {
t.Setenv(natMapperEnv, "")
require.NoError(t, os.Unsetenv(natMapperEnv))
require.NoError(t, disableNATMapperUnlessSet())
assert.Equal(t, "true", os.Getenv(natMapperEnv), "an unset mapper switch must be turned off")
t.Setenv(natMapperEnv, "false")
require.NoError(t, disableNATMapperUnlessSet())
assert.Equal(t, "false", os.Getenv(natMapperEnv), "an operator's explicit choice must be kept")
}
// The consumer parses this variable with strconv.ParseBool and reads anything
// it cannot parse as false, which enables the mapper. A value that would not
// survive that parse must not count as an operator's choice, or an orchestrator
// passing the name through empty turns the mapper on.
func TestDisableNATMapperIgnoresAValueTheConsumerCannotRead(t *testing.T) {
for _, value := range []string{"", "yes", "on", "0.5"} {
t.Run("value "+strconv.Quote(value), func(t *testing.T) {
t.Setenv(natMapperEnv, value)
require.NoError(t, disableNATMapperUnlessSet())
assert.Equal(t, "true", os.Getenv(natMapperEnv),
"%q does not parse as a bool, so the mapper must stay off", value)
})
}
// Everything ParseBool does accept is a real choice and is left alone.
for _, value := range []string{"0", "f", "FALSE", "1", "t", "True"} {
t.Run("value "+strconv.Quote(value), func(t *testing.T) {
t.Setenv(natMapperEnv, value)
require.NoError(t, disableNATMapperUnlessSet())
assert.Equal(t, value, os.Getenv(natMapperEnv), "a parseable value is the operator's choice")
})
}
}
func TestRunCheckReturnsWithoutConnecting(t *testing.T) {
fwd, err := ParseForward("http://127.0.0.1:0=https://grafana.internal")
require.NoError(t, err)
// An unroutable management URL and no setup key: a run that tried to log
// in would block on the browser flow or fail to connect, not return nil.
cfg := &Config{ManagementURL: "https://management.invalid", Check: true, Forwards: []Forward{fwd}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
assert.NoError(t, Run(ctx, cfg), "a check must return before any network access")
assert.NoError(t, ctx.Err(), "a check must return immediately")
}
e2e/agentnetwork/nblink_test.go
+670
View File
@@ -0,0 +1,670 @@
//go:build e2e
package agentnetwork
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/e2e/harness"
"github.com/netbirdio/netbird/shared/management/http/api"
)
const (
// nblinkSecondPort serves a second forward to the same endpoint from the
// same process, so one login is shown to serve more than one listener.
nblinkSecondPort = 8081
// nblinkDeadPort forwards to a name nothing in the overlay serves.
nblinkDeadPort = 8082
// nblinkPublicPort binds every interface, so it is reachable from other
// containers on the network, which the loopback forwards are not.
nblinkPublicPort = 8083
// nblinkDeadUpstream is under the overlay's own domain, so the lookup is
// answered by the in-process resolver rather than by anything outside.
nblinkDeadUpstream = "https://nothing-serves-this.netbird.local"
// nblinkUnreachableBody is what the forwarder itself answers when the dial
// over the overlay fails.
nblinkUnreachableBody = "upstream unreachable over the overlay"
)
// TestNBLinkReachesAgentNetwork proves an agent-network endpoint can be
// consumed from a process that has no privileges at all:
//
// curl --> nblink (loopback, userspace peer) --tunnel--> proxy --http--> vllm
//
// The regular agent needs NET_ADMIN and a TUN device to put the endpoint on
// the network. nblink runs the same client in netstack mode and hands the
// endpoint to whatever can reach a loopback port, so the container is granted
// no capabilities and mounts no device — the whole point of the binary, and
// what fails first if netstack mode regresses.
//
// Three things are being proved together, and none of them holds on its own:
//
// - The forward reaches the endpoint, which means the endpoint's name
// resolved inside the process. Nothing on the container can resolve it:
// the synthesized zone is served by the in-process DNS, /etc/resolv.conf
// is never touched, and a name that leaked to the host resolver would be
// NXDOMAIN.
// - The forwarder's peer is a peer like any other. Its request is authorized
// against the policy that names its group, metered, and written to the
// access log under its own identity.
// - The loopback listener still refuses what a browser can aim at it, which
// is the one new attack surface a forward on localhost creates.
//
// One forwarder carries most of the cases, because each one costs a session:
// it serves three forwards, reads its setup key from a file, and keeps its
// identity on a volume so the last case can restart it.
func TestNBLinkReachesAgentNetwork(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
defer cancel()
vllm, err := harness.StartVLLM(ctx, srv)
require.NoError(t, err, "start mock vLLM server")
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
// allowed holds the forwarder's peer and is named by the policy. denied
// holds a second forwarder's peer and is named by nothing, so the proxy
// has no policy to authorize it under.
allowed, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-nblink-allowed"})
require.NoError(t, err, "create allowed group")
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), allowed.Id) })
denied, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-nblink-denied"})
require.NoError(t, err, "create denied group")
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), denied.Id) })
allowedKey := mintSetupKey(ctx, t, "e2e-nblink-allowed-key", allowed.Id)
deniedKey := mintSetupKey(ctx, t, "e2e-nblink-denied-key", denied.Id)
dummyKey := "sk-nblink-e2e"
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
Name: "nblink-vllm",
ProviderId: "vllm",
UpstreamUrl: vllm.URL,
ApiKey: &dummyKey,
Enabled: ptr(true),
Models: &[]api.AgentNetworkProviderModel{
{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002},
},
})
require.NoError(t, err, "create provider")
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
// A second provider on the mock's streaming listener, which answers every
// request as server-sent events.
streamProv, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
Name: "nblink-stream",
ProviderId: "anthropic_api",
UpstreamUrl: vllm.StreamURL,
ApiKey: &dummyKey,
Enabled: ptr(true),
Models: &[]api.AgentNetworkProviderModel{
{Id: streamedModel, InputPer1k: streamInRate, OutputPer1k: streamOutRate},
},
})
require.NoError(t, err, "create streaming provider")
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), streamProv.Id) })
// Caps far above what this test drives, so the limit never blocks but
// usage metering is switched on — that is what writes consumption rows.
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
Name: "e2e-nblink-allow",
Enabled: ptr(true),
SourceGroups: []string{allowed.Id},
DestinationProviderIds: []string{prov.Id, streamProv.Id},
Limits: &api.AgentNetworkPolicyLimits{
TokenLimit: api.AgentNetworkPolicyTokenLimit{
Enabled: true,
GroupCap: 10_000_000,
UserCap: 10_000_000,
WindowSeconds: 60,
},
},
})
require.NoError(t, err, "create policy")
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
settings, err := srv.GetSettings(ctx)
require.NoError(t, err, "read settings")
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-nblink-proxy")
require.NoError(t, err, "mint proxy token")
px, err := harness.StartProxy(ctx, srv, proxyToken)
require.NoError(t, err, "start proxy")
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
stateVolume := fmt.Sprintf("e2e-nblink-state-%d", time.Now().UnixNano())
t.Cleanup(func() { _ = harness.RemoveDockerVolume(context.Background(), stateVolume) })
// The upstream is the endpoint itself, over HTTPS. nblink verifies the
// certificate like any Go client, so it needs the proxy's self-signed cert
// — the curl helpers elsewhere in this suite sidestep that with -k.
forward := nblinkForward(harness.NBLinkPort, "https://"+settings.Endpoint)
nbOpts := []harness.NBLinkOption{
harness.WithNBLinkForwards(
nblinkForward(nblinkSecondPort, "https://"+settings.Endpoint),
nblinkForward(nblinkDeadPort, nblinkDeadUpstream),
fmt.Sprintf("http://0.0.0.0:%d=https://%s", nblinkPublicPort, settings.Endpoint),
),
harness.WithNBLinkEnv("NB_ALLOW_PUBLIC_BIND", "true"),
// The name other containers use for the public forward.
harness.WithNBLinkEnv("NB_ALLOWED_HOST", "nblink"),
harness.WithNBLinkSetupKeyFile(),
harness.WithNBLinkStateVolume(stateVolume),
}
nb, err := harness.StartNBLink(ctx, srv, allowedKey, forward, px.CACertPath(), nbOpts...)
require.NoError(t, err, "start nblink")
// nb is replaced by the restart case, so the cleanup reads the variable
// rather than the first value.
t.Cleanup(func() { _ = nb.Terminate(context.Background()) })
diag := func() string {
return "\n=== nblink logs ===\n" + nb.Logs(context.Background()) +
"\n=== proxy logs ===\n" + px.Logs(context.Background())
}
before, _ := srv.ListAccessLogs(ctx)
sessionID := "e2e-session-nblink"
// Retry to absorb the beat between the session coming up and the proxy
// peer being reachable; the forward itself is already bound by now.
code, body := chatUntil(ctx, t, nb, 200, 120*time.Second, sessionID)
require.Equal(t, 200, code,
"a chat through the nblink forward must be served; body: %s%s", body, diag())
require.Contains(t, body, "chat.completion",
"the body should be the upstream's OpenAI-compatible completion; got: %s", body)
var peer api.Peer
t.Run("the peer registers under its hostname", func(t *testing.T) {
peers := peersWithHostname(ctx, t, nb.Hostname())
require.Len(t, peers, 1, "exactly one peer must carry the forwarder's hostname %q", nb.Hostname())
peer = peers[0]
assert.True(t, peerInGroup(peer, allowed.Id),
"the setup key read from the file must have joined the peer to its auto-group")
})
t.Run("the request is attributed to the forwarder's own peer", func(t *testing.T) {
require.NotEmpty(t, peer.Ip, "the peer lookup must have run first")
var row *api.AgentNetworkAccessLog
require.Eventually(t, func() bool {
logs, lerr := srv.ListAccessLogs(ctx)
if lerr != nil || logs.TotalRecords <= before.TotalRecords {
return false
}
for i, r := range logs.Data {
if r.SessionId != nil && *r.SessionId == sessionID {
row = &logs.Data[i]
return true
}
}
return false
}, 30*time.Second, 2*time.Second,
"session %q must reach the access log; a forward is not a way around accounting%s", sessionID, diag())
require.NotNil(t, row.SourceIp, "the access log row must carry the caller's overlay address")
assert.Equal(t, peer.Ip, *row.SourceIp,
"the access log must name the forwarder's peer as the caller")
require.Eventually(t, func() bool {
rows, lerr := srv.ListConsumption(ctx)
if lerr != nil {
return false
}
for _, r := range rows {
if r.TokensInput > 0 && r.TokensOutput > 0 {
return true
}
}
return false
}, 60*time.Second, 3*time.Second, "the upstream's usage must be metered into a consumption row")
})
t.Run("every forward in one process serves over the same session", func(t *testing.T) {
scode, sbody, serr := nb.PostOn(ctx, nblinkSecondPort, harness.ChatPath,
harness.ChatBody(harness.VLLMModel, "Reply with exactly: pong"), nil)
require.NoError(t, serr, "the probe must reach the second listener")
assert.Equal(t, 200, scode, "the second forward must be served too; body: %s%s", sbody, diag())
assert.Contains(t, sbody, "chat.completion", "the second forward must carry the upstream completion back")
})
t.Run("an upstream nothing serves answers 502 without hanging", func(t *testing.T) {
start := time.Now()
dcode, dbody, derr := nb.PostOn(ctx, nblinkDeadPort, harness.ChatPath,
harness.ChatBody(harness.VLLMModel, "Reply with exactly: pong"), nil)
elapsed := time.Since(start)
require.NoError(t, derr, "the probe must reach the third listener")
assert.Equal(t, 502, dcode, "a forward whose upstream cannot be dialled must answer 502; body: %s", dbody)
assert.Contains(t, dbody, nblinkUnreachableBody, "the 502 must be the forwarder's own answer")
// The forwarder bounds a dial at 30s; anything near curl's 90s limit
// means a request was left hanging.
assert.Less(t, elapsed, 40*time.Second, "an unreachable upstream must fail fast, took %s", elapsed)
})
t.Run("a streamed reply arrives as events and is metered", func(t *testing.T) {
streamSession := "e2e-session-nblink-stream"
body := fmt.Sprintf(`{"model":%q,"max_tokens":64,"stream":true,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}`, streamedModel)
scode, sbody, serr := nb.Post(ctx, "/v1/messages", body,
[]string{"anthropic-version: 2023-06-01", "x-session-id: " + streamSession})
require.NoError(t, serr, "the probe must reach the listener")
require.Equal(t, 200, scode, "a streamed request must be served; body: %s%s", sbody, diag())
assert.Contains(t, sbody, "event: message_start", "the caller must receive the event stream itself")
assert.Contains(t, sbody, "message_stop", "the stream must arrive complete")
row := findAccessLogBySession(t, ctx, streamSession)
assert.Equal(t, harness.VLLMStreamInputTokens, int(row.InputTokens),
"the proxy must meter a stream carried by the forwarder like any other")
assert.Equal(t, harness.VLLMStreamOutputTokens, int(row.OutputTokens),
"output tokens must be metered from the stream")
})
t.Run("only the public forward is reachable from the network", func(t *testing.T) {
chat := harness.ChatBody(harness.VLLMModel, "Reply with exactly: pong")
pcode, pbody, perr := nb.PostFromNetwork(ctx, nblinkPublicPort, harness.ChatPath, chat, nil)
require.NoError(t, perr, "the public forward must accept a connection from another container")
assert.Equal(t, 200, pcode, "the public forward must serve a caller under a listed name; body: %s", pbody)
// The shape a rebound page sends: its own name, with Origin and
// Sec-Fetch-Site agreeing that the request is same-origin.
rcode, rbody, rerr := nb.PostFromNetwork(ctx, nblinkPublicPort, harness.ChatPath, chat, []string{
"Host: attacker.example:8083", "Origin: http://attacker.example:8083", "Sec-Fetch-Site: same-origin",
})
require.NoError(t, rerr)
assert.Equal(t, 421, rcode, "a public forward must refuse a name it was not given; body: %s", rbody)
assert.Contains(t, nb.Logs(ctx), fmt.Sprintf("forwarding http://0.0.0.0:%d", nblinkPublicPort),
"a 0.0.0.0 forward must bind the IPv4 wildcard, not the dual-stack one")
// The Host check only protects loopback listeners, but the browser
// checks still hold on a public one.
ocode, _, oerr := nb.PostFromNetwork(ctx, nblinkPublicPort, harness.ChatPath, chat,
[]string{"Origin: https://attacker.example"})
require.NoError(t, oerr)
assert.Equal(t, 403, ocode, "a cross-origin browser request must be refused on a public forward too")
_, _, lerr := nb.PostFromNetwork(ctx, harness.NBLinkPort, harness.ChatPath, chat, nil)
assert.Error(t, lerr, "a loopback forward must refuse connections from other machines")
})
t.Run("the logs show no secrets and no side effects on the host", func(t *testing.T) {
// The suite runs at debug level, the most an operator can turn on.
logs := nb.Logs(ctx)
require.Contains(t, logs, "over the overlay", "the logs must have been read")
assert.NotContains(t, logs, allowedKey, "the setup key must not be logged")
assert.Contains(t, logs, "NAT port mapper disabled", "nblink must not ask the router for a port mapping")
assert.NotContains(t, logs, "/var/lib/netbird", "nblink must not touch an installed agent's state")
assert.NotContains(t, logs, "failed to login to Management Service",
"registering a new peer must not log an error")
})
t.Run("a peer no policy names cannot reach the endpoint", func(t *testing.T) {
other, oerr := harness.StartNBLink(ctx, srv, deniedKey, forward, px.CACertPath(),
harness.WithNBLinkName("nblink-denied"))
require.NoError(t, oerr, "start the second forwarder")
t.Cleanup(func() { _ = other.Terminate(context.Background()) })
otherDiag := func() string {
return "\n=== denied nblink logs ===\n" + other.Logs(context.Background()) +
"\n=== proxy logs ===\n" + px.Logs(context.Background())
}
// Management gives the endpoint's DNS record, the proxy peer and the
// rule allowing 443 to it only to peers in an enabled policy's source
// groups. This peer gets none of them, so it is refused in the overlay
// itself and the request never reaches the proxy's router. The
// forwarder reports that as its own 502.
//
// Keep asking for as long as the allowed forwarder needed to be served,
// so a 502 cannot be blamed on a tunnel that had not settled yet.
deniedSession := "e2e-session-nblink-denied"
deadline := time.Now().Add(60 * time.Second)
var ocode int
var obody string
for time.Now().Before(deadline) {
ocode, obody, _ = other.Chat(ctx, harness.VLLMModel, "Reply with exactly: pong", deniedSession)
if ocode != 502 {
break
}
time.Sleep(5 * time.Second)
}
assert.Equal(t, 502, ocode,
"a forwarder whose peer is in no policy's source group must not get through; body: %s%s", obody, otherDiag())
assert.Contains(t, obody, nblinkUnreachableBody,
"the refusal must be the forwarder failing to dial, not an answer from the proxy; body: %s", obody)
assert.NotContains(t, obody, "chat.completion", "the refusal must not carry an upstream completion")
// The forwarder logs why the dial failed at debug level, which this
// suite runs at. Finding the line ties the 502 to this upstream rather
// than to some other failure inside the forwarder.
dialFailure := fmt.Sprintf("dial %s:443 over the overlay failed", settings.Endpoint)
logs := other.Logs(ctx)
require.Contains(t, logs, dialFailure, "the 502 must come from the overlay dial; logs:\n%s", logs)
for _, line := range strings.Split(logs, "\n") {
if strings.Contains(line, dialFailure) {
t.Logf("denied forwarder: %s", strings.TrimSpace(line))
break
}
}
// The allowed forwarder is still served at the same moment, so the
// refusal is about this peer and not about the endpoint being down.
acode, abody := chatUntil(ctx, t, nb, 200, 30*time.Second, "")
assert.Equal(t, 200, acode, "the allowed forwarder must still be served; body: %s", abody)
rows, lerr := srv.ListAccessLogs(ctx)
require.NoError(t, lerr, "list access logs")
for _, r := range rows.Data {
assert.False(t, r.SessionId != nil && *r.SessionId == deniedSession,
"a request that never reached the proxy must leave no access log row")
}
})
t.Run("the loopback listener refuses browser-driven callers", func(t *testing.T) {
// A page the user visits can aim a request at a loopback listener in
// several shapes. Each must be refused before anything is proxied
// under the peer's identity, while an ordinary local caller is
// unaffected.
cases := []struct {
name string
headers []string
want int
}{
{
name: "a foreign Host, which is DNS rebinding",
headers: []string{"Host: attacker.example"},
want: 421,
},
{
name: "a cross-origin fetch",
headers: []string{"Origin: https://attacker.example"},
want: 403,
},
{
name: "a page on another local port, which is cross-origin too",
// Same machine, different origin: localhost:3000 has no more
// claim on this listener than any other site.
headers: []string{"Origin: http://localhost:3000"},
want: 403,
},
{
name: "an embedded no-cors GET, which carries no Origin at all",
// The shape the Host and Origin checks both miss: a loopback
// Host and no Origin. Sec-Fetch-Site is the only signal left.
headers: []string{"Sec-Fetch-Site: cross-site"},
want: 403,
},
}
// Every case sends the request already proven to return 200, so the
// only difference between a refusal and a success is the header under
// test.
body := harness.ChatBody(harness.VLLMModel, "Reply with exactly: pong")
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
gcode, gbody, gerr := nb.Post(ctx, harness.ChatPath, body, tc.headers)
require.NoError(t, gerr, "the probe itself must reach the listener")
assert.Equal(t, tc.want, gcode,
"%s must be refused by the listener, not proxied; body: %s", tc.name, gbody)
assert.NotContains(t, gbody, "chat.completion",
"a refused request must never carry an upstream completion back")
})
}
// The control: the same request with none of those headers is served,
// so the cases above prove a guard rather than a broken forward.
gcode, gbody, gerr := nb.Post(ctx, harness.ChatPath, body, nil)
require.NoError(t, gerr)
assert.Equal(t, 200, gcode,
"a plain local caller must still be served; body: %s%s", gbody, diag())
})
t.Run("an upstream whose certificate is not trusted is refused", func(t *testing.T) {
// No CA mounted: the proxy's self-signed certificate is unknown to the
// forwarder, which must refuse it rather than proxy in the clear.
untrusted, uerr := harness.StartNBLink(ctx, srv, allowedKey, forward, "",
harness.WithNBLinkName("nblink-untrusted"))
require.NoError(t, uerr, "start the forwarder without the proxy's CA")
t.Cleanup(func() { _ = untrusted.Terminate(context.Background()) })
// Until the tunnel settles the dial itself fails with the same 502, so
// wait for the handshake failure specifically.
var ucode int
var ubody string
require.Eventually(t, func() bool {
ucode, ubody, _ = untrusted.Chat(ctx, harness.VLLMModel, "Reply with exactly: pong", "")
return strings.Contains(untrusted.Logs(ctx), "x509")
}, 120*time.Second, 5*time.Second, "the forwarder must reach the TLS handshake and fail it; logs:\n%s", untrusted.Logs(ctx))
assert.Equal(t, 502, ucode, "an unverified upstream must not be served; body: %s", ubody)
assert.NotContains(t, ubody, "chat.completion", "no completion may come back over an unverified connection")
})
t.Run("an arbitrary UID in group 0 keeps its identity on a volume", func(t *testing.T) {
// OpenShift's restricted SCC runs the image as a UID from the
// namespace range with group 0. The state directory is group 0 and
// group-writable, and a fresh volume inherits that.
const name = "nblink-openshift"
volume := fmt.Sprintf("e2e-nblink-openshift-%d", time.Now().UnixNano())
t.Cleanup(func() { _ = harness.RemoveDockerVolume(context.Background(), volume) })
opts := []harness.NBLinkOption{
harness.WithNBLinkName(name),
harness.WithNBLinkUser("1000770000:0"),
harness.WithNBLinkStateVolume(volume),
harness.WithNBLinkSetupKeyFile(),
}
for range 2 {
fwdr, oerr := harness.StartNBLink(ctx, srv, allowedKey, forward, px.CACertPath(), opts...)
require.NoError(t, oerr, "start the forwarder as an arbitrary UID")
// Registered before the assertions below so a failure between here
// and the explicit Terminate cannot leave a container holding the
// volume the cleanup above tries to remove.
t.Cleanup(func() { _ = fwdr.Terminate(context.Background()) })
ocode, obody := chatUntil(ctx, t, fwdr, 200, 120*time.Second, "")
assert.Equal(t, 200, ocode, "the forwarder must serve as an arbitrary UID; body: %s", obody)
require.NoError(t, fwdr.Terminate(ctx), "stop the forwarder")
}
assert.Len(t, peersWithHostname(ctx, t, name), 1,
"an arbitrary UID must be able to persist the identity it registered")
})
t.Run("without a state dir every start is a new peer", func(t *testing.T) {
const name = "nblink-ephemeral"
for range 2 {
eph, eerr := harness.StartNBLink(ctx, srv, allowedKey, forward, px.CACertPath(), harness.WithNBLinkName(name))
require.NoError(t, eerr, "start the in-memory forwarder")
t.Cleanup(func() { _ = eph.Terminate(context.Background()) })
require.NoError(t, eph.Terminate(ctx), "stop the in-memory forwarder")
}
peers := peersWithHostname(ctx, t, name)
assert.Len(t, peers, 2, "each in-memory start must register its own peer, as the README warns")
})
// Last, because it replaces the forwarder every case above used.
t.Run("a state dir keeps the same peer across restarts", func(t *testing.T) {
require.NotEmpty(t, peer.Id, "the peer lookup must have run first")
// docker stop sends SIGTERM. The forwarder must drain and exit cleanly
// well inside the grace period rather than be killed at its end.
exitCode, took, serr := nb.Stop(ctx, 30*time.Second)
require.NoError(t, serr, "stop the first forwarder")
assert.Equal(t, 0, exitCode, "SIGTERM must be a clean exit")
assert.Less(t, took, 15*time.Second, "shutdown must not wait out the grace period")
assert.Contains(t, nb.Logs(ctx), "shutting down", "the forwarder must log the signal it handled")
require.NoError(t, nb.Terminate(ctx), "remove the first forwarder")
restarted, rerr := harness.StartNBLink(ctx, srv, allowedKey, forward, px.CACertPath(), nbOpts...)
require.NoError(t, rerr, "start the forwarder again on the same volume")
nb = restarted
peers := peersWithHostname(ctx, t, nb.Hostname())
require.Len(t, peers, 1,
"a restart on the same state dir must not register a second peer; got %d", len(peers))
assert.Equal(t, peer.Id, peers[0].Id, "the restarted forwarder must come back as the same peer")
rcode, rbody := chatUntil(ctx, t, nb, 200, 120*time.Second, "")
assert.Equal(t, 200, rcode, "the restarted forwarder must be served again; body: %s%s", rbody, diag())
})
}
// TestNBLinkWithoutManagement covers a forwarder that can never bring its
// session up. It must fail rather than hang, and a signal while it is trying
// must stop it straight away.
func TestNBLinkWithoutManagement(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
env := map[string]string{
"NB_MANAGEMENT_URL": "https://management.invalid",
"NB_SETUP_KEY": "11111111-2222-3333-4444-555555555555",
"NB_FORWARD": "http://8080=https://grafana.netbird.cloud",
}
t.Run("an unreachable management server fails the start", func(t *testing.T) {
start := time.Now()
res, err := harness.RunNBLinkIsolated(ctx, env)
took := time.Since(start)
require.NoError(t, err, "run the forwarder")
assert.NotEqual(t, 0, res.ExitCode, "a session that cannot come up must be an error; stderr: %s", res.Stderr)
assert.Contains(t, res.Stderr, "start client", "the error must say the session did not start")
assert.NotContains(t, res.Stderr, "over the overlay", "no forward may be announced without a session")
// startTimeout is 90s; past that something is waiting without a bound.
assert.Less(t, took, 120*time.Second, "the start must give up within its timeout")
})
t.Run("SIGTERM during startup exits at once", func(t *testing.T) {
exitCode, took, logs, err := harness.StopNBLinkDuringStartup(ctx, env, 3*time.Second, 30*time.Second)
require.NoError(t, err, "start and stop the forwarder")
assert.Equal(t, 0, exitCode, "a signal during startup is a clean exit; logs:\n%s", logs)
assert.Less(t, took, 5*time.Second, "the signal must not wait for the management dial to time out")
})
}
// TestNBLinkCheck runs the image with --check, which must validate the
// configuration and print it without starting a session. The container has no
// network, so a run that tried to reach management could not exit 0.
func TestNBLinkCheck(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
setupKey := "11111111-2222-3333-4444-555555555555"
t.Run("a valid configuration prints its forwards and exits 0", func(t *testing.T) {
res, err := harness.RunNBLinkCheck(ctx, map[string]string{
"NB_MANAGEMENT_URL": "https://management.invalid",
"NB_SETUP_KEY": setupKey,
"NB_FORWARD": "http://8080=https://one.netbird.cloud,http://127.0.0.1:0=http://two.netbird.cloud:8000/v1",
})
require.NoError(t, err, "run the check")
require.Equal(t, 0, res.ExitCode, "a valid configuration must pass; stderr: %s", res.Stderr)
assert.Contains(t, res.Stdout, "forwards: 2", "both comma-separated forwards must be parsed")
assert.Contains(t, res.Stdout, "127.0.0.1:8080 -> https://one.netbird.cloud",
"a port-only spec must bind loopback")
assert.Contains(t, res.Stdout, "127.0.0.1:0 -> http://two.netbird.cloud:8000/v1",
"the second forward must be printed as given")
assert.Contains(t, res.Stdout, "setup-key: true", "the check must report that a key is set")
assert.NotContains(t, res.Stdout+res.Stderr, setupKey, "the check must never print the key itself")
assert.NotContains(t, res.Stderr, "connecting to", "the check must not start a session")
})
t.Run("an unsupported forward exits non-zero and prints nothing", func(t *testing.T) {
res, err := harness.RunNBLinkCheck(ctx, map[string]string{
"NB_SETUP_KEY": setupKey,
"NB_FORWARD": "http://8080=https://one.netbird.cloud,tcp://5432=db.netbird.cloud:5432",
})
require.NoError(t, err, "run the check")
assert.NotEqual(t, 0, res.ExitCode, "a bad spec must fail the check")
assert.Empty(t, strings.TrimSpace(res.Stdout), "a failed check must not print a configuration")
assert.Contains(t, res.Stderr, "tcp forwarding is not supported yet",
"the error must name the spec that failed")
})
t.Run("a public bind without the opt-in exits non-zero", func(t *testing.T) {
res, err := harness.RunNBLinkCheck(ctx, map[string]string{
"NB_FORWARD": "http://0.0.0.0:8080=https://one.netbird.cloud",
})
require.NoError(t, err, "run the check")
assert.NotEqual(t, 0, res.ExitCode, "binding beyond loopback must need --allow-public-bind")
assert.Contains(t, res.Stderr, "NB_ALLOW_PUBLIC_BIND", "the error must say how to opt in")
})
}
// nblinkForward builds a forward spec: a loopback listener on port, proxying
// to upstream over the overlay.
//
// The listener stays on loopback, so --allow-public-bind is never passed and
// the default an operator gets is what the suite exercises.
func nblinkForward(port int, upstream string) string {
return fmt.Sprintf("http://127.0.0.1:%d=%s", port, upstream)
}
// peersWithHostname returns every peer registered under hostname.
func peersWithHostname(ctx context.Context, t *testing.T, hostname string) []api.Peer {
t.Helper()
peers, err := srv.API().Peers.List(ctx)
require.NoError(t, err, "list peers")
var matched []api.Peer
for _, p := range peers {
if p.Hostname == hostname {
matched = append(matched, p)
}
}
return matched
}
func peerInGroup(p api.Peer, groupID string) bool {
for _, g := range p.Groups {
if g.Id == groupID {
return true
}
}
return false
}
// mintSetupKey creates a reusable setup key that auto-joins its peer to one
// group, which is how a forwarder's peer lands in (or out of) a policy's
// source groups.
func mintSetupKey(ctx context.Context, t *testing.T, name, groupID string) string {
t.Helper()
ephemeral := false
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
Name: name,
Type: "reusable",
ExpiresIn: 86400,
UsageLimit: 0,
AutoGroups: []string{groupID},
Ephemeral: &ephemeral,
})
require.NoError(t, err, "mint setup key %s", name)
require.NotEmpty(t, sk.Key, "setup key plaintext")
return sk.Key
}
// chatUntil retries a chat through the forwarder until it answers with want,
// returning the last status and body either way.
func chatUntil(ctx context.Context, t *testing.T, nb *harness.NBLink, want int, timeout time.Duration, sessionID string) (int, string) {
t.Helper()
var code int
var body string
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
c, b, err := nb.Chat(ctx, harness.VLLMModel, "Reply with exactly: pong", sessionID)
if err == nil {
code, body = c, b
if code == want {
return code, body
}
}
if !waitBeforeRetry(ctx, 5*time.Second) {
break
}
}
return code, body
}
e2e/harness/Dockerfile.nblink
+37
View File
@@ -0,0 +1,37 @@
# Multistage build for nblink, used in e2e tests. The release image
# (client/Dockerfile-nblink) packages a goreleaser artifact, so this compiles
# from source the way Dockerfile.client does for the agent, while keeping the
# release image's runtime shape: distroless, non-root, numeric UID.
#
# The runtime stage deliberately grants nothing. nblink's claim is that it
# needs no TUN device and no capabilities, so an image or a container request
# that quietly hands it NET_ADMIN would make the suite prove the opposite of
# what it is for.
FROM golang:1.26.7-bookworm AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/go/pkg/mod go mod download
COPY . .
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux go build -o /out/nblink ./client/link/cmd/nblink \
&& mkdir -p /out/state/nblink && chmod 770 /out/state/nblink
FROM gcr.io/distroless/static-debian13:nonroot
# Same state directory as the release image, so a volume mounted over it
# starts out owned by 65532:0 and the identity-persistence case runs against
# the ownership the release image ships.
COPY --from=builder --chown=65532:0 /out/state/ /var/lib/
# Numeric, as in the release image: the runtime may substitute its own UID and
# there is no passwd entry to look up.
USER 65532:0
# No NB_STATE_DIR by default. State stays in memory and each run registers a
# fresh peer; the persistence case sets it explicitly with a volume.
ENV NB_LOG_LEVEL=info
COPY --from=builder /out/nblink /usr/local/bin/nblink
ENTRYPOINT ["/usr/local/bin/nblink"]
e2e/harness/nblink.go
+508
View File
@@ -0,0 +1,508 @@
//go:build e2e
package harness
import (
"context"
"errors"
"fmt"
"net/http"
"os/exec"
"strings"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
)
const (
nblinkDockerfile = "e2e/harness/Dockerfile.nblink"
// defaultNBLinkImage is the local tag nblink is built under from
// nblinkDockerfile. Override with NB_E2E_NBLINK_IMAGE: a value with a "/"
// is pulled as a published image; a bare tag is built under that name.
defaultNBLinkImage = "netbird-nblink:e2e"
nblinkAlias = "nblink"
// NBLinkPort is the loopback port every forwarder in this suite listens
// on. Tests reach it from a sidecar container sharing the forwarder's
// network namespace, so loopback is enough and --allow-public-bind stays
// off — the default an operator gets.
NBLinkPort = 8080
// nblinkCAPath is where the proxy's self-signed cert is mounted. nblink
// verifies its upstream like any Go HTTP client, so without this the
// forward fails the TLS handshake rather than reaching the endpoint.
nblinkCAPath = "/etc/nblink/upstream-ca.crt"
// nblinkReadyLog is the line startForwards writes once the overlay session
// is up and every forward is bound.
nblinkReadyLog = "over the overlay"
// nblinkSecretsDir holds the setup key for WithNBLinkSetupKeyFile, mounted
// read-only the way the README's container example mounts it.
nblinkSecretsDir = "/run/secrets"
nblinkKeyFile = "nb_setup_key"
nblinkKeyPath = nblinkSecretsDir + "/" + nblinkKeyFile
// keyWriterImage writes the key into its volume. The file is root-owned
// and readable by group 0 only, the ownership the README tells operators a
// key needs for UID 65532:0 to read it.
keyWriterImage = "alpine:3.22"
// nblinkStateDir is the state directory both images create, owned by
// 65532:0, so a volume mounted over it inherits that ownership.
nblinkStateDir = "/var/lib/nblink"
)
// ChatPath and ChatBody are the request the forwarder is driven with. They are
// exported so a test can send the very same request with one header changed
// and attribute the difference in the answer to that header alone.
const ChatPath = "/v1/chat/completions"
// NBLink is a running nblink container: a NetBird peer in userspace mode
// forwarding a local port to an upstream reached inside the tunnel.
type NBLink struct {
container testcontainers.Container
keyVolume string
network string
name string
port int
}
// nblinkOptions is what the NBLinkOption values assemble.
type nblinkOptions struct {
name string
port int
extra []string
keyAsFile bool
stateVolume string
env map[string]string
user string
}
// NBLinkOption adjusts how StartNBLink runs the forwarder.
type NBLinkOption func(*nblinkOptions)
// NBLinkCheckResult is what a --check run printed and how it exited.
type NBLinkCheckResult struct {
Stdout string
Stderr string
ExitCode int
}
// StartNBLink builds the nblink image and runs it on the combined server's
// network, joining with the given setup key and serving one HTTP forward.
//
// forward is a complete spec (scheme://[host:]port=upstream). caCertPath is a
// PEM file on the host holding the upstream's certificate, mounted in and
// pointed at with SSL_CERT_FILE; pass "" when the upstream needs no extra
// trust.
//
// The container is granted no capabilities and no TUN device. That is the
// assertion, not an oversight: nblink that needed either would not be able to
// run where it claims to.
func StartNBLink(ctx context.Context, c *Combined, setupKey, forward, caCertPath string, opts ...NBLinkOption) (*NBLink, error) {
o := nblinkOptions{name: nblinkAlias, port: NBLinkPort}
for _, opt := range opts {
opt(&o)
}
image, err := nblinkImage(ctx)
if err != nil {
return nil, err
}
env := map[string]string{
"NB_MANAGEMENT_URL": combinedExposedURL,
"NB_SETUP_KEY": setupKey,
"NB_FORWARD": strings.Join(append([]string{forward}, o.extra...), ","),
"NB_HOSTNAME": o.name,
"NB_LOG_LEVEL": "debug",
// Match the agent and the proxy: the combined relay is WebSocket-only,
// so the embedded client must use WS to hold a stable relay link.
"NB_RELAY_TRANSPORT": "ws",
}
if caCertPath != "" {
env["SSL_CERT_FILE"] = nblinkCAPath
}
var keyVolume string
if o.keyAsFile {
// The root filesystem is read-only, so the key cannot be copied in; it
// is mounted, as it would be in production.
keyVolume = fmt.Sprintf("e2e-nblink-key-%s-%d", o.name, time.Now().UnixNano())
if err := writeKeyVolume(ctx, keyVolume, setupKey); err != nil {
return nil, err
}
env["NB_SETUP_KEY"] = "file:" + nblinkKeyPath
}
if o.stateVolume != "" {
env["NB_STATE_DIR"] = nblinkStateDir
}
for k, v := range o.env {
env[k] = v
}
req := testcontainers.ContainerRequest{
Image: image,
// The embedded client reports the container hostname to management, so
// this is the name the peer is addressable by in the API.
Hostname: o.name,
User: o.user,
Networks: []string{c.network.Name},
NetworkAliases: map[string][]string{c.network.Name: {o.name}},
Env: env,
HostConfigModifier: func(hc *container.HostConfig) {
hardenNBLink(hc)
if caCertPath != "" {
hc.Binds = append(hc.Binds, caCertPath+":"+nblinkCAPath+":ro")
}
if keyVolume != "" {
hc.Mounts = append(hc.Mounts, mount.Mount{
Type: mount.TypeVolume,
Source: keyVolume,
Target: nblinkSecretsDir,
ReadOnly: true,
})
}
if o.stateVolume != "" {
hc.Mounts = append(hc.Mounts, mount.Mount{
Type: mount.TypeVolume,
Source: o.stateVolume,
Target: nblinkStateDir,
})
}
},
// The session has 90s of its own to come up, so the wait has to outlast
// that or a slow handshake reads as a missing log line.
WaitingFor: wait.ForLog(nblinkReadyLog).WithStartupTimeout(150 * time.Second),
}
ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
if err != nil {
if ctr != nil {
_ = ctr.Terminate(context.Background())
}
if keyVolume != "" {
_ = RemoveDockerVolume(context.Background(), keyVolume)
}
return nil, fmt.Errorf("start nblink container: %w", err)
}
return &NBLink{container: ctr, keyVolume: keyVolume, network: c.network.Name, name: o.name, port: o.port}, nil
}
// Hostname returns the container hostname the embedded client reports to
// management — the name the registered peer appears under in the peers API.
func (n *NBLink) Hostname() string {
return n.name
}
// Post issues a JSON POST to the forwarder's primary listener and returns the
// HTTP status and response body.
func (n *NBLink) Post(ctx context.Context, path, body string, extraHeaders []string) (int, string, error) {
return n.do(ctx, n.port, http.MethodPost, path, body, extraHeaders)
}
// PostOn issues a JSON POST to the forwarder's listener on port, for a forward
// added with WithNBLinkForwards.
func (n *NBLink) PostOn(ctx context.Context, port int, path, body string, extraHeaders []string) (int, string, error) {
return n.do(ctx, port, http.MethodPost, path, body, extraHeaders)
}
// Get issues a GET to the forwarder's primary listener.
func (n *NBLink) Get(ctx context.Context, path string, extraHeaders []string) (int, string, error) {
return n.do(ctx, n.port, http.MethodGet, path, "", extraHeaders)
}
// Chat issues an OpenAI-shaped chat completion to the forwarder, which proxies
// it to the agent-network endpoint over the tunnel. A non-empty sessionID is
// sent as the x-session-id header the proxy records.
func (n *NBLink) Chat(ctx context.Context, model, prompt, sessionID string) (int, string, error) {
return n.Post(ctx, ChatPath, ChatBody(model, prompt), withSessionID(nil, sessionID))
}
// do runs curl in a throwaway container sharing the forwarder's network
// namespace, so the request arrives on its loopback listener exactly as a
// process running beside it would send one.
//
// Nothing here pins or rewrites Host: the loopback listener refuses a Host
// that is not its own, and a test that papered over that would stop covering
// it. Callers that want a foreign Host pass it in extraHeaders on purpose.
func (n *NBLink) do(ctx context.Context, port int, method, path, body string, extraHeaders []string) (int, string, error) {
url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path)
return curlStatus(ctx, []string{"--network", "container:" + n.container.GetContainerID()}, url, method, body, extraHeaders)
}
// PostFromNetwork issues a JSON POST to the forwarder by its network alias from
// a separate container on the same network, the way another machine would
// reach a publicly bound forward. A connection that is refused comes back as
// an error, not a status.
func (n *NBLink) PostFromNetwork(ctx context.Context, port int, path, body string, extraHeaders []string) (int, string, error) {
url := fmt.Sprintf("http://%s:%d%s", n.name, port, path)
return curlStatus(ctx, []string{"--network", n.network}, url, http.MethodPost, body, extraHeaders)
}
// Stop sends the forwarder SIGTERM, the way docker stop and an orchestrator
// do, and reports how it exited and how long it took. The container is left
// stopped for Logs; Terminate removes it.
func (n *NBLink) Stop(ctx context.Context, grace time.Duration) (int, time.Duration, error) {
start := time.Now()
if err := n.container.Stop(ctx, &grace); err != nil {
return 0, 0, fmt.Errorf("stop nblink: %w", err)
}
took := time.Since(start)
state, err := n.container.State(ctx)
if err != nil {
return 0, took, fmt.Errorf("inspect nblink: %w", err)
}
return state.ExitCode, took, nil
}
// Logs returns the nblink container logs, for diagnostics on failure.
func (n *NBLink) Logs(ctx context.Context) string {
return containerLogs(ctx, n.container)
}
// Terminate stops the nblink container.
func (n *NBLink) Terminate(ctx context.Context) error {
if n.container == nil {
return nil
}
err := n.container.Terminate(ctx)
if n.keyVolume != "" {
// Not ctx: a cancelled or expired ctx is the likeliest reason the
// terminate above failed, and the volume has to go either way.
err = errors.Join(err, RemoveDockerVolume(context.Background(), n.keyVolume))
}
return err
}
// WithNBLinkName names the forwarder, setting both its network alias and its
// container hostname. The hostname is what the peer registers under, so it is
// the name the peer appears beneath in the peers API.
//
// Required to run a second forwarder against the same server: the default name
// is shared and two containers cannot hold one alias on a network.
func WithNBLinkName(name string) NBLinkOption {
return func(o *nblinkOptions) { o.name = name }
}
// WithNBLinkForwards adds forwards to the one StartNBLink is given. They are
// passed in the same comma-separated NB_FORWARD, so one process serves all of
// them over one session.
func WithNBLinkForwards(specs ...string) NBLinkOption {
return func(o *nblinkOptions) { o.extra = append(o.extra, specs...) }
}
// WithNBLinkSetupKeyFile delivers the setup key as a file in the container and
// points NB_SETUP_KEY at it with the file: prefix, instead of putting the key
// itself in the environment.
func WithNBLinkSetupKeyFile() NBLinkOption {
return func(o *nblinkOptions) { o.keyAsFile = true }
}
// WithNBLinkStateVolume mounts the named Docker volume as the state directory
// and sets NB_STATE_DIR, so the peer identity outlives the container. The
// volume is created on first use; RemoveDockerVolume deletes it.
func WithNBLinkStateVolume(volume string) NBLinkOption {
return func(o *nblinkOptions) { o.stateVolume = volume }
}
// WithNBLinkUser runs the forwarder as uid:gid instead of the image's user,
// the way OpenShift assigns an arbitrary UID in group 0.
func WithNBLinkUser(user string) NBLinkOption {
return func(o *nblinkOptions) { o.user = user }
}
// WithNBLinkEnv sets an extra environment variable on the forwarder, for
// settings such as NB_ALLOW_PUBLIC_BIND that have no option of their own.
func WithNBLinkEnv(key, value string) NBLinkOption {
return func(o *nblinkOptions) {
if o.env == nil {
o.env = map[string]string{}
}
o.env[key] = value
}
}
// ChatBody builds an OpenAI-shaped chat completion request body.
func ChatBody(model, prompt string) string {
return fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}]}`, model, prompt)
}
// RunNBLinkCheck runs the nblink image with --check and the given environment.
// The container gets no network at all, so a run that tried to reach
// management would fail rather than quietly succeed.
func RunNBLinkCheck(ctx context.Context, env map[string]string) (NBLinkCheckResult, error) {
return RunNBLinkIsolated(ctx, env, "--check")
}
// RunNBLinkIsolated runs the nblink image to completion with no network and
// the given environment and arguments, the way a forwarder behaves when
// management cannot be reached.
func RunNBLinkIsolated(ctx context.Context, env map[string]string, extraArgs ...string) (NBLinkCheckResult, error) {
image, err := nblinkImage(ctx)
if err != nil {
return NBLinkCheckResult{}, err
}
args := append(isolatedRunArgs("--rm"), envArgs(env)...)
args = append(args, image)
args = append(args, extraArgs...)
cmd := exec.CommandContext(ctx, "docker", args...)
var stdout, stderr strings.Builder
cmd.Stdout = &stdout
cmd.Stderr = &stderr
res := NBLinkCheckResult{}
err = cmd.Run()
res.Stdout, res.Stderr = stdout.String(), stderr.String()
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
res.ExitCode = exitErr.ExitCode()
return res, nil
}
if err != nil {
return res, fmt.Errorf("run nblink: %w", err)
}
return res, nil
}
// StopNBLinkDuringStartup starts the nblink image with no network, so the
// session can never come up, sends it SIGTERM after wait, and reports how it
// exited, how long the stop took, and what it logged.
func StopNBLinkDuringStartup(ctx context.Context, env map[string]string, wait, grace time.Duration) (int, time.Duration, string, error) {
image, err := nblinkImage(ctx)
if err != nil {
return 0, 0, "", err
}
args := append(isolatedRunArgs("-d"), envArgs(env)...)
out, err := exec.CommandContext(ctx, "docker", append(args, image)...).Output()
if err != nil {
return 0, 0, "", fmt.Errorf("start nblink: %w", err)
}
id := strings.TrimSpace(string(out))
defer func() { _ = exec.Command("docker", "rm", "-f", id).Run() }()
timer := time.NewTimer(wait)
defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
return 0, 0, "", ctx.Err()
}
start := time.Now()
stop := exec.CommandContext(ctx, "docker", "stop", "-t", fmt.Sprint(int(grace.Seconds())), id)
if out, err := stop.CombinedOutput(); err != nil {
return 0, 0, "", fmt.Errorf("stop nblink: %w: %s", err, out)
}
took := time.Since(start)
state, err := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.State.ExitCode}}", id).Output()
if err != nil {
return 0, took, "", fmt.Errorf("inspect nblink: %w", err)
}
exitCode := 0
_, _ = fmt.Sscanf(strings.TrimSpace(string(state)), "%d", &exitCode)
logs, err := exec.CommandContext(ctx, "docker", "logs", id).CombinedOutput()
if err != nil {
return exitCode, took, "", fmt.Errorf("read nblink logs: %w", err)
}
return exitCode, took, string(logs), nil
}
// RemoveDockerVolume deletes a volume created by WithNBLinkStateVolume.
func RemoveDockerVolume(ctx context.Context, volume string) error {
if out, err := exec.CommandContext(ctx, "docker", "volume", "rm", "-f", volume).CombinedOutput(); err != nil {
return fmt.Errorf("remove volume %s: %w: %s", volume, err, out)
}
return nil
}
// writeKeyVolume creates a volume holding the setup key as a root-owned file
// of mode 0440. The key goes in on stdin so it never appears in a process
// listing.
func writeKeyVolume(ctx context.Context, volume, key string) error {
if out, err := exec.CommandContext(ctx, "docker", "volume", "create", volume).CombinedOutput(); err != nil {
return fmt.Errorf("create key volume: %w: %s", err, out)
}
script := fmt.Sprintf("cat > /k/%[1]s && chown 0:0 /k/%[1]s && chmod 440 /k/%[1]s", nblinkKeyFile)
cmd := exec.CommandContext(ctx, "docker", "run", "--rm", "-i", "-v", volume+":/k", keyWriterImage, "sh", "-c", script)
cmd.Stdin = strings.NewReader(key + "\n")
if out, err := cmd.CombinedOutput(); err != nil {
_ = RemoveDockerVolume(context.Background(), volume)
return fmt.Errorf("write key volume: %w: %s", err, out)
}
return nil
}
// hardenNBLink takes away what Docker grants a container by default: every
// capability, a writable root filesystem, and the ability to gain privileges.
// nblink claims to need none of them, and this is where that claim is held to.
// State, when kept, goes to a mounted volume.
func hardenNBLink(hc *container.HostConfig) {
hc.CapDrop = []string{"ALL"}
hc.ReadonlyRootfs = true
hc.SecurityOpt = append(hc.SecurityOpt, "no-new-privileges")
}
func nblinkImage(ctx context.Context) (string, error) {
root, err := repoRoot(ctx)
if err != nil {
return "", err
}
return resolveImage(ctx, root, "NB_E2E_NBLINK_IMAGE", defaultNBLinkImage, nblinkDockerfile)
}
// curlStatus runs curl in a throwaway container with the given docker run
// network arguments and returns the status and body.
func curlStatus(ctx context.Context, network []string, url, method, body string, extraHeaders []string) (int, string, error) {
args := append([]string{"run", "--rm"}, network...)
args = append(args,
curlImage,
"-s", "--connect-timeout", "5", "--max-time", "90",
"-o", "/dev/stderr", "-w", "%{http_code}",
"-X", method, url,
"-H", "Content-Type: application/json",
)
for _, h := range extraHeaders {
args = append(args, "-H", h)
}
if body != "" {
args = append(args, "--data", body)
}
cmd := exec.CommandContext(ctx, "docker", args...)
// -w writes the status to stdout, -o /dev/stderr the body to stderr, so
// the two come back separately.
var stdout, stderr strings.Builder
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return 0, stderr.String(), fmt.Errorf("curl through nblink: %w", err)
}
code := 0
_, _ = fmt.Sscanf(strings.TrimSpace(stdout.String()), "%d", &code)
return code, stderr.String(), nil
}
// isolatedRunArgs is docker run with no network and the same hardening
// StartNBLink applies.
func isolatedRunArgs(mode string) []string {
return []string{"run", mode, "--network", "none", "--cap-drop", "ALL", "--read-only", "--security-opt", "no-new-privileges"}
}
func envArgs(env map[string]string) []string {
var args []string
for k, v := range env {
args = append(args, "-e", k+"="+v)
}
return args
}
e2e/harness/proxy.go
+7
View File
@@ -242,6 +242,13 @@ func (p *Proxy) DebugClients(ctx context.Context) ([]ProxyDebugClient, error) {
return resp.Clients, nil
}
// CACertPath returns the host path of the self-signed certificate the proxy
// serves for the agent-network endpoint. A client that verifies TLS properly —
// nblink does, where the curl helpers pass -k — needs it in its trust store.
func (p *Proxy) CACertPath() string {
return filepath.Join(p.workDir, "tls.crt")
}
// Logs returns the proxy container logs, for diagnostics on failure.
func (p *Proxy) Logs(ctx context.Context) string {
return containerLogs(ctx, p.container)
shared/management/client/grpc.go
+7 -1
View File
@@ -621,7 +621,13 @@ func (c *GrpcClient) login(req *proto.LoginRequest) (*proto.LoginResponse, error
err = backoff.Retry(operation, nbgrpc.Backoff(c.ctx))
if err != nil {
log.Errorf("failed to login to Management Service: %v", err)
// PermissionDenied is how management asks an unknown peer to register,
// which the caller does next, so it is not an error worth reporting.
if s, ok := gstatus.FromError(err); ok && s.Code() == codes.PermissionDenied {
log.Debugf("login to Management Service refused: %v", err)
} else {
log.Errorf("failed to login to Management Service: %v", err)
}
return nil, err
}