sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-04-16 07:16:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Share kernel Wireguard port with raw socket (#826)

Browse Source 
This PR brings support of a shared port between stun (ICE agent) and
the kernel WireGuard

It implements a single port mode for execution with kernel WireGuard
interface using a raw socket listener.

BPF filters ensure that only STUN packets hit the NetBird userspace app

Removed a lot of the proxy logic and direct mode exchange.

Now we are doing an extra hole punch to the remote WireGuard 
port for best-effort cases and support to old client's direct mode.
This commit is contained in:
Maycon Santos
2023-05-03 14:47:44 +02:00
committed by GitHub
parent 59372ee159
commit ecac82a5ae
15 changed files with 663 additions and 563 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
client/internal/engine.go
View File

@@ -3,6 +3,7 @@ package internal
import (
"context"
"fmt"
"io"
"math/rand"
"net"
"net/netip"
@@ -23,9 +24,11 @@ import (
nbssh "github.com/netbirdio/netbird/client/ssh"
nbdns "github.com/netbirdio/netbird/dns"
"github.com/netbirdio/netbird/iface"
"github.com/netbirdio/netbird/iface/bind"
mgm "github.com/netbirdio/netbird/management/client"
mgmProto "github.com/netbirdio/netbird/management/proto"
"github.com/netbirdio/netbird/route"
"github.com/netbirdio/netbird/sharedsock"
signal "github.com/netbirdio/netbird/signal/client"
sProto "github.com/netbirdio/netbird/signal/proto"
"github.com/netbirdio/netbird/util"
@@ -99,10 +102,8 @@ type Engine struct {
wgInterface *iface.WGIface
udpMux ice.UDPMux
udpMuxSrflx ice.UniversalUDPMux
udpMuxConn *net.UDPConn
udpMuxConnSrflx *net.UDPConn
udpMux *bind.UniversalUDPMuxDefault
udpMuxConn io.Closer
// networkSerial is the latest CurrentSerial (state ID) of the network sent by the Management service
networkSerial uint64
@@ -206,33 +207,17 @@ func (e *Engine) Start() error {
e.close()
return err
}
e.udpMux = udpMux.UDPMuxDefault
e.udpMuxSrflx = udpMux
e.udpMux = udpMux
log.Infof("using userspace bind mode %s", udpMux.LocalAddr().String())
} else {
networkName := "udp"
if e.config.DisableIPv6Discovery {
networkName = "udp4"
}
e.udpMuxConn, err = net.ListenUDP(networkName, &net.UDPAddr{Port: e.config.UDPMuxPort})
rawSock, err := sharedsock.Listen(e.config.WgPort, sharedsock.NewSTUNFilter())
if err != nil {
log.Errorf("failed listening on UDP port %d: [%s]", e.config.UDPMuxPort, err.Error())
e.close()
return err
}
udpMuxParams := ice.UDPMuxParams{
UDPConn: e.udpMuxConn,
Net: transportNet,
}
e.udpMux = ice.NewUDPMuxDefault(udpMuxParams)
e.udpMuxConnSrflx, err = net.ListenUDP(networkName, &net.UDPAddr{Port: e.config.UDPMuxSrflxPort})
if err != nil {
log.Errorf("failed listening on UDP port %d: [%s]", e.config.UDPMuxSrflxPort, err.Error())
e.close()
return err
}
e.udpMuxSrflx = ice.NewUniversalUDPMuxDefault(ice.UniversalUDPMuxParams{UDPConn: e.udpMuxConnSrflx, Net: transportNet})
mux := bind.NewUniversalUDPMuxDefault(bind.UniversalUDPMuxParams{UDPConn: rawSock, Net: transportNet})
go mux.ReadFromConn(e.ctx)
e.udpMuxConn = rawSock
e.udpMux = mux
}
e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.wgInterface, e.statusRecorder)
@@ -394,9 +379,6 @@ func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKe
return err
}
// indicates message support in gRPC
msg.Body.FeaturesSupported = []uint32{signal.DirectCheck}
err = s.Send(msg)
if err != nil {
return err
@@ -824,8 +806,8 @@ func (e Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, er
InterfaceBlackList: e.config.IFaceBlackList,
DisableIPv6Discovery: e.config.DisableIPv6Discovery,
Timeout: timeout,
UDPMux: e.udpMux,
UDPMuxSrflx: e.udpMuxSrflx,
UDPMux: e.udpMux.UDPMuxDefault,
UDPMuxSrflx: e.udpMux,
ProxyConfig: proxyConfig,
LocalWgPort: e.config.WgPort,
NATExternalIPs: e.parseNATExternalIPMappings(),
@@ -918,18 +900,6 @@ func (e *Engine) receiveSignalEvents() {
}
conn.OnRemoteCandidate(candidate)
case sProto.Body_MODE:
protoMode := msg.GetBody().GetMode()
if protoMode == nil {
return fmt.Errorf("received an empty mode message")
}
err := conn.OnModeMessage(peer.ModeMessage{
Direct: protoMode.GetDirect(),
})
if err != nil {
log.Errorf("failed processing a mode message -> %s", err)
return err
}
}
return nil
@@ -1020,12 +990,6 @@ func (e *Engine) close() {
}
}
if e.udpMuxConnSrflx != nil {
if err := e.udpMuxConnSrflx.Close(); err != nil {
log.Debugf("close server reflexive udp mux connection: %v", err)
}
}
if !isNil(e.sshServer) {
err := e.sshServer.Stop()
if err != nil {

160
client/internal/peer/conn.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/netbirdio/netbird/client/internal/proxy"
"github.com/netbirdio/netbird/client/internal/stdnet"
"github.com/netbirdio/netbird/iface"
"github.com/netbirdio/netbird/iface/bind"
signal "github.com/netbirdio/netbird/signal/client"
sProto "github.com/netbirdio/netbird/signal/proto"
"github.com/netbirdio/netbird/version"
@@ -358,88 +359,10 @@ func (conn *Conn) Open() error {
}
}
// useProxy determines whether a direct connection (without a go proxy) is possible
//
// There are 3 cases:
//
// * When neither candidate is from hard nat and one of the peers has a public IP
//
// * both peers are in the same private network
//
// * Local peer uses userspace interface with bind.ICEBind and is not relayed
//
// Please note, that this check happens when peers were already able to ping each other using ICE layer.
func shouldUseProxy(pair *ice.CandidatePair, userspaceBind bool) bool {
if !isRelayCandidate(pair.Local) && userspaceBind {
log.Debugf("shouldn't use proxy because using Bind and the connection is not relayed")
return false
}
if !isHardNATCandidate(pair.Local) && isHostCandidateWithPublicIP(pair.Remote) {
log.Debugf("shouldn't use proxy because the local peer is not behind a hard NAT and the remote one has a public IP")
return false
}
if !isHardNATCandidate(pair.Remote) && isHostCandidateWithPublicIP(pair.Local) {
log.Debugf("shouldn't use proxy because the remote peer is not behind a hard NAT and the local one has a public IP")
return false
}
if isHostCandidateWithPrivateIP(pair.Local) && isHostCandidateWithPrivateIP(pair.Remote) && isSameNetworkPrefix(pair) {
log.Debugf("shouldn't use proxy because peers are in the same private /16 network")
return false
}
if (isPeerReflexiveCandidateWithPrivateIP(pair.Local) && isHostCandidateWithPrivateIP(pair.Remote) ||
isHostCandidateWithPrivateIP(pair.Local) && isPeerReflexiveCandidateWithPrivateIP(pair.Remote)) && isSameNetworkPrefix(pair) {
log.Debugf("shouldn't use proxy because peers are in the same private /16 network and one peer is peer reflexive")
return false
}
return true
}
func isSameNetworkPrefix(pair *ice.CandidatePair) bool {
localIP := net.ParseIP(pair.Local.Address())
remoteIP := net.ParseIP(pair.Remote.Address())
if localIP == nil || remoteIP == nil {
return false
}
// only consider /16 networks
mask := net.IPMask{255, 255, 0, 0}
return localIP.Mask(mask).Equal(remoteIP.Mask(mask))
}
func isRelayCandidate(candidate ice.Candidate) bool {
return candidate.Type() == ice.CandidateTypeRelay
}
func isHardNATCandidate(candidate ice.Candidate) bool {
return candidate.Type() == ice.CandidateTypeRelay || candidate.Type() == ice.CandidateTypePeerReflexive
}
func isHostCandidateWithPublicIP(candidate ice.Candidate) bool {
return candidate.Type() == ice.CandidateTypeHost && isPublicIP(candidate.Address())
}
func isHostCandidateWithPrivateIP(candidate ice.Candidate) bool {
return candidate.Type() == ice.CandidateTypeHost && !isPublicIP(candidate.Address())
}
func isPeerReflexiveCandidateWithPrivateIP(candidate ice.Candidate) bool {
return candidate.Type() == ice.CandidateTypePeerReflexive && !isPublicIP(candidate.Address())
}
func isPublicIP(address string) bool {
ip := net.ParseIP(address)
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() {
return false
}
return true
}
// startProxy starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected
func (conn *Conn) startProxy(remoteConn net.Conn, remoteWgPort int) error {
conn.mu.Lock()
@@ -452,7 +375,7 @@ func (conn *Conn) startProxy(remoteConn net.Conn, remoteWgPort int) error {
}
peerState := State{PubKey: conn.config.Key}
p := conn.getProxyWithMessageExchange(pair, remoteWgPort)
p := conn.getProxy(pair, remoteWgPort)
conn.proxy = p
err = p.Start(remoteConn)
if err != nil {
@@ -478,62 +401,35 @@ func (conn *Conn) startProxy(remoteConn net.Conn, remoteWgPort int) error {
return nil
}
func (conn *Conn) getProxyWithMessageExchange(pair *ice.CandidatePair, remoteWgPort int) proxy.Proxy {
useProxy := shouldUseProxy(pair, conn.config.UserspaceBind)
localDirectMode := !useProxy
remoteDirectMode := localDirectMode
if conn.meta.protoSupport.DirectCheck {
go conn.sendLocalDirectMode(localDirectMode)
// will block until message received or timeout
remoteDirectMode = conn.receiveRemoteDirectMode()
// todo rename this method and the proxy package to something more appropriate
func (conn *Conn) getProxy(pair *ice.CandidatePair, remoteWgPort int) proxy.Proxy {
if isRelayCandidate(pair.Local) {
return proxy.NewWireGuardProxy(conn.config.ProxyConfig)
}
if conn.config.UserspaceBind && localDirectMode {
return proxy.NewNoProxy(conn.config.ProxyConfig)
}
// To support old version's with direct mode we attempt to punch an additional role with the remote wireguard port
go conn.punchRemoteWGPort(pair, remoteWgPort)
if localDirectMode && remoteDirectMode {
return proxy.NewDirectNoProxy(conn.config.ProxyConfig, remoteWgPort)
}
log.Debugf("falling back to local proxy mode with peer %s", conn.config.Key)
return proxy.NewWireGuardProxy(conn.config.ProxyConfig)
return proxy.NewNoProxy(conn.config.ProxyConfig)
}
func (conn *Conn) sendLocalDirectMode(localMode bool) {
// todo what happens when we couldn't deliver this message?
// we could retry, etc but there is no guarantee
err := conn.sendSignalMessage(&sProto.Message{
Key: conn.config.LocalKey,
RemoteKey: conn.config.Key,
Body: &sProto.Body{
Type: sProto.Body_MODE,
Mode: &sProto.Mode{
Direct: &localMode,
},
NetBirdVersion: version.NetbirdVersion(),
},
})
func (conn *Conn) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
// wait local endpoint configuration
time.Sleep(time.Second)
addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort))
if err != nil {
log.Errorf("failed to send local proxy mode to remote peer %s, error: %s", conn.config.Key, err)
log.Warnf("got an error while resolving the udp address, err: %s", err)
return
}
}
func (conn *Conn) receiveRemoteDirectMode() bool {
timeout := time.Second
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case receivedMSG := <-conn.remoteModeCh:
return receivedMSG.Direct
case <-timer.C:
// we didn't receive a message from remote so we assume that it supports the direct mode to keep the old behaviour
log.Debugf("timeout after %s while waiting for remote direct mode message from remote peer %s",
timeout, conn.config.Key)
return true
mux, ok := conn.config.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault)
if !ok {
log.Warn("invalid udp mux conversion")
return
}
_, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr)
if err != nil {
log.Warnf("got an error while sending the punch packet, err: %s", err)
}
}
@@ -757,16 +653,6 @@ func (conn *Conn) GetKey() string {
return conn.config.Key
}
// OnModeMessage unmarshall the payload message and send it to the mode message channel
func (conn *Conn) OnModeMessage(message ModeMessage) error {
select {
case conn.remoteModeCh <- message:
return nil
default:
return fmt.Errorf("unable to process mode message: channel busy")
}
}
// RegisterProtoSupportMeta register supported proto message in the connection metadata
func (conn *Conn) RegisterProtoSupportMeta(support []uint32) {
protoSupport := signal.ParseFeaturesSupported(support)

309
client/internal/peer/conn_test.go
View File

@@ -9,11 +9,9 @@ import (
"github.com/magiconair/properties/assert"
"github.com/pion/ice/v2"
"golang.org/x/sync/errgroup"
"github.com/netbirdio/netbird/client/internal/proxy"
"github.com/netbirdio/netbird/iface"
sproto "github.com/netbirdio/netbird/signal/proto"
)
var connConf = ConnConfig{
@@ -170,310 +168,3 @@ func TestConn_Close(t *testing.T) {
wg.Wait()
}
type mockICECandidate struct {
ice.Candidate
AddressFunc func() string
TypeFunc func() ice.CandidateType
}
// Address mocks and overwrite ice.Candidate Address method
func (m *mockICECandidate) Address() string {
if m.AddressFunc != nil {
return m.AddressFunc()
}
return ""
}
// Type mocks and overwrite ice.Candidate Type method
func (m *mockICECandidate) Type() ice.CandidateType {
if m.TypeFunc != nil {
return m.TypeFunc()
}
return ice.CandidateTypeUnspecified
}
func TestConn_ShouldUseProxy(t *testing.T) {
publicHostCandidate := &mockICECandidate{
AddressFunc: func() string {
return "8.8.8.8"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeHost
},
}
privateHostCandidate := &mockICECandidate{
AddressFunc: func() string {
return "10.0.0.1"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeHost
},
}
srflxCandidate := &mockICECandidate{
AddressFunc: func() string {
return "1.1.1.1"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeServerReflexive
},
}
prflxCandidate := &mockICECandidate{
AddressFunc: func() string {
return "1.1.1.1"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypePeerReflexive
},
}
relayCandidate := &mockICECandidate{
AddressFunc: func() string {
return "1.1.1.1"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeRelay
},
}
testCases := []struct {
name string
candatePair *ice.CandidatePair
expected bool
}{
{
name: "Use Proxy When Local Candidate Is Relay",
candatePair: &ice.CandidatePair{
Local: relayCandidate,
Remote: privateHostCandidate,
},
expected: true,
},
{
name: "Use Proxy When Remote Candidate Is Relay",
candatePair: &ice.CandidatePair{
Local: privateHostCandidate,
Remote: relayCandidate,
},
expected: true,
},
{
name: "Use Proxy When Local Candidate Is Peer Reflexive",
candatePair: &ice.CandidatePair{
Local: prflxCandidate,
Remote: privateHostCandidate,
},
expected: true,
},
{
name: "Use Proxy When Remote Candidate Is Peer Reflexive",
candatePair: &ice.CandidatePair{
Local: privateHostCandidate,
Remote: prflxCandidate,
},
expected: true,
},
{
name: "Don't Use Proxy When Local Candidate Is Public And Remote Is Private",
candatePair: &ice.CandidatePair{
Local: publicHostCandidate,
Remote: privateHostCandidate,
},
expected: false,
},
{
name: "Don't Use Proxy When Remote Candidate Is Public And Local Is Private",
candatePair: &ice.CandidatePair{
Local: privateHostCandidate,
Remote: publicHostCandidate,
},
expected: false,
},
{
name: "Don't Use Proxy When Local Candidate is Public And Remote Is Server Reflexive",
candatePair: &ice.CandidatePair{
Local: publicHostCandidate,
Remote: srflxCandidate,
},
expected: false,
},
{
name: "Don't Use Proxy When Remote Candidate is Public And Local Is Server Reflexive",
candatePair: &ice.CandidatePair{
Local: srflxCandidate,
Remote: publicHostCandidate,
},
expected: false,
},
{
name: "Don't Use Proxy When Both Candidates Are Public",
candatePair: &ice.CandidatePair{
Local: publicHostCandidate,
Remote: publicHostCandidate,
},
expected: false,
},
{
name: "Don't Use Proxy When Both Candidates Are Private",
candatePair: &ice.CandidatePair{
Local: privateHostCandidate,
Remote: privateHostCandidate,
},
expected: false,
},
{
name: "Don't Use Proxy When Both Candidates are in private network and one is peer reflexive",
candatePair: &ice.CandidatePair{
Local: &mockICECandidate{AddressFunc: func() string {
return "10.16.102.168"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeHost
}},
Remote: &mockICECandidate{AddressFunc: func() string {
return "10.16.101.96"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypePeerReflexive
}},
},
expected: false,
},
{
name: "Should Use Proxy When Both Candidates are in private network and both are peer reflexive",
candatePair: &ice.CandidatePair{
Local: &mockICECandidate{AddressFunc: func() string {
return "10.16.102.168"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypePeerReflexive
}},
Remote: &mockICECandidate{AddressFunc: func() string {
return "10.16.101.96"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypePeerReflexive
}},
},
expected: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
result := shouldUseProxy(testCase.candatePair, false)
if result != testCase.expected {
t.Errorf("got a different result. Expected %t Got %t", testCase.expected, result)
}
})
}
}
func TestGetProxyWithMessageExchange(t *testing.T) {
publicHostCandidate := &mockICECandidate{
AddressFunc: func() string {
return "8.8.8.8"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeHost
},
}
relayCandidate := &mockICECandidate{
AddressFunc: func() string {
return "1.1.1.1"
},
TypeFunc: func() ice.CandidateType {
return ice.CandidateTypeRelay
},
}
testCases := []struct {
name string
candatePair *ice.CandidatePair
inputDirectModeSupport bool
inputRemoteModeMessage bool
expected proxy.Type
}{
{
name: "Should Result In Using Wireguard Proxy When Local Eval Is Use Proxy",
candatePair: &ice.CandidatePair{
Local: relayCandidate,
Remote: publicHostCandidate,
},
inputDirectModeSupport: true,
inputRemoteModeMessage: true,
expected: proxy.TypeWireGuard,
},
{
name: "Should Result In Using Wireguard Proxy When Remote Eval Is Use Proxy",
candatePair: &ice.CandidatePair{
Local: publicHostCandidate,
Remote: publicHostCandidate,
},
inputDirectModeSupport: true,
inputRemoteModeMessage: false,
expected: proxy.TypeWireGuard,
},
{
name: "Should Result In Using Wireguard Proxy When Remote Direct Mode Support Is False And Local Eval Is Use Proxy",
candatePair: &ice.CandidatePair{
Local: relayCandidate,
Remote: publicHostCandidate,
},
inputDirectModeSupport: false,
inputRemoteModeMessage: false,
expected: proxy.TypeWireGuard,
},
{
name: "Should Result In Using Direct When Remote Direct Mode Support Is False And Local Eval Is No Use Proxy",
candatePair: &ice.CandidatePair{
Local: publicHostCandidate,
Remote: publicHostCandidate,
},
inputDirectModeSupport: false,
inputRemoteModeMessage: false,
expected: proxy.TypeDirectNoProxy,
},
{
name: "Should Result In Using Direct When Local And Remote Eval Is No Proxy",
candatePair: &ice.CandidatePair{
Local: publicHostCandidate,
Remote: publicHostCandidate,
},
inputDirectModeSupport: true,
inputRemoteModeMessage: true,
expected: proxy.TypeDirectNoProxy,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
g := errgroup.Group{}
conn, err := NewConn(connConf, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
conn.meta.protoSupport.DirectCheck = testCase.inputDirectModeSupport
conn.SetSendSignalMessage(func(message *sproto.Message) error {
return nil
})
g.Go(func() error {
return conn.OnModeMessage(ModeMessage{
Direct: testCase.inputRemoteModeMessage,
})
})
resultProxy := conn.getProxyWithMessageExchange(testCase.candatePair, 1000)
err = g.Wait()
if err != nil {
t.Error(err)
}
if resultProxy.Type() != testCase.expected {
t.Errorf("result didn't match expected value: Expected: %s, Got: %s", testCase.expected, resultProxy.Type())
}
})
}
}

57
client/internal/proxy/direct.go
View File

@@ -1,57 +0,0 @@
package proxy
import (
log "github.com/sirupsen/logrus"
"net"
)
// DirectNoProxy is used when there is no need for a proxy between ICE and WireGuard.
// This is possible in either of these cases:
// - peers are in the same local network
// - one of the peers has a public static IP (host)
// DirectNoProxy will just update remote peer with a remote host and fixed WireGuard port (r.g. 51820).
// In order DirectNoProxy to work, WireGuard port has to be fixed for the time being.
type DirectNoProxy struct {
config Config
// RemoteWgListenPort is a WireGuard port of a remote peer.
// It is used instead of the hardcoded 51820 port.
RemoteWgListenPort int
}
// NewDirectNoProxy creates a new DirectNoProxy with a provided config and remote peer's WireGuard listen port
func NewDirectNoProxy(config Config, remoteWgPort int) *DirectNoProxy {
return &DirectNoProxy{config: config, RemoteWgListenPort: remoteWgPort}
}
// Close removes peer from the WireGuard interface
func (p *DirectNoProxy) Close() error {
err := p.config.WgInterface.RemovePeer(p.config.RemoteKey)
if err != nil {
return err
}
return nil
}
// Start just updates WireGuard peer with the remote IP and default WireGuard port
func (p *DirectNoProxy) Start(remoteConn net.Conn) error {
log.Debugf("using DirectNoProxy while connecting to peer %s", p.config.RemoteKey)
addr, err := net.ResolveUDPAddr("udp", remoteConn.RemoteAddr().String())
if err != nil {
return err
}
addr.Port = p.RemoteWgListenPort
err = p.config.WgInterface.UpdatePeer(p.config.RemoteKey, p.config.AllowedIps, DefaultWgKeepAlive,
addr, p.config.PreSharedKey)
if err != nil {
return err
}
return nil
}
// Type returns the type of this proxy
func (p *DirectNoProxy) Type() Type {
return TypeDirectNoProxy
}