mirror of
https://github.com/netbirdio/netbird.git
synced 2026-04-27 20:56:44 +00:00
Compare commits
6 Commits
cli-ws-pro
...
unsymmetri
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a6a8d139c7 | ||
|
|
c987cddc85 | ||
|
|
4d33567888 | ||
|
|
88467883fc | ||
|
|
954f40991f | ||
|
|
34341d95a9 |
@@ -29,7 +29,8 @@ func Backoff(ctx context.Context) backoff.BackOff {
|
||||
// The component parameter specifies the WebSocket proxy component path (e.g., "/management", "/signal").
|
||||
func CreateConnection(ctx context.Context, addr string, tlsEnabled bool, component string) (*grpc.ClientConn, error) {
|
||||
transportOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
if tlsEnabled {
|
||||
// for js, the outer websocket layer takes care of tls
|
||||
if tlsEnabled && runtime.GOOS != "js" {
|
||||
certPool, err := x509.SystemCertPool()
|
||||
if err != nil || certPool == nil {
|
||||
log.Debugf("System cert pool not available; falling back to embedded cert, error: %v", err)
|
||||
@@ -37,9 +38,7 @@ func CreateConnection(ctx context.Context, addr string, tlsEnabled bool, compone
|
||||
}
|
||||
|
||||
transportOption = grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
|
||||
// for js, outer websocket layer takes care of tls verification via WithCustomDialer
|
||||
InsecureSkipVerify: runtime.GOOS == "js",
|
||||
RootCAs: certPool,
|
||||
RootCAs: certPool,
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -73,6 +73,44 @@ func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *KernelConfigurer) RemoveEndpointAddress(peerKey string) error {
|
||||
peerKeyParsed, err := wgtypes.ParseKey(peerKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get the existing peer to preserve its allowed IPs
|
||||
existingPeer, err := c.getPeer(c.deviceName, peerKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get peer: %w", err)
|
||||
}
|
||||
|
||||
removePeerCfg := wgtypes.PeerConfig{
|
||||
PublicKey: peerKeyParsed,
|
||||
Remove: true,
|
||||
}
|
||||
|
||||
if err := c.configure(wgtypes.Config{Peers: []wgtypes.PeerConfig{removePeerCfg}}); err != nil {
|
||||
return fmt.Errorf(`error removing peer %s from interface %s: %w`, peerKey, c.deviceName, err)
|
||||
}
|
||||
|
||||
//Re-add the peer without the endpoint but same AllowedIPs
|
||||
reAddPeerCfg := wgtypes.PeerConfig{
|
||||
PublicKey: peerKeyParsed,
|
||||
AllowedIPs: existingPeer.AllowedIPs,
|
||||
ReplaceAllowedIPs: true,
|
||||
}
|
||||
|
||||
if err := c.configure(wgtypes.Config{Peers: []wgtypes.PeerConfig{reAddPeerCfg}}); err != nil {
|
||||
return fmt.Errorf(
|
||||
`error re-adding peer %s to interface %s with allowed IPs %v: %w`,
|
||||
peerKey, c.deviceName, existingPeer.AllowedIPs, err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *KernelConfigurer) RemovePeer(peerKey string) error {
|
||||
peerKeyParsed, err := wgtypes.ParseKey(peerKey)
|
||||
if err != nil {
|
||||
|
||||
@@ -106,6 +106,67 @@ func (c *WGUSPConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *WGUSPConfigurer) RemoveEndpointAddress(peerKey string) error {
|
||||
peerKeyParsed, err := wgtypes.ParseKey(peerKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse peer key: %w", err)
|
||||
}
|
||||
|
||||
ipcStr, err := c.device.IpcGet()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get IPC config: %w", err)
|
||||
}
|
||||
|
||||
// Parse current status to get allowed IPs for the peer
|
||||
stats, err := parseStatus(c.deviceName, ipcStr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse IPC config: %w", err)
|
||||
}
|
||||
|
||||
var allowedIPs []net.IPNet
|
||||
found := false
|
||||
for _, peer := range stats.Peers {
|
||||
if peer.PublicKey == peerKey {
|
||||
allowedIPs = peer.AllowedIPs
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("peer %s not found", peerKey)
|
||||
}
|
||||
|
||||
// remove the peer from the WireGuard configuration
|
||||
peer := wgtypes.PeerConfig{
|
||||
PublicKey: peerKeyParsed,
|
||||
Remove: true,
|
||||
}
|
||||
|
||||
config := wgtypes.Config{
|
||||
Peers: []wgtypes.PeerConfig{peer},
|
||||
}
|
||||
if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil {
|
||||
return fmt.Errorf("failed to remove peer: %s", ipcErr)
|
||||
}
|
||||
|
||||
// Build the peer config
|
||||
peer = wgtypes.PeerConfig{
|
||||
PublicKey: peerKeyParsed,
|
||||
ReplaceAllowedIPs: true,
|
||||
AllowedIPs: allowedIPs,
|
||||
}
|
||||
|
||||
config = wgtypes.Config{
|
||||
Peers: []wgtypes.PeerConfig{peer},
|
||||
}
|
||||
|
||||
if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil {
|
||||
return fmt.Errorf("remove endpoint address: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *WGUSPConfigurer) RemovePeer(peerKey string) error {
|
||||
peerKeyParsed, err := wgtypes.ParseKey(peerKey)
|
||||
if err != nil {
|
||||
|
||||
@@ -21,4 +21,5 @@ type WGConfigurer interface {
|
||||
GetStats() (map[string]configurer.WGStats, error)
|
||||
FullStats() (*configurer.Stats, error)
|
||||
LastActivities() map[string]monotime.Time
|
||||
RemoveEndpointAddress(peerKey string) error
|
||||
}
|
||||
|
||||
@@ -148,6 +148,17 @@ func (w *WGIface) UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAliv
|
||||
return w.configurer.UpdatePeer(peerKey, allowedIps, keepAlive, endpoint, preSharedKey)
|
||||
}
|
||||
|
||||
func (w *WGIface) RemoveEndpointAddress(peerKey string) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.configurer == nil {
|
||||
return ErrIfaceNotFound
|
||||
}
|
||||
|
||||
log.Debugf("Removing endpoint address: %s", peerKey)
|
||||
return w.configurer.RemoveEndpointAddress(peerKey)
|
||||
}
|
||||
|
||||
// RemovePeer removes a Wireguard Peer from the interface iface
|
||||
func (w *WGIface) RemovePeer(peerKey string) error {
|
||||
w.mu.Lock()
|
||||
|
||||
@@ -388,7 +388,8 @@ func (d *DefaultManager) squashAcceptRules(
|
||||
|
||||
// trace which type of protocols was squashed
|
||||
squashedRules := []*mgmProto.FirewallRule{}
|
||||
squashedProtocols := map[mgmProto.RuleProtocol]struct{}{}
|
||||
squashedProtocolsIn := map[mgmProto.RuleProtocol]struct{}{}
|
||||
squashedProtocolsOut := map[mgmProto.RuleProtocol]struct{}{}
|
||||
|
||||
// this function we use to do calculation, can we squash the rules by protocol or not.
|
||||
// We summ amount of Peers IP for given protocol we found in original rules list.
|
||||
@@ -397,7 +398,7 @@ func (d *DefaultManager) squashAcceptRules(
|
||||
// 2. Any of rule contains Port.
|
||||
//
|
||||
// We zeroed this to notify squash function that this protocol can't be squashed.
|
||||
addRuleToCalculationMap := func(i int, r *mgmProto.FirewallRule, protocols map[mgmProto.RuleProtocol]*protoMatch) {
|
||||
addRuleToCalculationMap := func(i int, r *mgmProto.FirewallRule, protocols map[mgmProto.RuleProtocol]*protoMatch, squashedProtocols map[mgmProto.RuleProtocol]struct{}) {
|
||||
hasPortRestrictions := r.Action == mgmProto.RuleAction_DROP ||
|
||||
r.Port != "" || !portInfoEmpty(r.PortInfo)
|
||||
|
||||
@@ -435,9 +436,9 @@ func (d *DefaultManager) squashAcceptRules(
|
||||
for i, r := range networkMap.FirewallRules {
|
||||
// calculate squash for different directions
|
||||
if r.Direction == mgmProto.RuleDirection_IN {
|
||||
addRuleToCalculationMap(i, r, in)
|
||||
addRuleToCalculationMap(i, r, in, squashedProtocolsIn)
|
||||
} else {
|
||||
addRuleToCalculationMap(i, r, out)
|
||||
addRuleToCalculationMap(i, r, out, squashedProtocolsOut)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,7 +451,7 @@ func (d *DefaultManager) squashAcceptRules(
|
||||
mgmProto.RuleProtocol_UDP,
|
||||
}
|
||||
|
||||
squash := func(matches map[mgmProto.RuleProtocol]*protoMatch, direction mgmProto.RuleDirection) {
|
||||
squash := func(matches map[mgmProto.RuleProtocol]*protoMatch, direction mgmProto.RuleDirection, squashedProtocols map[mgmProto.RuleProtocol]struct{}) {
|
||||
for _, protocol := range protocolOrders {
|
||||
match, ok := matches[protocol]
|
||||
if !ok || len(match.ips) != totalIPs || len(match.ips) < 2 {
|
||||
@@ -478,11 +479,22 @@ func (d *DefaultManager) squashAcceptRules(
|
||||
}
|
||||
}
|
||||
|
||||
squash(in, mgmProto.RuleDirection_IN)
|
||||
squash(out, mgmProto.RuleDirection_OUT)
|
||||
squash(in, mgmProto.RuleDirection_IN, squashedProtocolsIn)
|
||||
squash(out, mgmProto.RuleDirection_OUT, squashedProtocolsOut)
|
||||
|
||||
// if all protocol was squashed everything is allow and we can ignore all other rules
|
||||
if _, ok := squashedProtocols[mgmProto.RuleProtocol_ALL]; ok {
|
||||
_, inSquashed := squashedProtocolsIn[mgmProto.RuleProtocol_ALL]
|
||||
_, outSquashed := squashedProtocolsOut[mgmProto.RuleProtocol_ALL]
|
||||
|
||||
squashedProtocols := make(map[mgmProto.RuleProtocol]struct{})
|
||||
for k := range squashedProtocolsIn {
|
||||
squashedProtocols[k] = struct{}{}
|
||||
}
|
||||
for k := range squashedProtocolsOut {
|
||||
squashedProtocols[k] = struct{}{}
|
||||
}
|
||||
|
||||
if inSquashed && outSquashed {
|
||||
return squashedRules, squashedProtocols
|
||||
}
|
||||
|
||||
@@ -494,10 +506,15 @@ func (d *DefaultManager) squashAcceptRules(
|
||||
// filter out rules which was squashed from final list
|
||||
// if we also have other not squashed rules.
|
||||
for i, r := range networkMap.FirewallRules {
|
||||
squashedProtocols := squashedProtocolsIn
|
||||
protocols := in
|
||||
if r.Direction == mgmProto.RuleDirection_OUT {
|
||||
squashedProtocols = squashedProtocolsOut
|
||||
protocols = out
|
||||
}
|
||||
|
||||
if _, ok := squashedProtocols[r.Protocol]; ok {
|
||||
if m, ok := in[r.Protocol]; ok && m.ips[r.PeerIP] == i {
|
||||
continue
|
||||
} else if m, ok := out[r.Protocol]; ok && m.ips[r.PeerIP] == i {
|
||||
if m, ok := protocols[r.Protocol]; ok && m.ips[r.PeerIP] == i {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -758,6 +758,129 @@ func TestPortInfoEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultManagerSquashRulesMixed(t *testing.T) {
|
||||
networkMap := &mgmProto.NetworkMap{
|
||||
RemotePeers: []*mgmProto.RemotePeerConfig{
|
||||
{AllowedIps: []string{"100.66.152.160"}},
|
||||
},
|
||||
FirewallRules: []*mgmProto.FirewallRule{
|
||||
{
|
||||
PeerIP: "0.0.0.0",
|
||||
Direction: mgmProto.RuleDirection_OUT,
|
||||
Action: mgmProto.RuleAction_ACCEPT,
|
||||
Protocol: mgmProto.RuleProtocol_ALL,
|
||||
},
|
||||
{
|
||||
PeerIP: "100.66.152.160",
|
||||
Direction: mgmProto.RuleDirection_IN,
|
||||
Action: mgmProto.RuleAction_ACCEPT,
|
||||
Protocol: mgmProto.RuleProtocol_ALL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manager := &DefaultManager{}
|
||||
rules, _ := manager.squashAcceptRules(networkMap)
|
||||
|
||||
assert.Equal(t, 2, len(rules))
|
||||
|
||||
var inRules, outRules []*mgmProto.FirewallRule
|
||||
for _, r := range rules {
|
||||
if r.Direction == mgmProto.RuleDirection_IN {
|
||||
inRules = append(inRules, r)
|
||||
} else {
|
||||
outRules = append(outRules, r)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, len(inRules))
|
||||
assert.Equal(t, 1, len(outRules))
|
||||
assert.Equal(t, "0.0.0.0", outRules[0].PeerIP)
|
||||
assert.Equal(t, "100.66.152.160", inRules[0].PeerIP)
|
||||
}
|
||||
|
||||
func TestDefaultManagerSquashRulesBothOptimized(t *testing.T) {
|
||||
networkMap := &mgmProto.NetworkMap{
|
||||
RemotePeers: []*mgmProto.RemotePeerConfig{
|
||||
{AllowedIps: []string{"100.66.152.160"}},
|
||||
},
|
||||
FirewallRules: []*mgmProto.FirewallRule{
|
||||
{
|
||||
PeerIP: "0.0.0.0",
|
||||
Direction: mgmProto.RuleDirection_OUT,
|
||||
Action: mgmProto.RuleAction_ACCEPT,
|
||||
Protocol: mgmProto.RuleProtocol_ALL,
|
||||
},
|
||||
{
|
||||
PeerIP: "0.0.0.0",
|
||||
Direction: mgmProto.RuleDirection_IN,
|
||||
Action: mgmProto.RuleAction_ACCEPT,
|
||||
Protocol: mgmProto.RuleProtocol_ALL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manager := &DefaultManager{}
|
||||
rules, _ := manager.squashAcceptRules(networkMap)
|
||||
|
||||
assert.Equal(t, 2, len(rules))
|
||||
|
||||
var inRules, outRules []*mgmProto.FirewallRule
|
||||
for _, r := range rules {
|
||||
if r.Direction == mgmProto.RuleDirection_IN {
|
||||
inRules = append(inRules, r)
|
||||
} else {
|
||||
outRules = append(outRules, r)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, len(inRules))
|
||||
assert.Equal(t, 1, len(outRules))
|
||||
assert.Equal(t, "0.0.0.0", outRules[0].PeerIP)
|
||||
assert.Equal(t, "0.0.0.0", inRules[0].PeerIP)
|
||||
}
|
||||
|
||||
func TestDefaultManagerSquashRulesBothSpecific(t *testing.T) {
|
||||
networkMap := &mgmProto.NetworkMap{
|
||||
RemotePeers: []*mgmProto.RemotePeerConfig{
|
||||
{AllowedIps: []string{"100.66.152.160"}},
|
||||
},
|
||||
FirewallRules: []*mgmProto.FirewallRule{
|
||||
{
|
||||
PeerIP: "100.66.152.160",
|
||||
Direction: mgmProto.RuleDirection_OUT,
|
||||
Action: mgmProto.RuleAction_ACCEPT,
|
||||
Protocol: mgmProto.RuleProtocol_ALL,
|
||||
},
|
||||
{
|
||||
PeerIP: "100.66.152.160",
|
||||
Direction: mgmProto.RuleDirection_IN,
|
||||
Action: mgmProto.RuleAction_ACCEPT,
|
||||
Protocol: mgmProto.RuleProtocol_ALL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
manager := &DefaultManager{}
|
||||
rules, _ := manager.squashAcceptRules(networkMap)
|
||||
|
||||
assert.Equal(t, 2, len(rules))
|
||||
|
||||
var inRules, outRules []*mgmProto.FirewallRule
|
||||
for _, r := range rules {
|
||||
if r.Direction == mgmProto.RuleDirection_IN {
|
||||
inRules = append(inRules, r)
|
||||
} else {
|
||||
outRules = append(outRules, r)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, len(inRules))
|
||||
assert.Equal(t, 1, len(outRules))
|
||||
assert.Equal(t, "100.66.152.160", outRules[0].PeerIP)
|
||||
assert.Equal(t, "100.66.152.160", inRules[0].PeerIP)
|
||||
}
|
||||
|
||||
func TestDefaultManagerEnableSSHRules(t *testing.T) {
|
||||
networkMap := &mgmProto.NetworkMap{
|
||||
PeerConfig: &mgmProto.PeerConfig{
|
||||
|
||||
@@ -105,6 +105,10 @@ type MockWGIface struct {
|
||||
LastActivitiesFunc func() map[string]monotime.Time
|
||||
}
|
||||
|
||||
func (m *MockWGIface) RemoveEndpointAddress(_ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockWGIface) FullStats() (*configurer.Stats, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ type wgIfaceBase interface {
|
||||
UpdateAddr(newAddr string) error
|
||||
GetProxy() wgproxy.Proxy
|
||||
UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
|
||||
RemoveEndpointAddress(key string) error
|
||||
RemovePeer(peerKey string) error
|
||||
AddAllowedIP(peerKey string, allowedIP netip.Prefix) error
|
||||
RemoveAllowedIP(peerKey string, allowedIP netip.Prefix) error
|
||||
|
||||
@@ -430,6 +430,9 @@ func (conn *Conn) onICEStateDisconnected() {
|
||||
} else {
|
||||
conn.Log.Infof("ICE disconnected, do not switch to Relay. Reset priority to: %s", conntype.None.String())
|
||||
conn.currentConnPriority = conntype.None
|
||||
if err := conn.config.WgConfig.WgInterface.RemoveEndpointAddress(conn.config.WgConfig.RemoteKey); err != nil {
|
||||
conn.Log.Errorf("failed to remove wg endpoint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
changed := conn.statusICE.Get() != worker.StatusDisconnected
|
||||
@@ -523,6 +526,9 @@ func (conn *Conn) onRelayDisconnected() {
|
||||
if conn.currentConnPriority == conntype.Relay {
|
||||
conn.Log.Debugf("clean up WireGuard config")
|
||||
conn.currentConnPriority = conntype.None
|
||||
if err := conn.config.WgConfig.WgInterface.RemoveEndpointAddress(conn.config.WgConfig.RemoteKey); err != nil {
|
||||
conn.Log.Errorf("failed to remove wg endpoint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if conn.wgProxyRelay != nil {
|
||||
|
||||
@@ -18,4 +18,5 @@ type WGIface interface {
|
||||
GetStats() (map[string]configurer.WGStats, error)
|
||||
GetProxy() wgproxy.Proxy
|
||||
Address() wgaddr.Address
|
||||
RemoveEndpointAddress(key string) error
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.netbird-wsproxy-signal.rule=Host(`$NETBIRD_DOMAIN`) && PathPrefix(`/ws-proxy/signal`)
|
||||
- traefik.http.routers.netbird-wsproxy-signal.service=netbird-wsproxy-signal
|
||||
- traefik.http.services.netbird-wsproxy-signal.loadbalancer.server.port=10000
|
||||
- traefik.http.services.netbird-wsproxy-signal.loadbalancer.server.port=80
|
||||
- traefik.http.routers.netbird-signal.rule=Host(`$NETBIRD_DOMAIN`) && PathPrefix(`/signalexchange.SignalExchange/`)
|
||||
- traefik.http.services.netbird-signal.loadbalancer.server.port=10000
|
||||
- traefik.http.services.netbird-signal.loadbalancer.server.scheme=h2c
|
||||
|
||||
@@ -621,7 +621,7 @@ renderCaddyfile() {
|
||||
# relay
|
||||
reverse_proxy /relay* relay:80
|
||||
# Signal
|
||||
reverse_proxy /ws-proxy/signal* signal:10000
|
||||
reverse_proxy /ws-proxy/signal* signal:80
|
||||
reverse_proxy /signalexchange.SignalExchange/* h2c://signal:10000
|
||||
# Management
|
||||
reverse_proxy /api/* management:80
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -252,7 +251,7 @@ func updateMgmtConfig(ctx context.Context, path string, config *nbconfig.Config)
|
||||
}
|
||||
|
||||
func (s *BaseServer) handlerFunc(gRPCHandler *grpc.Server, httpHandler http.Handler, meter metric.Meter) http.Handler {
|
||||
wsProxy := wsproxyserver.New(netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), ManagementLegacyPort), wsproxyserver.WithOTelMeter(meter))
|
||||
wsProxy := wsproxyserver.New(gRPCHandler, wsproxyserver.WithOTelMeter(meter))
|
||||
|
||||
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
switch {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"net/http"
|
||||
// nolint:gosec
|
||||
_ "net/http/pprof"
|
||||
"net/netip"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
||||
@@ -63,10 +62,10 @@ var (
|
||||
Use: "run",
|
||||
Short: "start NetBird Signal Server daemon",
|
||||
SilenceUsage: true,
|
||||
PreRun: func(cmd *cobra.Command, args []string) {
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
err := util.InitLog(logLevel, logFile)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing log %v", err)
|
||||
return fmt.Errorf("failed initializing log: %w", err)
|
||||
}
|
||||
|
||||
flag.Parse()
|
||||
@@ -87,6 +86,8 @@ var (
|
||||
signalPort = 80
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
flag.Parse()
|
||||
@@ -254,7 +255,7 @@ func startServerWithCertManager(certManager *autocert.Manager, grpcRootHandler h
|
||||
}
|
||||
|
||||
func grpcHandlerFunc(grpcServer *grpc.Server, meter metric.Meter) http.Handler {
|
||||
wsProxy := wsproxyserver.New(netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), legacyGRPCPort), wsproxyserver.WithOTelMeter(meter))
|
||||
wsProxy := wsproxyserver.New(grpcServer, wsproxyserver.WithOTelMeter(meter))
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
|
||||
@@ -2,42 +2,41 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
"github.com/netbirdio/netbird/util/wsproxy"
|
||||
)
|
||||
|
||||
const (
|
||||
dialTimeout = 10 * time.Second
|
||||
bufferSize = 32 * 1024
|
||||
bufferSize = 32 * 1024
|
||||
ioTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// Config contains the configuration for the WebSocket proxy.
|
||||
type Config struct {
|
||||
LocalGRPCAddr netip.AddrPort
|
||||
Handler http.Handler
|
||||
Path string
|
||||
MetricsRecorder MetricsRecorder
|
||||
}
|
||||
|
||||
// Proxy handles WebSocket to TCP proxying for gRPC connections.
|
||||
// Proxy handles WebSocket to gRPC handler proxying.
|
||||
type Proxy struct {
|
||||
config Config
|
||||
metrics MetricsRecorder
|
||||
}
|
||||
|
||||
// New creates a new WebSocket proxy instance with optional configuration
|
||||
func New(localGRPCAddr netip.AddrPort, opts ...Option) *Proxy {
|
||||
func New(handler http.Handler, opts ...Option) *Proxy {
|
||||
config := Config{
|
||||
LocalGRPCAddr: localGRPCAddr,
|
||||
Handler: handler,
|
||||
Path: wsproxy.ProxyPath,
|
||||
MetricsRecorder: NoOpMetricsRecorder{}, // Default to no-op
|
||||
}
|
||||
@@ -63,7 +62,7 @@ func (p *Proxy) handleWebSocket(w http.ResponseWriter, r *http.Request) {
|
||||
p.metrics.RecordConnection(ctx)
|
||||
defer p.metrics.RecordDisconnection(ctx)
|
||||
|
||||
log.Debugf("WebSocket proxy handling connection from %s, forwarding to %s", r.RemoteAddr, p.config.LocalGRPCAddr)
|
||||
log.Debugf("WebSocket proxy handling connection from %s, forwarding to internal gRPC handler", r.RemoteAddr)
|
||||
acceptOptions := &websocket.AcceptOptions{
|
||||
OriginPatterns: []string{"*"},
|
||||
}
|
||||
@@ -75,71 +74,41 @@ func (p *Proxy) handleWebSocket(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := wsConn.Close(websocket.StatusNormalClosure, ""); err != nil {
|
||||
log.Debugf("Failed to close WebSocket: %v", err)
|
||||
}
|
||||
_ = wsConn.Close(websocket.StatusNormalClosure, "")
|
||||
}()
|
||||
|
||||
log.Debugf("WebSocket proxy attempting to connect to local gRPC at %s", p.config.LocalGRPCAddr)
|
||||
tcpConn, err := net.DialTimeout("tcp", p.config.LocalGRPCAddr.String(), dialTimeout)
|
||||
if err != nil {
|
||||
p.metrics.RecordError(ctx, "tcp_dial_failed")
|
||||
log.Warnf("Failed to connect to local gRPC server at %s: %v", p.config.LocalGRPCAddr, err)
|
||||
if err := wsConn.Close(websocket.StatusInternalError, "Backend unavailable"); err != nil {
|
||||
log.Debugf("Failed to close WebSocket after connection failure: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
clientConn, serverConn := net.Pipe()
|
||||
defer func() {
|
||||
if err := tcpConn.Close(); err != nil {
|
||||
log.Debugf("Failed to close TCP connection: %v", err)
|
||||
}
|
||||
_ = clientConn.Close()
|
||||
_ = serverConn.Close()
|
||||
}()
|
||||
|
||||
log.Debugf("WebSocket proxy established: client %s -> local gRPC %s", r.RemoteAddr, p.config.LocalGRPCAddr)
|
||||
log.Debugf("WebSocket proxy established: %s -> gRPC handler", r.RemoteAddr)
|
||||
|
||||
p.proxyData(ctx, wsConn, tcpConn)
|
||||
go func() {
|
||||
(&http2.Server{}).ServeConn(serverConn, &http2.ServeConnOpts{
|
||||
Context: ctx,
|
||||
Handler: p.config.Handler,
|
||||
})
|
||||
}()
|
||||
|
||||
p.proxyData(ctx, wsConn, clientConn, r.RemoteAddr)
|
||||
}
|
||||
|
||||
func (p *Proxy) proxyData(ctx context.Context, wsConn *websocket.Conn, tcpConn net.Conn) {
|
||||
func (p *Proxy) proxyData(ctx context.Context, wsConn *websocket.Conn, pipeConn net.Conn, clientAddr string) {
|
||||
proxyCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go p.wsToTCP(proxyCtx, cancel, &wg, wsConn, tcpConn)
|
||||
go p.tcpToWS(proxyCtx, cancel, &wg, wsConn, tcpConn)
|
||||
go p.wsToPipe(proxyCtx, cancel, &wg, wsConn, pipeConn, clientAddr)
|
||||
go p.pipeToWS(proxyCtx, cancel, &wg, wsConn, pipeConn, clientAddr)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
log.Tracef("Proxy data transfer completed, both goroutines terminated")
|
||||
case <-proxyCtx.Done():
|
||||
log.Tracef("Proxy data transfer cancelled, forcing connection closure")
|
||||
|
||||
if err := wsConn.Close(websocket.StatusGoingAway, "proxy cancelled"); err != nil {
|
||||
log.Tracef("Error closing WebSocket during cancellation: %v", err)
|
||||
}
|
||||
if err := tcpConn.Close(); err != nil {
|
||||
log.Tracef("Error closing TCP connection during cancellation: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
log.Tracef("Goroutines terminated after forced connection closure")
|
||||
case <-time.After(2 * time.Second):
|
||||
log.Tracef("Goroutines did not terminate within timeout after connection closure")
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (p *Proxy) wsToTCP(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, wsConn *websocket.Conn, tcpConn net.Conn) {
|
||||
func (p *Proxy) wsToPipe(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, wsConn *websocket.Conn, pipeConn net.Conn, clientAddr string) {
|
||||
defer wg.Done()
|
||||
defer cancel()
|
||||
|
||||
@@ -148,80 +117,73 @@ func (p *Proxy) wsToTCP(ctx context.Context, cancel context.CancelFunc, wg *sync
|
||||
if err != nil {
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
log.Debugf("wsToTCP goroutine terminating due to context cancellation")
|
||||
case websocket.CloseStatus(err) == websocket.StatusNormalClosure:
|
||||
log.Debugf("WebSocket closed normally")
|
||||
log.Debugf("WebSocket from %s terminating due to context cancellation", clientAddr)
|
||||
case websocket.CloseStatus(err) != -1:
|
||||
log.Debugf("WebSocket from %s disconnected", clientAddr)
|
||||
default:
|
||||
p.metrics.RecordError(ctx, "websocket_read_error")
|
||||
log.Errorf("WebSocket read error: %v", err)
|
||||
log.Debugf("WebSocket read error from %s: %v", clientAddr, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if msgType != websocket.MessageBinary {
|
||||
log.Warnf("Unexpected WebSocket message type: %v", msgType)
|
||||
log.Warnf("Unexpected WebSocket message type from %s: %v", clientAddr, msgType)
|
||||
continue
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
log.Tracef("wsToTCP goroutine terminating due to context cancellation before TCP write")
|
||||
log.Tracef("wsToPipe goroutine terminating due to context cancellation before pipe write")
|
||||
return
|
||||
}
|
||||
|
||||
if err := tcpConn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil {
|
||||
log.Debugf("Failed to set TCP write deadline: %v", err)
|
||||
if err := pipeConn.SetWriteDeadline(time.Now().Add(ioTimeout)); err != nil {
|
||||
log.Debugf("Failed to set pipe write deadline: %v", err)
|
||||
}
|
||||
|
||||
n, err := tcpConn.Write(data)
|
||||
n, err := pipeConn.Write(data)
|
||||
if err != nil {
|
||||
p.metrics.RecordError(ctx, "tcp_write_error")
|
||||
log.Errorf("TCP write error: %v", err)
|
||||
p.metrics.RecordError(ctx, "pipe_write_error")
|
||||
log.Warnf("Pipe write error for %s: %v", clientAddr, err)
|
||||
return
|
||||
}
|
||||
|
||||
p.metrics.RecordBytesTransferred(ctx, "ws_to_tcp", int64(n))
|
||||
p.metrics.RecordBytesTransferred(ctx, "ws_to_grpc", int64(n))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy) tcpToWS(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, wsConn *websocket.Conn, tcpConn net.Conn) {
|
||||
func (p *Proxy) pipeToWS(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, wsConn *websocket.Conn, pipeConn net.Conn, clientAddr string) {
|
||||
defer wg.Done()
|
||||
defer cancel()
|
||||
|
||||
buf := make([]byte, bufferSize)
|
||||
for {
|
||||
if err := tcpConn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
|
||||
log.Debugf("Failed to set TCP read deadline: %v", err)
|
||||
}
|
||||
n, err := tcpConn.Read(buf)
|
||||
|
||||
n, err := pipeConn.Read(buf)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
log.Tracef("tcpToWS goroutine terminating due to context cancellation")
|
||||
log.Tracef("pipeToWS goroutine terminating due to context cancellation")
|
||||
return
|
||||
}
|
||||
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
continue
|
||||
}
|
||||
|
||||
if err != io.EOF {
|
||||
log.Errorf("TCP read error: %v", err)
|
||||
log.Debugf("Pipe read error for %s: %v", clientAddr, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
log.Tracef("tcpToWS goroutine terminating due to context cancellation before WebSocket write")
|
||||
log.Tracef("pipeToWS goroutine terminating due to context cancellation before WebSocket write")
|
||||
return
|
||||
}
|
||||
|
||||
if err := wsConn.Write(ctx, websocket.MessageBinary, buf[:n]); err != nil {
|
||||
p.metrics.RecordError(ctx, "websocket_write_error")
|
||||
log.Errorf("WebSocket write error: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
if err := wsConn.Write(ctx, websocket.MessageBinary, buf[:n]); err != nil {
|
||||
p.metrics.RecordError(ctx, "websocket_write_error")
|
||||
log.Warnf("WebSocket write error for %s: %v", clientAddr, err)
|
||||
return
|
||||
}
|
||||
|
||||
p.metrics.RecordBytesTransferred(ctx, "tcp_to_ws", int64(n))
|
||||
p.metrics.RecordBytesTransferred(ctx, "grpc_to_ws", int64(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user