diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml index a993293d4..cc7629534 100644 --- a/.github/workflows/buf.yml +++ b/.github/workflows/buf.yml @@ -27,7 +27,22 @@ jobs: push: false archive: false pr_comment: false - build: false lint: false format: false - breaking: true + # A push that creates a branch carries no `before` commit, so the + # action's default baseline is the all-zero SHA and `buf breaking` + # dies cloning it. Skipping costs nothing: every commit on a freshly + # cut release branch should have already passed this check on main. + breaking: ${{ !github.event.created }} + # The alternative is to compare against the default branch instead of + # skipping. Not used: buf clones the baseline when the job runs, so a + # main that has moved on since the branch was cut reads as protos + # deleted on the release branch. Resolving to an empty string on every + # other event is what keeps the action's own default in place, which + # stacked pull requests need. + # breaking_against: >- + # ${{ github.event.created + # && format('{0}#format=git,branch={1}', + # github.event.repository.clone_url, + # github.event.repository.default_branch) + # || '' }} diff --git a/client/android/preferences.go b/client/android/preferences.go index de66e0059..d8d81932b 100644 --- a/client/android/preferences.go +++ b/client/android/preferences.go @@ -325,6 +325,27 @@ func (p *Preferences) SetDisableIPv6(disable bool) { p.configInput.DisableIPv6 = &disable } +// GetRemoteJobsAllowed reads the remote jobs opt-in from config file +func (p *Preferences) GetRemoteJobsAllowed() (bool, error) { + if p.configInput.RemoteJobsAllowed != nil { + return *p.configInput.RemoteJobsAllowed, nil + } + + cfg, err := profilemanager.ReadOrGenerateConfig(p.configInput.ConfigPath) + if err != nil { + return false, err + } + if cfg.RemoteJobsAllowed == nil { + return false, nil + } + return *cfg.RemoteJobsAllowed, err +} + +// SetRemoteJobsAllowed stores the given value and waits for commit +func (p *Preferences) SetRemoteJobsAllowed(allowed bool) { + p.configInput.RemoteJobsAllowed = &allowed +} + // Commit writes out the changes to the config file func (p *Preferences) Commit() error { _, err := profilemanager.UpdateOrCreateConfig(p.configInput) diff --git a/client/cmd/service_controller.go b/client/cmd/service_controller.go index b187a7b87..e9a0e055f 100644 --- a/client/cmd/service_controller.go +++ b/client/cmd/service_controller.go @@ -41,13 +41,15 @@ func daemonServerOptions(network string) []grpc.ServerOption { if network == "tcp" { log.Warnf("daemon is listening on TCP (%s): callers carry no verifiable identity over TCP, "+ "so privileged operations (SSH root login, SSH auth, enabling the SSH server, management URL changes, "+ - "deregistration) will be denied. Use a unix socket, or npipe:// on Windows", daemonAddr) + "deregistration) will be denied, and the SSH JWT cache is neither filled nor served. "+ + "Use a unix socket, or npipe:// on Windows", daemonAddr) return nil } creds := ipcauth.NewTransportCredentials() //nolint:staticcheck if creds == nil { //nolint:staticcheck // nil only on platforms without a peer-identity primitive - log.Warnf("daemon IPC has no peer-identity primitive on %s: privileged operations will be denied", runtime.GOOS) + log.Warnf("daemon IPC has no peer-identity primitive on %s: privileged operations will be denied "+ + "and the SSH JWT cache is neither filled nor served", runtime.GOOS) return nil } diff --git a/client/firewall/allower_other.go b/client/firewall/allower_other.go new file mode 100644 index 000000000..4d2ec9094 --- /dev/null +++ b/client/firewall/allower_other.go @@ -0,0 +1,11 @@ +//go:build android || (!linux && !windows) + +package firewall + +import "github.com/netbirdio/netbird/client/firewall/uspfilter" + +// interfaceAllower returns no allower: these platforms have no host firewall to +// open for the interface. +func interfaceAllower(IFaceMapper, uint16) uspfilter.InterfaceAllower { + return nil +} diff --git a/client/firewall/allower_windows.go b/client/firewall/allower_windows.go new file mode 100644 index 000000000..b9efa18a4 --- /dev/null +++ b/client/firewall/allower_windows.go @@ -0,0 +1,10 @@ +//go:build windows + +package firewall + +import "github.com/netbirdio/netbird/client/firewall/uspfilter" + +// interfaceAllower returns the Windows netsh-based interface allower. +func interfaceAllower(iface IFaceMapper, _ uint16) uspfilter.InterfaceAllower { + return uspfilter.NewWindowsInterfaceAllower(iface) +} diff --git a/client/firewall/create.go b/client/firewall/create.go index 24f12bc6d..cb68a0d04 100644 --- a/client/firewall/create.go +++ b/client/firewall/create.go @@ -6,8 +6,6 @@ import ( "fmt" "runtime" - log "github.com/sirupsen/logrus" - firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/firewall/uspfilter" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" @@ -21,13 +19,11 @@ func NewFirewall(iface IFaceMapper, _ *statemanager.Manager, flowLogger nftypes. } // use userspace packet filtering firewall - fm, err := uspfilter.Create(iface, disableServerRoutes, flowLogger, mtu) - if err != nil { - return nil, err - } - err = fm.AllowNetbird() - if err != nil { - log.Warnf("failed to allow netbird interface traffic: %v", err) - } - return fm, nil + return uspfilter.Create(uspfilter.Config{ + IFace: iface, + DisableServerRoutes: disableServerRoutes, + FlowLogger: flowLogger, + MTU: mtu, + InterfaceAllower: interfaceAllower(iface, mtu), + }) } diff --git a/client/firewall/create_linux.go b/client/firewall/create_linux.go index d916ebad4..d585e85d7 100644 --- a/client/firewall/create_linux.go +++ b/client/firewall/create_linux.go @@ -16,6 +16,7 @@ import ( firewall "github.com/netbirdio/netbird/client/firewall/manager" nbnftables "github.com/netbirdio/netbird/client/firewall/nftables" "github.com/netbirdio/netbird/client/firewall/uspfilter" + "github.com/netbirdio/netbird/client/iface/netstack" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" "github.com/netbirdio/netbird/client/internal/statemanager" ) @@ -29,47 +30,107 @@ const ( NFTABLES ) -// SKIP_NFTABLES_ENV is the environment variable to skip nftables check -const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK" +// SkipNftablesEnv is the environment variable to skip nftables check +const SkipNftablesEnv = "NB_SKIP_NFTABLES_CHECK" + +// errNoFirewallManager indicates no kernel firewall backend is present, +// as opposed to a backend that exists but failed to create or initialize. +var errNoFirewallManager = errors.New("no firewall manager found") // FWType is the type for the firewall type type FWType int func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager, flowLogger nftypes.FlowLogger, disableServerRoutes bool, mtu uint16) (firewall.Manager, error) { - // We run in userspace mode and force userspace firewall was requested. We don't attempt native firewall. - if iface.IsUserspaceBind() && forceUserspaceFirewall() { - log.Info("forcing userspace firewall") - return createUserspaceFirewall(iface, nil, disableServerRoutes, flowLogger, mtu) + // Userspace firewall without a native counterpart: routing is handled + // entirely in userspace. The interface is opened in the kernel's foreign + // filter chains via a table-less allower, except in netstack mode where no + // kernel interface exists. + if netstack.IsEnabled() || (iface.IsUserspaceBind() && forceUserspaceFirewall()) { + if netstack.IsEnabled() { + log.Info("netstack mode, using userspace firewall") + } else { + log.Info("forcing userspace firewall") + } + cfg := uspfilter.Config{ + IFace: iface, + DisableServerRoutes: disableServerRoutes, + FlowLogger: flowLogger, + MTU: mtu, + InterfaceAllower: interfaceAllower(iface, mtu), + } + + return uspfilter.Create(cfg) } // Use native firewall for either kernel or userspace, the interface appears identical to netfilter - fm, err := createNativeFirewall(iface, stateManager, disableServerRoutes, mtu) - - // Kernel cannot fall back to anything else, need to return error - if !iface.IsUserspaceBind() { - return fm, err - } - - // Fall back to the userspace packet filter if native is unavailable - if err != nil { - log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err) - return createUserspaceFirewall(iface, nil, disableServerRoutes, flowLogger, mtu) - } - - // Native firewall handles packet filtering, but the userspace WireGuard bind - // needs a device filter for DNS interception hooks. Install a minimal - // hooks-only filter that passes all traffic through to the kernel firewall. - if err := iface.SetFilter(&uspfilter.HooksFilter{}); err != nil { - log.Warnf("failed to set hooks filter, DNS via memory hooks will not work: %v", err) + fm, err := createNativeFirewall(iface, stateManager, mtu) + switch { + case err == nil && !iface.IsUserspaceBind(): + // Nothing to do, fall through + case err == nil && iface.IsUserspaceBind(): + // Native firewall handles packet filtering, but the userspace WireGuard bind + // needs a device filter for DNS interception hooks. Install a minimal + // hooks-only filter that passes all traffic through to the kernel firewall. + if err := iface.SetFilter(&uspfilter.HooksFilter{}); err != nil { + log.Warnf("failed to set hooks filter, DNS via memory hooks will not work: %v", err) + } + case err != nil && !iface.IsUserspaceBind(): + // Kernel cannot fall back to anything else, need to return error + return nil, err + case err != nil && iface.IsUserspaceBind(): + // Fall back to the userspace packet filter if native is unavailable + logNativeFirewallUnavailable(err) + return uspfilter.Create(uspfilter.Config{ + IFace: iface, + DisableServerRoutes: disableServerRoutes, + FlowLogger: flowLogger, + MTU: mtu, + InterfaceAllower: interfaceAllower(iface, mtu), + }) } return fm, nil } -func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, routes bool, mtu uint16) (firewall.Manager, error) { +// interfaceAllower selects how the userspace firewall opens the interface in +// foreign kernel chains: nftables when available (which also opens foreign nft +// tables), else iptables (the legacy fallback, filter INPUT only), else nil. +// firewalld trust is applied separately by the manager. Netstack has no kernel +// interface to open. +func interfaceAllower(iface IFaceMapper, mtu uint16) uspfilter.InterfaceAllower { + if netstack.IsEnabled() { + return nil + } + + nftAllower, err := nbnftables.NewInterfaceAllower(iface, mtu) + if err == nil { + return nftAllower + } + log.Infof("no nftables interface allower: %v", err) + + iptAllower, err := nbiptables.NewInterfaceAllower(iface) + if err == nil { + return iptAllower + } + log.Infof("no iptables interface allower: %v", err) + + return nil +} + +// logNativeFirewallUnavailable logs the fallback to userspace at info level +// when no kernel firewall backend exists, and at warn level otherwise. +func logNativeFirewallUnavailable(err error) { + if errors.Is(err, errNoFirewallManager) { + log.Infof("no native firewall backend available: %v. Proceeding with userspace", err) + } else { + log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err) + } +} + +func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, mtu uint16) (firewall.Manager, error) { fm, err := createFW(iface, mtu) if err != nil { - return nil, fmt.Errorf("create firewall: %s", err) + return nil, fmt.Errorf("create firewall: %w", err) } if err = fm.Init(stateManager); err != nil { @@ -88,29 +149,10 @@ func createFW(iface IFaceMapper, mtu uint16) (firewall.Manager, error) { log.Info("creating an nftables firewall manager") return nbnftables.Create(iface, mtu) default: - log.Info("no firewall manager found, trying to use userspace packet filtering firewall") - return nil, errors.New("no firewall manager found") + return nil, errNoFirewallManager } } -func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (firewall.Manager, error) { - var errUsp error - if fm != nil { - fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm, disableServerRoutes, flowLogger, mtu) - } else { - fm, errUsp = uspfilter.Create(iface, disableServerRoutes, flowLogger, mtu) - } - - if errUsp != nil { - return nil, fmt.Errorf("create userspace firewall: %s", errUsp) - } - - if err := fm.AllowNetbird(); err != nil { - log.Errorf("failed to allow netbird interface traffic: %v", err) - } - return fm, nil -} - // check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found. func check() FWType { useIPTABLES := false @@ -132,35 +174,38 @@ func check() FWType { } } - nf := nftables.Conn{} - if chains, err := nf.ListChains(); err == nil && os.Getenv(SKIP_NFTABLES_ENV) != "true" { - if !useIPTABLES { - return NFTABLES - } - - // search for chains where table is filter - // if we find one, we assume that nftables manager can be used with iptables - for _, chain := range chains { - if chain.Table.Name == "filter" { + // Honor the skip env before probing nftables at all. + if os.Getenv(SkipNftablesEnv) != "true" { + nf := nftables.Conn{} + if chains, err := nf.ListChains(); err == nil { + if !useIPTABLES { return NFTABLES } - } - // check tables for the following constraints: - // 1. there is no chain in nftables for the filter table and there is at least one chain in iptables, we assume that nftables manager can not be used - // 2. there is no tables or more than one table, we assume that nftables manager can be used - // 3. there is only one table and its name is filter, we assume that nftables manager can not be used, since there was no chain in it - // 4. if we find an error we log and continue with iptables check - nbTablesList, err := nf.ListTables() - switch { - case err == nil && len(iptablesChains) > 0: - return IPTABLES - case err == nil && len(nbTablesList) != 1: - return NFTABLES - case err == nil && len(nbTablesList) == 1 && nbTablesList[0].Name == "filter": - return IPTABLES - case err != nil: - log.Errorf("failed to list nftables tables on fw manager discovery: %s", err) + // search for chains where table is filter + // if we find one, we assume that nftables manager can be used with iptables + for _, chain := range chains { + if chain.Table.Name == "filter" { + return NFTABLES + } + } + + // check tables for the following constraints: + // 1. there is no chain in nftables for the filter table and there is at least one chain in iptables, we assume that nftables manager can not be used + // 2. there is no tables or more than one table, we assume that nftables manager can be used + // 3. there is only one table and its name is filter, we assume that nftables manager can not be used, since there was no chain in it + // 4. if we find an error we log and continue with iptables check + nbTablesList, err := nf.ListTables() + switch { + case err == nil && len(iptablesChains) > 0: + return IPTABLES + case err == nil && len(nbTablesList) != 1: + return NFTABLES + case err == nil && len(nbTablesList) == 1 && nbTablesList[0].Name == "filter": + return IPTABLES + case err != nil: + log.Errorf("failed to list nftables tables on fw manager discovery: %s", err) + } } } @@ -176,15 +221,21 @@ func isIptablesClientAvailable(client *iptables.IPTables) bool { return err == nil } +// forceUserspaceFirewall reports whether the userspace firewall is forced. +// NB_FORCE_USERSPACE_ROUTER is an alias: forcing userspace routing implies the +// userspace firewall, since the two are no longer separable. func forceUserspaceFirewall() bool { - val := os.Getenv(EnvForceUserspaceFirewall) + return envForceBool(EnvForceUserspaceFirewall) || envForceBool(uspfilter.EnvForceUserspaceRouter) +} + +func envForceBool(name string) bool { + val := os.Getenv(name) if val == "" { return false } - force, err := strconv.ParseBool(val) if err != nil { - log.Warnf("failed to parse %s: %v", EnvForceUserspaceFirewall, err) + log.Warnf("failed to parse %s: %v", name, err) return false } return force diff --git a/client/firewall/iptables/acl_linux.go b/client/firewall/iptables/acl_linux.go deleted file mode 100644 index 89d1ebf7c..000000000 --- a/client/firewall/iptables/acl_linux.go +++ /dev/null @@ -1,603 +0,0 @@ -package iptables - -import ( - "errors" - "fmt" - "maps" - "net" - "slices" - - "github.com/coreos/go-iptables/iptables" - "github.com/google/uuid" - ipset "github.com/lrh3321/ipset-go" - log "github.com/sirupsen/logrus" - - firewall "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/internal/statemanager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - tableName = "filter" - - // rules chains contains the effective ACL rules - chainNameInputRules = "NETBIRD-ACL-INPUT" - - // mangleFwdKey is the entries map key for mangle FORWARD guard rules that prevent - // external DNAT from bypassing ACL rules. - mangleFwdKey = "MANGLE-FORWARD" -) - -type aclEntries map[string][][]string - -type entry struct { - spec []string - position int -} - -type aclManager struct { - iptablesClient *iptables.IPTables - wgIface iFaceMapper - entries aclEntries - optionalEntries map[string][]entry - ipsetStore *ipsetStore - v6 bool - ipsetSupported bool - - stateManager *statemanager.Manager -} - -func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*aclManager, error) { - return &aclManager{ - iptablesClient: iptablesClient, - wgIface: wgIface, - entries: make(map[string][][]string), - optionalEntries: make(map[string][]entry), - ipsetStore: newIpsetStore(), - v6: iptablesClient.Proto() == iptables.ProtocolIPv6, - }, nil -} - -func (m *aclManager) init(stateManager *statemanager.Manager) error { - m.stateManager = stateManager - - m.ipsetSupported = m.probeIPSetSupport() - - m.seedInitialEntries() - m.seedInitialOptionalEntries() - - if err := m.cleanChains(); err != nil { - return fmt.Errorf("clean chains: %w", err) - } - - if err := m.createDefaultChains(); err != nil { - return fmt.Errorf("create default chains: %w", err) - } - - m.updateState() - - return nil -} - -func (m *aclManager) AddPeerFiltering( - id []byte, - ip net.IP, - protocol firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - chain := chainNameInputRules - - ipsetName = transformIPsetName(ipsetName, sPort, dPort, action) - if m.v6 && ipsetName != "" { - ipsetName += "-v6" - } - // When the kernel lacks the required ipset hash module, fall back to - // per-IP iptables rules (pre-0.68 behavior) so ACLs keep working instead - // of silently leaving the chain empty. - if ipsetName != "" && !m.ipsetSupported { - ipsetName = "" - } - proto := protoForFamily(protocol, m.v6) - specs := filterRuleSpecs(ip, proto, sPort, dPort, action, ipsetName) - - mangleSpecs := slices.Clone(specs) - mangleSpecs = append(mangleSpecs, - "-i", m.wgIface.Name(), - "-m", "addrtype", "--dst-type", "LOCAL", - "-j", "MARK", "--set-xmark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), - ) - - specs = append(specs, "-j", actionToStr(action)) - if ipsetName != "" { - if ipList, ipsetExists := m.ipsetStore.ipset(ipsetName); ipsetExists { - if err := m.addToIPSet(ipsetName, ip); err != nil { - return nil, fmt.Errorf("add IP to ipset: %w", err) - } - // if ruleset already exists it means we already have the firewall rule - // so we need to update IPs in the ruleset and return new fw.Rule object for ACL manager. - ipList.addIP(ip.String()) - return []firewall.Rule{&Rule{ - ruleID: uuid.New().String(), - ipsetName: ipsetName, - ip: ip.String(), - chain: chain, - specs: specs, - v6: m.v6, - }}, nil - } - - if err := m.flushIPSet(ipsetName); err != nil { - if errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("flush ipset %s before use: %v", ipsetName, err) - } else { - log.Errorf("flush ipset %s before use: %v", ipsetName, err) - } - } - if err := m.createIPSet(ipsetName); err != nil { - return nil, fmt.Errorf("create ipset: %w", err) - } - if err := m.addToIPSet(ipsetName, ip); err != nil { - return nil, fmt.Errorf("add IP to ipset: %w", err) - } - - ipList := newIpList(ip.String()) - m.ipsetStore.addIpList(ipsetName, ipList) - } - - ok, err := m.iptablesClient.Exists(tableFilter, chain, specs...) - if err != nil { - return nil, fmt.Errorf("failed to check rule: %w", err) - } - if ok { - return nil, fmt.Errorf("rule already exists") - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - if action == firewall.ActionDrop { - // Insert at the beginning of the chain (position 1) - err = m.iptablesClient.Insert(tableFilter, chain, 1, specs...) - } else { - err = m.iptablesClient.Append(tableFilter, chain, specs...) - } - if err != nil { - return nil, err - } - - if err := m.iptablesClient.Append(tableMangle, chainRTPRE, mangleSpecs...); err != nil { - log.Errorf("failed to add mangle rule: %v", err) - mangleSpecs = nil - } - - rule := &Rule{ - ruleID: uuid.New().String(), - specs: specs, - mangleSpecs: mangleSpecs, - ipsetName: ipsetName, - ip: ip.String(), - chain: chain, - v6: m.v6, - } - - m.updateState() - - return []firewall.Rule{rule}, nil -} - -// DeletePeerRule from the firewall by rule definition -func (m *aclManager) DeletePeerRule(rule firewall.Rule) error { - r, ok := rule.(*Rule) - if !ok { - return fmt.Errorf("invalid rule type") - } - - shouldDestroyIpset := false - if ipsetList, ok := m.ipsetStore.ipset(r.ipsetName); ok { - // delete IP from ruleset IPs list and ipset - if _, ok := ipsetList.ips[r.ip]; ok { - ip := net.ParseIP(r.ip) - if ip == nil { - return fmt.Errorf("parse IP %s", r.ip) - } - if err := m.delFromIPSet(r.ipsetName, ip); err != nil { - return fmt.Errorf("delete ip from ipset: %w", err) - } - delete(ipsetList.ips, r.ip) - } - - // if after delete, set still contains other IPs, - // no need to delete firewall rule and we should exit here - if len(ipsetList.ips) != 0 { - return nil - } - - // we delete last IP from the set, that means we need to delete - // set itself and associated firewall rule too - m.ipsetStore.deleteIpset(r.ipsetName) - shouldDestroyIpset = true - } - - if err := m.iptablesClient.Delete(tableName, r.chain, r.specs...); err != nil { - return fmt.Errorf("failed to delete rule: %s, %v: %w", r.chain, r.specs, err) - } - - if r.mangleSpecs != nil { - if err := m.iptablesClient.Delete(tableMangle, chainRTPRE, r.mangleSpecs...); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - - if shouldDestroyIpset { - if err := m.destroyIPSet(r.ipsetName); err != nil { - if errors.Is(err, ipset.ErrBusy) || errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("destroy empty ipset: %v", err) - } else { - log.Errorf("destroy empty ipset: %v", err) - } - } - } - - m.updateState() - - return nil -} - -func (m *aclManager) Reset() error { - if err := m.cleanChains(); err != nil { - return fmt.Errorf("clean chains: %w", err) - } - - m.updateState() - - return nil -} - -// todo write less destructive cleanup mechanism -func (m *aclManager) cleanChains() error { - ok, err := m.iptablesClient.ChainExists(tableName, chainNameInputRules) - if err != nil { - log.Debugf("failed to list chains: %s", err) - return err - } - if ok { - for _, rule := range m.entries["INPUT"] { - err := m.iptablesClient.DeleteIfExists(tableName, "INPUT", rule...) - if err != nil { - log.Errorf("failed to delete rule: %v, %s", rule, err) - } - } - - for _, rule := range m.entries["FORWARD"] { - err := m.iptablesClient.DeleteIfExists(tableName, "FORWARD", rule...) - if err != nil { - log.Errorf("failed to delete rule: %v, %s", rule, err) - } - } - - err = m.iptablesClient.ClearAndDeleteChain(tableName, chainNameInputRules) - if err != nil { - log.Debugf("failed to clear and delete %s chain: %s", chainNameInputRules, err) - return err - } - } - - ok, err = m.iptablesClient.ChainExists("mangle", "PREROUTING") - if err != nil { - return fmt.Errorf("list chains: %w", err) - } - if ok { - for _, rule := range m.entries["PREROUTING"] { - err := m.iptablesClient.DeleteIfExists("mangle", "PREROUTING", rule...) - if err != nil { - log.Errorf("failed to delete rule: %v, %s", rule, err) - } - } - } - - for _, rule := range m.entries[mangleFwdKey] { - if err := m.iptablesClient.DeleteIfExists(tableMangle, chainFORWARD, rule...); err != nil { - log.Errorf("failed to delete mangle FORWARD guard rule: %v, %s", rule, err) - } - } - - for _, ipsetName := range m.ipsetStore.ipsetNames() { - if err := m.flushIPSet(ipsetName); err != nil { - if errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("flush ipset %q during reset: %v", ipsetName, err) - } else { - log.Errorf("flush ipset %q during reset: %v", ipsetName, err) - } - } - if err := m.destroyIPSet(ipsetName); err != nil { - if errors.Is(err, ipset.ErrBusy) || errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("destroy ipset %q during reset: %v", ipsetName, err) - } else { - log.Errorf("destroy ipset %q during reset: %v", ipsetName, err) - } - } - m.ipsetStore.deleteIpset(ipsetName) - } - - return nil -} - -func (m *aclManager) createDefaultChains() error { - // chain netbird-acl-input-rules - if err := m.iptablesClient.NewChain(tableName, chainNameInputRules); err != nil { - log.Debugf("failed to create '%s' chain: %s", chainNameInputRules, err) - return err - } - - for chainName, rules := range m.entries { - // mangle FORWARD guard rules are handled separately below - if chainName == mangleFwdKey { - continue - } - for _, rule := range rules { - if err := m.iptablesClient.InsertUnique(tableName, chainName, 1, rule...); err != nil { - log.Debugf("failed to create input chain jump rule: %s", err) - return err - } - } - } - - for chainName, entries := range m.optionalEntries { - for _, entry := range entries { - if err := m.iptablesClient.InsertUnique(tableName, chainName, entry.position, entry.spec...); err != nil { - log.Errorf("failed to insert optional entry %v: %v", entry.spec, err) - continue - } - m.entries[chainName] = append(m.entries[chainName], entry.spec) - } - } - clear(m.optionalEntries) - - // Insert mangle FORWARD guard rules to prevent external DNAT bypass. - for _, rule := range m.entries[mangleFwdKey] { - if err := m.iptablesClient.AppendUnique(tableMangle, chainFORWARD, rule...); err != nil { - log.Errorf("failed to add mangle FORWARD guard rule: %v", err) - } - } - - return nil -} - -// seedInitialEntries adds default rules to the entries map, rules are inserted on pos 1, hence the order is reversed. -// We want to make sure our traffic is not dropped by existing rules. - -// The existing FORWARD rules/policies decide outbound traffic towards our interface. -// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule. -func (m *aclManager) seedInitialEntries() { - established := getConntrackEstablished() - - m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", "DROP"}) - m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", chainNameInputRules}) - m.appendToEntries("INPUT", append([]string{"-i", m.wgIface.Name()}, established...)) - - // Inbound is handled by our ACLs, the rest is dropped. - // For outbound we respect the FORWARD policy. However, we need to allow established/related traffic for inbound rules. - m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", "DROP"}) - - m.appendToEntries("FORWARD", []string{"-o", m.wgIface.Name(), "-j", chainRTFWDOUT}) - m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", chainRTFWDIN}) - - // Mangle FORWARD guard: when external DNAT redirects traffic from the wg interface, it - // traverses FORWARD instead of INPUT, bypassing ACL rules. ACCEPT rules in filter FORWARD - // can be inserted above ours. Mangle runs before filter, so these guard rules enforce the - // ACL mark check where it cannot be overridden. - m.appendToEntries(mangleFwdKey, []string{ - "-i", m.wgIface.Name(), - "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", - "-j", "ACCEPT", - }) - m.appendToEntries(mangleFwdKey, []string{ - "-i", m.wgIface.Name(), - "-m", "conntrack", "--ctstate", "DNAT", - "-m", "mark", "!", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), - "-j", "DROP", - }) -} - -func (m *aclManager) seedInitialOptionalEntries() { - m.optionalEntries["FORWARD"] = []entry{ - { - spec: []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", "ACCEPT"}, - position: 2, - }, - } -} - -func (m *aclManager) appendToEntries(chainName string, spec []string) { - m.entries[chainName] = append(m.entries[chainName], spec) -} - -func (m *aclManager) updateState() { - if m.stateManager == nil { - return - } - - var currentState *ShutdownState - if existing := m.stateManager.GetState(currentState); existing != nil { - if existingState, ok := existing.(*ShutdownState); ok { - currentState = existingState - } - } - if currentState == nil { - currentState = &ShutdownState{} - } - - currentState.Lock() - defer currentState.Unlock() - - // Clone the maps so the persisted state holds a private snapshot. The - // live maps keep being mutated by subsequent rule operations while the - // state manager marshals the state from its periodic-save goroutine. - // Sharing them by reference races the two and aborts the process with a - // concurrent map iteration and write. - if m.v6 { - currentState.ACLEntries6 = maps.Clone(m.entries) - currentState.ACLIPsetStore6 = m.ipsetStore.clone() - } else { - currentState.ACLEntries = maps.Clone(m.entries) - currentState.ACLIPsetStore = m.ipsetStore.clone() - } - - if err := m.stateManager.UpdateState(currentState); err != nil { - log.Errorf("failed to update state: %v", err) - } -} - -// filterRuleSpecs returns the specs of a filtering rule -// protoForFamily translates ICMP to ICMPv6 for ip6tables. -// ip6tables requires "ipv6-icmp" (or "icmpv6") instead of "icmp". -func protoForFamily(protocol firewall.Protocol, v6 bool) string { - if v6 && protocol == firewall.ProtocolICMP { - return "ipv6-icmp" - } - return string(protocol) -} - -func filterRuleSpecs(ip net.IP, protocol string, sPort, dPort *firewall.Port, action firewall.Action, ipsetName string) (specs []string) { - // don't use IP matching if IP is 0.0.0.0 - matchByIP := !ip.IsUnspecified() - - if matchByIP { - if ipsetName != "" { - specs = append(specs, "-m", "set", "--match-set", ipsetName, "src") - } else { - specs = append(specs, "-s", ip.String()) - } - } - if protocol != "all" { - specs = append(specs, "-p", protocol) - } - specs = append(specs, applyPort("--sport", sPort)...) - specs = append(specs, applyPort("--dport", dPort)...) - return specs -} - -func actionToStr(action firewall.Action) string { - if action == firewall.ActionAccept { - return "ACCEPT" - } - return "DROP" -} - -func transformIPsetName(ipsetName string, sPort, dPort *firewall.Port, action firewall.Action) string { - if ipsetName == "" { - return "" - } - - actionSuffix := "" - if action == firewall.ActionDrop { - actionSuffix = "-drop" - } - - switch { - case sPort != nil && dPort != nil: - return ipsetName + "-sport-dport" + actionSuffix - case sPort != nil: - return ipsetName + "-sport" + actionSuffix - case dPort != nil: - return ipsetName + "-dport" + actionSuffix - default: - return ipsetName + actionSuffix - } -} - -// probeIPSetSupport checks whether the kernel can create the ipset type used for -// ACL rules. On kernels lacking the required ipset hash module, ipset creation -// fails (e.g. "invalid argument"), which would otherwise leave the ACL chain -// empty and silently drop all policy-permitted inbound traffic. When unsupported, -// the manager falls back to per-IP iptables rules. -func (m *aclManager) probeIPSetSupport() bool { - // Use a unique name so concurrent processes don't collide and we only ever - // destroy the set we created ourselves. ipset names are limited to 31 chars, - // so use a short random suffix. - probeName := "nb-probe-" + uuid.New().String()[:8] - - opts := ipset.CreateOptions{ - Replace: true, - } - if m.v6 { - opts.Family = ipset.FamilyIPV6 - } - - if err := ipset.Create(probeName, ipset.TypeHashNet, opts); err != nil { - log.Warnf("ipset is not available (failed to create probe set: %v); "+ - "falling back to per-IP iptables ACL rules. Ensure the kernel provides "+ - "the ipset hash:net module (ip_set_hash_net) for better performance with large rule sets", err) - return false - } - - defer func() { - if err := ipset.Destroy(probeName); err != nil { - log.Debugf("destroy ipset probe set %q: %v", probeName, err) - } - }() - - return true -} - -func (m *aclManager) createIPSet(name string) error { - opts := ipset.CreateOptions{ - Replace: true, - } - if m.v6 { - opts.Family = ipset.FamilyIPV6 - } - - if err := ipset.Create(name, ipset.TypeHashNet, opts); err != nil { - return fmt.Errorf("create ipset %s: %w", name, err) - } - - log.Debugf("created ipset %s with type hash:net", name) - return nil -} - -func (m *aclManager) addToIPSet(name string, ip net.IP) error { - cidr := uint8(32) - if ip.To4() == nil { - cidr = 128 - } - - entry := &ipset.Entry{ - IP: ip, - CIDR: cidr, - Replace: true, - } - - if err := ipset.Add(name, entry); err != nil { - return fmt.Errorf("add IP to ipset %s: %w", name, err) - } - - return nil -} - -func (m *aclManager) delFromIPSet(name string, ip net.IP) error { - cidr := uint8(32) - if ip.To4() == nil { - cidr = 128 - } - - entry := &ipset.Entry{ - IP: ip, - CIDR: cidr, - } - - if err := ipset.Del(name, entry); err != nil { - return fmt.Errorf("delete IP from ipset %s: %w", name, err) - } - - return nil -} - -func (m *aclManager) flushIPSet(name string) error { - return ipset.Flush(name) -} - -func (m *aclManager) destroyIPSet(name string) error { - return ipset.Destroy(name) -} diff --git a/client/firewall/iptables/chains_linux.go b/client/firewall/iptables/chains_linux.go new file mode 100644 index 000000000..58bfa8c6a --- /dev/null +++ b/client/firewall/iptables/chains_linux.go @@ -0,0 +1,346 @@ +//go:build !android + +package iptables + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) createContainers() error { + for _, chainInfo := range []struct { + chain string + table string + }{ + {chainRTFwdIn, tableFilter}, + {chainRTFwdOut, tableFilter}, + {chainRTPre, tableMangle}, + {chainRTNAT, tableNat}, + {chainRTRdr, tableNat}, + {chainRTMSSClamp, tableMangle}, + } { + // Fallback: clear chains that survived an unclean shutdown. + if ok, _ := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain); ok { + if err := r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { + log.Warnf("clear stale chain %s in %s: %v", chainInfo.chain, chainInfo.table, err) + } + } + if err := r.iptablesClient.NewChain(chainInfo.table, chainInfo.chain); err != nil { + return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) + } + } + + if err := r.insertEstablishedRule(chainRTFwdIn); err != nil { + return fmt.Errorf("insert established rule: %w", err) + } + + if err := r.insertEstablishedRule(chainRTFwdOut); err != nil { + return fmt.Errorf("insert established rule: %w", err) + } + + if err := r.addPostroutingRules(); err != nil { + return fmt.Errorf("add static nat rules: %w", err) + } + + if err := r.addJumpRules(); err != nil { + return fmt.Errorf("add jump rules: %w", err) + } + + if err := r.addMSSClampingRules(); err != nil { + log.Errorf("failed to add MSS clamping rules: %s", err) + } + + return nil +} + +func (r *family) addJumpRules() error { + // Jump to nat chain + natRule := jumpRuleSpec(chainRTNAT) + if err := r.iptablesClient.Insert(tableNat, chainPostrouting, 1, natRule...); err != nil { + return fmt.Errorf("add nat postrouting jump rule: %w", err) + } + r.rules[jumpNATPost] = natRule + + // Jump to mangle prerouting chain + preRule := jumpRuleSpec(chainRTPre) + if err := r.iptablesClient.Insert(tableMangle, chainPrerouting, 1, preRule...); err != nil { + return fmt.Errorf("add mangle prerouting jump rule: %w", err) + } + r.rules[jumpManglePre] = preRule + + // Jump to nat prerouting chain + rdrRule := jumpRuleSpec(chainRTRdr) + if err := r.iptablesClient.Insert(tableNat, chainPrerouting, 1, rdrRule...); err != nil { + return fmt.Errorf("add nat prerouting jump rule: %w", err) + } + r.rules[jumpNATPre] = rdrRule + + return nil +} + +func (r *family) setupDataPlaneMark() error { + var merr *multierror.Error + preRule := []string{ + "-i", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "NEW", + "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkIn), + } + + if err := r.iptablesClient.AppendUnique(tableMangle, chainPrerouting, preRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add mangle prerouting rule: %w", err)) + } else { + r.rules[markManglePre] = preRule + } + + postRule := []string{ + "-o", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "NEW", + "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkOut), + } + + if err := r.iptablesClient.AppendUnique(tableMangle, chainPostrouting, postRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add mangle postrouting rule: %w", err)) + } else { + r.rules[markManglePost] = postRule + } + + return nberrors.FormatErrorOrNil(merr) +} + +// seedInitialEntries adds default rules to the entries map. Rules are +// inserted at position 1, so the order here is reversed. +// +// Existing FORWARD policy decides outbound traffic towards our +// interface. If FORWARD policy is "drop", we add an +// established/related rule to allow return traffic for inbound rules. +func (r *family) seedInitialEntries() { + established := getConntrackEstablished() + + r.appendToEntries(chainInput, []string{"-i", r.wgIface.Name(), "-j", "DROP"}) + r.appendToEntries(chainInput, []string{"-i", r.wgIface.Name(), "-j", chainACLInput}) + r.appendToEntries(chainInput, append([]string{"-i", r.wgIface.Name()}, established...)) + + r.appendToEntries(chainForward, []string{"-i", r.wgIface.Name(), "-j", "DROP"}) + r.appendToEntries(chainForward, []string{"-o", r.wgIface.Name(), "-j", chainRTFwdOut}) + r.appendToEntries(chainForward, []string{"-i", r.wgIface.Name(), "-j", chainRTFwdIn}) + + // Mangle FORWARD guard: when external DNAT redirects traffic from + // the wg interface, it traverses FORWARD instead of INPUT, + // bypassing ACL rules. ACCEPT rules in filter FORWARD can be + // inserted above ours. Mangle runs before filter, so these guard + // rules enforce the ACL mark check where it cannot be overridden. + r.appendToEntries(mangleForwardKey, []string{ + "-i", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", + "-j", "ACCEPT", + }) + r.appendToEntries(mangleForwardKey, []string{ + "-i", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "DNAT", + "-m", "mark", "!", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), + "-j", "DROP", + }) +} + +func (r *family) seedInitialOptionalEntries() { + r.optionalEntries[chainForward] = []entry{ + { + spec: []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", "ACCEPT"}, + position: 2, + }, + } +} + +func (r *family) appendToEntries(chain chainKey, spec ruleSpec) { + r.entries[chain] = append(r.entries[chain], spec) +} + +func (r *family) createDefaultChains() error { + if err := r.iptablesClient.NewChain(tableFilter, chainACLInput); err != nil { + return fmt.Errorf("create %s chain: %w", chainACLInput, err) + } + + for chain, rules := range r.entries { + // mangle FORWARD guard rules are handled separately below + if chain == mangleForwardKey { + continue + } + for _, rule := range rules { + if err := r.iptablesClient.InsertUnique(tableFilter, string(chain), 1, rule...); err != nil { + return fmt.Errorf("insert jump rule into %s: %w", chain, err) + } + } + } + + for chain, entries := range r.optionalEntries { + for _, entry := range entries { + if err := r.iptablesClient.InsertUnique(tableFilter, string(chain), entry.position, entry.spec...); err != nil { + log.Errorf("failed to insert optional entry %v: %v", entry.spec, err) + continue + } + r.entries[chain] = append(r.entries[chain], entry.spec) + } + } + clear(r.optionalEntries) + + // Insert mangle FORWARD guard rules to prevent external DNAT bypass. + for _, rule := range r.entries[mangleForwardKey] { + if err := r.iptablesClient.AppendUnique(tableMangle, chainForward, rule...); err != nil { + log.Errorf("failed to add mangle FORWARD guard rule: %v", err) + } + } + + return nil +} + +func (r *family) cleanUpDefaultForwardRules() error { + var merr *multierror.Error + + // cleanJumpRules removes the OUTPUT jump to NETBIRD-NAT-OUTPUT among + // the others, so the chain below deletes cleanly instead of failing + // with "device or resource busy". + if err := r.cleanJumpRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clean jump rules: %w", err)) + } + + for _, chainInfo := range []struct { + chain string + table string + }{ + {chainRTFwdIn, tableFilter}, + {chainRTFwdOut, tableFilter}, + {chainRTPre, tableMangle}, + {chainRTNAT, tableNat}, + {chainRTRdr, tableNat}, + {chainNATOutput, tableNat}, + {chainRTMSSClamp, tableMangle}, + } { + ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)) + continue + } + if ok { + if err := r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)) + } + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) cleanJumpRules() error { + // locations maps each jump rule to the built-in table and chain it + // was inserted into, plus the netbird chain it targets. + locations := map[firewall.RuleID]struct{ table, chain, target string }{ + jumpNATPost: {tableNat, chainPostrouting, chainRTNAT}, + jumpManglePre: {tableMangle, chainPrerouting, chainRTPre}, + jumpNATPre: {tableNat, chainPrerouting, chainRTRdr}, + jumpMSSClamp: {tableMangle, chainForward, chainRTMSSClamp}, + jumpNATOutput: {tableNat, chainOutput, chainNATOutput}, + } + + var merr *multierror.Error + for ruleID, loc := range locations { + rule, exists := r.rules[ruleID] + if !exists { + // Untracked (e.g. fresh start after an unclean shutdown with no + // restored state): if the target chain survived, remove the stale + // jump to it so the chain can be deleted. + ok, err := r.iptablesClient.ChainExists(loc.table, loc.target) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("check chain %s in table %s: %w", loc.target, loc.table, err)) + continue + } + if !ok { + continue + } + rule = jumpRuleSpec(loc.target) + } + if err := r.iptablesClient.DeleteIfExists(loc.table, loc.chain, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete rule from chain %s in table %s: %w", loc.chain, loc.table, err)) + continue + } + delete(r.rules, ruleID) + } + return nberrors.FormatErrorOrNil(merr) +} + +// jumpRuleSpec builds the iptables rule spec that jumps to target. Create +// and cleanup sites share it so the installed and deleted specs cannot drift. +func jumpRuleSpec(target string) []string { + return []string{"-j", target} +} + +func (r *family) cleanAclChains() error { + var merr *multierror.Error + + if err := r.cleanInputAclChain(); err != nil { + merr = multierror.Append(merr, err) + } + + for _, rule := range r.entries[mangleForwardKey] { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainForward, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete mangle %s guard rule %v: %w", chainForward, rule, err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) cleanInputAclChain() error { + ok, err := r.iptablesClient.ChainExists(tableFilter, chainACLInput) + if err != nil { + return fmt.Errorf("check chain %s: %w", chainACLInput, err) + } + if !ok { + return nil + } + + var merr *multierror.Error + for _, rule := range r.entries[chainInput] { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainInput, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete %s rule %v: %w", chainInput, rule, err)) + } + } + + for _, rule := range r.entries[chainForward] { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainForward, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete %s rule %v: %w", chainForward, rule, err)) + } + } + + if err := r.iptablesClient.ClearAndDeleteChain(tableFilter, chainACLInput); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clear and delete %s chain: %w", chainACLInput, err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) cleanupDataPlaneMark() error { + var merr *multierror.Error + if preRule, exists := r.rules[markManglePre]; exists { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPrerouting, preRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove mangle prerouting rule: %w", err)) + } else { + delete(r.rules, markManglePre) + } + } + + if postRule, exists := r.rules[markManglePost]; exists { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPostrouting, postRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove mangle postrouting rule: %w", err)) + } else { + delete(r.rules, markManglePost) + } + } + + return nberrors.FormatErrorOrNil(merr) +} diff --git a/client/firewall/iptables/dnat_linux.go b/client/firewall/iptables/dnat_linux.go new file mode 100644 index 000000000..eca8386c0 --- /dev/null +++ b/client/firewall/iptables/dnat_linux.go @@ -0,0 +1,302 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "net/netip" + "strconv" + "strings" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { + ruleID := rule.ID() + if _, exists := r.rules[ruleID+dnatSuffix]; exists { + return rule, nil + } + + toDestination := rule.TranslatedAddress.String() + switch { + case len(rule.TranslatedPort.Values) == 0: + // no translated port, use original port + case len(rule.TranslatedPort.Values) == 1: + toDestination += fmt.Sprintf(":%d", rule.TranslatedPort.Values[0]) + case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: + // need the "/originalport" suffix to avoid dnat port randomization + toDestination += fmt.Sprintf(":%d-%d/%d", rule.TranslatedPort.Values[0], rule.TranslatedPort.Values[1], rule.DestinationPort.Values[0]) + default: + return nil, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) + } + + proto := strings.ToLower(string(rule.Protocol)) + + rules := make(map[firewall.RuleID]ruleInfo, 3) + + // DNAT rule + dnatRule := []string{ + "!", "-i", r.wgIface.Name(), + "-p", proto, + "-j", "DNAT", + "--to-destination", toDestination, + } + dnatRule = append(dnatRule, applyPort("--dport", &rule.DestinationPort)...) + rules[ruleID+dnatSuffix] = ruleInfo{ + table: tableNat, + chain: chainRTRdr, + rule: dnatRule, + } + + // SNAT rule + snatRule := []string{ + "-o", r.wgIface.Name(), + "-p", proto, + "-d", rule.TranslatedAddress.String(), + "-j", "MASQUERADE", + } + snatRule = append(snatRule, applyPort("--dport", &rule.TranslatedPort)...) + rules[ruleID+snatSuffix] = ruleInfo{ + table: tableNat, + chain: chainRTNAT, + rule: snatRule, + } + + // Forward filtering rule, if fwd policy is DROP + forwardRule := []string{ + "-o", r.wgIface.Name(), + "-p", proto, + "-d", rule.TranslatedAddress.String(), + "-j", "ACCEPT", + } + forwardRule = append(forwardRule, applyPort("--dport", &rule.TranslatedPort)...) + rules[ruleID+fwdSuffix] = ruleInfo{ + table: tableFilter, + chain: chainRTFwdOut, + rule: forwardRule, + } + + for key, ruleInfo := range rules { + if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { + r.cleanupFailedDNATAdd(rules) + return nil, fmt.Errorf("add rule %s: %w", key, err) + } + r.rules[key] = ruleInfo.rule + } + + if err := r.ipFwdState.RequestForwarding(r.v6); err != nil { + r.cleanupFailedDNATAdd(rules) + return nil, fmt.Errorf("enable forwarding: %w", err) + } + + r.updateState() + return rule, nil +} + +// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied +// AddDNATRule before rolling back the kernel rules, so no entries remain that +// never got a forwarding refcount. rollbackRules re-adds entries it failed to +// remove from the kernel. +func (r *family) cleanupFailedDNATAdd(rules map[firewall.RuleID]ruleInfo) { + for key := range rules { + delete(r.rules, key) + } + if err := r.rollbackRules(rules); err != nil { + log.Errorf("rollback failed: %v", err) + } +} + +func (r *family) rollbackRules(rules map[firewall.RuleID]ruleInfo) error { + var merr *multierror.Error + for key, ruleInfo := range rules { + if err := r.iptablesClient.DeleteIfExists(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("rollback rule %s: %w", key, err)) + // On rollback error, add to rules map for next cleanup + r.rules[key] = ruleInfo.rule + } + } + if merr != nil { + r.updateState() + } + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) DeleteDNATRule(rule firewall.Rule) error { + ruleID := rule.ID() + + _, hadDNAT := r.rules[ruleID+dnatSuffix] + _, hadSNAT := r.rules[ruleID+snatSuffix] + _, hadFWD := r.rules[ruleID+fwdSuffix] + if !hadDNAT && !hadSNAT && !hadFWD { + return nil + } + + var merr *multierror.Error + if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists { + if err := r.iptablesClient.Delete(tableNat, chainRTRdr, dnatRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err)) + } else { + delete(r.rules, ruleID+dnatSuffix) + } + } + + if snatRule, exists := r.rules[ruleID+snatSuffix]; exists { + if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err)) + } else { + delete(r.rules, ruleID+snatSuffix) + } + } + + if fwdRule, exists := r.rules[ruleID+fwdSuffix]; exists { + if err := r.iptablesClient.Delete(tableFilter, chainRTFwdOut, fwdRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err)) + } else { + delete(r.rules, ruleID+fwdSuffix) + } + } + + // Release the refcount only once all rules are gone from the kernel. On + // partial failure the failed entries stay in r.rules so a retry can remove + // them and release then. + if merr == nil { + r.releaseForwarding() + } + + r.updateState() + + return nberrors.FormatErrorOrNil(merr) +} + +// releaseForwarding drops one IP forwarding reference, logging any error. +func (r *family) releaseForwarding() { + if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil { + log.Errorf("release IP forwarding: %v", err) + } +} + +func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + dnatRule := []string{ + "-i", r.wgIface.Name(), + "-p", strings.ToLower(protoForFamily(protocol, r.v6)), + "--dport", strconv.Itoa(int(originalPort)), + "-d", localAddr.String(), + "-m", "addrtype", "--dst-type", "LOCAL", + "-j", "DNAT", + "--to-destination", ":" + strconv.Itoa(int(translatedPort)), + } + + info := ruleInfo{ + table: tableNat, + chain: chainRTRdr, + rule: dnatRule, + } + + if err := r.iptablesClient.Append(info.table, info.chain, info.rule...); err != nil { + return fmt.Errorf("add inbound DNAT rule: %w", err) + } + r.rules[ruleID] = info.rule + + r.updateState() + return nil +} + +// RemoveInboundDNAT removes an inbound DNAT rule. +func (r *family) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if dnatRule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.Delete(tableNat, chainRTRdr, dnatRule...); err != nil { + return fmt.Errorf("delete inbound DNAT rule: %w", err) + } + delete(r.rules, ruleID) + } + + r.updateState() + return nil +} + +// ensureNATOutputChain lazily creates the OUTPUT NAT chain and jump rule on first use. +func (r *family) ensureNATOutputChain() error { + if _, exists := r.rules[jumpNATOutput]; exists { + return nil + } + + chainExists, err := r.iptablesClient.ChainExists(tableNat, chainNATOutput) + if err != nil { + return fmt.Errorf("check chain %s: %w", chainNATOutput, err) + } + if !chainExists { + if err := r.iptablesClient.NewChain(tableNat, chainNATOutput); err != nil { + return fmt.Errorf("create chain %s: %w", chainNATOutput, err) + } + } + + jumpRule := jumpRuleSpec(chainNATOutput) + if err := r.iptablesClient.Insert(tableNat, chainOutput, 1, jumpRule...); err != nil { + if !chainExists { + if delErr := r.iptablesClient.ClearAndDeleteChain(tableNat, chainNATOutput); delErr != nil { + log.Warnf("failed to rollback chain %s: %v", chainNATOutput, delErr) + } + } + return fmt.Errorf("add OUTPUT jump rule: %w", err) + } + r.rules[jumpNATOutput] = jumpRule + + r.updateState() + return nil +} + +// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. +func (r *family) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + if err := r.ensureNATOutputChain(); err != nil { + return err + } + + dnatRule := []string{ + "-p", strings.ToLower(protoForFamily(protocol, localAddr.Is6())), + "--dport", strconv.Itoa(int(originalPort)), + "-d", localAddr.String(), + "-j", "DNAT", + "--to-destination", ":" + strconv.Itoa(int(translatedPort)), + } + + if err := r.iptablesClient.Append(tableNat, chainNATOutput, dnatRule...); err != nil { + return fmt.Errorf("add output DNAT rule: %w", err) + } + r.rules[ruleID] = dnatRule + + r.updateState() + return nil +} + +// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. +func (r *family) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if dnatRule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.Delete(tableNat, chainNATOutput, dnatRule...); err != nil { + return fmt.Errorf("delete output DNAT rule: %w", err) + } + delete(r.rules, ruleID) + } + + r.updateState() + return nil +} diff --git a/client/firewall/iptables/dnat_refcount_linux_test.go b/client/firewall/iptables/dnat_refcount_linux_test.go index 681bc0b99..40ebc6cc3 100644 --- a/client/firewall/iptables/dnat_refcount_linux_test.go +++ b/client/firewall/iptables/dnat_refcount_linux_test.go @@ -80,7 +80,7 @@ func iptDnatV6(port uint16) fw.ForwardRule { // and a single DisableRouting drops both back to zero. func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) { m := newIptRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState require.NoError(t, m.EnableRouting(), "first enable") require.NoError(t, m.EnableRouting(), "second enable") @@ -99,7 +99,7 @@ func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) { // DisableRouting does not release references held by active DNAT rules. func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) { m := newIptRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState r1, err := m.AddDNATRule(iptDnatV6(9095)) require.NoError(t, err, "add v6 dnat") @@ -116,7 +116,7 @@ func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) { // TestIptablesDNAT_RefcountBalancedV4 covers a Balanced Add/Delete pair on v4. func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) { m := newIptRefcountManager(t, false) - state := m.router.ipFwdState + state := m.family4.ipFwdState r1, err := m.AddDNATRule(iptDnatV4(7081)) require.NoError(t, err, "add v4 dnat 1") @@ -145,9 +145,9 @@ func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) { // decrements back to zero. func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) { m := newIptRefcountManager(t, true) - require.NotNil(t, m.router6, "v6 router") - require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state") - state := m.router.ipFwdState + require.NotNil(t, m.family6, "v6 family") + require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state") + state := m.family4.ipFwdState r1, err := m.AddDNATRule(iptDnatV6(9081)) require.NoError(t, err, "add v6 dnat 1") @@ -176,7 +176,7 @@ func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) { // without bumping the refcount. func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) { m := newIptRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState rule := iptDnatV4(7083) r1, err := m.AddDNATRule(rule) @@ -198,7 +198,7 @@ func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) { // neither errors nor releases the refcount. func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { m := newIptRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState phantom := iptDnatV4(7099) require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4") @@ -223,7 +223,7 @@ func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { // rule is a no-op. func TestIptablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { m := newIptRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState r1, err := m.AddDNATRule(iptDnatV6(9083)) require.NoError(t, err) diff --git a/client/firewall/iptables/family_linux.go b/client/firewall/iptables/family_linux.go new file mode 100644 index 000000000..c5ed8cc20 --- /dev/null +++ b/client/firewall/iptables/family_linux.go @@ -0,0 +1,258 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "maps" + "net/netip" + + "github.com/coreos/go-iptables/iptables" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbid "github.com/netbirdio/netbird/client/internal/acl/id" + "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" + "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" + "github.com/netbirdio/netbird/client/internal/statemanager" +) + +// constants needed to manage and create iptable rules +const ( + tableFilter = "filter" + tableNat = "nat" + tableMangle = "mangle" + + // chainACLInput is the peer ACL chain that holds installed + // peer-filtering rules. + chainACLInput = "NETBIRD-ACL-INPUT" + + // mangleForwardKey is the entries map key for mangle FORWARD guard + // rules that prevent external DNAT from bypassing ACL rules. + mangleForwardKey chainKey = "MANGLE-FORWARD" + + chainInput = "INPUT" + chainPostrouting = "POSTROUTING" + chainPrerouting = "PREROUTING" + chainForward = "FORWARD" + chainRTNAT = "NETBIRD-RT-NAT" + chainRTFwdIn = "NETBIRD-RT-FWD-IN" + chainRTFwdOut = "NETBIRD-RT-FWD-OUT" + chainRTPre = "NETBIRD-RT-PRE" + chainRTRdr = "NETBIRD-RT-RDR" + chainNATOutput = "NETBIRD-NAT-OUTPUT" + chainRTMSSClamp = "NETBIRD-RT-MSSCLAMP" + + jumpManglePre = "jump-mangle-pre" + jumpNATPre = "jump-nat-pre" + jumpNATPost = "jump-nat-post" + jumpNATOutput = "jump-nat-output" + jumpMSSClamp = "jump-mss-clamp" + markManglePre = "mark-mangle-pre" + markManglePost = "mark-mangle-post" + matchSet = "--match-set" + + dnatSuffix firewall.RuleID = "_dnat" + snatSuffix firewall.RuleID = "_snat" + fwdSuffix firewall.RuleID = "_fwd" + + // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. + ipv4TCPHeaderSize = 40 + // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. + ipv6TCPHeaderSize = 60 +) + +type ruleInfo struct { + chain string + table string + rule []string +} + +type routeRules map[firewall.RuleID][]string + +// ruleSpec is a single iptables rule expressed as its argument list +// (e.g. {"-i", "wg0", "-j", "DROP"}). +type ruleSpec []string + +// chainKey identifies the chain a seeded entry belongs to. It holds +// built-in chain names ("INPUT", "FORWARD", "PREROUTING") plus the +// synthetic mangleForwardKey bucket for the mangle FORWARD guard rules. +type chainKey string + +// aclEntries maps a chain to the rules seeded into it to jump into or +// guard the netbird ACL chains. +type aclEntries map[chainKey][]ruleSpec + +type entry struct { + spec ruleSpec + position int +} + +// ipsetCounter is the shared hash:net refcounter used by peer and +// route ACLs alike. The ipset library does not support comments, so +// the key is just the set name (string). +type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}] + +// family holds the per-address-family iptables state. One instance +// handles route ACLs, peer ACLs, NAT, DNAT, and MSS clamping for a +// single family; the top-level Manager owns one for v4 and another +// for v6. +type family struct { + iptablesClient *iptables.IPTables + wgIface iFaceMapper + v6 bool + + // Peer ACL chain bookkeeping. + entries aclEntries + optionalEntries map[chainKey][]entry + + // filters holds peer + route filter rules keyed by content hash. + // AddFilterRule writes here; DeleteFilterRule looks up by id. + filters map[nbid.RuleID]*Rule + ipsetCounter *ipsetCounter + // ipsetSupported records whether the kernel can create the hash:net + // sets the source matches rely on; probed once at init. When false, + // multi-source rules expand to one rule per source prefix. + ipsetSupported bool + + // rules holds NAT, jump, and MSS-clamping rules (auxiliary + // plumbing that isn't a filter rule). + rules routeRules + + // Routing / NAT. + legacyManagement bool + mtu uint16 + ipFwdState *ipfwdstate.IPForwardingState + + stateManager *statemanager.Manager +} + +func newFamily(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16) (*family, error) { + r := &family{ + iptablesClient: iptablesClient, + wgIface: wgIface, + v6: iptablesClient.Proto() == iptables.ProtocolIPv6, + entries: make(aclEntries), + optionalEntries: make(map[chainKey][]entry), + filters: make(map[nbid.RuleID]*Rule), + rules: make(routeRules), + mtu: mtu, + ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), + } + + r.ipsetCounter = refcounter.New( + func(name string, sources []netip.Prefix) (struct{}, error) { + return struct{}{}, r.createIpSet(name, sources) + }, + func(name string, _ struct{}) error { + return r.deleteIpSet(name) + }, + ) + + return r, nil +} + +// init wires the family to the state manager and installs both the +// route ACL containers and the peer ACL chain skeleton. +func (r *family) init(stateManager *statemanager.Manager) error { + r.stateManager = stateManager + + r.ipsetSupported = r.probeIPSetSupport() + + if err := r.cleanUpDefaultForwardRules(); err != nil { + log.Errorf("failed to clean up rules from FORWARD chain: %s", err) + } + + if err := r.createContainers(); err != nil { + return fmt.Errorf("create containers: %w", err) + } + + if err := r.setupDataPlaneMark(); err != nil { + log.Errorf("failed to set up data plane mark: %v", err) + } + + r.seedInitialEntries() + r.seedInitialOptionalEntries() + + if err := r.cleanAclChains(); err != nil { + return fmt.Errorf("clean acl chains: %w", err) + } + if err := r.createDefaultChains(); err != nil { + return fmt.Errorf("create default chains: %w", err) + } + + r.updateState() + + return nil +} + +// Reset tears down all firewall state owned by this family. ACL +// chain cleanup runs before route-chain cleanup because the route +// chains are still referenced by FORWARD jumps installed during +// seedInitialEntries; deleting them first would trip EBUSY. +func (r *family) Reset() error { + var merr *multierror.Error + + if err := r.cleanAclChains(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.cleanUpDefaultForwardRules(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.ipsetCounter.Flush(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.cleanupDataPlaneMark(); err != nil { + merr = multierror.Append(merr, err) + } + + clear(r.rules) + clear(r.filters) + r.updateState() + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) updateState() { + if r.stateManager == nil { + return + } + + var currentState *ShutdownState + if existing := r.stateManager.GetState(currentState); existing != nil { + if existingState, ok := existing.(*ShutdownState); ok { + currentState = existingState + } + } + if currentState == nil { + currentState = &ShutdownState{} + } + + currentState.Lock() + defer currentState.Unlock() + + // Clone the rule maps so the persisted state holds a private snapshot. + // The live maps keep being mutated by subsequent rule operations while + // the state manager marshals the state from its periodic-save goroutine. + // Sharing the maps by reference races the two and aborts the process with + // a concurrent map iteration and write. The ipset counter guards itself + // during marshaling, so it can be shared directly. + if r.v6 { + currentState.RouteRules6 = maps.Clone(r.rules) + currentState.RouteIPsetCounter6 = r.ipsetCounter + currentState.ACLEntries6 = maps.Clone(r.entries) + } else { + currentState.RouteRules = maps.Clone(r.rules) + currentState.RouteIPsetCounter = r.ipsetCounter + currentState.ACLEntries = maps.Clone(r.entries) + } + + if err := r.stateManager.UpdateState(currentState); err != nil { + log.Errorf("failed to update state: %v", err) + } +} diff --git a/client/firewall/iptables/filter_linux.go b/client/firewall/iptables/filter_linux.go new file mode 100644 index 000000000..dc606da2d --- /dev/null +++ b/client/firewall/iptables/filter_linux.go @@ -0,0 +1,430 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "net/netip" + "slices" + "strconv" + "strings" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbid "github.com/netbirdio/netbird/client/internal/acl/id" + nbnet "github.com/netbirdio/netbird/client/net" +) + +// AddFilterRule installs a packet-filtering rule. With destination +// empty, the rule goes to the peer ACL input chain plus a paired +// mangle PREROUTING rule for the redirect mark. With destination set +// (prefix or named set), it goes to the route ACL forward chain. +// Multi-source rules collapse to one iptables rule via the shared +// hash:net ipset. +func (r *family) AddFilterRule( + id []byte, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort *firewall.Port, + dPort *firewall.Port, + action firewall.Action, +) (firewall.Rule, error) { + ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action) + if existing, ok := r.filters[ruleID]; ok { + return existing, nil + } + + rule, err := r.installFilterRules(ruleID, sources, destination, proto, sPort, dPort, action, r.ipsetSupported) + if err != nil { + return nil, err + } + + r.filters[ruleID] = rule + r.updateState() + return rule, nil +} + +// installFilterRules resolves the source matches and installs one +// iptables rule per match. It is more than one rule only when useIPSet +// is false and a multi-source rule has to be expanded per prefix. +func (r *family) installFilterRules( + ruleID nbid.RuleID, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort *firewall.Port, + dPort *firewall.Port, + action firewall.Action, + useIPSet bool, +) (*Rule, error) { + srcMatches, err := r.applySourceMatches(sources, useIPSet) + if err != nil { + return nil, fmt.Errorf("apply source match: %w", err) + } + + rule, err := r.installFilterRule(ruleID, srcMatches, destination, proto, sPort, dPort, action) + if err != nil { + for _, srcMatch := range srcMatches { + r.dropSourceMatch(srcMatch) + } + return nil, err + } + return rule, nil +} + +func (r *family) hasRule(id nbid.RuleID) bool { + _, ok := r.filters[id] + return ok +} + +// hasDNATRule reports whether this family owns the DNAT rule set for +// the given user id. DNAT rules live in r.rules under the well-known +// "_dnat" key; the lookup here is used by Manager.DeleteDNATRule +// to pick the right family. +func (r *family) hasDNATRule(id firewall.RuleID) bool { + _, ok := r.rules[id+dnatSuffix] + return ok +} + +// DeleteFilterRule removes a previously installed filter rule. The +// rule's stored chain/table identify where to delete from; source set +// references are recovered from the spec via findSets and dropped +// from the shared ipset counter. +func (r *family) DeleteFilterRule(rule firewall.Rule) error { + ruleID := rule.ID() + pr, ok := r.filters[ruleID] + if !ok { + log.Debugf("filter rule %s not found", ruleID) + return nil + } + + // DeleteIfExists keeps the deletes idempotent so a retry after a + // partial failure does not error on the parts already removed. + var merr *multierror.Error + for _, fs := range pr.allSpecs() { + if err := r.iptablesClient.DeleteIfExists(tableFilter, pr.chain, fs.specs...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete rule from %s: %w", pr.chain, err)) + } + if fs.mangleSpecs != nil { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, fs.mangleSpecs...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete mangle rule: %w", err)) + } + } + } + if merr != nil { + // Leave the rule tracked so the caller retries the remaining part. + return nberrors.FormatErrorOrNil(merr) + } + + // The rule is gone from iptables, so untrack it regardless of how the + // refcount decrement goes, but surface decrement failures so callers + // see the ipset desync. Only the primary spec can reference sets: the + // per-prefix expansion never uses them. + delete(r.filters, ruleID) + r.updateState() + if err := r.decrementSetCounter(pr.specs); err != nil { + return fmt.Errorf("drop source set references: %w", err) + } + return nil +} + +// findSets scans an iptables rule spec for "-m set --match-set +// " fragments and returns the named sets in occurrence order. +// Used at delete time to drop ipsetCounter references. +func findSets(rule []string) []string { + var sets []string + for i, arg := range rule { + if arg == "-m" && i+3 < len(rule) && rule[i+1] == "set" && rule[i+2] == matchSet { + sets = append(sets, rule[i+3]) + } + } + return sets +} + +// sourceNetwork classifies a source-prefix list into the firewall.Network +// shape the rest of the spec-builder consumes: empty for match-any, a +// single prefix inline, or an ipset for multiple sources. +func sourceNetwork(sources []netip.Prefix) firewall.Network { + switch { + case len(sources) == 0: + return firewall.Network{} + case len(sources) == 1 && sources[0].Bits() == 0: + return firewall.Network{} + case len(sources) == 1: + return firewall.Network{Prefix: sources[0]} + default: + return firewall.Network{Set: firewall.NewPrefixSet(sources)} + } +} + +// applySourceMatches returns one source match fragment per iptables +// rule needed for the sources: normally a single fragment (a set match, +// a direct -s match, or nil for match-any), and one -s fragment per +// prefix when a multi-source rule cannot use ipset. Per-prefix rules +// are the only form a kernel without the ipset modules can express. +func (r *family) applySourceMatches(sources []netip.Prefix, useIPSet bool) ([][]string, error) { + network := sourceNetwork(sources) + if !network.IsSet() || useIPSet { + match, err := r.applySourceMatch(network, sources) + if err != nil { + return nil, err + } + return [][]string{match}, nil + } + + matches := make([][]string, 0, len(sources)) + for _, source := range sources { + matches = append(matches, []string{"-s", source.String()}) + } + return matches, nil +} + +// applySourceMatch returns the iptables match fragment for the rule's +// source. For a Set it increments the shared ipset's refcount; for a +// Prefix it emits a direct -s match; for the wildcard it returns nil. +func (r *family) applySourceMatch(network firewall.Network, prefixes []netip.Prefix) ([]string, error) { + switch { + case network.IsSet(): + if r.ipsetCounter == nil { + return nil, fmt.Errorf("multi-source peer rule requires shared ipset counter") + } + name := r.ipsetName(network.Set.HashedName()) + if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil { + return nil, fmt.Errorf("ipset increment %s: %w", name, err) + } + return []string{"-m", "set", matchSet, name, "src"}, nil + case network.IsPrefix(): + return []string{"-s", network.Prefix.String()}, nil + default: + return nil, nil + } +} + +// dropSourceMatch undoes whatever applySourceMatch reserved when +// installing a rule fails. Safe to call when the spec is empty or holds +// only inline matchers. Decrement errors are logged but not returned: +// the install error is what the caller needs to see. +func (r *family) dropSourceMatch(srcMatch []string) { + if r.ipsetCounter == nil { + return + } + for _, name := range findSets(srcMatch) { + if _, err := r.ipsetCounter.Decrement(name); err != nil { + log.Errorf("rollback ipset decrement %s: %v", name, err) + } + } +} + +// decrementSetCounter drops ipset references owned by a raw rule spec +// stored in r.rules (NAT / legacy route entries). It returns an error +// aggregate so the caller surfaces decrement failures. +func (r *family) decrementSetCounter(rule []string) error { + if r.ipsetCounter == nil { + return nil + } + var merr *multierror.Error + for _, name := range findSets(rule) { + if _, err := r.ipsetCounter.Decrement(name); err != nil { + merr = multierror.Append(merr, fmt.Errorf("decrement counter: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +// installFilterRule assembles and writes the iptables filter-chain +// rules for one filter rule, one per source match fragment. With +// destination empty the rules land in the peer ACL input chain and each +// gets a paired mangle PREROUTING rule for the redirect mark. With +// destination set the rules land in the route ACL forward chain and +// there is no mangle pairing. +func (r *family) installFilterRule( + ruleID nbid.RuleID, + srcMatches [][]string, + destination firewall.Network, + protocol firewall.Protocol, + sPort, dPort *firewall.Port, + action firewall.Action, +) (*Rule, error) { + isRoute := !destination.IsZero() + + proto := protoForFamily(protocol, r.v6) + + var destExp []string + if isRoute { + var err error + destExp, err = r.applyNetwork("-d", destination, nil) + if err != nil { + return nil, fmt.Errorf("apply network -d: %w", err) + } + } + matchSpecs := filterMatchSpecs(proto, sPort, dPort) + + chain := chainACLInput + if isRoute { + chain = chainRTFwdIn + } + + var installed []filterSpecs + for _, srcMatch := range srcMatches { + specs := slices.Clone(srcMatch) + specs = append(specs, destExp...) + specs = append(specs, matchSpecs...) + + var mangleSpecs []string + if !isRoute { + mangleSpecs = slices.Clone(specs) + mangleSpecs = append(mangleSpecs, + "-i", r.wgIface.Name(), + "-m", "addrtype", "--dst-type", "LOCAL", + "-j", "MARK", "--set-xmark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), + ) + } + + specs = append(specs, "-j", actionToStr(action)) + + if err := r.insertFilterRule(chain, action, specs); err != nil { + // Leave nothing half-installed: the caller sees an error, so a + // partial rule would silently keep matching without being tracked. + r.removeFilterSpecs(chain, installed) + r.dropSourceMatch(destExp) + return nil, fmt.Errorf("install filter rule on %s: %w", chain, err) + } + + // The mangle redirect-mark rule is best effort: the filter rule itself + // is what enforces the ACL, so a mangle failure must not undo it. Drop + // the spec so teardown does not try to remove a rule that was not added. + if mangleSpecs != nil { + if err := r.iptablesClient.Append(tableMangle, chainRTPre, mangleSpecs...); err != nil { + log.Errorf("add mangle rule: %v", err) + mangleSpecs = nil + } + } + + installed = append(installed, filterSpecs{specs: specs, mangleSpecs: mangleSpecs}) + } + + return &Rule{ + id: ruleID, + specs: installed[0].specs, + mangleSpecs: installed[0].mangleSpecs, + extraRules: installed[1:], + chain: chain, + v6: r.v6, + }, nil +} + +// insertFilterRule writes one assembled rule spec into the given ACL +// chain. Peer ACL drops are inserted at position 1 so they precede the +// chain's catch-all; route ACL drops are inserted at position 2 to sit +// immediately after the established/related accept rule. +func (r *family) insertFilterRule(chain string, action firewall.Action, specs []string) error { + if action == firewall.ActionDrop { + pos := 1 + if chain == chainRTFwdIn { + pos = 2 + } + return r.iptablesClient.Insert(tableFilter, chain, pos, specs...) + } + return r.iptablesClient.Append(tableFilter, chain, specs...) +} + +// removeFilterSpecs deletes the already-installed rules of a partially +// applied filter rule. +func (r *family) removeFilterSpecs(chain string, installed []filterSpecs) { + for _, fs := range installed { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chain, fs.specs...); err != nil { + log.Debugf("delete partial filter rule: %v", err) + } + if fs.mangleSpecs != nil { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, fs.mangleSpecs...); err != nil { + log.Debugf("delete partial mangle rule: %v", err) + } + } + } +} + +// applyNetwork resolves a firewall.Network into the iptables match +// fragment for the given direction flag (-s or -d). Set networks +// increment the shared ipset refcount; prefixes emit a direct match; +// an empty network returns no spec ("match any"). +func (r *family) applyNetwork(flag string, network firewall.Network, prefixes []netip.Prefix) ([]string, error) { + direction := "src" + if flag == "-d" { + direction = "dst" + } + + if network.IsSet() { + // A destination set is populated later from DNS results, so unlike a + // source set it cannot be expanded into per-prefix rules. Without + // ipset such a rule is not expressible; report it instead of + // installing something broader than the policy allows. + if flag == "-d" && !r.ipsetSupported { + return nil, fmt.Errorf("destination set %s requires ipset (ip_set_hash_net and xt_set)", network.Set.HashedName()) + } + + name := r.ipsetName(network.Set.HashedName()) + if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil { + return nil, fmt.Errorf("create or get ipset: %w", err) + } + + return []string{"-m", "set", matchSet, name, direction}, nil + } + if network.IsPrefix() { + return []string{flag, network.Prefix.String()}, nil + } + + // nolint:nilnil + return nil, nil +} + +// protoForFamily translates ICMP to ICMPv6 for ip6tables. +// ip6tables requires "ipv6-icmp" (or "icmpv6") instead of "icmp". +func protoForFamily(protocol firewall.Protocol, v6 bool) string { + if v6 && protocol == firewall.ProtocolICMP { + return "ipv6-icmp" + } + return string(protocol) +} + +// filterMatchSpecs returns the proto/port match fragment for a +// filtering rule. The source match (-s or -m set) is built by the +// caller and prepended. +func filterMatchSpecs(protocol string, sPort, dPort *firewall.Port) (specs []string) { + if protocol != "all" { + specs = append(specs, "-p", protocol) + } + specs = append(specs, applyPort("--sport", sPort)...) + specs = append(specs, applyPort("--dport", dPort)...) + return specs +} + +func actionToStr(action firewall.Action) string { + if action == firewall.ActionAccept { + return "ACCEPT" + } + return "DROP" +} + +func applyPort(flag string, port *firewall.Port) []string { + if port == nil { + return nil + } + + if port.IsRange && len(port.Values) == 2 { + return []string{flag, fmt.Sprintf("%d:%d", port.Values[0], port.Values[1])} + } + + if len(port.Values) > 1 { + portList := make([]string, len(port.Values)) + for i, p := range port.Values { + portList[i] = strconv.Itoa(int(p)) + } + return []string{"-m", "multiport", flag, strings.Join(portList, ",")} + } + + return []string{flag, strconv.Itoa(int(port.Values[0]))} +} diff --git a/client/firewall/iptables/interface_allower_linux.go b/client/firewall/iptables/interface_allower_linux.go new file mode 100644 index 000000000..40e9728e2 --- /dev/null +++ b/client/firewall/iptables/interface_allower_linux.go @@ -0,0 +1,93 @@ +package iptables + +import ( + "fmt" + + "github.com/coreos/go-iptables/iptables" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" +) + +// InterfaceAllower opens the NetBird interface on the iptables filter INPUT +// chain so the host firewall doesn't drop traffic the userspace firewall +// handles. It is the fallback used when nftables is unavailable (an +// iptables-legacy host). +// +// It opens INPUT only: the userspace router never forwards in the kernel. +// firewalld trust is handled by the uspfilter manager, not here. +type InterfaceAllower struct { + ifaceName string + ipt4 *iptables.IPTables + // ipt6 is nil when the interface has no IPv6 overlay address. + ipt6 *iptables.IPTables +} + +// NewInterfaceAllower builds an iptables allower for the interface. It returns +// an error when iptables is unavailable, so the caller can fall back to +// firewalld trust. +func NewInterfaceAllower(wgIface iFaceMapper) (*InterfaceAllower, error) { + ipt4, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err != nil { + return nil, fmt.Errorf("iptables not available: %w", err) + } + if _, err := ipt4.ListChains(tableFilter); err != nil { + return nil, fmt.Errorf("iptables filter table not available: %w", err) + } + + a := &InterfaceAllower{ifaceName: wgIface.Name(), ipt4: ipt4} + + // Missing v6 must not break the v4 path: open v4 only and continue. + if wgIface.Address().HasIPv6() { + ipt6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + if err != nil { + log.Warnf("ip6tables not available, opening interface on v4 only: %v", err) + } else if _, err := ipt6.ListChains(tableFilter); err != nil { + log.Warnf("ip6tables filter table not available, opening interface on v4 only: %v", err) + } else { + a.ipt6 = ipt6 + } + } + + return a, nil +} + +// Apply inserts the interface accept rule on the filter INPUT chain. It removes +// any stale rule first so an unclean exit (e.g. SIGKILL, where Close never ran) +// is recovered deterministically rather than accumulating duplicates. +func (a *InterfaceAllower) Apply() error { + var merr *multierror.Error + for _, ipt := range a.clients() { + if err := ipt.DeleteIfExists(tableFilter, chainInput, a.inputRule()...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clean stale interface accept rule: %w", err)) + } + if err := ipt.Insert(tableFilter, chainInput, 1, a.inputRule()...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add interface accept rule: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +// Close removes the interface accept rule. +func (a *InterfaceAllower) Close() error { + var merr *multierror.Error + for _, ipt := range a.clients() { + if err := ipt.DeleteIfExists(tableFilter, chainInput, a.inputRule()...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove interface accept rule: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +func (a *InterfaceAllower) inputRule() []string { + return []string{"-i", a.ifaceName, "-j", "ACCEPT"} +} + +func (a *InterfaceAllower) clients() []*iptables.IPTables { + clients := []*iptables.IPTables{a.ipt4} + if a.ipt6 != nil { + clients = append(clients, a.ipt6) + } + return clients +} diff --git a/client/firewall/iptables/ipset_linux.go b/client/firewall/iptables/ipset_linux.go new file mode 100644 index 000000000..2a3685af7 --- /dev/null +++ b/client/firewall/iptables/ipset_linux.go @@ -0,0 +1,131 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "net/netip" + + "github.com/google/uuid" + "github.com/hashicorp/go-multierror" + "github.com/lrh3321/ipset-go" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +// probeIPSetSupport checks whether the kernel can create the ipset type +// used for source and destination matches. On kernels lacking the +// required ipset hash module, set creation fails (e.g. "invalid +// argument"), which would otherwise fail every multi-source rule and +// leave traffic the policy permits blocked by the catch-all drop. When +// unsupported, multi-source rules fall back to one rule per prefix. +func (r *family) probeIPSetSupport() bool { + // Use a unique name so concurrent processes don't collide and we only ever + // destroy the set we created ourselves. ipset names are limited to 31 chars, + // so use a short random suffix. + probeName := "nb-probe-" + uuid.New().String()[:8] + + if err := r.createIPSet(probeName); err != nil { + log.Warnf("ipset is not available (failed to create probe set: %v); "+ + "falling back to per-IP iptables ACL rules. Ensure the kernel provides "+ + "the ipset hash:net module (ip_set_hash_net) for better performance with large rule sets", err) + return false + } + + if err := r.destroyIPSet(probeName); err != nil { + log.Debugf("destroy ipset probe set %q: %v", probeName, err) + } + + return true +} + +func (r *family) createIpSet(setName string, sources []netip.Prefix) error { + if err := r.createIPSet(setName); err != nil { + return fmt.Errorf("create set %s: %w", setName, err) + } + + for _, prefix := range sources { + if err := r.addPrefixToIPSet(setName, prefix); err != nil { + // The refcounter records nothing when this callback errors, + // so destroy the set or it leaks in the kernel. A partial + // source set would also fail-open for deny rules, so the + // rule must fail rather than install with a missing source. + if derr := r.destroyIPSet(setName); derr != nil { + log.Warnf("rollback ipset %s after add failure: %v", setName, derr) + } + return fmt.Errorf("add element to set %s: %w", setName, err) + } + } + + return nil +} + +func (r *family) deleteIpSet(setName string) error { + if err := r.destroyIPSet(setName); err != nil { + return fmt.Errorf("destroy set %s: %w", setName, err) + } + + log.Debugf("deleted unused ipset %s", setName) + return nil +} + +func (r *family) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { + name := r.ipsetName(set.HashedName()) + var merr *multierror.Error + for _, prefix := range prefixes { + if err := r.addPrefixToIPSet(name, prefix); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add prefix to ipset: %w", err)) + } + } + if merr == nil { + log.Debugf("updated set %s with prefixes %v", name, prefixes) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) ipsetName(name string) string { + if r.v6 { + return name + "-v6" + } + return name +} + +func (r *family) createIPSet(name string) error { + opts := ipset.CreateOptions{ + Replace: true, + } + if r.v6 { + opts.Family = ipset.FamilyIPV6 + } + + if err := ipset.Create(name, ipset.TypeHashNet, opts); err != nil { + return fmt.Errorf("create ipset %s: %w", name, err) + } + + log.Debugf("created ipset %s with type hash:net", name) + return nil +} + +func (r *family) addPrefixToIPSet(name string, prefix netip.Prefix) error { + addr := prefix.Addr() + ip := addr.AsSlice() + + entry := &ipset.Entry{ + IP: ip, + CIDR: uint8(prefix.Bits()), + Replace: true, + } + + if err := ipset.Add(name, entry); err != nil { + return fmt.Errorf("add prefix to ipset %s: %w", name, err) + } + + return nil +} + +func (r *family) destroyIPSet(name string) error { + return ipset.Destroy(name) +} diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index aa052d933..49b88f1ea 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -3,7 +3,6 @@ package iptables import ( "context" "fmt" - "net" "net/netip" "sync" @@ -18,25 +17,21 @@ import ( "github.com/netbirdio/netbird/client/internal/statemanager" ) -type resetter interface { - Reset() error -} - -// Manager of iptables firewall +// Manager of iptables firewall. Per-family state (peer ACLs, route +// ACLs, NAT, DNAT, MSS clamping) lives on family; Manager dispatches +// by family and provides the public firewall.Manager surface. type Manager struct { mutex sync.Mutex wgIface iFaceMapper ipv4Client *iptables.IPTables - aclMgr *aclManager - router *router + family4 *family rawSupported bool // IPv6 counterparts, nil when no v6 overlay ipv6Client *iptables.IPTables - aclMgr6 *aclManager - router6 *router + family6 *family } // iFaceMapper defines subset methods of interface required for manager @@ -57,14 +52,9 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) { ipv4Client: iptablesClient, } - m.router, err = newRouter(iptablesClient, wgIface, mtu) + m.family4, err = newFamily(iptablesClient, wgIface, mtu) if err != nil { - return nil, fmt.Errorf("create router: %w", err) - } - - m.aclMgr, err = newAclManager(iptablesClient, wgIface) - if err != nil { - return nil, fmt.Errorf("create acl manager: %w", err) + return nil, fmt.Errorf("create family: %w", err) } if wgIface.Address().HasIPv6() { @@ -81,21 +71,18 @@ func (m *Manager) createIPv6Components(wgIface iFaceMapper, mtu uint16) error { if err != nil { return fmt.Errorf("init ip6tables: %w", err) } + + family6, err := newFamily(ip6Client, wgIface, mtu) + if err != nil { + return fmt.Errorf("create v6 family: %w", err) + } + + // Share the same IP forwarding state with the v4 family, since the + // forwarding refcounter is per-family but shared between both families. + family6.ipFwdState = m.family4.ipFwdState + m.ipv6Client = ip6Client - - m.router6, err = newRouter(ip6Client, wgIface, mtu) - if err != nil { - return fmt.Errorf("create v6 router: %w", err) - } - - // Share the same IP forwarding state with the v4 router, since - // Forwarding refcounter is per-family but shared between v4 and v6 routers. - m.router6.ipFwdState = m.router.ipFwdState - - m.aclMgr6, err = newAclManager(ip6Client, wgIface) - if err != nil { - return fmt.Errorf("create v6 acl manager: %w", err) - } + m.family6 = family6 return nil } @@ -109,7 +96,7 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { InterfaceState: &InterfaceState{ NameStr: m.wgIface.Name(), WGAddress: m.wgIface.Address(), - MTU: m.router.mtu, + MTU: m.family4.mtu, }, } stateManager.RegisterState(state) @@ -141,31 +128,24 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { return nil } -// initChains initializes router and ACL chains for both address families, -// rolling back on failure. +// initChains initializes the per-family firewall state for both +// address families, rolling back on failure. func (m *Manager) initChains(stateManager *statemanager.Manager) error { type initStep struct { name string - init func(*statemanager.Manager) error - mgr resetter + r *family } - steps := []initStep{ - {"router", m.router.init, m.router}, - {"acl manager", m.aclMgr.init, m.aclMgr}, - } + steps := []initStep{{"v4", m.family4}} if m.hasIPv6() { - steps = append(steps, - initStep{"v6 router", m.router6.init, m.router6}, - initStep{"v6 acl manager", m.aclMgr6.init, m.aclMgr6}, - ) + steps = append(steps, initStep{"v6", m.family6}) } var initialized []initStep for _, s := range steps { - if err := s.init(stateManager); err != nil { + if err := s.r.init(stateManager); err != nil { for i := len(initialized) - 1; i >= 0; i-- { - if rerr := initialized[i].mgr.Reset(); rerr != nil { + if rerr := initialized[i].r.Reset(); rerr != nil { log.Warnf("rollback %s: %v", initialized[i].name, rerr) } } @@ -176,84 +156,50 @@ func (m *Manager) initChains(stateManager *statemanager.Manager) error { return nil } -// AddPeerFiltering adds a rule to the firewall -// -// Comment will be ignored because some system this feature is not supported -func (m *Manager) AddPeerFiltering( - id []byte, - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if ip.To4() != nil { - return m.aclMgr.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) - } - if !m.hasIPv6() { - return nil, fmt.Errorf("add peer filtering for %s: %w", ip, firewall.ErrIPv6NotInitialized) - } - return m.aclMgr6.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) -} - -func (m *Manager) AddRouteFiltering( +// AddFilterRule installs a packet-filtering rule. See firewall.Manager +// docs for destination semantics. Sources are a single address family; +// the rule is dispatched to the matching v4 / v6 backend. +func (m *Manager) AddFilterRule( id []byte, sources []netip.Prefix, destination firewall.Network, proto firewall.Protocol, - sPort, dPort *firewall.Port, + sPort *firewall.Port, + dPort *firewall.Port, action firewall.Action, ) (firewall.Rule, error) { + if len(sources) == 0 { + return nil, firewall.ErrNoSources + } + m.mutex.Lock() defer m.mutex.Unlock() - if isIPv6RouteRule(sources, destination) { + fam := m.family4 + if isIPv6Rule(sources, destination) { if !m.hasIPv6() { - return nil, fmt.Errorf("add route filtering: %w", firewall.ErrIPv6NotInitialized) + return nil, fmt.Errorf("add filtering: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + fam = m.family6 } - - return m.router.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + return fam.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) } -func isIPv6RouteRule(sources []netip.Prefix, destination firewall.Network) bool { - if destination.IsPrefix() { - return destination.Prefix.Addr().Is6() - } - return len(sources) > 0 && sources[0].Addr().Is6() -} - -// DeletePeerRule from the firewall by rule definition -func (m *Manager) DeletePeerRule(rule firewall.Rule) error { +// DeleteFilterRule removes a rule previously added via AddFilterRule. +// The rule is looked up by id in each family's filter cache. +func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - if m.hasIPv6() && isIPv6IptRule(rule) { - return m.aclMgr6.DeletePeerRule(rule) + id := rule.ID() + if m.family4.hasRule(id) { + return m.family4.DeleteFilterRule(rule) } - return m.aclMgr.DeletePeerRule(rule) -} - -func isIPv6IptRule(rule firewall.Rule) bool { - r, ok := rule.(*Rule) - return ok && r.v6 -} - -// DeleteRouteRule deletes a routing rule. -// Route rules are keyed by content hash. Check v4 first, try v6 if not found. -func (m *Manager) DeleteRouteRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.hasIPv6() && !m.router.hasRule(rule.ID()) { - return m.router6.DeleteRouteRule(rule) + if m.hasIPv6() && m.family6.hasRule(id) { + return m.family6.DeleteFilterRule(rule) } - return m.router.DeleteRouteRule(rule) + log.Debugf("filter rule %s not found in any family", id) + return nil } func (m *Manager) IsServerRouteSupported() bool { @@ -272,10 +218,10 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return fmt.Errorf("add NAT rule: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddNatRule(pair) + return m.family6.AddNatRule(pair) } - if err := m.router.AddNatRule(pair); err != nil { + if err := m.family4.AddNatRule(pair); err != nil { return err } @@ -284,7 +230,7 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { // wildcard 0.0.0.0/0 destination where the client resolves DNS. if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.AddNatRule(v6Pair); err != nil { + if err := m.family6.AddNatRule(v6Pair); err != nil { return fmt.Errorf("add v6 NAT rule: %w", err) } } @@ -300,18 +246,18 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return nil } - return m.router6.RemoveNatRule(pair) + return m.family6.RemoveNatRule(pair) } var merr *multierror.Error - if err := m.router.RemoveNatRule(pair); err != nil { + if err := m.family4.RemoveNatRule(pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v4 NAT rule: %w", err)) } if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.RemoveNatRule(v6Pair); err != nil { + if err := m.family6.RemoveNatRule(v6Pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v6 NAT rule: %w", err)) } } @@ -320,11 +266,14 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { } func (m *Manager) SetLegacyManagement(isLegacy bool) error { - if err := firewall.SetLegacyManagement(m.router, isLegacy); err != nil { + m.mutex.Lock() + defer m.mutex.Unlock() + + if err := firewall.SetLegacyManagement(m.family4, isLegacy); err != nil { return err } if m.hasIPv6() { - return firewall.SetLegacyManagement(m.router6, isLegacy) + return firewall.SetLegacyManagement(m.family6, isLegacy) } return nil } @@ -341,19 +290,13 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { } if m.hasIPv6() { - if err := m.aclMgr6.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset v6 acl manager: %w", err)) - } - if err := m.router6.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset v6 router: %w", err)) + if err := m.family6.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) } } - if err := m.aclMgr.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset acl manager: %w", err)) - } - if err := m.router.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset router: %w", err)) + if err := m.family4.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset family: %w", err)) } // Appending to merr intentionally blocks DeleteState below so ShutdownState @@ -372,27 +315,6 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { return nberrors.FormatErrorOrNil(merr) } -// AllowNetbird allows netbird interface traffic. -// This is called when USPFilter wraps the native firewall, adding blanket accept -// rules so that packet filtering is handled in userspace instead of by netfilter. -func (m *Manager) AllowNetbird() error { - var merr *multierror.Error - if _, err := m.AddPeerFiltering(nil, net.IP{0, 0, 0, 0}, firewall.ProtocolALL, nil, nil, firewall.ActionAccept, ""); err != nil { - merr = multierror.Append(merr, fmt.Errorf("allow netbird v4 interface traffic: %w", err)) - } - if m.hasIPv6() { - if _, err := m.AddPeerFiltering(nil, net.IPv6zero, firewall.ProtocolALL, nil, nil, firewall.ActionAccept, ""); err != nil { - merr = multierror.Append(merr, fmt.Errorf("allow netbird v6 interface traffic: %w", err)) - } - } - - if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - - return nberrors.FormatErrorOrNil(merr) -} - // Flush doesn't need to be implemented for this manager func (m *Manager) Flush() error { return nil } @@ -403,11 +325,11 @@ func (m *Manager) SetLogLevel(log.Level) { func (m *Manager) EnableRouting() error { // v6 only when the overlay actually has v6. - return m.router.ipFwdState.RequestRouting(m.router6 != nil) + return m.family4.ipFwdState.RequestRouting(m.hasIPv6()) } func (m *Manager) DisableRouting() error { - return m.router.ipFwdState.ReleaseRouting() + return m.family4.ipFwdState.ReleaseRouting() } // AddDNATRule adds a DNAT rule @@ -419,9 +341,9 @@ func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) if !m.hasIPv6() { return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddDNATRule(rule) + return m.family6.AddDNATRule(rule) } - return m.router.AddDNATRule(rule) + return m.family4.AddDNATRule(rule) } // DeleteDNATRule deletes a DNAT rule @@ -429,10 +351,10 @@ func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - if m.hasIPv6() && !m.router.hasRule(rule.ID()+dnatSuffix) { - return m.router6.DeleteDNATRule(rule) + if m.hasIPv6() && !m.family4.hasDNATRule(rule.ID()) { + return m.family6.DeleteDNATRule(rule) } - return m.router.DeleteDNATRule(rule) + return m.family4.DeleteDNATRule(rule) } // UpdateSet updates the set with the given prefixes @@ -449,12 +371,12 @@ func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { } } - if err := m.router.UpdateSet(set, v4Prefixes); err != nil { + if err := m.family4.UpdateSet(set, v4Prefixes); err != nil { return err } if m.hasIPv6() && len(v6Prefixes) > 0 { - if err := m.router6.UpdateSet(set, v6Prefixes); err != nil { + if err := m.family6.UpdateSet(set, v6Prefixes); err != nil { return fmt.Errorf("update v6 set: %w", err) } } @@ -471,9 +393,9 @@ func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protoco if !m.hasIPv6() { return fmt.Errorf("add inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveInboundDNAT removes an inbound DNAT rule. @@ -485,9 +407,9 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot if !m.hasIPv6() { return fmt.Errorf("remove inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. @@ -499,9 +421,9 @@ func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol if !m.hasIPv6() { return fmt.Errorf("add output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. @@ -513,14 +435,14 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto if !m.hasIPv6() { return fmt.Errorf("remove output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } const ( chainNameRaw = "NETBIRD-RAW" - chainOUTPUT = "OUTPUT" + chainOutput = "OUTPUT" tableRaw = "raw" ) @@ -595,15 +517,15 @@ func (m *Manager) initNoTrackChain() error { jumpRule := []string{"-j", chainNameRaw} - if err := m.ipv4Client.InsertUnique(tableRaw, chainOUTPUT, 1, jumpRule...); err != nil { + if err := m.ipv4Client.InsertUnique(tableRaw, chainOutput, 1, jumpRule...); err != nil { if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { log.Debugf("delete orphan chain: %v", delErr) } return fmt.Errorf("add output jump rule: %w", err) } - if err := m.ipv4Client.InsertUnique(tableRaw, chainPREROUTING, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOUTPUT, jumpRule...); delErr != nil { + if err := m.ipv4Client.InsertUnique(tableRaw, chainPrerouting, 1, jumpRule...); err != nil { + if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); delErr != nil { log.Debugf("delete output jump rule: %v", delErr) } if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { @@ -630,11 +552,11 @@ func (m *Manager) cleanupNoTrackChain() error { jumpRule := []string{"-j", chainNameRaw} - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOUTPUT, jumpRule...); err != nil { + if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); err != nil { return fmt.Errorf("remove output jump rule: %w", err) } - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPREROUTING, jumpRule...); err != nil { + if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPrerouting, jumpRule...); err != nil { return fmt.Errorf("remove prerouting jump rule: %w", err) } @@ -649,3 +571,13 @@ func (m *Manager) cleanupNoTrackChain() error { func getConntrackEstablished() []string { return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} } + +// isIPv6Rule reports whether the rule belongs to the IPv6 family, from +// the destination prefix when set, otherwise from the (single-family) +// sources. +func isIPv6Rule(sources []netip.Prefix, destination firewall.Network) bool { + if destination.IsPrefix() { + return destination.Prefix.Addr().Is6() + } + return len(sources) > 0 && sources[0].Addr().Is6() +} diff --git a/client/firewall/iptables/manager_linux_test.go b/client/firewall/iptables/manager_linux_test.go index 2c3c1a08e..9f53352e1 100644 --- a/client/firewall/iptables/manager_linux_test.go +++ b/client/firewall/iptables/manager_linux_test.go @@ -5,16 +5,19 @@ package iptables import ( "fmt" "net/netip" + "slices" "strings" "testing" "time" "github.com/coreos/go-iptables/iptables" + "github.com/lrh3321/ipset-go" "github.com/stretchr/testify/require" fw "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface" "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/shared/management/domain" ) var ifaceMock = &iFaceMock{ @@ -67,47 +70,37 @@ func TestIptablesManager(t *testing.T) { time.Sleep(time.Second) }() - var rule2 []fw.Rule + var rule2 fw.Rule t.Run("add second rule", func(t *testing.T) { ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{ IsRange: true, Values: []uint16{8043, 8046}, } - rule2, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", port, nil, fw.ActionAccept, "") + rule2, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", port, nil, fw.ActionAccept) require.NoError(t, err, "failed to add rule") - for _, r := range rule2 { - rr := r.(*Rule) - checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) - } + rr := rule2.(*Rule) + checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) }) t.Run("delete second rule", func(t *testing.T) { - for _, r := range rule2 { - err := manager.DeletePeerRule(r) - require.NoError(t, err, "failed to delete rule") - } - - require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty") + require.NoError(t, manager.DeleteFilterRule(rule2), "failed to delete rule") }) t.Run("reset check", func(t *testing.T) { // add second rule ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{Values: []uint16{5353}} - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "udp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "udp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") err = manager.Close(nil) require.NoError(t, err, "failed to reset") - ok, err := ipv4Client.ChainExists("filter", chainNameInputRules) + ok, err := ipv4Client.ChainExists("filter", chainACLInput) require.NoError(t, err, "failed check chain exists") - - if ok { - require.NoErrorf(t, err, "chain '%v' still exists after Close", chainNameInputRules) - } + require.Falsef(t, ok, "chain %q still exists after Close", chainACLInput) }) } @@ -128,15 +121,13 @@ func TestIptablesManagerDenyRules(t *testing.T) { ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{Values: []uint16{22}} - rule, err := manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionDrop, "deny-ssh") + rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionDrop) require.NoError(t, err, "failed to add deny rule") - require.NotEmpty(t, rule, "deny rule should not be empty") + require.NotNil(t, rule, "deny rule should not be nil") // Verify the rule was added by checking iptables - for _, r := range rule { - rr := r.(*Rule) - checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) - } + rr := rule.(*Rule) + checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) }) t.Run("deny rule precedence test", func(t *testing.T) { @@ -144,36 +135,40 @@ func TestIptablesManagerDenyRules(t *testing.T) { port := &fw.Port{Values: []uint16{80}} // Add accept rule first - _, err := manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "accept-http") + _, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add accept rule") // Add deny rule second for same IP/port - this should take precedence - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionDrop, "deny-http") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionDrop) require.NoError(t, err, "failed to add deny rule") // Inspect the actual iptables rules to verify deny rule comes before accept rule - rules, err := ipv4Client.List("filter", chainNameInputRules) + rules, err := ipv4Client.List("filter", chainACLInput) require.NoError(t, err, "failed to list iptables rules") // Debug: print all rules - t.Logf("All iptables rules in chain %s:", chainNameInputRules) + t.Logf("All iptables rules in chain %s:", chainACLInput) for i, rule := range rules { t.Logf(" [%d] %s", i, rule) } + // Single-source rules emit a direct `-s /32 ... --dport 80` + // match. Match on that shape instead of the legacy + // per-(action,port) ipset names ("deny-http"/"accept-http") + // that this test predates. + srcMatch := fmt.Sprintf("-s %s/32", ip) var denyRuleIndex, acceptRuleIndex = -1, -1 for i, rule := range rules { - if strings.Contains(rule, "DROP") { - t.Logf("Found DROP rule at index %d: %s", i, rule) - if strings.Contains(rule, "deny-http") && strings.Contains(rule, "80") { - denyRuleIndex = i - } + if !strings.Contains(rule, srcMatch) || !strings.Contains(rule, "--dport 80") { + continue } - if strings.Contains(rule, "ACCEPT") { + if strings.Contains(rule, "-j DROP") { + t.Logf("Found DROP rule at index %d: %s", i, rule) + denyRuleIndex = i + } + if strings.Contains(rule, "-j ACCEPT") { t.Logf("Found ACCEPT rule at index %d: %s", i, rule) - if strings.Contains(rule, "accept-http") && strings.Contains(rule, "80") { - acceptRuleIndex = i - } + acceptRuleIndex = i } } @@ -198,7 +193,6 @@ func TestIptablesManagerIPSet(t *testing.T) { }, } - // just check on the local interface manager, err := Create(mock, iface.DefaultMTU) require.NoError(t, err) require.NoError(t, manager.Init(nil)) @@ -212,27 +206,39 @@ func TestIptablesManagerIPSet(t *testing.T) { time.Sleep(time.Second) }() - var rule2 []fw.Rule - t.Run("add second rule", func(t *testing.T) { + var rule2 fw.Rule + t.Run("single source uses direct -s match (no ipset)", func(t *testing.T) { ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{ Values: []uint16{443}, } - rule2, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", port, nil, fw.ActionAccept, "default") - for _, r := range rule2 { - require.NoError(t, err, "failed to add rule") - require.Equal(t, r.(*Rule).ipsetName, "default-sport", "ipset name must be set") - require.Equal(t, r.(*Rule).ip, "10.20.0.3", "ipset IP must be set") - } + rule2, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", port, nil, fw.ActionAccept) + require.NoError(t, err, "failed to add rule") + require.NotNil(t, rule2) + require.Contains(t, rule2.(*Rule).specs, "-s", + "single-source rule should use direct -s match, not an ipset") + require.Empty(t, findSets(rule2.(*Rule).specs), + "single-source rule should not allocate a shared ipset") }) - t.Run("delete second rule", func(t *testing.T) { - for _, r := range rule2 { - err := manager.DeletePeerRule(r) - require.NoError(t, err, "failed to delete rule") + t.Run("delete single-source rule", func(t *testing.T) { + require.NoError(t, manager.DeleteFilterRule(rule2), "failed to delete rule") + }) - require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty") + t.Run("multi-source uses shared ipset", func(t *testing.T) { + sources := []netip.Prefix{ + netip.PrefixFrom(netip.MustParseAddr("10.20.0.3"), 32), + netip.PrefixFrom(netip.MustParseAddr("10.20.0.4"), 32), + netip.PrefixFrom(netip.MustParseAddr("10.20.0.5"), 32), } + port := &fw.Port{Values: []uint16{8080}} + multi, err := manager.AddFilterRule(nil, sources, fw.Network{}, "tcp", nil, port, fw.ActionAccept) + require.NoError(t, err, "failed to add multi-source rule") + require.NotNil(t, multi, "multi-source rule must produce one iptables rule") + sets := findSets(multi.(*Rule).specs) + require.Len(t, sets, 1, "multi-source rule must reference exactly one ipset") + + require.NoError(t, manager.DeleteFilterRule(multi)) }) t.Run("reset check", func(t *testing.T) { @@ -241,9 +247,324 @@ func TestIptablesManagerIPSet(t *testing.T) { }) } +// TestIptablesFilterIPSetFallback verifies that when the kernel lacks +// ipset support, a multi-source rule falls back to one iptables rule +// per source prefix instead of silently leaving the chain empty. See +// discussion #6125. +func TestIptablesFilterIPSetFallback(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + // Simulate a kernel without the ipset hash module. + manager.family4.ipsetSupported = false + + sources := []netip.Prefix{ + netip.MustParsePrefix("10.20.0.42/32"), + netip.MustParsePrefix("10.20.0.43/32"), + } + port := &fw.Port{Values: []uint16{22}} + + rule, err := manager.AddFilterRule(nil, sources, fw.Network{}, "tcp", nil, port, fw.ActionAccept) + require.NoError(t, err, "AddFilterRule should succeed via fallback") + + rr := rule.(*Rule) + all := rr.allSpecs() + require.Len(t, all, len(sources), "each source prefix needs its own rule") + for i, fs := range all { + joined := strings.Join(fs.specs, " ") + require.Contains(t, joined, "-s "+sources[i].String(), "fallback rule must match by source prefix") + require.NotContains(t, joined, matchSet, "fallback rule must not use ipset matching") + + // The rule must actually be present in the ACL chain (not silently dropped). + checkRuleSpecs(t, ipv4Client, rr.chain, true, fs.specs...) + + // Every expanded peer rule keeps its own redirect-mark pairing. + require.NotNil(t, fs.mangleSpecs, "peer rule must carry a mangle pairing") + checkTableRuleSpecs(t, ipv4Client, tableMangle, chainRTPre, true, fs.mangleSpecs...) + } + + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete fallback rule") + for _, fs := range all { + checkRuleSpecs(t, ipv4Client, rr.chain, false, fs.specs...) + checkTableRuleSpecs(t, ipv4Client, tableMangle, chainRTPre, false, fs.mangleSpecs...) + } +} + +// TestIptablesFilterDestinationSetRequiresIPSet documents that a dynamic +// (domain) destination cannot be expressed without ipset: its prefixes are only +// known after DNS resolution, so there is nothing to expand into per-prefix +// rules. The call must report that rather than install a broader rule than the +// policy allows. +func TestIptablesFilterDestinationSetRequiresIPSet(t *testing.T) { + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + manager.family4.ipsetSupported = false + + destination := fw.Network{Set: fw.NewDomainSet(domain.List{"example.com"})} + + _, err = manager.AddFilterRule(nil, []netip.Prefix{netip.MustParsePrefix("172.16.0.0/16")}, + destination, fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.Error(t, err, "a domain destination is not expressible without ipset") + require.ErrorContains(t, err, "requires ipset") +} + +// TestIptablesNatRuleDropsSourceSetOnDestinationFailure covers a marking rule +// whose source set is created but whose destination set is not: the source +// reference has to go back, or the set it created stays in the kernel with a +// count nothing will ever drop. +func TestIptablesNatRuleDropsSourceSetOnDestinationFailure(t *testing.T) { + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + sourceSet := fw.NewPrefixSet([]netip.Prefix{ + netip.MustParsePrefix("100.0.0.0/16"), + netip.MustParsePrefix("10.10.0.0/16"), + }) + destSet := fw.NewDomainSet(domain.List{"example.org"}) + + // Poison the destination set's name so its hash:net creation fails after + // the source set has already been created. + poisoned := manager.family4.ipsetName(destSet.HashedName()) + require.NoError(t, ipset.Create(poisoned, ipset.TypeHashIP, ipset.CreateOptions{})) + t.Cleanup(func() { + if err := ipset.Destroy(poisoned); err != nil { + t.Logf("destroy poisoned set %s: %v", poisoned, err) + } + }) + + pair := fw.RouterPair{ + ID: "nat-source-set-test", + Source: fw.Network{Set: sourceSet}, + Destination: fw.Network{Set: destSet}, + Masquerade: true, + Dynamic: true, + } + + require.Error(t, manager.AddNatRule(pair), "the destination set must fail to be created") + + _, ok := manager.family4.ipsetCounter.Get(manager.family4.ipsetName(sourceSet.HashedName())) + require.False(t, ok, "the source set reference must be released") +} + +// TestIptablesNatRuleReAddKeepsSetReferences re-adds the same NAT rule the way +// a repeated network-map update does. The marking rule's set references must not +// grow, or RemoveNatRule can never drop the count to zero and the set stays in +// the kernel for the rest of the process lifetime. +func TestIptablesNatRuleReAddKeepsSetReferences(t *testing.T) { + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + set := fw.NewDomainSet(domain.List{"example.com"}) + pair := fw.RouterPair{ + ID: "nat-reference-test", + Source: fw.Network{Prefix: netip.MustParsePrefix("100.0.0.0/16")}, + Destination: fw.Network{Set: set}, + Masquerade: true, + Dynamic: true, + } + + require.NoError(t, manager.AddNatRule(pair), "add nat rule") + name := manager.family4.ipsetName(set.HashedName()) + first, ok := manager.family4.ipsetCounter.Get(name) + require.True(t, ok, "the marking rule must hold a reference to its set") + + require.NoError(t, manager.AddNatRule(pair), "re-add nat rule") + second, ok := manager.family4.ipsetCounter.Get(name) + require.True(t, ok, "the set must still be referenced") + require.Equal(t, first.Count, second.Count, "re-adding the same rule must not add references") + + require.NoError(t, manager.RemoveNatRule(pair), "remove nat rule") + _, ok = manager.family4.ipsetCounter.Get(name) + require.False(t, ok, "removing the rule must drop the last reference") +} + +// TestIptablesRouteFilterIPSetFallback covers the route ACL side of the +// fallback: with a destination set, the expanded per-source rules land +// in the route forward chain and are all removed on delete. +func TestIptablesRouteFilterIPSetFallback(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + manager.family4.ipsetSupported = false + + sources := []netip.Prefix{ + netip.MustParsePrefix("172.16.0.0/16"), + netip.MustParsePrefix("192.168.0.0/16"), + } + destination := fw.Network{Prefix: netip.MustParsePrefix("10.0.0.0/8")} + port := &fw.Port{Values: []uint16{443}} + + rule, err := manager.AddFilterRule(nil, sources, destination, "tcp", nil, port, fw.ActionAccept) + require.NoError(t, err, "route ACL must install without ipset") + + rr := rule.(*Rule) + require.Equal(t, chainRTFwdIn, rr.chain, "route rule must land in the forward chain") + + all := rr.allSpecs() + require.Len(t, all, len(sources), "each source prefix needs its own rule") + for i, fs := range all { + joined := strings.Join(fs.specs, " ") + require.Contains(t, joined, "-s "+sources[i].String(), "fallback rule must match by source prefix") + require.NotContains(t, joined, matchSet, "fallback rule must not use ipset matching") + require.Nil(t, fs.mangleSpecs, "route rules have no mangle pairing") + + checkRuleSpecs(t, ipv4Client, rr.chain, true, fs.specs...) + } + + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete fallback rule") + for _, fs := range all { + checkRuleSpecs(t, ipv4Client, rr.chain, false, fs.specs...) + } +} + +// TestIptablesCloseRemovesAllState exercises a spread of rule kinds and then +// asserts Close puts every table it touches back exactly as it found it. A +// leaked chain, jump, or ipset survives the daemon and nothing can remove it +// afterwards, since the tracking that knew about it is gone. +func TestIptablesCloseRemovesAllState(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + before := snapshotIptables(t, ipv4Client) + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + // A failed assertion below returns before the Close under test, which would + // leave this test's chains and sets in the kernel for the next one. + t.Cleanup(func() { + if err := manager.Close(nil); err != nil { + t.Logf("close after failure: %v", err) + } + }) + + sources := []netip.Prefix{ + netip.MustParsePrefix("10.20.0.42/32"), + netip.MustParsePrefix("10.20.0.43/32"), + } + + // A multi-source peer rule: shared ipset plus the mangle redirect pairing. + _, err = manager.AddFilterRule(nil, sources, fw.Network{}, "tcp", + nil, &fw.Port{Values: []uint16{22}}, fw.ActionAccept) + require.NoError(t, err, "add peer rule") + + // A route rule with a dynamic destination: a second set, in the forward chain. + _, err = manager.AddFilterRule(nil, sources, + fw.Network{Set: fw.NewDomainSet(domain.List{"example.com"})}, + fw.ProtocolALL, nil, nil, fw.ActionDrop) + require.NoError(t, err, "add route rule") + + // NAT marking for a routed destination, both directions. + pair := fw.RouterPair{ + ID: "cleanup-test", + Source: fw.Network{Prefix: netip.MustParsePrefix("100.0.0.0/16")}, + Destination: fw.Network{Prefix: netip.MustParsePrefix("192.168.55.0/24")}, + Masquerade: true, + } + require.NoError(t, manager.AddNatRule(pair), "add nat rule") + require.NoError(t, manager.EnableRouting(), "enable routing") + + // A DNAT redirect, which also holds a forwarding reference. + dnat := fw.ForwardRule{ + Protocol: fw.ProtocolTCP, + DestinationPort: fw.Port{Values: []uint16{8080}}, + TranslatedAddress: netip.MustParseAddr("10.20.0.44"), + TranslatedPort: fw.Port{Values: []uint16{80}}, + } + _, err = manager.AddDNATRule(dnat) + require.NoError(t, err, "add dnat rule") + + require.NotEqual(t, before, snapshotIptables(t, ipv4Client), "the manager must have installed state") + + // Everything above stays in place, so Close is what has to remove it. + require.NoError(t, manager.Close(nil), "close") + + after := snapshotIptables(t, ipv4Client) + require.Equal(t, before.chains, after.chains, "Close must remove every chain it created") + require.Equal(t, before.rules, after.rules, "Close must remove every rule it created") + require.Equal(t, before.sets, after.sets, "Close must destroy every ipset it created") +} + +// iptablesState is a snapshot of the tables the manager writes to, used to +// compare the kernel before and after a manager lifetime. +type iptablesState struct { + chains map[string][]string + rules map[string][]string + sets []string +} + +func snapshotIptables(t *testing.T, client *iptables.IPTables) iptablesState { + t.Helper() + + state := iptablesState{ + chains: map[string][]string{}, + rules: map[string][]string{}, + } + + for _, table := range []string{tableFilter, tableNat, tableMangle, tableRaw} { + chains, err := client.ListChains(table) + require.NoErrorf(t, err, "list chains in %s", table) + slices.Sort(chains) + state.chains[table] = chains + + for _, chain := range chains { + rules, err := client.List(table, chain) + require.NoErrorf(t, err, "list rules in %s/%s", table, chain) + state.rules[table+"/"+chain] = rules + } + } + + sets, err := ipset.ListAll() + require.NoError(t, err, "list ipsets") + for _, set := range sets { + state.sets = append(state.sets, set.SetName) + } + slices.Sort(state.sets) + + return state +} + func checkRuleSpecs(t *testing.T, ipv4Client *iptables.IPTables, chainName string, mustExists bool, rulespec ...string) { t.Helper() - exists, err := ipv4Client.Exists("filter", chainName, rulespec...) + checkTableRuleSpecs(t, ipv4Client, tableFilter, chainName, mustExists, rulespec...) +} + +func checkTableRuleSpecs(t *testing.T, ipv4Client *iptables.IPTables, table, chainName string, mustExists bool, rulespec ...string) { + t.Helper() + exists, err := ipv4Client.Exists(table, chainName, rulespec...) require.NoError(t, err, "failed to check rule") require.Falsef(t, !exists && mustExists, "rule '%v' does not exist", rulespec) require.Falsef(t, exists && !mustExists, "rule '%v' exist", rulespec) @@ -283,7 +604,7 @@ func TestIptablesCreatePerformance(t *testing.T) { start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") } @@ -291,40 +612,3 @@ func TestIptablesCreatePerformance(t *testing.T) { }) } } - -// TestIptablesACLIPSetFallback verifies that when the kernel lacks ipset support, -// the ACL manager falls back to per-IP iptables rules (-s ) instead of -// silently leaving the chain empty. See discussion #6125. -func TestIptablesACLIPSetFallback(t *testing.T) { - ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) - require.NoError(t, err) - - // Use Create()/Init() so the router-owned chains (chainRTFWDIN/OUT) are - // created before the ACL manager's createDefaultChains() references them. - manager, err := Create(ifaceMock, iface.DefaultMTU) - require.NoError(t, err) - require.NoError(t, manager.Init(nil)) - - aclMgr := manager.aclMgr - // Simulate a kernel without the ipset hash module. - aclMgr.ipsetSupported = false - - defer func() { - require.NoError(t, manager.Close(nil)) - }() - - ip := netip.MustParseAddr("10.20.0.42") - port := &fw.Port{Values: []uint16{22}} - - rules, err := aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "nb0000001") - require.NoError(t, err, "AddPeerFiltering should succeed via fallback") - require.NotEmpty(t, rules) - - rule := rules[0].(*Rule) - require.Empty(t, rule.ipsetName, "fallback rule must not reference an ipset") - require.Contains(t, strings.Join(rule.specs, " "), "-s 10.20.0.42", "fallback rule must match by source IP") - require.NotContains(t, strings.Join(rule.specs, " "), "--match-set", "fallback rule must not use ipset matching") - - // The rule must actually be present in the ACL chain (not silently dropped). - checkRuleSpecs(t, ipv4Client, rule.chain, true, rule.specs...) -} diff --git a/client/firewall/iptables/router_linux.go b/client/firewall/iptables/router_linux.go deleted file mode 100644 index 01b18570c..000000000 --- a/client/firewall/iptables/router_linux.go +++ /dev/null @@ -1,1181 +0,0 @@ -//go:build !android - -package iptables - -import ( - "fmt" - "maps" - "net/netip" - "strconv" - "strings" - - "github.com/coreos/go-iptables/iptables" - "github.com/hashicorp/go-multierror" - ipset "github.com/lrh3321/ipset-go" - log "github.com/sirupsen/logrus" - - nberrors "github.com/netbirdio/netbird/client/errors" - firewall "github.com/netbirdio/netbird/client/firewall/manager" - nbid "github.com/netbirdio/netbird/client/internal/acl/id" - "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" - "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" - "github.com/netbirdio/netbird/client/internal/statemanager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -// constants needed to manage and create iptable rules -const ( - tableFilter = "filter" - tableNat = "nat" - tableMangle = "mangle" - - chainPOSTROUTING = "POSTROUTING" - chainPREROUTING = "PREROUTING" - chainFORWARD = "FORWARD" - chainRTNAT = "NETBIRD-RT-NAT" - chainRTFWDIN = "NETBIRD-RT-FWD-IN" - chainRTFWDOUT = "NETBIRD-RT-FWD-OUT" - chainRTPRE = "NETBIRD-RT-PRE" - chainRTRDR = "NETBIRD-RT-RDR" - chainNATOutput = "NETBIRD-NAT-OUTPUT" - chainRTMSSCLAMP = "NETBIRD-RT-MSSCLAMP" - routingFinalForwardJump = "ACCEPT" - routingFinalNatJump = "MASQUERADE" - - jumpManglePre = "jump-mangle-pre" - jumpNatPre = "jump-nat-pre" - jumpNatPost = "jump-nat-post" - jumpNatOutput = "jump-nat-output" - jumpMSSClamp = "jump-mss-clamp" - markManglePre = "mark-mangle-pre" - markManglePost = "mark-mangle-post" - matchSet = "--match-set" - - dnatSuffix = "_dnat" - snatSuffix = "_snat" - fwdSuffix = "_fwd" - - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. - ipv4TCPHeaderSize = 40 - // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. - ipv6TCPHeaderSize = 60 -) - -type ruleInfo struct { - chain string - table string - rule []string -} - -type routeFilteringRuleParams struct { - Source firewall.Network - Destination firewall.Network - Proto firewall.Protocol - SPort *firewall.Port - DPort *firewall.Port - Direction firewall.RuleDirection - Action firewall.Action -} - -type routeRules map[string][]string - -// the ipset library currently does not support comments, so we use the name only (string) -type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}] - -type router struct { - iptablesClient *iptables.IPTables - rules routeRules - ipsetCounter *ipsetCounter - wgIface iFaceMapper - legacyManagement bool - mtu uint16 - v6 bool - - stateManager *statemanager.Manager - ipFwdState *ipfwdstate.IPForwardingState -} - -func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16) (*router, error) { - r := &router{ - iptablesClient: iptablesClient, - rules: make(map[string][]string), - wgIface: wgIface, - mtu: mtu, - v6: iptablesClient.Proto() == iptables.ProtocolIPv6, - ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), - } - - r.ipsetCounter = refcounter.New( - func(name string, sources []netip.Prefix) (struct{}, error) { - return struct{}{}, r.createIpSet(name, sources) - }, - func(name string, _ struct{}) error { - return r.deleteIpSet(name) - }, - ) - - return r, nil -} - -func (r *router) init(stateManager *statemanager.Manager) error { - r.stateManager = stateManager - - if err := r.cleanUpDefaultForwardRules(); err != nil { - log.Errorf("failed to clean up rules from FORWARD chain: %s", err) - } - - if err := r.createContainers(); err != nil { - return fmt.Errorf("create containers: %w", err) - } - - if err := r.setupDataPlaneMark(); err != nil { - log.Errorf("failed to set up data plane mark: %v", err) - } - - r.updateState() - - return nil -} - -func (r *router) AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination firewall.Network, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, -) (firewall.Rule, error) { - ruleKey := nbid.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action) - if _, ok := r.rules[string(ruleKey)]; ok { - return ruleKey, nil - } - - var source firewall.Network - if len(sources) > 1 { - source.Set = firewall.NewPrefixSet(sources) - } else if len(sources) > 0 { - source.Prefix = sources[0] - } - - params := routeFilteringRuleParams{ - Source: source, - Destination: destination, - Proto: proto, - SPort: sPort, - DPort: dPort, - Action: action, - } - - rule, err := r.genRouteRuleSpec(params, sources) - if err != nil { - return nil, fmt.Errorf("generate route rule spec: %w", err) - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - if action == firewall.ActionDrop { - // after the established rule - err = r.iptablesClient.Insert(tableFilter, chainRTFWDIN, 2, rule...) - } else { - err = r.iptablesClient.Append(tableFilter, chainRTFWDIN, rule...) - } - - if err != nil { - return nil, fmt.Errorf("add route rule: %v", err) - } - - r.rules[string(ruleKey)] = rule - - r.updateState() - - return ruleKey, nil -} - -func (r *router) hasRule(id string) bool { - _, ok := r.rules[id] - return ok -} - -func (r *router) DeleteRouteRule(rule firewall.Rule) error { - ruleKey := rule.ID() - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.Delete(tableFilter, chainRTFWDIN, rule...); err != nil { - return fmt.Errorf("delete route rule: %v", err) - } - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement ipset counter: %w", err) - } - } else { - log.Debugf("route rule %s not found", ruleKey) - } - - r.updateState() - - return nil -} - -func (r *router) decrementSetCounter(rule []string) error { - sets := r.findSets(rule) - var merr *multierror.Error - for _, setName := range sets { - if _, err := r.ipsetCounter.Decrement(setName); err != nil { - merr = multierror.Append(merr, fmt.Errorf("decrement counter: %w", err)) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) findSets(rule []string) []string { - var sets []string - for i, arg := range rule { - if arg == "-m" && i+3 < len(rule) && rule[i+1] == "set" && rule[i+2] == matchSet { - sets = append(sets, rule[i+3]) - } - } - return sets -} - -func (r *router) createIpSet(setName string, sources []netip.Prefix) error { - if err := r.createIPSet(setName); err != nil { - return fmt.Errorf("create set %s: %w", setName, err) - } - - for _, prefix := range sources { - if err := r.addPrefixToIPSet(setName, prefix); err != nil { - return fmt.Errorf("add element to set %s: %w", setName, err) - } - } - - return nil -} - -func (r *router) deleteIpSet(setName string) error { - if err := r.destroyIPSet(setName); err != nil { - return fmt.Errorf("destroy set %s: %w", setName, err) - } - - log.Debugf("Deleted unused ipset %s", setName) - return nil -} - -// AddNatRule inserts an iptables rule pair into the nat chain -func (r *router) AddNatRule(pair firewall.RouterPair) error { - if r.legacyManagement { - log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) - if err := r.addLegacyRouteRule(pair); err != nil { - return fmt.Errorf("add legacy routing rule: %w", err) - } - } - - if !pair.Masquerade { - return nil - } - - if err := r.addNatRule(pair); err != nil { - return fmt.Errorf("add nat rule: %w", err) - } - - if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil { - return fmt.Errorf("add inverse nat rule: %w", err) - } - - r.updateState() - - return nil -} - -// RemoveNatRule removes an iptables rule pair from forwarding and nat chains -func (r *router) RemoveNatRule(pair firewall.RouterPair) error { - if pair.Masquerade { - if err := r.removeNatRule(pair); err != nil { - return fmt.Errorf("remove nat rule: %w", err) - } - - if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { - return fmt.Errorf("remove inverse nat rule: %w", err) - } - } - - if err := r.removeLegacyRouteRule(pair); err != nil { - return fmt.Errorf("remove legacy routing rule: %w", err) - } - - r.updateState() - - return nil -} - -// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls -func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - if err := r.removeLegacyRouteRule(pair); err != nil { - return err - } - - rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", routingFinalForwardJump} - if err := r.iptablesClient.Append(tableFilter, chainRTFWDIN, rule...); err != nil { - return fmt.Errorf("add legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err) - } - - r.rules[ruleKey] = rule - - return nil -} - -func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWDIN, rule...); err != nil { - return fmt.Errorf("remove legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err) - } - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement ipset counter: %w", err) - } - } - - return nil -} - -// GetLegacyManagement returns the current legacy management mode -func (r *router) GetLegacyManagement() bool { - return r.legacyManagement -} - -// SetLegacyManagement sets the route manager to use legacy management mode -func (r *router) SetLegacyManagement(isLegacy bool) { - r.legacyManagement = isLegacy -} - -// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls -func (r *router) RemoveAllLegacyRouteRules() error { - var merr *multierror.Error - for k, rule := range r.rules { - if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) { - continue - } - if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWDIN, rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err)) - } else { - delete(r.rules, k) - } - } - - r.updateState() - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) Reset() error { - var merr *multierror.Error - if err := r.cleanUpDefaultForwardRules(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.ipsetCounter.Flush(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.cleanupDataPlaneMark(); err != nil { - merr = multierror.Append(merr, err) - } - - r.rules = make(map[string][]string) - r.updateState() - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) cleanUpDefaultForwardRules() error { - if err := r.cleanJumpRules(); err != nil { - return fmt.Errorf("clean jump rules: %w", err) - } - - log.Debug("flushing routing related tables") - - // Remove jump rules from built-in chains before deleting custom chains, - // otherwise the chain deletion fails with "device or resource busy". - if ok, err := r.iptablesClient.ChainExists(tableNat, chainNATOutput); err != nil { - return fmt.Errorf("check chain %s: %w", chainNATOutput, err) - } else if ok { - jumpRule := []string{"-j", chainNATOutput} - if err := r.iptablesClient.Delete(tableNat, "OUTPUT", jumpRule...); err != nil { - log.Debugf("clean OUTPUT jump rule: %v", err) - } - } - - for _, chainInfo := range []struct { - chain string - table string - }{ - {chainRTFWDIN, tableFilter}, - {chainRTFWDOUT, tableFilter}, - {chainRTPRE, tableMangle}, - {chainRTNAT, tableNat}, - {chainRTRDR, tableNat}, - {chainNATOutput, tableNat}, - {chainRTMSSCLAMP, tableMangle}, - } { - ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain) - if err != nil { - return fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) - } else if ok { - if err = r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { - return fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) - } - } - } - - return nil -} - -func (r *router) createContainers() error { - for _, chainInfo := range []struct { - chain string - table string - }{ - {chainRTFWDIN, tableFilter}, - {chainRTFWDOUT, tableFilter}, - {chainRTPRE, tableMangle}, - {chainRTNAT, tableNat}, - {chainRTRDR, tableNat}, - {chainRTMSSCLAMP, tableMangle}, - } { - // Fallback: clear chains that survived an unclean shutdown. - if ok, _ := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain); ok { - if err := r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { - log.Warnf("clear stale chain %s in %s: %v", chainInfo.chain, chainInfo.table, err) - } - } - if err := r.iptablesClient.NewChain(chainInfo.table, chainInfo.chain); err != nil { - return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) - } - } - - if err := r.insertEstablishedRule(chainRTFWDIN); err != nil { - return fmt.Errorf("insert established rule: %w", err) - } - - if err := r.insertEstablishedRule(chainRTFWDOUT); err != nil { - return fmt.Errorf("insert established rule: %w", err) - } - - if err := r.addPostroutingRules(); err != nil { - return fmt.Errorf("add static nat rules: %w", err) - } - - if err := r.addJumpRules(); err != nil { - return fmt.Errorf("add jump rules: %w", err) - } - - if err := r.addMSSClampingRules(); err != nil { - log.Errorf("failed to add MSS clamping rules: %s", err) - } - - return nil -} - -// setupDataPlaneMark configures the fwmark for the data plane -func (r *router) setupDataPlaneMark() error { - var merr *multierror.Error - preRule := []string{ - "-i", r.wgIface.Name(), - "-m", "conntrack", "--ctstate", "NEW", - "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkIn), - } - - if err := r.iptablesClient.AppendUnique(tableMangle, chainPREROUTING, preRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add mangle prerouting rule: %w", err)) - } else { - r.rules[markManglePre] = preRule - } - - postRule := []string{ - "-o", r.wgIface.Name(), - "-m", "conntrack", "--ctstate", "NEW", - "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkOut), - } - - if err := r.iptablesClient.AppendUnique(tableMangle, chainPOSTROUTING, postRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add mangle postrouting rule: %w", err)) - } else { - r.rules[markManglePost] = postRule - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) cleanupDataPlaneMark() error { - var merr *multierror.Error - if preRule, exists := r.rules[markManglePre]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPREROUTING, preRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove mangle prerouting rule: %w", err)) - } else { - delete(r.rules, markManglePre) - } - } - - if postRule, exists := r.rules[markManglePost]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPOSTROUTING, postRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove mangle postrouting rule: %w", err)) - } else { - delete(r.rules, markManglePost) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) addPostroutingRules() error { - // First rule for outbound masquerade - rule1 := []string{ - "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), - "!", "-o", "lo", - "-j", routingFinalNatJump, - } - if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil { - return fmt.Errorf("add outbound masquerade rule: %v", err) - } - r.rules["static-nat-outbound"] = rule1 - - // Second rule for return traffic masquerade - rule2 := []string{ - "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), - "-o", r.wgIface.Name(), - "-j", routingFinalNatJump, - } - if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil { - return fmt.Errorf("add return masquerade rule: %v", err) - } - r.rules["static-nat-return"] = rule2 - - return nil -} - -// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. -func (r *router) addMSSClampingRules() error { - overhead := uint16(ipv4TCPHeaderSize) - if r.v6 { - overhead = ipv6TCPHeaderSize - } - mss := r.mtu - overhead - - // Add jump rule from FORWARD chain in mangle table to our custom chain - jumpRule := []string{ - "-j", chainRTMSSCLAMP, - } - if err := r.iptablesClient.Insert(tableMangle, chainFORWARD, 1, jumpRule...); err != nil { - return fmt.Errorf("add jump to MSS clamp chain: %w", err) - } - r.rules[jumpMSSClamp] = jumpRule - - ruleOut := []string{ - "-o", r.wgIface.Name(), - "-p", "tcp", - "--tcp-flags", "SYN,RST", "SYN", - "-j", "TCPMSS", - "--set-mss", fmt.Sprintf("%d", mss), - } - if err := r.iptablesClient.Append(tableMangle, chainRTMSSCLAMP, ruleOut...); err != nil { - return fmt.Errorf("add outbound MSS clamp rule: %w", err) - } - r.rules["mss-clamp-out"] = ruleOut - - return nil -} - -func (r *router) insertEstablishedRule(chain string) error { - establishedRule := getConntrackEstablished() - - err := r.iptablesClient.Insert(tableFilter, chain, 1, establishedRule...) - if err != nil { - return fmt.Errorf("failed to insert established rule: %v", err) - } - - ruleKey := "established-" + chain - r.rules[ruleKey] = establishedRule - - return nil -} - -func (r *router) addJumpRules() error { - // Jump to nat chain - natRule := []string{"-j", chainRTNAT} - if err := r.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, natRule...); err != nil { - return fmt.Errorf("add nat postrouting jump rule: %v", err) - } - r.rules[jumpNatPost] = natRule - - // Jump to mangle prerouting chain - preRule := []string{"-j", chainRTPRE} - if err := r.iptablesClient.Insert(tableMangle, chainPREROUTING, 1, preRule...); err != nil { - return fmt.Errorf("add mangle prerouting jump rule: %v", err) - } - r.rules[jumpManglePre] = preRule - - // Jump to nat prerouting chain - rdrRule := []string{"-j", chainRTRDR} - if err := r.iptablesClient.Insert(tableNat, chainPREROUTING, 1, rdrRule...); err != nil { - return fmt.Errorf("add nat prerouting jump rule: %v", err) - } - r.rules[jumpNatPre] = rdrRule - - return nil -} - -func (r *router) cleanJumpRules() error { - for _, ruleKey := range []string{jumpNatPost, jumpManglePre, jumpNatPre, jumpMSSClamp} { - if rule, exists := r.rules[ruleKey]; exists { - var table, chain string - switch ruleKey { - case jumpNatPost: - table = tableNat - chain = chainPOSTROUTING - case jumpManglePre: - table = tableMangle - chain = chainPREROUTING - case jumpNatPre: - table = tableNat - chain = chainPREROUTING - case jumpMSSClamp: - table = tableMangle - chain = chainFORWARD - default: - return fmt.Errorf("unknown jump rule: %s", ruleKey) - } - - if err := r.iptablesClient.DeleteIfExists(table, chain, rule...); err != nil { - return fmt.Errorf("delete rule from chain %s in table %s, err: %v", chain, table, err) - } - delete(r.rules, ruleKey) - } - } - return nil -} - -func (r *router) addNatRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.NatFormat, pair) - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil { - return fmt.Errorf("error while removing existing marking rule for %s: %v", pair.Destination, err) - } - delete(r.rules, ruleKey) - } - - markValue := nbnet.PreroutingFwmarkMasquerade - if pair.Inverse { - markValue = nbnet.PreroutingFwmarkMasqueradeReturn - } - - rule := []string{"-i", r.wgIface.Name()} - if pair.Inverse { - rule = []string{"!", "-i", r.wgIface.Name()} - } - - rule = append(rule, - "-m", "conntrack", - "--ctstate", "NEW", - ) - sourceExp, err := r.applyNetwork("-s", pair.Source, nil) - if err != nil { - return fmt.Errorf("apply network -s: %w", err) - } - destExp, err := r.applyNetwork("-d", pair.Destination, nil) - if err != nil { - return fmt.Errorf("apply network -d: %w", err) - } - - rule = append(rule, sourceExp...) - rule = append(rule, destExp...) - rule = append(rule, - "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue), - ) - - // Ensure nat rules come first, so the mark can be overwritten. - // Currently overwritten by the dst-type LOCAL rules for redirected traffic. - if err := r.iptablesClient.Insert(tableMangle, chainRTPRE, 1, rule...); err != nil { - // TODO: rollback ipset counter - return fmt.Errorf("error while adding marking rule for %s: %v", pair.Destination, err) - } - - r.rules[ruleKey] = rule - - r.updateState() - return nil -} - -func (r *router) removeNatRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.NatFormat, pair) - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil { - return fmt.Errorf("error while removing marking rule for %s: %v", pair.Destination, err) - } - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement ipset counter: %w", err) - } - } else { - log.Debugf("marking rule %s not found", ruleKey) - } - - r.updateState() - return nil -} - -func (r *router) updateState() { - if r.stateManager == nil { - return - } - - var currentState *ShutdownState - if existing := r.stateManager.GetState(currentState); existing != nil { - if existingState, ok := existing.(*ShutdownState); ok { - currentState = existingState - } - } - if currentState == nil { - currentState = &ShutdownState{} - } - - currentState.Lock() - defer currentState.Unlock() - - // Clone the rule map so the persisted state holds a private snapshot. The - // live map keeps being mutated by subsequent rule operations while the - // state manager marshals the state from its periodic-save goroutine. - // Sharing it by reference races the two and aborts the process with a - // concurrent map iteration and write. The ipset counter guards itself - // during marshaling, so it can be shared directly. - if r.v6 { - currentState.RouteRules6 = maps.Clone(r.rules) - currentState.RouteIPsetCounter6 = r.ipsetCounter - } else { - currentState.RouteRules = maps.Clone(r.rules) - currentState.RouteIPsetCounter = r.ipsetCounter - } - - if err := r.stateManager.UpdateState(currentState); err != nil { - log.Errorf("failed to update state: %v", err) - } -} - -func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - ruleKey := rule.ID() - if _, exists := r.rules[ruleKey+dnatSuffix]; exists { - return rule, nil - } - - toDestination := rule.TranslatedAddress.String() - switch { - case len(rule.TranslatedPort.Values) == 0: - // no translated port, use original port - case len(rule.TranslatedPort.Values) == 1: - toDestination += fmt.Sprintf(":%d", rule.TranslatedPort.Values[0]) - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - // need the "/originalport" suffix to avoid dnat port randomization - toDestination += fmt.Sprintf(":%d-%d/%d", rule.TranslatedPort.Values[0], rule.TranslatedPort.Values[1], rule.DestinationPort.Values[0]) - default: - return nil, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } - - proto := strings.ToLower(string(rule.Protocol)) - - rules := make(map[string]ruleInfo, 3) - - // DNAT rule - dnatRule := []string{ - "!", "-i", r.wgIface.Name(), - "-p", proto, - "-j", "DNAT", - "--to-destination", toDestination, - } - dnatRule = append(dnatRule, applyPort("--dport", &rule.DestinationPort)...) - rules[ruleKey+dnatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTRDR, - rule: dnatRule, - } - - // SNAT rule - snatRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "MASQUERADE", - } - snatRule = append(snatRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleKey+snatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTNAT, - rule: snatRule, - } - - // Forward filtering rule, if fwd policy is DROP - forwardRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "ACCEPT", - } - forwardRule = append(forwardRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleKey+fwdSuffix] = ruleInfo{ - table: tableFilter, - chain: chainRTFWDOUT, - rule: forwardRule, - } - - for key, ruleInfo := range rules { - if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - r.cleanupFailedDNATAdd(rules) - return nil, fmt.Errorf("add rule %s: %w", key, err) - } - r.rules[key] = ruleInfo.rule - } - - if err := r.ipFwdState.RequestForwarding(r.v6); err != nil { - r.cleanupFailedDNATAdd(rules) - return nil, fmt.Errorf("enable forwarding: %w", err) - } - - r.updateState() - return rule, nil -} - -// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied -// AddDNATRule before rolling back the kernel rules, so no entries remain that -// never got a forwarding refcount. rollbackRules re-adds entries it failed to -// remove from the kernel. -func (r *router) cleanupFailedDNATAdd(rules map[string]ruleInfo) { - for key := range rules { - delete(r.rules, key) - } - if err := r.rollbackRules(rules); err != nil { - log.Errorf("rollback failed: %v", err) - } -} - -func (r *router) rollbackRules(rules map[string]ruleInfo) error { - var merr *multierror.Error - for key, ruleInfo := range rules { - if err := r.iptablesClient.DeleteIfExists(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("rollback rule %s: %w", key, err)) - // On rollback error, add to rules map for next cleanup - r.rules[key] = ruleInfo.rule - } - } - if merr != nil { - r.updateState() - } - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) DeleteDNATRule(rule firewall.Rule) error { - ruleKey := rule.ID() - - _, hadDNAT := r.rules[ruleKey+dnatSuffix] - _, hadSNAT := r.rules[ruleKey+snatSuffix] - _, hadFWD := r.rules[ruleKey+fwdSuffix] - if !hadDNAT && !hadSNAT && !hadFWD { - return nil - } - - var merr *multierror.Error - if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err)) - } else { - delete(r.rules, ruleKey+dnatSuffix) - } - } - - if snatRule, exists := r.rules[ruleKey+snatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err)) - } else { - delete(r.rules, ruleKey+snatSuffix) - } - } - - if fwdRule, exists := r.rules[ruleKey+fwdSuffix]; exists { - if err := r.iptablesClient.Delete(tableFilter, chainRTFWDOUT, fwdRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err)) - } else { - delete(r.rules, ruleKey+fwdSuffix) - } - } - - // Release the refcount only once all rules are gone from the kernel. On - // partial failure the failed entries stay in r.rules so a retry can remove - // them and release then. - if merr == nil { - if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil { - log.Errorf("%v", err) - } - } - - r.updateState() - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) genRouteRuleSpec(params routeFilteringRuleParams, sources []netip.Prefix) ([]string, error) { - var rule []string - - sourceExp, err := r.applyNetwork("-s", params.Source, sources) - if err != nil { - return nil, fmt.Errorf("apply network -s: %w", err) - - } - destExp, err := r.applyNetwork("-d", params.Destination, nil) - if err != nil { - return nil, fmt.Errorf("apply network -d: %w", err) - } - - rule = append(rule, sourceExp...) - rule = append(rule, destExp...) - - if params.Proto != firewall.ProtocolALL { - rule = append(rule, "-p", strings.ToLower(protoForFamily(params.Proto, r.v6))) - rule = append(rule, applyPort("--sport", params.SPort)...) - rule = append(rule, applyPort("--dport", params.DPort)...) - } - - rule = append(rule, "-j", actionToStr(params.Action)) - - return rule, nil -} - -func (r *router) applyNetwork(flag string, network firewall.Network, prefixes []netip.Prefix) ([]string, error) { - direction := "src" - if flag == "-d" { - direction = "dst" - } - - if network.IsSet() { - name := r.ipsetName(network.Set.HashedName()) - if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil { - return nil, fmt.Errorf("create or get ipset: %w", err) - } - - return []string{"-m", "set", matchSet, name, direction}, nil - } - if network.IsPrefix() { - return []string{flag, network.Prefix.String()}, nil - } - - // nolint:nilnil - return nil, nil -} - -func (r *router) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { - name := r.ipsetName(set.HashedName()) - var merr *multierror.Error - for _, prefix := range prefixes { - if err := r.addPrefixToIPSet(name, prefix); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add prefix to ipset: %w", err)) - } - } - if merr == nil { - log.Debugf("updated set %s with prefixes %v", name, prefixes) - } - - return nberrors.FormatErrorOrNil(merr) -} - -// AddInboundDNAT adds an inbound DNAT rule redirecting traffic from NetBird peers to local services. -func (r *router) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - dnatRule := []string{ - "-i", r.wgIface.Name(), - "-p", strings.ToLower(protoForFamily(protocol, r.v6)), - "--dport", strconv.Itoa(int(originalPort)), - "-d", localAddr.String(), - "-m", "addrtype", "--dst-type", "LOCAL", - "-j", "DNAT", - "--to-destination", ":" + strconv.Itoa(int(translatedPort)), - } - - ruleInfo := ruleInfo{ - table: tableNat, - chain: chainRTRDR, - rule: dnatRule, - } - - if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - return fmt.Errorf("add inbound DNAT rule: %w", err) - } - r.rules[ruleID] = ruleInfo.rule - - r.updateState() - return nil -} - -// RemoveInboundDNAT removes an inbound DNAT rule. -func (r *router) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if dnatRule, exists := r.rules[ruleID]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil { - return fmt.Errorf("delete inbound DNAT rule: %w", err) - } - delete(r.rules, ruleID) - } - - r.updateState() - return nil -} - -// ensureNATOutputChain lazily creates the OUTPUT NAT chain and jump rule on first use. -func (r *router) ensureNATOutputChain() error { - if _, exists := r.rules[jumpNatOutput]; exists { - return nil - } - - chainExists, err := r.iptablesClient.ChainExists(tableNat, chainNATOutput) - if err != nil { - return fmt.Errorf("check chain %s: %w", chainNATOutput, err) - } - if !chainExists { - if err := r.iptablesClient.NewChain(tableNat, chainNATOutput); err != nil { - return fmt.Errorf("create chain %s: %w", chainNATOutput, err) - } - } - - jumpRule := []string{"-j", chainNATOutput} - if err := r.iptablesClient.Insert(tableNat, "OUTPUT", 1, jumpRule...); err != nil { - if !chainExists { - if delErr := r.iptablesClient.ClearAndDeleteChain(tableNat, chainNATOutput); delErr != nil { - log.Warnf("failed to rollback chain %s: %v", chainNATOutput, delErr) - } - } - return fmt.Errorf("add OUTPUT jump rule: %w", err) - } - r.rules[jumpNatOutput] = jumpRule - - r.updateState() - return nil -} - -// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. -func (r *router) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - if err := r.ensureNATOutputChain(); err != nil { - return err - } - - dnatRule := []string{ - "-p", strings.ToLower(protoForFamily(protocol, localAddr.Is6())), - "--dport", strconv.Itoa(int(originalPort)), - "-d", localAddr.String(), - "-j", "DNAT", - "--to-destination", ":" + strconv.Itoa(int(translatedPort)), - } - - if err := r.iptablesClient.Append(tableNat, chainNATOutput, dnatRule...); err != nil { - return fmt.Errorf("add output DNAT rule: %w", err) - } - r.rules[ruleID] = dnatRule - - r.updateState() - return nil -} - -// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. -func (r *router) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if dnatRule, exists := r.rules[ruleID]; exists { - if err := r.iptablesClient.Delete(tableNat, chainNATOutput, dnatRule...); err != nil { - return fmt.Errorf("delete output DNAT rule: %w", err) - } - delete(r.rules, ruleID) - } - - r.updateState() - return nil -} - -func applyPort(flag string, port *firewall.Port) []string { - if port == nil { - return nil - } - - if port.IsRange && len(port.Values) == 2 { - return []string{flag, fmt.Sprintf("%d:%d", port.Values[0], port.Values[1])} - } - - if len(port.Values) > 1 { - portList := make([]string, len(port.Values)) - for i, p := range port.Values { - portList[i] = strconv.Itoa(int(p)) - } - return []string{"-m", "multiport", flag, strings.Join(portList, ",")} - } - - return []string{flag, strconv.Itoa(int(port.Values[0]))} -} - -// ipsetName returns the ipset name, suffixed with "-v6" for the v6 router -// to avoid collisions since ipsets are global in the kernel. -func (r *router) ipsetName(name string) string { - if r.v6 { - return name + "-v6" - } - return name -} - -func (r *router) createIPSet(name string) error { - opts := ipset.CreateOptions{ - Replace: true, - } - if r.v6 { - opts.Family = ipset.FamilyIPV6 - } - - if err := ipset.Create(name, ipset.TypeHashNet, opts); err != nil { - return fmt.Errorf("create ipset %s: %w", name, err) - } - - log.Debugf("created ipset %s with type hash:net", name) - return nil -} - -func (r *router) addPrefixToIPSet(name string, prefix netip.Prefix) error { - addr := prefix.Addr() - ip := addr.AsSlice() - - entry := &ipset.Entry{ - IP: ip, - CIDR: uint8(prefix.Bits()), - Replace: true, - } - - if err := ipset.Add(name, entry); err != nil { - return fmt.Errorf("add prefix to ipset %s: %w", name, err) - } - - return nil -} - -func (r *router) destroyIPSet(name string) error { - return ipset.Destroy(name) -} diff --git a/client/firewall/iptables/router_linux_test.go b/client/firewall/iptables/router_linux_test.go index 9ca6b9f7e..6c4ae9425 100644 --- a/client/firewall/iptables/router_linux_test.go +++ b/client/firewall/iptables/router_linux_test.go @@ -31,7 +31,7 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) { iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "failed to init iptables client") - manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) + manager, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) require.NoError(t, err, "should return a valid iptables manager") require.NoError(t, manager.init(nil)) @@ -52,12 +52,12 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) { // 11. MSS clamping rule for outbound traffic require.Len(t, manager.rules, 11, "should have created rules map") - exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, "-j", chainRTNAT) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPOSTROUTING) + exists, err := manager.iptablesClient.Exists(tableNat, chainPostrouting, "-j", chainRTNAT) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPostrouting) require.True(t, exists, "postrouting jump rule should exist") - exists, err = manager.iptablesClient.Exists(tableMangle, chainPREROUTING, "-j", chainRTPRE) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPREROUTING) + exists, err = manager.iptablesClient.Exists(tableMangle, chainPrerouting, "-j", chainRTPre) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPrerouting) require.True(t, exists, "prerouting jump rule should exist") pair := firewall.RouterPair{ @@ -84,7 +84,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) { iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "failed to init iptables client") - manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) + manager, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) require.NoError(t, err, "shouldn't return error") require.NoError(t, manager.init(nil)) @@ -95,7 +95,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) { err = manager.AddNatRule(testCase.InputPair) require.NoError(t, err, "marking rule should be inserted") - natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.NatFormat) markingRule := []string{ "-i", ifaceMock.Name(), "-m", "conntrack", @@ -106,8 +106,8 @@ func TestIptablesManager_AddNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), } - exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err := iptablesClient.Exists(tableMangle, chainRTPre, markingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) if testCase.InputPair.Masquerade { require.True(t, exists, "marking rule should be created") foundRule, found := manager.rules[natRuleKey] @@ -121,7 +121,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) { // Check inverse rule inversePair := firewall.GetInversePair(testCase.InputPair) - inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair) + inverseRuleKey := inversePair.GenKey(firewall.NatFormat) inverseMarkingRule := []string{ "!", "-i", ifaceMock.Name(), "-m", "conntrack", @@ -132,8 +132,8 @@ func TestIptablesManager_AddNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), } - exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err = iptablesClient.Exists(tableMangle, chainRTPre, inverseMarkingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) if testCase.InputPair.Masquerade { require.True(t, exists, "inverse marking rule should be created") foundRule, found := manager.rules[inverseRuleKey] @@ -157,7 +157,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { t.Run(testCase.Name, func(t *testing.T) { iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4) - manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) + manager, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) require.NoError(t, err, "shouldn't return error") require.NoError(t, manager.init(nil)) defer func() { @@ -170,7 +170,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { err = manager.RemoveNatRule(testCase.InputPair) require.NoError(t, err, "shouldn't return error") - natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.NatFormat) markingRule := []string{ "-i", ifaceMock.Name(), "-m", "conntrack", @@ -181,8 +181,8 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), } - exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err := iptablesClient.Exists(tableMangle, chainRTPre, markingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) require.False(t, exists, "marking rule should not exist") _, found := manager.rules[natRuleKey] @@ -190,7 +190,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { // Check inverse rule removal inversePair := firewall.GetInversePair(testCase.InputPair) - inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair) + inverseRuleKey := inversePair.GenKey(firewall.NatFormat) inverseMarkingRule := []string{ "!", "-i", ifaceMock.Name(), "-m", "conntrack", @@ -201,8 +201,8 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), } - exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err = iptablesClient.Exists(tableMangle, chainRTPre, inverseMarkingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) require.False(t, exists, "inverse marking rule should not exist") _, found = manager.rules[inverseRuleKey] @@ -219,13 +219,13 @@ func TestRouter_AddRouteFiltering(t *testing.T) { iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "Failed to create iptables client") - r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router manager") + r, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) + require.NoError(t, err, "Failed to create family manager") require.NoError(t, r.init(nil)) defer func() { err := r.Reset() - require.NoError(t, err, "Failed to reset router") + require.NoError(t, err, "Failed to reset family") }() tests := []struct { @@ -334,62 +334,30 @@ func TestRouter_AddRouteFiltering(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ruleKey, err := r.AddRouteFiltering(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) - require.NoError(t, err, "AddRouteFiltering failed") + ruleKey, err := r.AddFilterRule(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) + require.NoError(t, err, "AddFilterRule failed") - // Check if the rule is in the internal map - rule, ok := r.rules[ruleKey.ID()] - assert.True(t, ok, "Rule not found in internal map") + stored, ok := r.filters[ruleKey.ID()] + require.True(t, ok, "rule not stored in filters") + t.Logf("Internal rule: %v", stored.specs) - // Log the internal rule - t.Logf("Internal rule: %v", rule) - - // Check if the rule exists in iptables - exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, rule...) + exists, err := iptablesClient.Exists(tableFilter, chainRTFwdIn, stored.specs...) assert.NoError(t, err, "Failed to check rule existence") assert.True(t, exists, "Rule not found in iptables") - var source firewall.Network - if len(tt.sources) > 1 { - source.Set = firewall.NewPrefixSet(tt.sources) - } else if len(tt.sources) > 0 { - source.Prefix = tt.sources[0] - } - // Verify rule content - params := routeFilteringRuleParams{ - Source: source, - Destination: firewall.Network{Prefix: tt.destination}, - Proto: tt.proto, - SPort: tt.sPort, - DPort: tt.dPort, - Action: tt.action, - } - - expectedRule, err := r.genRouteRuleSpec(params, nil) - require.NoError(t, err, "Failed to generate expected rule spec") - if tt.expectSet { setName := firewall.NewPrefixSet(tt.sources).HashedName() - expectedRule, err = r.genRouteRuleSpec(params, nil) - require.NoError(t, err, "Failed to generate expected rule spec with set") - - // Check if the set was created _, exists := r.ipsetCounter.Get(setName) assert.True(t, exists, "IPSet not created") + assert.NotEmpty(t, findSets(stored.specs), "Rule should reference an ipset") } - assert.Equal(t, expectedRule, rule, "Rule content mismatch") - - // Clean up - err = r.DeleteRouteRule(ruleKey) - require.NoError(t, err, "Failed to delete rule") + require.NoError(t, r.DeleteFilterRule(ruleKey), "Failed to delete rule") }) } } func TestFindSetNameInRule(t *testing.T) { - r := &router{} - testCases := []struct { name string rule []string @@ -430,7 +398,7 @@ func TestFindSetNameInRule(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result := r.findSets(tc.rule) + result := findSets(tc.rule) if len(result) != len(tc.expected) { t.Errorf("Expected %d sets, got %d. Sets found: %v", len(tc.expected), len(result), result) diff --git a/client/firewall/iptables/routing_linux.go b/client/firewall/iptables/routing_linux.go new file mode 100644 index 000000000..c63be4e43 --- /dev/null +++ b/client/firewall/iptables/routing_linux.go @@ -0,0 +1,273 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "strings" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) AddNatRule(pair firewall.RouterPair) error { + if r.legacyManagement { + log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) + if err := r.addLegacyRouteRule(pair); err != nil { + return fmt.Errorf("add legacy routing rule: %w", err) + } + } + + if pair.Masquerade { + if err := r.addNatRule(pair); err != nil { + return fmt.Errorf("add nat rule: %w", err) + } + + if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil { + return fmt.Errorf("add inverse nat rule: %w", err) + } + } + + r.updateState() + + return nil +} + +// RemoveNatRule removes an iptables rule pair from forwarding and nat chains +func (r *family) RemoveNatRule(pair firewall.RouterPair) error { + if pair.Masquerade { + if err := r.removeNatRule(pair); err != nil { + return fmt.Errorf("remove nat rule: %w", err) + } + + if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { + return fmt.Errorf("remove inverse nat rule: %w", err) + } + } + + if err := r.removeLegacyRouteRule(pair); err != nil { + return fmt.Errorf("remove legacy routing rule: %w", err) + } + + r.updateState() + + return nil +} + +// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls +func (r *family) addLegacyRouteRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + if err := r.removeLegacyRouteRule(pair); err != nil { + return err + } + + rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", "ACCEPT"} + if err := r.iptablesClient.Append(tableFilter, chainRTFwdIn, rule...); err != nil { + return fmt.Errorf("add legacy forwarding rule %s -> %s: %w", pair.Source, pair.Destination, err) + } + + r.rules[ruleID] = rule + + return nil +} + +func (r *family) removeLegacyRouteRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + if rule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFwdIn, rule...); err != nil { + return fmt.Errorf("remove legacy forwarding rule %s -> %s: %w", pair.Source, pair.Destination, err) + } + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement ipset counter: %w", err) + } + } + + return nil +} + +// GetLegacyManagement returns the current legacy management mode +func (r *family) GetLegacyManagement() bool { + return r.legacyManagement +} + +// SetLegacyManagement sets the route manager to use legacy management mode +func (r *family) SetLegacyManagement(isLegacy bool) { + r.legacyManagement = isLegacy +} + +// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls +func (r *family) RemoveAllLegacyRouteRules() error { + var merr *multierror.Error + for k, rule := range r.rules { + if !strings.HasPrefix(string(k), firewall.ForwardingFormatPrefix) { + continue + } + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFwdIn, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %w", err)) + } else { + delete(r.rules, k) + } + } + + r.updateState() + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) addPostroutingRules() error { + // First rule for outbound masquerade + rule1 := []string{ + "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), + "!", "-o", "lo", + "-j", "MASQUERADE", + } + if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil { + return fmt.Errorf("add outbound masquerade rule: %w", err) + } + r.rules["static-nat-outbound"] = rule1 + + // Second rule for return traffic masquerade + rule2 := []string{ + "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), + "-o", r.wgIface.Name(), + "-j", "MASQUERADE", + } + if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil { + return fmt.Errorf("add return masquerade rule: %w", err) + } + r.rules["static-nat-return"] = rule2 + + return nil +} + +// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. +func (r *family) addMSSClampingRules() error { + overhead := uint16(ipv4TCPHeaderSize) + if r.v6 { + overhead = ipv6TCPHeaderSize + } + mss := r.mtu - overhead + + // Add jump rule from FORWARD chain in mangle table to our custom chain + jumpRule := jumpRuleSpec(chainRTMSSClamp) + if err := r.iptablesClient.Insert(tableMangle, chainForward, 1, jumpRule...); err != nil { + return fmt.Errorf("add jump to MSS clamp chain: %w", err) + } + r.rules[jumpMSSClamp] = jumpRule + + ruleOut := []string{ + "-o", r.wgIface.Name(), + "-p", "tcp", + "--tcp-flags", "SYN,RST", "SYN", + "-j", "TCPMSS", + "--set-mss", fmt.Sprintf("%d", mss), + } + if err := r.iptablesClient.Append(tableMangle, chainRTMSSClamp, ruleOut...); err != nil { + return fmt.Errorf("add outbound MSS clamp rule: %w", err) + } + r.rules["mss-clamp-out"] = ruleOut + + return nil +} + +func (r *family) insertEstablishedRule(chain string) error { + establishedRule := getConntrackEstablished() + + err := r.iptablesClient.Insert(tableFilter, chain, 1, establishedRule...) + if err != nil { + return fmt.Errorf("insert established rule: %w", err) + } + + ruleID := firewall.RuleID("established-" + chain) + r.rules[ruleID] = establishedRule + + return nil +} + +func (r *family) addNatRule(pair firewall.RouterPair) (err error) { + ruleID := pair.GenKey(firewall.NatFormat) + + if rule, exists := r.rules[ruleID]; exists { + if derr := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, rule...); derr != nil { + return fmt.Errorf("remove existing marking rule for %s: %w", pair.Destination, derr) + } + delete(r.rules, ruleID) + + // Drop the replaced spec's set references only once the new spec has + // taken its own, so a set both specs share is not destroyed and + // recreated, which would lose the prefixes UpdateSet put in it. + defer func() { + if derr := r.decrementSetCounter(rule); derr != nil && err == nil { + err = fmt.Errorf("decrement ipset counter: %w", derr) + } + }() + } + + markValue := nbnet.PreroutingFwmarkMasquerade + if pair.Inverse { + markValue = nbnet.PreroutingFwmarkMasqueradeReturn + } + + rule := []string{"-i", r.wgIface.Name()} + if pair.Inverse { + rule = []string{"!", "-i", r.wgIface.Name()} + } + + rule = append(rule, + "-m", "conntrack", + "--ctstate", "NEW", + ) + sourceExp, err := r.applyNetwork("-s", pair.Source, nil) + if err != nil { + return fmt.Errorf("apply network -s: %w", err) + } + destExp, err := r.applyNetwork("-d", pair.Destination, nil) + if err != nil { + r.dropSourceMatch(sourceExp) + return fmt.Errorf("apply network -d: %w", err) + } + + rule = append(rule, sourceExp...) + rule = append(rule, destExp...) + rule = append(rule, + "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue), + ) + + // Ensure nat rules come first, so the mark can be overwritten. + // Currently overwritten by the dst-type LOCAL rules for redirected traffic. + if err := r.iptablesClient.Insert(tableMangle, chainRTPre, 1, rule...); err != nil { + r.dropSourceMatch(rule) + return fmt.Errorf("add marking rule for %s: %w", pair.Destination, err) + } + + r.rules[ruleID] = rule + + return nil +} + +func (r *family) removeNatRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.NatFormat) + + if rule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, rule...); err != nil { + return fmt.Errorf("remove marking rule for %s: %w", pair.Destination, err) + } + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement ipset counter: %w", err) + } + } else { + log.Debugf("marking rule %s not found", ruleID) + } + + return nil +} diff --git a/client/firewall/iptables/rule.go b/client/firewall/iptables/rule.go index 4f4eab167..33bcbd1d2 100644 --- a/client/firewall/iptables/rule.go +++ b/client/firewall/iptables/rule.go @@ -1,18 +1,37 @@ package iptables -// Rule to handle management of rules -type Rule struct { - ruleID string - ipsetName string +import "github.com/netbirdio/netbird/client/firewall/manager" +// Rule to handle management of rules. Source set membership (when the +// rule was built against a shared hash:net ipset) is encoded in specs; +// DeleteFilterRule recovers it via findSets so the refcounter can drop +// the right reference. +type Rule struct { + id manager.RuleID specs []string mangleSpecs []string - ip string - chain string - v6 bool + // extraRules holds the rules beyond the first when the ipset + // fallback expands a multi-source rule into one rule per prefix. + extraRules []filterSpecs + chain string + v6 bool } -// GetRuleID returns the rule id -func (r *Rule) ID() string { - return r.ruleID +// filterSpecs is one installed iptables rule: its filter-table spec and +// the paired mangle redirect-mark spec (nil for route rules or when the +// mangle rule could not be added). +type filterSpecs struct { + specs []string + mangleSpecs []string +} + +// allSpecs returns the spec pairs of every iptables rule backing this +// Rule, the primary one first. +func (r *Rule) allSpecs() []filterSpecs { + return append([]filterSpecs{{specs: r.specs, mangleSpecs: r.mangleSpecs}}, r.extraRules...) +} + +// ID returns the rule id +func (r *Rule) ID() manager.RuleID { + return r.id } diff --git a/client/firewall/iptables/rulestore_linux.go b/client/firewall/iptables/rulestore_linux.go deleted file mode 100644 index a6d36540e..000000000 --- a/client/firewall/iptables/rulestore_linux.go +++ /dev/null @@ -1,127 +0,0 @@ -package iptables - -import ( - "encoding/json" - "maps" -) - -type ipList struct { - ips map[string]struct{} -} - -func newIpList(ip string) *ipList { - ips := make(map[string]struct{}) - ips[ip] = struct{}{} - - return &ipList{ - ips: ips, - } -} - -func (s *ipList) addIP(ip string) { - s.ips[ip] = struct{}{} -} - -// clone returns a deep copy of the ipList with its own ips map. -func (s *ipList) clone() *ipList { - if s == nil { - return nil - } - return &ipList{ips: maps.Clone(s.ips)} -} - -// MarshalJSON implements json.Marshaler -func (s *ipList) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - IPs map[string]struct{} `json:"ips"` - }{ - IPs: s.ips, - }) -} - -// UnmarshalJSON implements json.Unmarshaler -func (s *ipList) UnmarshalJSON(data []byte) error { - temp := struct { - IPs map[string]struct{} `json:"ips"` - }{} - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - s.ips = temp.IPs - - if temp.IPs == nil { - temp.IPs = make(map[string]struct{}) - } - - return nil -} - -type ipsetStore struct { - ipsets map[string]*ipList -} - -func newIpsetStore() *ipsetStore { - return &ipsetStore{ - ipsets: make(map[string]*ipList), - } -} - -// clone returns a deep copy of the ipsetStore with its own ipsets map and -// independent ipList entries. -func (s *ipsetStore) clone() *ipsetStore { - if s == nil { - return nil - } - cloned := &ipsetStore{ipsets: make(map[string]*ipList, len(s.ipsets))} - for name, list := range s.ipsets { - cloned.ipsets[name] = list.clone() - } - return cloned -} - -func (s *ipsetStore) ipset(ipsetName string) (*ipList, bool) { - r, ok := s.ipsets[ipsetName] - return r, ok -} - -func (s *ipsetStore) addIpList(ipsetName string, list *ipList) { - s.ipsets[ipsetName] = list -} - -func (s *ipsetStore) deleteIpset(ipsetName string) { - delete(s.ipsets, ipsetName) -} - -func (s *ipsetStore) ipsetNames() []string { - names := make([]string, 0, len(s.ipsets)) - for name := range s.ipsets { - names = append(names, name) - } - return names -} - -// MarshalJSON implements json.Marshaler -func (s *ipsetStore) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - IPSets map[string]*ipList `json:"ipsets"` - }{ - IPSets: s.ipsets, - }) -} - -// UnmarshalJSON implements json.Unmarshaler -func (s *ipsetStore) UnmarshalJSON(data []byte) error { - temp := struct { - IPSets map[string]*ipList `json:"ipsets"` - }{} - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - s.ipsets = temp.IPSets - - if temp.IPSets == nil { - temp.IPSets = make(map[string]*ipList) - } - - return nil -} diff --git a/client/firewall/iptables/state_linux.go b/client/firewall/iptables/state_linux.go index f4be37d01..00bf1cebd 100644 --- a/client/firewall/iptables/state_linux.go +++ b/client/firewall/iptables/state_linux.go @@ -29,17 +29,13 @@ type ShutdownState struct { InterfaceState *InterfaceState `json:"interface_state,omitempty"` - RouteRules routeRules `json:"route_rules,omitempty"` - RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"` - - ACLEntries aclEntries `json:"acl_entries,omitempty"` - ACLIPsetStore *ipsetStore `json:"acl_ipset_store,omitempty"` - - // IPv6 counterparts + RouteRules routeRules `json:"route_rules,omitempty"` RouteRules6 routeRules `json:"route_rules_v6,omitempty"` + RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"` RouteIPsetCounter6 *ipsetCounter `json:"route_ipset_counter_v6,omitempty"` - ACLEntries6 aclEntries `json:"acl_entries_v6,omitempty"` - ACLIPsetStore6 *ipsetStore `json:"acl_ipset_store_v6,omitempty"` + + ACLEntries aclEntries `json:"acl_entries,omitempty"` + ACLEntries6 aclEntries `json:"acl_entries_v6,omitempty"` } func (s *ShutdownState) Name() string { @@ -57,17 +53,14 @@ func (s *ShutdownState) Cleanup() error { } if s.RouteRules != nil { - ipt.router.rules = s.RouteRules + ipt.family4.rules = s.RouteRules } if s.RouteIPsetCounter != nil { - ipt.router.ipsetCounter.LoadData(s.RouteIPsetCounter) + ipt.family4.ipsetCounter.LoadData(s.RouteIPsetCounter) } if s.ACLEntries != nil { - ipt.aclMgr.entries = s.ACLEntries - } - if s.ACLIPsetStore != nil { - ipt.aclMgr.ipsetStore = s.ACLIPsetStore + ipt.family4.entries = s.ACLEntries } // Clean up v6 state even if the current run has no IPv6. @@ -79,16 +72,13 @@ func (s *ShutdownState) Cleanup() error { } if ipt.hasIPv6() { if s.RouteRules6 != nil { - ipt.router6.rules = s.RouteRules6 + ipt.family6.rules = s.RouteRules6 } if s.RouteIPsetCounter6 != nil { - ipt.router6.ipsetCounter.LoadData(s.RouteIPsetCounter6) + ipt.family6.ipsetCounter.LoadData(s.RouteIPsetCounter6) } if s.ACLEntries6 != nil { - ipt.aclMgr6.entries = s.ACLEntries6 - } - if s.ACLIPsetStore6 != nil { - ipt.aclMgr6.ipsetStore = s.ACLIPsetStore6 + ipt.family6.entries = s.ACLEntries6 } } diff --git a/client/firewall/iptables/testhelpers_linux_test.go b/client/firewall/iptables/testhelpers_linux_test.go new file mode 100644 index 000000000..fe44f7cc3 --- /dev/null +++ b/client/firewall/iptables/testhelpers_linux_test.go @@ -0,0 +1,27 @@ +//go:build privileged + +package iptables + +import ( + "fmt" + "net" + "net/netip" +) + +func pfx(ip net.IP) []netip.Prefix { + if ip == nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + if ip.IsUnspecified() { + if ip.To4() != nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + } + a, ok := netip.AddrFromSlice(ip) + if !ok { + panic(fmt.Sprintf("invalid IP length: %d", len(ip))) + } + a = a.Unmap() + return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())} +} diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 149c6db83..97a94d0f5 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -3,7 +3,6 @@ package manager import ( "errors" "fmt" - "net" "net/netip" "sort" @@ -16,6 +15,12 @@ import ( // method but the IPv6 firewall components were not initialized. var ErrIPv6NotInitialized = errors.New("IPv6 firewall not initialized") +// ErrNoSources is returned when AddFilterRule is called with an empty +// source list. "Match any source" must be expressed explicitly with a +// /0 prefix; an empty list is a caller error and is rejected rather +// than silently widening the rule to every source. +var ErrNoSources = errors.New("rule has no sources") + const ( ForwardingFormatPrefix = "netbird-fwd-" ForwardingFormat = "netbird-fwd-%s-%t" @@ -23,13 +28,18 @@ const ( NatFormat = "netbird-nat-%s-%t" ) +// RuleID identifies a firewall rule. It is a typed string so the +// compiler catches accidental mixing with arbitrary string keys. It is +// only an identifier and does not implement Rule. +type RuleID string + // Rule abstraction should be implemented by each firewall manager // // Each firewall type for different OS can use different type // of the properties to hold data of the created rule type Rule interface { // ID returns the rule id - ID() string + ID() RuleID } // RuleDirection is the traffic direction which a rule is applied @@ -91,6 +101,13 @@ func (d Network) IsPrefix() bool { return d.Prefix.IsValid() } +// IsZero returns true if the network designates no destination, i.e. it +// is the zero value. A zero Network is the peer-rule sentinel; a non-zero +// one carries a prefix or set destination. +func (d Network) IsZero() bool { + return !d.IsPrefix() && !d.IsSet() +} + // Manager is the high level abstraction of a firewall manager // // It declares methods which handle actions required by the @@ -98,46 +115,42 @@ func (d Network) IsPrefix() bool { type Manager interface { Init(stateManager *statemanager.Manager) error - // AllowNetbird allows netbird interface traffic - AllowNetbird() error - - // AddPeerFiltering adds a rule to the firewall + // AddFilterRule adds a packet-filtering rule to the firewall. // - // If comment argument is empty firewall manager should set - // rule ID as comment for the rule + // If destination is the zero Network, the rule applies to traffic + // inbound to this node, i.e. peer ACL semantics, installed in + // the kernel's input chain. If destination is set (prefix or + // set), the rule applies to forwarded traffic with that + // destination, route ACL semantics, installed in the forward + // chain. // - // Note: Callers should call Flush() after adding rules to ensure - // they are applied to the kernel and rule handles are refreshed. - AddPeerFiltering( + // sources must be a single address family; the caller splits mixed + // families and calls once per family. "Match any source" must be + // expressed with an explicit /0 prefix; an empty sources list is + // rejected with ErrNoSources so a zeroed list can never widen a + // rule to every source. + // + // Note: callers should call Flush() after adding rules. + AddFilterRule( id []byte, - ip net.IP, + sources []netip.Prefix, + destination Network, proto Protocol, sPort *Port, dPort *Port, action Action, - ipsetName string, - ) ([]Rule, error) + ) (Rule, error) - // DeletePeerRule from the firewall by rule definition - DeletePeerRule(rule Rule) error + // DeleteFilterRule removes a filtering rule previously added via + // AddFilterRule. The rule's own type identifies whether it lives + // in the peer (input) or route (forward) path. + DeleteFilterRule(rule Rule) error // IsServerRouteSupported returns true if the firewall supports server side routing operations IsServerRouteSupported() bool IsStateful() bool - AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination Network, - proto Protocol, - sPort, dPort *Port, - action Action, - ) (Rule, error) - - // DeleteRouteRule deletes a routing rule - DeleteRouteRule(rule Rule) error - // AddNatRule inserts a routing NAT rule AddNatRule(pair RouterPair) error @@ -185,8 +198,9 @@ type Manager interface { SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error } -func GenKey(format string, pair RouterPair) string { - return fmt.Sprintf(format, pair.ID, pair.Inverse) +// GenKey builds the rule id for this pair from the given format. +func (p RouterPair) GenKey(format string) RuleID { + return RuleID(fmt.Sprintf(format, p.ID, p.Inverse)) } // LegacyManager defines the interface for legacy management operations @@ -242,6 +256,20 @@ func MergeIPRanges(prefixes []netip.Prefix) []netip.Prefix { return merged } +// UnmapPrefix normalizes a v4-mapped v6 prefix (::ffff:a.b.c.d) to its +// plain v4 form, shifting the prefix length out of the 96-bit mapped +// range. Other prefixes are returned unchanged. Keeping prefixes +// unmapped ensures v4 rules match consistently and the match builders +// read the correct address length. +func UnmapPrefix(p netip.Prefix) netip.Prefix { + addr := p.Addr() + if !addr.Is4In6() { + return p + } + bits := max(p.Bits()-96, 0) + return netip.PrefixFrom(addr.Unmap(), bits) +} + // SortPrefixes sorts the given slice of netip.Prefix in place. // It sorts first by IP address, then by prefix length (most specific to least specific). func SortPrefixes(prefixes []netip.Prefix) { diff --git a/client/firewall/manager/forward_rule.go b/client/firewall/manager/forward_rule.go index 21a43520e..c2e9e5c60 100644 --- a/client/firewall/manager/forward_rule.go +++ b/client/firewall/manager/forward_rule.go @@ -13,13 +13,13 @@ type ForwardRule struct { TranslatedPort Port } -func (r ForwardRule) ID() string { +func (r ForwardRule) ID() RuleID { id := fmt.Sprintf("%s;%s;%s;%s", r.Protocol, r.DestinationPort.String(), r.TranslatedAddress.String(), r.TranslatedPort.String()) - return id + return RuleID(id) } func (r ForwardRule) String() string { diff --git a/client/firewall/manager/set.go b/client/firewall/manager/set.go index dda93bf47..fa55471ea 100644 --- a/client/firewall/manager/set.go +++ b/client/firewall/manager/set.go @@ -40,7 +40,7 @@ func (h Set) Comment() string { // NewPrefixSet generates a unique name for an ipset based on the given prefixes. func NewPrefixSet(prefixes []netip.Prefix) Set { - // sort for consistent naming + prefixes = slices.Clone(prefixes) SortPrefixes(prefixes) hash := sha256.New() diff --git a/client/firewall/nftables/acl_linux.go b/client/firewall/nftables/acl_linux.go deleted file mode 100644 index 9d2ea7264..000000000 --- a/client/firewall/nftables/acl_linux.go +++ /dev/null @@ -1,713 +0,0 @@ -package nftables - -import ( - "bytes" - "fmt" - "net" - "slices" - "strconv" - "strings" - "time" - - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" - log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - - firewall "github.com/netbirdio/netbird/client/firewall/manager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - - // rules chains contains the effective ACL rules - chainNameInputRules = "netbird-acl-input-rules" - - // filter chains contains the rules that jump to the rules chains - chainNameInputFilter = "netbird-acl-input-filter" - chainNameForwardFilter = "netbird-acl-forward-filter" - chainNameManglePrerouting = "netbird-mangle-prerouting" - chainNameManglePostrouting = "netbird-mangle-postrouting" -) - -const flushError = "flush: %w" - -type AclManager struct { - rConn *nftables.Conn - sConn *nftables.Conn - wgIface iFaceMapper - routingFwChainName string - af addrFamily - - workTable *nftables.Table - chainInputRules *nftables.Chain - chainPrerouting *nftables.Chain - - ipsetStore *ipsetStore - rules map[string]*Rule -} - -func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) { - // sConn is used for creating sets and adding/removing elements from them - // it's differ then rConn (which does create new conn for each flush operation) - // and is permanent. Using same connection for both type of operations - // overloads netlink with high amount of rules ( > 10000) - sConn, err := nftables.New(nftables.AsLasting()) - if err != nil { - return nil, fmt.Errorf("create nf conn: %w", err) - } - - return &AclManager{ - rConn: &nftables.Conn{}, - sConn: sConn, - wgIface: wgIface, - workTable: table, - routingFwChainName: routingFwChainName, - af: familyForAddr(table.Family == nftables.TableFamilyIPv4), - - ipsetStore: newIpsetStore(), - rules: make(map[string]*Rule), - }, nil -} - -func (m *AclManager) init(workTable *nftables.Table) error { - m.workTable = workTable - return m.createDefaultChains() -} - -// AddPeerFiltering rule to the firewall -// -// If comment argument is empty firewall manager should set -// rule ID as comment for the rule -func (m *AclManager) AddPeerFiltering( - id []byte, - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - var ipset *nftables.Set - if ipsetName != "" { - var err error - ipset, err = m.addIpToSet(ipsetName, ip) - if err != nil { - return nil, err - } - } - - newRules := make([]firewall.Rule, 0, 2) - ioRule, err := m.addIOFiltering(ip, proto, sPort, dPort, action, ipset) - if err != nil { - return nil, err - } - - newRules = append(newRules, ioRule) - return newRules, nil -} - -// DeletePeerRule from the firewall by rule definition -func (m *AclManager) DeletePeerRule(rule firewall.Rule) error { - r, ok := rule.(*Rule) - if !ok { - return fmt.Errorf("invalid rule type") - } - - if r.nftSet == nil { - if err := m.rConn.DelRule(r.nftRule); err != nil { - log.Errorf("failed to delete rule: %v", err) - } - if r.mangleRule != nil { - if err := m.rConn.DelRule(r.mangleRule); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - delete(m.rules, r.ID()) - return m.rConn.Flush() - } - - ips, ok := m.ipsetStore.ips(r.nftSet.Name) - if !ok { - if err := m.rConn.DelRule(r.nftRule); err != nil { - log.Errorf("failed to delete rule: %v", err) - } - if r.mangleRule != nil { - if err := m.rConn.DelRule(r.mangleRule); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - delete(m.rules, r.ID()) - return m.rConn.Flush() - } - - if _, ok := ips[r.ip.String()]; ok { - err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: ipToBytes(r.ip, m.af)}}) - if err != nil { - log.Errorf("delete elements for set %q: %v", r.nftSet.Name, err) - } - if err := m.sConn.Flush(); err != nil { - log.Debugf("flush error of set delete element, %s", r.nftSet.Name) - return err - } - m.ipsetStore.DeleteIpFromSet(r.nftSet.Name, r.ip) - } - - // if after delete, set still contains other IPs, - // no need to delete firewall rule and we should exit here - if len(ips) > 0 { - return nil - } - - if err := m.rConn.DelRule(r.nftRule); err != nil { - log.Errorf("failed to delete rule: %v", err) - } - if r.mangleRule != nil { - if err := m.rConn.DelRule(r.mangleRule); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - - if err := m.rConn.Flush(); err != nil { - return err - } - - delete(m.rules, r.ID()) - m.ipsetStore.DeleteReferenceFromIpSet(r.nftSet.Name) - - if m.ipsetStore.HasReferenceToSet(r.nftSet.Name) { - return nil - } - - // we delete last IP from the set, that means we need to delete - // set itself and associated firewall rule too - m.rConn.FlushSet(r.nftSet) - m.rConn.DelSet(r.nftSet) - m.ipsetStore.deleteIpset(r.nftSet.Name) - return nil -} - -// createDefaultAllowRules creates default allow rules for the input and output chains -func (m *AclManager) createDefaultAllowRules() error { - expIn := []expr.Any{ - &expr.Verdict{ - Kind: expr.VerdictAccept, - }, - } - - _ = m.rConn.InsertRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainInputRules, - Position: 0, - Exprs: expIn, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - return nil -} - -// Flush rule/chain/set operations from the buffer -// -// Method also get all rules after flush and refreshes handle values in the rulesets -func (m *AclManager) Flush() error { - if err := m.flushWithBackoff(); err != nil { - return err - } - - if err := m.refreshRuleHandles(m.chainInputRules, false); err != nil { - log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err) - } - if err := m.refreshRuleHandles(m.chainPrerouting, true); err != nil { - log.Errorf("failed to refresh rule handles prerouting chain: %v", err) - } - - return nil -} - -func (m *AclManager) addIOFiltering( - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipset *nftables.Set, -) (*Rule, error) { - ruleId := generatePeerRuleId(ip, proto, sPort, dPort, action, ipset) - if r, ok := m.rules[ruleId]; ok { - return &Rule{ - nftRule: r.nftRule, - mangleRule: r.mangleRule, - nftSet: r.nftSet, - ruleID: r.ruleID, - ip: ip, - }, nil - } - - var expressions []expr.Any - - if proto != firewall.ProtocolALL { - expressions = append(expressions, &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: m.af.protoOffset, - Len: uint32(1), - }) - - protoData, err := m.af.protoNum(proto) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %v", err) - } - - expressions = append(expressions, &expr.Cmp{ - Register: 1, - Op: expr.CmpOpEq, - Data: []byte{protoData}, - }) - } - - rawIP := ipToBytes(ip, m.af) - // check if rawIP contains zeroed IPv4 0.0.0.0 value - // in that case not add IP match expression into the rule definition - if slices.ContainsFunc(rawIP, func(v byte) bool { return v != 0 }) { - expressions = append(expressions, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: m.af.srcAddrOffset, - Len: m.af.addrLen, - }, - ) - // add individual IP for match if no ipset defined - if ipset == nil { - expressions = append(expressions, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: rawIP, - }, - ) - } else { - expressions = append(expressions, - &expr.Lookup{ - SourceRegister: 1, - SetName: ipset.Name, - SetID: ipset.ID, - }, - ) - } - } - - expressions = append(expressions, applyPort(sPort, true)...) - expressions = append(expressions, applyPort(dPort, false)...) - - mainExpressions := slices.Clone(expressions) - - switch action { - case firewall.ActionAccept: - mainExpressions = append(mainExpressions, &expr.Verdict{Kind: expr.VerdictAccept}) - case firewall.ActionDrop: - mainExpressions = append(mainExpressions, &expr.Verdict{Kind: expr.VerdictDrop}) - } - - userData := []byte(ruleId) - - chain := m.chainInputRules - rule := &nftables.Rule{ - Table: m.workTable, - Chain: chain, - Exprs: mainExpressions, - UserData: userData, - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - var nftRule *nftables.Rule - if action == firewall.ActionDrop { - nftRule = m.rConn.InsertRule(rule) - } else { - nftRule = m.rConn.AddRule(rule) - } - - if err := m.rConn.Flush(); err != nil { - return nil, fmt.Errorf("flush input rule %s: %v", ruleId, err) - } - - ruleStruct := &Rule{ - nftRule: nftRule, - // best effort mangle rule - mangleRule: m.createPreroutingRule(expressions, userData), - nftSet: ipset, - ruleID: ruleId, - ip: ip, - } - m.rules[ruleId] = ruleStruct - if ipset != nil { - m.ipsetStore.AddReferenceToIpset(ipset.Name) - } - - return ruleStruct, nil -} - -func (m *AclManager) createPreroutingRule(expressions []expr.Any, userData []byte) *nftables.Rule { - if m.chainPrerouting == nil { - log.Warn("prerouting chain is not created") - return nil - } - - preroutingExprs := slices.Clone(expressions) - - // interface - preroutingExprs = append([]expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyIIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - }, preroutingExprs...) - - // local destination and mark - preroutingExprs = append(preroutingExprs, - &expr.Fib{ - Register: 1, - ResultADDRTYPE: true, - FlagDADDR: true, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL), - }, - - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), - }, - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - SourceRegister: true, - }, - ) - - nfRule := m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainPrerouting, - Exprs: preroutingExprs, - UserData: userData, - }) - - if err := m.rConn.Flush(); err != nil { - log.Errorf("failed to flush mangle rule %s: %v", string(userData), err) - return nil - } - - return nfRule -} - -func (m *AclManager) createDefaultChains() (err error) { - // chainNameInputRules - chain := m.createChain(chainNameInputRules) - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to create chain (%s): %s", chain.Name, err) - return fmt.Errorf(flushError, err) - } - m.chainInputRules = chain - - // netbird-acl-input-filter - // type filter hook input priority filter; policy accept; - chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput) - m.addJumpRule(chain, m.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules - m.addDropExpressions(chain, expr.MetaKeyIIFNAME) - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to create chain (%s): %s", chain.Name, err) - return err - } - - // netbird-acl-forward-filter - chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward) - m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd - m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME) - - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err) - return fmt.Errorf(flushError, err) - } - - if err := m.allowRedirectedTraffic(chainFwFilter); err != nil { - log.Errorf("failed to allow redirected traffic: %s", err) - } - - return nil -} - -// Makes redirected traffic originally destined for the host itself (now subject to the forward filter) -// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the -// netbird peer IP. -func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error { - // Chain is created by route manager - // TODO: move creation to a common place - m.chainPrerouting = &nftables.Chain{ - Name: chainNameManglePrerouting, - Table: m.workTable, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityMangle, - } - - m.addFwmarkToForward(chainFwFilter) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - return nil -} - -func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) { - m.rConn.InsertRule(&nftables.Rule{ - Table: m.workTable, - Chain: chainFwFilter, - Exprs: []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), - }, - &expr.Verdict{ - Kind: expr.VerdictAccept, - }, - }, - }) -} - -func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) { - expressions := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - &expr.Verdict{ - Kind: expr.VerdictJump, - Chain: m.routingFwChainName, - }, - } - - _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: chainFwFilter, - Exprs: expressions, - }) -} - -func (m *AclManager) createChain(name string) *nftables.Chain { - chain := &nftables.Chain{ - Name: name, - Table: m.workTable, - } - - chain = m.rConn.AddChain(chain) - - insertReturnTrafficRule(m.rConn, m.workTable, chain) - - return chain -} - -func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain { - polAccept := nftables.ChainPolicyAccept - chain := &nftables.Chain{ - Name: name, - Table: m.workTable, - Hooknum: hookNum, - Priority: nftables.ChainPriorityFilter, - Type: nftables.ChainTypeFilter, - Policy: &polAccept, - } - - return m.rConn.AddChain(chain) -} - -func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any { - expressions := []expr.Any{ - &expr.Meta{Key: ifaceKey, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - &expr.Verdict{Kind: expr.VerdictDrop}, - } - _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: chain, - Exprs: expressions, - }) - return nil -} - -func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) { - expressions := []expr.Any{ - &expr.Meta{Key: ifaceKey, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - &expr.Verdict{ - Kind: expr.VerdictJump, - Chain: to, - }, - } - - _ = m.rConn.AddRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: expressions, - }) -} - -func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, error) { - ipset, err := m.rConn.GetSetByName(m.workTable, ipsetName) - rawIP := ipToBytes(ip, m.af) - if err != nil { - if ipset, err = m.createSet(m.workTable, ipsetName); err != nil { - return nil, fmt.Errorf("get set name: %v", err) - } - - m.ipsetStore.newIpset(ipset.Name) - } - - if m.ipsetStore.IsIpInSet(ipset.Name, ip) { - return ipset, nil - } - - if err := m.sConn.SetAddElements(ipset, []nftables.SetElement{{Key: rawIP}}); err != nil { - return nil, fmt.Errorf("add set element for the first time: %v", err) - } - - m.ipsetStore.AddIpToSet(ipset.Name, ip) - - if err := m.sConn.Flush(); err != nil { - return nil, fmt.Errorf("flush add elements: %v", err) - } - - return ipset, nil -} - -// createSet in given table by name -func (m *AclManager) createSet(table *nftables.Table, name string) (*nftables.Set, error) { - ipset := &nftables.Set{ - Name: name, - Table: table, - Dynamic: true, - KeyType: m.af.setKeyType, - } - - if err := m.rConn.AddSet(ipset, nil); err != nil { - return nil, fmt.Errorf("create set: %v", err) - } - - if err := m.rConn.Flush(); err != nil { - return nil, fmt.Errorf("flush created set: %v", err) - } - - return ipset, nil -} - -func (m *AclManager) flushWithBackoff() (err error) { - backoff := 4 - backoffTime := 1000 * time.Millisecond - for i := 0; ; i++ { - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to flush nftables: %v", err) - if !strings.Contains(err.Error(), "busy") { - return - } - log.Error("failed to flush nftables, retrying...") - if i == backoff-1 { - return err - } - time.Sleep(backoffTime) - backoffTime *= 2 - continue - } - break - } - return -} - -func (m *AclManager) refreshRuleHandles(chain *nftables.Chain, mangle bool) error { - if m.workTable == nil || chain == nil { - return nil - } - - list, err := m.rConn.GetRules(m.workTable, chain) - if err != nil { - return err - } - - for _, rule := range list { - if len(rule.UserData) == 0 { - continue - } - split := bytes.Split(rule.UserData, []byte(" ")) - r, ok := m.rules[string(split[0])] - if ok { - if mangle { - *r.mangleRule = *rule - } else { - *r.nftRule = *rule - } - } - } - - return nil -} - -func generatePeerRuleId(ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action, ipset *nftables.Set) string { - rulesetID := ":" + string(proto) + ":" - if sPort != nil { - rulesetID += sPort.String() - } - rulesetID += ":" - if dPort != nil { - rulesetID += dPort.String() - } - rulesetID += ":" - rulesetID += strconv.Itoa(int(action)) - if ipset == nil { - return "ip:" + ip.String() + rulesetID - } - return "set:" + ipset.Name + rulesetID -} - -func ifname(n string) []byte { - b := make([]byte, 16) - copy(b, n+"\x00") - return b -} - - -// ipToBytes converts net.IP to the correct byte length for the address family. -func ipToBytes(ip net.IP, af addrFamily) []byte { - if af.addrLen == 4 { - return ip.To4() - } - return ip.To16() -} - diff --git a/client/firewall/nftables/chains_linux.go b/client/firewall/nftables/chains_linux.go new file mode 100644 index 000000000..71f0c60f2 --- /dev/null +++ b/client/firewall/nftables/chains_linux.go @@ -0,0 +1,880 @@ +//go:build !android + +package nftables + +import ( + "bytes" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/coreos/go-iptables/iptables" + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/firewall/firewalld" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) createContainers() error { + r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameRoutingFw, + Table: r.workTable, + }) + + prio := *nftables.ChainPriorityNATSource - 1 + r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameRoutingNat, + Table: r.workTable, + Hooknum: nftables.ChainHookPostrouting, + Priority: &prio, + Type: nftables.ChainTypeNAT, + }) + + r.chains[chainNameRoutingRdr] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameRoutingRdr, + Table: r.workTable, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + Type: nftables.ChainTypeNAT, + }) + + r.chains[chainNameManglePostrouting] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameManglePostrouting, + Table: r.workTable, + Hooknum: nftables.ChainHookPostrouting, + Priority: nftables.ChainPriorityMangle, + Type: nftables.ChainTypeFilter, + }) + + r.chains[chainNameManglePrerouting] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameManglePrerouting, + Table: r.workTable, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityMangle, + Type: nftables.ChainTypeFilter, + }) + + r.chains[chainNameMangleForward] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameMangleForward, + Table: r.workTable, + Hooknum: nftables.ChainHookForward, + Priority: nftables.ChainPriorityMangle, + Type: nftables.ChainTypeFilter, + }) + + insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw]) + + r.addPostroutingRules() + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("initialize tables: %v", err) + } + + if err := r.addMSSClampingRules(); err != nil { + log.Errorf("failed to add MSS clamping rules: %s", err) + } + + // Kernel routing opens both INPUT and FORWARD. + if err := r.openInterface(true); err != nil { + log.Errorf("failed to open interface in foreign chains: %s", err) + } + + if err := firewalld.TrustInterface(r.wgIface.Name()); err != nil { + log.Warnf("failed to trust interface in firewalld: %v", err) + } + + if err := r.refreshRulesMap(); err != nil { + log.Errorf("failed to refresh rules: %s", err) + } + + return nil +} + +// setupDataPlaneMark configures the fwmark for the data plane +func (r *family) setupDataPlaneMark() error { + if r.chains[chainNameManglePrerouting] == nil || r.chains[chainNameManglePostrouting] == nil { + return errors.New("no mangle chains found") + } + + ctNew := getCtNewExprs() + preExprs := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + } + preExprs = append(preExprs, ctNew...) + preExprs = append(preExprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkIn), + }, + &expr.Ct{ + Key: expr.CtKeyMARK, + Register: 1, + SourceRegister: true, + }, + ) + + preNftRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameManglePrerouting], + Exprs: preExprs, + } + r.conn.AddRule(preNftRule) + + postExprs := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + } + postExprs = append(postExprs, ctNew...) + postExprs = append(postExprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkOut), + }, + &expr.Ct{ + Key: expr.CtKeyMARK, + Register: 1, + SourceRegister: true, + }, + ) + + postNftRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameManglePostrouting], + Exprs: postExprs, + } + r.conn.AddRule(postNftRule) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush: %w", err) + } + + return nil +} + +// openInterface adds passthrough accept rules for the NetBird interface to the +// kernel's filter table and external chains so they don't drop our traffic. +// includeForward also opens the FORWARD chains (kernel routing); when false only +// INPUT is opened, which is all the userspace router needs since it never +// forwards in the kernel. +func (r *family) openInterface(includeForward bool) error { + var merr *multierror.Error + + if err := r.acceptFilterTableRules(includeForward); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.acceptExternalChainsRules(includeForward); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add accept rules to external chains: %w", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) acceptFilterTableRules(includeForward bool) error { + if r.filterTable == nil { + return nil + } + + fw := "iptables" + + defer func() { + log.Debugf("Used %s to add accept input/forward rules", fw) + }() + + // Try iptables first and fallback to nftables if iptables is not available. + // Use the correct protocol (iptables vs ip6tables) for the address family. + ipt, err := iptables.NewWithProtocol(r.iptablesProto()) + if err != nil { + log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err) + + fw = "nftables" + return r.acceptFilterRulesNftables(r.filterTable, includeForward) + } + + if err := r.acceptFilterRulesIptables(ipt, includeForward); err != nil { + log.Warnf("iptables failed (table may be incompatible), falling back to nftables: %v", err) + fw = "nftables" + return r.acceptFilterRulesNftables(r.filterTable, includeForward) + } + return nil +} + +func (r *family) acceptFilterRulesIptables(ipt *iptables.IPTables, includeForward bool) error { + var merr *multierror.Error + + if includeForward { + for _, rule := range r.getAcceptForwardRules() { + if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add iptables forward rule: %v", err)) + } else { + log.Debugf("added iptables forward rule: %v", rule) + } + } + } + + inputRule := r.getAcceptInputRule() + if err := ipt.Insert("filter", chainNameInput, 1, inputRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add iptables input rule: %v", err)) + } else { + log.Debugf("added iptables input rule: %v", inputRule) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) getAcceptForwardRules() [][]string { + intf := r.wgIface.Name() + return [][]string{ + {"-i", intf, "-j", "ACCEPT"}, + {"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, + } +} + +func (r *family) getAcceptInputRule() []string { + return []string{"-i", r.wgIface.Name(), "-j", "ACCEPT"} +} + +// acceptFilterRulesNftables adds accept rules to the ip filter table using nftables. +// This is used when iptables is not available. +func (r *family) acceptFilterRulesNftables(table *nftables.Table, includeForward bool) error { + intf := ifname(r.wgIface.Name()) + + if includeForward { + forwardChain := &nftables.Chain{ + Name: chainNameForward, + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookForward, + Priority: nftables.ChainPriorityFilter, + } + r.insertForwardAcceptRules(forwardChain, intf) + } + + inputChain := &nftables.Chain{ + Name: chainNameInput, + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookInput, + Priority: nftables.ChainPriorityFilter, + } + r.insertInputAcceptRule(inputChain, intf) + + return r.conn.Flush() +} + +// acceptExternalChainsRules adds accept rules to external chains (non-netbird, non-iptables tables). +// It dynamically finds chains at call time to handle chains that may have been created after startup. +func (r *family) acceptExternalChainsRules(includeForward bool) error { + chains := r.findExternalChains() + if len(chains) == 0 { + return nil + } + + intf := ifname(r.wgIface.Name()) + for _, chain := range chains { + r.applyExternalChainAccept(chain, intf, includeForward) + } + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush external chain rules: %w", err) + } + return nil +} + +func (r *family) applyExternalChainAccept(chain *nftables.Chain, intf []byte, includeForward bool) { + if chain.Hooknum == nil { + log.Debugf("skipping external chain %s/%s: hooknum is nil", chain.Table.Name, chain.Name) + return + } + + log.Debugf("adding accept rules to external %s chain: %s %s/%s", + hookName(chain.Hooknum), familyName(chain.Table.Family), chain.Table.Name, chain.Name) + + switch *chain.Hooknum { + case *nftables.ChainHookForward: + if includeForward { + r.insertForwardAcceptRules(chain, intf) + } + case *nftables.ChainHookInput: + r.insertInputAcceptRule(chain, intf) + } +} + +func (r *family) insertForwardAcceptRules(chain *nftables.Chain, intf []byte) { + existing, err := r.existingNetbirdRulesInChain(chain) + if err != nil { + log.Warnf("skip forward accept rules in %s/%s: %v", chain.Table.Name, chain.Name, err) + return + } + r.insertForwardIifRule(chain, intf, existing) + r.insertForwardOifEstablishedRule(chain, intf, existing) +} + +func (r *family) insertForwardIifRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { + if existing[userDataAcceptForwardRuleIif] { + return + } + r.conn.InsertRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, + &expr.Counter{}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + UserData: []byte(userDataAcceptForwardRuleIif), + }) +} + +func (r *family) insertForwardOifEstablishedRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { + if existing[userDataAcceptForwardRuleOif] { + return + } + exprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, + } + r.conn.InsertRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: append(exprs, getEstablishedExprs(2)...), + UserData: []byte(userDataAcceptForwardRuleOif), + }) +} + +func (r *family) insertInputAcceptRule(chain *nftables.Chain, intf []byte) { + existing, err := r.existingNetbirdRulesInChain(chain) + if err != nil { + log.Warnf("skip input accept rule in %s/%s: %v", chain.Table.Name, chain.Name, err) + return + } + if existing[userDataAcceptInputRule] { + return + } + r.conn.InsertRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, + &expr.Counter{}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + UserData: []byte(userDataAcceptInputRule), + }) +} + +// existingNetbirdRulesInChain returns the set of netbird-owned UserData tags present in a chain; callers must bail on error since InsertRule is additive. +func (r *family) existingNetbirdRulesInChain(chain *nftables.Chain) (map[string]bool, error) { + rules, err := r.conn.GetRules(chain.Table, chain) + if err != nil { + return nil, fmt.Errorf("list rules: %w", err) + } + present := map[string]bool{} + for _, rule := range rules { + if !isNetbirdAcceptRuleTag(rule.UserData) { + continue + } + present[string(rule.UserData)] = true + } + return present, nil +} + +func isNetbirdAcceptRuleTag(userData []byte) bool { + switch string(userData) { + case userDataAcceptForwardRuleIif, + userDataAcceptForwardRuleOif, + userDataAcceptInputRule: + return true + } + return false +} + +func (r *family) removeAcceptFilterRules() error { + var merr *multierror.Error + + if err := r.removeFilterTableRules(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.removeExternalChainsRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove external chain rules: %w", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) removeFilterTableRules() error { + if r.filterTable == nil { + return nil + } + + ipt, err := iptables.NewWithProtocol(r.iptablesProto()) + if err != nil { + log.Debugf("iptables not available, using nftables to remove filter rules: %v", err) + return r.removeAcceptRulesFromTable(r.filterTable) + } + + if err := r.removeAcceptFilterRulesIptables(ipt); err != nil { + log.Debugf("iptables removal failed (table may be incompatible), falling back to nftables: %v", err) + return r.removeAcceptRulesFromTable(r.filterTable) + } + return nil +} + +func (r *family) removeAcceptRulesFromTable(table *nftables.Table) error { + chains, err := r.conn.ListChainsOfTableFamily(table.Family) + if err != nil { + return fmt.Errorf("list chains: %v", err) + } + + for _, chain := range chains { + if chain.Table.Name != table.Name { + continue + } + + if chain.Name != chainNameForward && chain.Name != chainNameInput { + continue + } + + if err := r.removeAcceptRulesFromChain(table, chain); err != nil { + return err + } + } + + return r.conn.Flush() +} + +func (r *family) removeAcceptRulesFromChain(table *nftables.Table, chain *nftables.Chain) error { + rules, err := r.conn.GetRules(table, chain) + if err != nil { + return fmt.Errorf("get rules from %s/%s: %v", table.Name, chain.Name, err) + } + + for _, rule := range rules { + if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) || + bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) || + bytes.Equal(rule.UserData, []byte(userDataAcceptInputRule)) { + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("delete rule from %s/%s: %v", table.Name, chain.Name, err) + } + } + } + return nil +} + +// removeExternalChainsRules removes our accept rules from all external chains. +// This is deterministic - it scans for chains at removal time rather than relying on saved state, +// ensuring cleanup works even after a crash or if chains changed. +func (r *family) removeExternalChainsRules() error { + chains := r.findExternalChains() + if len(chains) == 0 { + return nil + } + + var merr *multierror.Error + for _, chain := range chains { + if err := r.removeAcceptRulesFromChain(chain.Table, chain); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove rules from external chain %s/%s: %w", chain.Table.Name, chain.Name, err)) + continue + } + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("flush external chain %s/%s: %w", chain.Table.Name, chain.Name, err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +// findExternalChains scans for chains from non-netbird tables that have FORWARD or INPUT hooks. +// This is used both at startup (to know where to add rules) and at cleanup (to ensure deterministic removal). +func (r *family) findExternalChains() []*nftables.Chain { + var chains []*nftables.Chain + + families := []nftables.TableFamily{r.af.tableFamily, nftables.TableFamilyINet} + + for _, family := range families { + allChains, err := r.conn.ListChainsOfTableFamily(family) + if err != nil { + log.Debugf("list chains for family %d: %v", family, err) + continue + } + + for _, chain := range allChains { + if r.isExternalChain(chain) { + chains = append(chains, chain) + } + } + } + + return chains +} + +func (r *family) isExternalChain(chain *nftables.Chain) bool { + if r.workTable != nil && chain.Table.Name == r.workTable.Name { + return false + } + + // Skip firewalld-owned chains. Firewalld creates its chains with the + // NFT_CHAIN_OWNER flag, so inserting rules into them returns EPERM. + // We delegate acceptance to firewalld by trusting the interface instead. + if chain.Table.Name == firewalldTableName { + return false + } + + // Skip iptables/ip6tables-managed tables (adding nft-native rules breaks iptables-save compat) + if (chain.Table.Family == nftables.TableFamilyIPv4 || chain.Table.Family == nftables.TableFamilyIPv6) && isIptablesTable(chain.Table.Name) { + return false + } + + if chain.Type != nftables.ChainTypeFilter { + return false + } + + if chain.Hooknum == nil { + return false + } + + return *chain.Hooknum == *nftables.ChainHookForward || *chain.Hooknum == *nftables.ChainHookInput +} + +func isIptablesTable(name string) bool { + switch name { + case tableNameFilter, tableNat, tableMangle, tableRaw, tableSecurity: + return true + } + return false +} + +func (r *family) removeAcceptFilterRulesIptables(ipt *iptables.IPTables) error { + var merr *multierror.Error + + for _, rule := range r.getAcceptForwardRules() { + if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove iptables forward rule: %v", err)) + } + } + + inputRule := r.getAcceptInputRule() + if err := ipt.DeleteIfExists("filter", chainNameInput, inputRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove iptables input rule: %v", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +// Flush rule/chain/set operations from the buffer +// +// Method also get all rules after flush and refreshes handle values in the rulesets +func (r *family) Flush() error { + if err := r.flushWithBackoff(); err != nil { + return err + } + + if err := r.refreshRuleHandles(r.chainInputRules, false); err != nil { + log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err) + } + if err := r.refreshRuleHandles(r.chainPrerouting, true); err != nil { + log.Errorf("failed to refresh rule handles prerouting chain: %v", err) + } + + return nil +} + +// queuePreroutingRule builds the prerouting mangle rule that marks +// redirected traffic and queues it on the connection without flushing, +// so the caller can commit it in the same transaction as the rule it +// pairs with. Returns nil when the prerouting chain is absent, in which +// case nothing is queued. +func (r *family) queuePreroutingRule(expressions []expr.Any, userData []byte) *nftables.Rule { + if r.chainPrerouting == nil { + log.Warn("prerouting chain is not created") + return nil + } + + preroutingExprs := slices.Clone(expressions) + + // interface + preroutingExprs = append([]expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + }, preroutingExprs...) + + // local destination and mark + preroutingExprs = append(preroutingExprs, + &expr.Fib{ + Register: 1, + ResultADDRTYPE: true, + FlagDADDR: true, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL), + }, + + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), + }, + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + SourceRegister: true, + }, + ) + + return r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chainPrerouting, + Exprs: preroutingExprs, + UserData: userData, + }) +} + +func (r *family) createDefaultChains() (err error) { + // chainNameInputRules + chain := r.createChain(chainNameInputRules) + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to create chain (%s): %s", chain.Name, err) + return fmt.Errorf(flushError, err) + } + r.chainInputRules = chain + + // netbird-acl-input-filter + // type filter hook input priority filter; policy accept; + chain = r.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput) + r.addJumpRule(chain, r.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules + r.addDropExpressions(chain, expr.MetaKeyIIFNAME) + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to create chain (%s): %s", chain.Name, err) + return err + } + + // netbird-acl-forward-filter + chainFwFilter := r.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward) + r.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd + r.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME) + + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err) + return fmt.Errorf(flushError, err) + } + + if err := r.allowRedirectedTraffic(chainFwFilter); err != nil { + log.Errorf("failed to allow redirected traffic: %s", err) + } + + return nil +} + +// Makes redirected traffic originally destined for the host itself (now subject to the forward filter) +// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the +// netbird peer IP. +func (r *family) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error { + r.chainPrerouting = r.chains[chainNameManglePrerouting] + + r.addFwmarkToForward(chainFwFilter) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf(flushError, err) + } + + return nil +} + +func (r *family) addFwmarkToForward(chainFwFilter *nftables.Chain) { + r.conn.InsertRule(&nftables.Rule{ + Table: r.workTable, + Chain: chainFwFilter, + Exprs: []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), + }, + &expr.Verdict{ + Kind: expr.VerdictAccept, + }, + }, + }) +} + +func (r *family) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) { + expressions := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Verdict{ + Kind: expr.VerdictJump, + Chain: r.routingFwChainName, + }, + } + + _ = r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: chainFwFilter, + Exprs: expressions, + }) +} + +func (r *family) createChain(name string) *nftables.Chain { + chain := &nftables.Chain{ + Name: name, + Table: r.workTable, + } + + chain = r.conn.AddChain(chain) + + insertReturnTrafficRule(r.conn, r.workTable, chain) + + return chain +} + +func (r *family) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain { + polAccept := nftables.ChainPolicyAccept + chain := &nftables.Chain{ + Name: name, + Table: r.workTable, + Hooknum: hookNum, + Priority: nftables.ChainPriorityFilter, + Type: nftables.ChainTypeFilter, + Policy: &polAccept, + } + + return r.conn.AddChain(chain) +} + +func (r *family) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any { + expressions := []expr.Any{ + &expr.Meta{Key: ifaceKey, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Verdict{Kind: expr.VerdictDrop}, + } + _ = r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: chain, + Exprs: expressions, + }) + return nil +} + +func (r *family) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) { + expressions := []expr.Any{ + &expr.Meta{Key: ifaceKey, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Verdict{ + Kind: expr.VerdictJump, + Chain: to, + }, + } + + _ = r.conn.AddRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: expressions, + }) +} + +func (r *family) flushWithBackoff() (err error) { + backoff := 4 + backoffTime := 1000 * time.Millisecond + for i := 0; ; i++ { + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to flush nftables: %v", err) + if !strings.Contains(err.Error(), "busy") { + return + } + log.Error("failed to flush nftables, retrying...") + if i == backoff-1 { + return err + } + time.Sleep(backoffTime) + backoffTime *= 2 + continue + } + break + } + return +} + +func (r *family) refreshRuleHandles(chain *nftables.Chain, mangle bool) error { + if r.workTable == nil || chain == nil { + return nil + } + + list, err := r.conn.GetRules(r.workTable, chain) + if err != nil { + return err + } + + for _, rule := range list { + if len(rule.UserData) == 0 { + continue + } + pr, ok := r.filters[firewall.RuleID(rule.UserData)] + if !ok { + continue + } + if mangle { + if pr.mangleRule != nil { + *pr.mangleRule = *rule + } + } else { + *pr.nftRule = *rule + } + } + + return nil +} diff --git a/client/firewall/nftables/dnat_linux.go b/client/firewall/nftables/dnat_linux.go new file mode 100644 index 000000000..8eae694a2 --- /dev/null +++ b/client/firewall/nftables/dnat_linux.go @@ -0,0 +1,573 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "net/netip" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/google/nftables/xt" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { + ruleID := rule.ID() + if _, exists := r.rules[ruleID+dnatSuffix]; exists { + return rule, nil + } + + protoNum, err := r.af.protoNum(rule.Protocol) + if err != nil { + return nil, fmt.Errorf("convert protocol to number: %w", err) + } + + // Request forwarding before queueing rules: addDnatRedirect/addDnatMasq + // buffer netlink messages on r.conn that the next caller's Flush would + // commit if we returned without flushing them ourselves. + if err := r.ipFwdState.RequestForwarding(r.isV6()); err != nil { + return nil, fmt.Errorf("enable forwarding: %w", err) + } + + if err := r.addDnatRedirect(rule, protoNum, ruleID); err != nil { + r.releaseForwarding() + return nil, err + } + + if err := r.addDnatMasq(rule, protoNum, ruleID); err != nil { + r.releaseForwarding() + delete(r.rules, ruleID+dnatSuffix) + return nil, err + } + + // Unlike iptables, there's no point in adding "out" rules in the forward chain here as our policy is ACCEPT. + // To overcome DROP policies in other chains, we'd have to add rules to the chains there. + // We also cannot just add "oif accept" there and filter in our own table as we don't know what is supposed to be allowed. + // TODO: find chains with drop policies and add rules there + + if err := r.conn.Flush(); err != nil { + r.releaseForwarding() + delete(r.rules, ruleID+dnatSuffix) + delete(r.rules, ruleID+snatSuffix) + return nil, fmt.Errorf("flush rules: %w", err) + } + + return &rule, nil +} + +func (r *family) addDnatRedirect(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error { + dnatExprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{protoNum}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + } + portExprs, err := r.applyPort(&rule.DestinationPort, false) + if err != nil { + return fmt.Errorf("apply destination port: %w", err) + } + dnatExprs = append(dnatExprs, portExprs...) + + // shifted translated port is not supported in nftables, so we hand this over to xtables + if rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2 { + if rule.TranslatedPort.Values[0] != rule.DestinationPort.Values[0] || + rule.TranslatedPort.Values[1] != rule.DestinationPort.Values[1] { + return r.addXTablesRedirect(dnatExprs, ruleID, rule) + } + } + + additionalExprs, regProtoMin, regProtoMax, err := r.handleTranslatedPort(rule) + if err != nil { + return err + } + dnatExprs = append(dnatExprs, additionalExprs...) + + dnatExprs = append(dnatExprs, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: uint32(r.af.tableFamily), + RegAddrMin: 1, + RegProtoMin: regProtoMin, + RegProtoMax: regProtoMax, + }, + ) + + dnatRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingRdr], + Exprs: dnatExprs, + UserData: []byte(ruleID + dnatSuffix), + } + r.conn.AddRule(dnatRule) + r.rules[ruleID+dnatSuffix] = dnatRule + + return nil +} + +func (r *family) handleTranslatedPort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { + switch { + case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: + return r.handlePortRange(rule) + case len(rule.TranslatedPort.Values) == 0: + return r.handleAddressOnly(rule) + case len(rule.TranslatedPort.Values) == 1: + return r.handleSinglePort(rule) + default: + return nil, 0, 0, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) + } +} + +func (r *family) handlePortRange(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { + exprs := []expr.Any{ + &expr.Immediate{ + Register: 1, + Data: rule.TranslatedAddress.AsSlice(), + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), + }, + &expr.Immediate{ + Register: 3, + Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[1]), + }, + } + return exprs, 2, 3, nil +} + +func (r *family) handleAddressOnly(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { + exprs := []expr.Any{ + &expr.Immediate{ + Register: 1, + Data: rule.TranslatedAddress.AsSlice(), + }, + } + return exprs, 0, 0, nil +} + +func (r *family) handleSinglePort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { + exprs := []expr.Any{ + &expr.Immediate{ + Register: 1, + Data: rule.TranslatedAddress.AsSlice(), + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), + }, + } + return exprs, 2, 0, nil +} + +func (r *family) addXTablesRedirect(dnatExprs []expr.Any, ruleID firewall.RuleID, rule firewall.ForwardRule) error { + dnatExprs = append(dnatExprs, + &expr.Counter{}, + &expr.Target{ + Name: "DNAT", + Rev: 2, + Info: &xt.NatRange2{ + NatRange: xt.NatRange{ + Flags: uint(xt.NatRangeMapIPs | xt.NatRangeProtoSpecified | xt.NatRangeProtoOffset), + MinIP: rule.TranslatedAddress.AsSlice(), + MaxIP: rule.TranslatedAddress.AsSlice(), + MinPort: rule.TranslatedPort.Values[0], + MaxPort: rule.TranslatedPort.Values[1], + }, + BasePort: rule.DestinationPort.Values[0], + }, + }, + ) + + natTable := &nftables.Table{ + Name: tableNat, + Family: r.af.tableFamily, + } + dnatRule := &nftables.Rule{ + Table: natTable, + Chain: &nftables.Chain{ + Name: chainNameNatPrerouting, + Table: natTable, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + }, + Exprs: dnatExprs, + UserData: []byte(ruleID + dnatSuffix), + } + r.conn.AddRule(dnatRule) + r.rules[ruleID+dnatSuffix] = dnatRule + + return nil +} + +func (r *family) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error { + portExprs, err := r.applyPort(&rule.TranslatedPort, false) + if err != nil { + return fmt.Errorf("apply translated port: %w", err) + } + + masqExprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{protoNum}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: r.af.dstAddrOffset, + Len: r.af.addrLen, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: rule.TranslatedAddress.AsSlice(), + }, + } + + masqExprs = append(masqExprs, portExprs...) + masqExprs = append(masqExprs, &expr.Masq{}) + + masqRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingNat], + Exprs: masqExprs, + UserData: []byte(ruleID + snatSuffix), + } + r.conn.AddRule(masqRule) + r.rules[ruleID+snatSuffix] = masqRule + + return nil +} + +func (r *family) DeleteDNATRule(rule firewall.Rule) error { + ruleID := rule.ID() + + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + var merr *multierror.Error + var needsFlush bool + var found bool + + if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists { + found = true + if dnatRule.Handle == 0 { + log.Warnf("dnat rule %s has no handle, removing stale entry", ruleID+dnatSuffix) + delete(r.rules, ruleID+dnatSuffix) + } else if err := r.conn.DelRule(dnatRule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete dnat rule: %w", err)) + } else { + needsFlush = true + } + } + + if masqRule, exists := r.rules[ruleID+snatSuffix]; exists { + found = true + if masqRule.Handle == 0 { + log.Warnf("snat rule %s has no handle, removing stale entry", ruleID+snatSuffix) + delete(r.rules, ruleID+snatSuffix) + } else if err := r.conn.DelRule(masqRule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete snat rule: %w", err)) + } else { + needsFlush = true + } + } + + if needsFlush { + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf(flushError, err)) + } + } + + if merr != nil { + return nberrors.FormatErrorOrNil(merr) + } + + delete(r.rules, ruleID+dnatSuffix) + delete(r.rules, ruleID+snatSuffix) + + // Release once, only if the rule was present and removed. + if found { + r.releaseForwarding() + } + + return nil +} + +// releaseForwarding drops one IP forwarding reference, logging any error. +func (r *family) releaseForwarding() { + if err := r.ipFwdState.ReleaseForwarding(r.isV6()); err != nil { + log.Errorf("release IP forwarding: %v", err) + } +} + +// isV6 reports whether this family handles the IPv6 table. +func (r *family) isV6() bool { + return r.af.tableFamily == nftables.TableFamilyIPv6 +} + +func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + protoNum, err := r.af.protoNum(protocol) + if err != nil { + return fmt.Errorf("convert protocol to number: %w", err) + } + + exprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 2}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 2, + Data: []byte{protoNum}, + }, + &expr.Payload{ + DestRegister: 3, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 3, + Data: binaryutil.BigEndian.PutUint16(originalPort), + }, + } + + bits := 32 + if localAddr.Is6() { + bits = 128 + } + exprs = append(exprs, prefixMatchExprs(r.af, netip.PrefixFrom(localAddr, bits), false)...) + + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: localAddr.AsSlice(), + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(translatedPort), + }, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: uint32(r.af.tableFamily), + RegAddrMin: 1, + RegProtoMin: 2, + RegProtoMax: 0, + }, + ) + + dnatRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingRdr], + Exprs: exprs, + UserData: []byte(ruleID), + } + r.conn.AddRule(dnatRule) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("add inbound DNAT rule: %w", err) + } + + r.rules[ruleID] = dnatRule + + return nil +} + +// RemoveInboundDNAT removes an inbound DNAT rule. +func (r *family) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + rule, exists := r.rules[ruleID] + if !exists { + return nil + } + + if rule.Handle == 0 { + log.Warnf("inbound DNAT rule %s has no handle, removing stale entry", ruleID) + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("delete inbound DNAT rule %s: %w", ruleID, err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush delete inbound DNAT rule: %w", err) + } + delete(r.rules, ruleID) + + return nil +} + +// ensureNATOutputChain lazily creates the OUTPUT NAT chain on first use. +func (r *family) ensureNATOutputChain() error { + if _, exists := r.chains[chainNameNATOutput]; exists { + return nil + } + + r.chains[chainNameNATOutput] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameNATOutput, + Table: r.workTable, + Hooknum: nftables.ChainHookOutput, + Priority: nftables.ChainPriorityNATDest, + Type: nftables.ChainTypeNAT, + }) + + if err := r.conn.Flush(); err != nil { + delete(r.chains, chainNameNATOutput) + return fmt.Errorf("create NAT output chain: %w", err) + } + return nil +} + +// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. +func (r *family) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + if err := r.ensureNATOutputChain(); err != nil { + return err + } + + protoNum, err := r.af.protoNum(protocol) + if err != nil { + return fmt.Errorf("convert protocol to number: %w", err) + } + + exprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{protoNum}, + }, + &expr.Payload{ + DestRegister: 2, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 2, + Data: binaryutil.BigEndian.PutUint16(originalPort), + }, + } + + bits := 32 + if localAddr.Is6() { + bits = 128 + } + exprs = append(exprs, prefixMatchExprs(r.af, netip.PrefixFrom(localAddr, bits), false)...) + + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: localAddr.AsSlice(), + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(translatedPort), + }, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: uint32(r.af.tableFamily), + RegAddrMin: 1, + RegProtoMin: 2, + }, + ) + + dnatRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameNATOutput], + Exprs: exprs, + UserData: []byte(ruleID), + } + r.conn.AddRule(dnatRule) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("add output DNAT rule: %w", err) + } + + r.rules[ruleID] = dnatRule + + return nil +} + +// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. +func (r *family) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + rule, exists := r.rules[ruleID] + if !exists { + return nil + } + + if rule.Handle == 0 { + log.Warnf("output DNAT rule %s has no handle, removing stale entry", ruleID) + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("delete output DNAT rule %s: %w", ruleID, err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush delete output DNAT rule: %w", err) + } + delete(r.rules, ruleID) + + return nil +} diff --git a/client/firewall/nftables/dnat_refcount_linux_test.go b/client/firewall/nftables/dnat_refcount_linux_test.go index 86079676f..cdc24e77f 100644 --- a/client/firewall/nftables/dnat_refcount_linux_test.go +++ b/client/firewall/nftables/dnat_refcount_linux_test.go @@ -82,7 +82,7 @@ func dnatV6(port uint16) fw.ForwardRule { // v4 refcount at zero. func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) { m := newNftRefcountManager(t, false) - state := m.router.ipFwdState + state := m.family4.ipFwdState r1, err := m.AddDNATRule(dnatV4(8081)) require.NoError(t, err, "add v4 dnat 1") @@ -111,9 +111,9 @@ func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) { // and decrements back to zero on Delete. func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) { m := newNftRefcountManager(t, true) - require.NotNil(t, m.router6, "v6 router") - require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state") - state := m.router.ipFwdState + require.NotNil(t, m.family6, "v6 family") + require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state") + state := m.family4.ipFwdState r1, err := m.AddDNATRule(dnatV6(9091)) require.NoError(t, err, "add v6 dnat 1") @@ -142,7 +142,7 @@ func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) { // ForwardRule) does not double-increment the refcount. func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) { m := newNftRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState rule := dnatV4(8083) r1, err := m.AddDNATRule(rule) @@ -165,7 +165,7 @@ func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) { // never added does not underflow the refcount. func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { m := newNftRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState // Construct a Rule reference for something never added. The router stores // rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing @@ -195,7 +195,7 @@ func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { // and a single DisableRouting drops both back to zero. func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) { m := newNftRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState require.NoError(t, m.EnableRouting(), "first enable") require.NoError(t, m.EnableRouting(), "second enable") @@ -214,7 +214,7 @@ func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) { // DisableRouting does not release references held by active DNAT rules. func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) { m := newNftRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState r1, err := m.AddDNATRule(dnatV6(9095)) require.NoError(t, err, "add v6 dnat") @@ -232,7 +232,7 @@ func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) { // twice does not underflow the refcount (the second delete is a no-op). func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { m := newNftRefcountManager(t, true) - state := m.router.ipFwdState + state := m.family4.ipFwdState r1, err := m.AddDNATRule(dnatV6(9093)) require.NoError(t, err) diff --git a/client/firewall/nftables/family_linux.go b/client/firewall/nftables/family_linux.go new file mode 100644 index 000000000..7a5df3ed7 --- /dev/null +++ b/client/firewall/nftables/family_linux.go @@ -0,0 +1,249 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "net/netip" + + "github.com/coreos/go-iptables/iptables" + "github.com/google/nftables" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/firewall/firewalld" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" + "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" +) + +const ( + tableNat = "nat" + tableMangle = "mangle" + tableRaw = "raw" + tableSecurity = "security" + + chainNameNatPrerouting = "PREROUTING" + chainNameRoutingFw = "netbird-rt-fwd" + chainNameRoutingNat = "netbird-rt-postrouting" + chainNameRoutingRdr = "netbird-rt-redirect" + chainNameNATOutput = "netbird-nat-output" + chainNameForward = "FORWARD" + chainNameMangleForward = "netbird-mangle-forward" + + // Peer ACL chain names. + chainNameInputRules = "netbird-acl-input-rules" + chainNameInputFilter = "netbird-acl-input-filter" + chainNameForwardFilter = "netbird-acl-forward-filter" + chainNameManglePrerouting = "netbird-mangle-prerouting" + chainNameManglePostrouting = "netbird-mangle-postrouting" + + flushError = "flush: %w" + + firewalldTableName = "firewalld" + + userDataAcceptForwardRuleIif = "frwacceptiif" + userDataAcceptForwardRuleOif = "frwacceptoif" + userDataAcceptInputRule = "inputaccept" + + dnatSuffix firewall.RuleID = "_dnat" + snatSuffix firewall.RuleID = "_snat" + + // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. + ipv4TCPHeaderSize = 40 + // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. + ipv6TCPHeaderSize = 60 + + // maxPrefixesSet 1638 prefixes start to fail, taking some margin + maxPrefixesSet = 1500 + refreshRulesMapError = "refresh rules map: %w" +) + +var ( + errFilterTableNotFound = fmt.Errorf("'filter' table not found") +) + +type setInput struct { + set firewall.Set + prefixes []netip.Prefix +} + +// family holds the per-address-family nftables state. One instance +// handles route ACLs, peer ACLs, NAT, DNAT, and MSS clamping for a +// single family; the top-level Manager owns one for v4 and another +// for v6. The name predates the peer-ACL absorption; it's effectively +// the per-family backend now. +type family struct { + conn *nftables.Conn + workTable *nftables.Table + filterTable *nftables.Table + chains map[string]*nftables.Chain + + // filters holds peer + route filter rules keyed by content hash. + // AddFilterRule writes here; DeleteFilterRule looks up by id. + filters map[firewall.RuleID]*Rule + + // rules holds NAT, DNAT, and external accept rules (auxiliary + // plumbing that isn't a filter rule). + rules map[firewall.RuleID]*nftables.Rule + + // Peer ACL chain handles. + chainInputRules *nftables.Chain + chainPrerouting *nftables.Chain + routingFwChainName string + + ipsetCounter *refcounter.Counter[string, setInput, *nftables.Set] + + af addrFamily + wgIface iFaceMapper + ipFwdState *ipfwdstate.IPForwardingState + legacyManagement bool + mtu uint16 +} + +func newFamily(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) *family { + r := &family{ + conn: &nftables.Conn{}, + workTable: workTable, + chains: make(map[string]*nftables.Chain), + filters: make(map[firewall.RuleID]*Rule), + rules: make(map[firewall.RuleID]*nftables.Rule), + routingFwChainName: chainNameRoutingFw, + af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4), + wgIface: wgIface, + ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), + mtu: mtu, + } + + r.ipsetCounter = refcounter.New( + r.createIpSet, + r.deleteIpSet, + ) + + var err error + r.filterTable, err = r.loadFilterTable() + if err != nil { + log.Debugf("ip filter table not found: %v", err) + } + + return r +} + +func (r *family) init(workTable *nftables.Table) error { + r.workTable = workTable + + if err := r.removeAcceptFilterRules(); err != nil { + log.Errorf("failed to clean up rules from filter table: %s", err) + } + + if err := r.createContainers(); err != nil { + return fmt.Errorf("create containers: %w", err) + } + + if err := r.setupDataPlaneMark(); err != nil { + log.Errorf("failed to set up data plane mark: %v", err) + } + + if err := r.createDefaultChains(); err != nil { + return fmt.Errorf("create default acl chains: %w", err) + } + + return nil +} + +// Reset cleans existing nftables filter table rules from the system +func (r *family) Reset() error { + // clear without deleting the ipsets, the nf table will be deleted by the caller + r.ipsetCounter.Clear() + + var merr *multierror.Error + + if err := r.removeAcceptFilterRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove accept filter rules: %w", err)) + } + + if err := firewalld.UntrustInterface(r.wgIface.Name()); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.removeNatPreroutingRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove filter prerouting rules: %w", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) loadFilterTable() (*nftables.Table, error) { + tables, err := r.conn.ListTablesOfFamily(r.af.tableFamily) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + + for _, table := range tables { + if table.Name == "filter" { + return table, nil + } + } + + return nil, errFilterTableNotFound +} + +func hookName(hook *nftables.ChainHook) string { + if hook == nil { + return "unknown" + } + switch *hook { + case *nftables.ChainHookForward: + return chainNameForward + case *nftables.ChainHookInput: + return chainNameInput + default: + return fmt.Sprintf("hook(%d)", *hook) + } +} + +func familyName(family nftables.TableFamily) string { + switch family { + case nftables.TableFamilyIPv4: + return "ip" + case nftables.TableFamilyIPv6: + return "ip6" + case nftables.TableFamilyINet: + return "inet" + default: + return fmt.Sprintf("family(%d)", family) + } +} + +func (r *family) iptablesProto() iptables.Protocol { + if r.af.tableFamily == nftables.TableFamilyIPv6 { + return iptables.ProtocolIPv6 + } + return iptables.ProtocolIPv4 +} + +func (r *family) refreshRulesMap() error { + var merr *multierror.Error + newRules := make(map[firewall.RuleID]*nftables.Rule) + for _, chain := range r.chains { + rules, err := r.conn.GetRules(chain.Table, chain) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("list rules for chain %s: %w", chain.Name, err)) + // preserve existing entries for this chain since we can't verify their state + for k, v := range r.rules { + if v.Chain != nil && v.Chain.Name == chain.Name { + newRules[k] = v + } + } + continue + } + for _, rule := range rules { + if len(rule.UserData) > 0 { + newRules[firewall.RuleID(rule.UserData)] = rule + } + } + } + r.rules = newRules + return nberrors.FormatErrorOrNil(merr) +} diff --git a/client/firewall/nftables/filter_linux.go b/client/firewall/nftables/filter_linux.go new file mode 100644 index 000000000..ebd238063 --- /dev/null +++ b/client/firewall/nftables/filter_linux.go @@ -0,0 +1,540 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "net" + "net/netip" + "slices" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbid "github.com/netbirdio/netbird/client/internal/acl/id" +) + +// AddFilterRule installs one nftables packet-filter rule. With +// destination empty the rule goes to the peer ACL input chain plus a +// paired prerouting mangle rule for the redirect mark. With +// destination set (prefix or named set) it goes to the route ACL +// forward chain. Multi-source rules collapse to one nftables rule +// backed by the shared refcounted hash:net set. +func (r *family) AddFilterRule( + id []byte, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort *firewall.Port, + dPort *firewall.Port, + action firewall.Action, +) (firewall.Rule, error) { + isRoute := !destination.IsZero() + + ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action) + if existing, ok := r.filters[ruleID]; ok { + return existing, nil + } + + srcExprs, err := r.applyNetwork(sourceNetwork(sources), sources, true) + if err != nil { + return nil, fmt.Errorf("apply source: %w", err) + } + + var exprs []expr.Any + if isRoute { + exprs, err = r.buildRouteFilterExprs(srcExprs, destination, proto, sPort, dPort) + } else { + exprs, err = r.buildPeerFilterExprs(srcExprs, proto, sPort, dPort) + } + if err != nil { + r.dropNetworkMatch(srcExprs) + return nil, err + } + + mainExprs := slices.Clone(exprs) + verdict := expr.VerdictAccept + if action == firewall.ActionDrop { + verdict = expr.VerdictDrop + } + mainExprs = append(mainExprs, &expr.Verdict{Kind: verdict}) + + chain := r.chainInputRules + if isRoute { + chain = r.chains[chainNameRoutingFw] + } + + userData := []byte(ruleID) + + // Build the paired prerouting mangle rule before flushing so both + // rules commit in one transaction. An anonymous port set binds to + // exactly one rule, so the mangle rule needs its own expression list + // with fresh sets, not a clone of the main rule's. Guard on the + // prerouting chain first: building the expressions queues the port + // set, so skipping the build when there is no chain to bind it to + // keeps an unbound set out of the connection batch. + var mangleRule *nftables.Rule + if !isRoute && r.chainPrerouting != nil { + mangleExprs, err := r.buildPeerFilterExprs(srcExprs, proto, sPort, dPort) + if err != nil { + r.dropNetworkMatch(exprs) + return nil, fmt.Errorf("build mangle rule: %w", err) + } + mangleRule = r.queuePreroutingRule(mangleExprs, userData) + } + + nftRule := &nftables.Rule{ + Table: r.workTable, + Chain: chain, + Exprs: mainExprs, + UserData: userData, + } + if action == firewall.ActionDrop { + nftRule = r.conn.InsertRule(nftRule) + } else { + nftRule = r.conn.AddRule(nftRule) + } + if err := r.conn.Flush(); err != nil { + r.dropNetworkMatch(exprs) + return nil, fmt.Errorf(flushError, err) + } + + rule := &Rule{ + nftRule: nftRule, + mangleRule: mangleRule, + sources: sources, + id: ruleID, + } + r.filters[ruleID] = rule + + log.Debugf("added filter rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", + sources, destination, proto, sPort, dPort, action) + return rule, nil +} + +// buildPeerFilterExprs assembles the input-chain (peer ACL) match: the +// IP-header protocol byte read via Payload, then source, then ports +// (no counter), matching the historical peer shape so per-rule kernel +// state is identical to pre-unification. +func (r *family) buildPeerFilterExprs( + srcExprs []expr.Any, + proto firewall.Protocol, + sPort, dPort *firewall.Port, +) ([]expr.Any, error) { + var exprs []expr.Any + + if proto != firewall.ProtocolALL { + protoNum, err := r.af.protoNum(proto) + if err != nil { + return nil, fmt.Errorf("convert protocol to number: %w", err) + } + exprs = append(exprs, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: r.af.protoOffset, + Len: 1, + }, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoNum}}, + ) + } + exprs = append(exprs, srcExprs...) + + portExprs, err := r.applyPorts(sPort, dPort) + if err != nil { + return nil, err + } + exprs = append(exprs, portExprs...) + return exprs, nil +} + +// buildRouteFilterExprs assembles the forward-chain (route ACL) match: +// source, then destination, then optional proto/ports, then a counter. +func (r *family) buildRouteFilterExprs( + srcExprs []expr.Any, + destination firewall.Network, + proto firewall.Protocol, + sPort, dPort *firewall.Port, +) ([]expr.Any, error) { + exprs := append([]expr.Any{}, srcExprs...) + + destExprs, err := r.applyNetwork(destination, nil, false) + if err != nil { + return nil, fmt.Errorf("apply destination: %w", err) + } + exprs = append(exprs, destExprs...) + + if proto != firewall.ProtocolALL { + protoNum, err := r.af.protoNum(proto) + if err != nil { + r.dropNetworkMatch(destExprs) + return nil, fmt.Errorf("convert protocol to number: %w", err) + } + exprs = append(exprs, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoNum}}, + ) + + portExprs, err := r.applyPorts(sPort, dPort) + if err != nil { + r.dropNetworkMatch(destExprs) + return nil, err + } + exprs = append(exprs, portExprs...) + } + + exprs = append(exprs, &expr.Counter{}) + return exprs, nil +} + +func (r *family) hasRule(id firewall.RuleID) bool { + _, ok := r.filters[id] + return ok +} + +func (r *family) hasDNATRule(id firewall.RuleID) bool { + _, ok := r.rules[id+dnatSuffix] + return ok +} + +// DeleteFilterRule removes a previously installed filter rule. Source +// set references are recovered from the stored rule's expressions via +// findSets and dropped from the shared refcounter. +func (r *family) DeleteFilterRule(rule firewall.Rule) error { + ruleID := rule.ID() + pr, ok := r.filters[ruleID] + if !ok { + log.Debugf("filter rule %s not found", ruleID) + return nil + } + + // A freshly added rule carries no handle until it is read back from + // the kernel, and Flush only refreshes the peer chains. Pull live + // handles for this rule's chain before deciding it is stale so route + // rules (which Flush never refreshes) can actually be deleted. A + // refresh failure aborts the delete without touching tracking state, + // so the caller can retry while the rule may still exist in the kernel. + if pr.nftRule.Handle == 0 { + if err := r.refreshRuleHandles(pr.nftRule.Chain, false); err != nil { + return fmt.Errorf("refresh handles for chain %s: %w", pr.nftRule.Chain.Name, err) + } + } + // Refresh the mangle handle independently: the main rule's handle can + // be populated while the prerouting refresh during Flush failed, and + // gating the mangle refresh on the main handle would leak the mangle + // rule on delete. + if pr.mangleRule != nil && pr.mangleRule.Handle == 0 { + if err := r.refreshRuleHandles(r.chainPrerouting, true); err != nil { + return fmt.Errorf("refresh mangle handles: %w", err) + } + } + + if pr.nftRule.Handle == 0 { + log.Warnf("filter rule %s has no handle, removing stale entry", ruleID) + // The paired mangle rule can still be in the kernel with a live + // handle. Dropping the tracking entry without removing it would + // leave a prerouting rule that nothing can find again. + if err := r.deleteMangleRule(pr, ruleID); err != nil { + return err + } + r.dropNetworkMatch(pr.nftRule.Exprs) + delete(r.filters, ruleID) + return nil + } + + if err := r.conn.DelRule(pr.nftRule); err != nil { + log.Errorf("queue rule delete: %v", err) + } + r.queueMangleDelete(pr) + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush delete %s: %w", ruleID, err) + } + + r.dropNetworkMatch(pr.nftRule.Exprs) + delete(r.filters, ruleID) + return nil +} + +// deleteMangleRule removes the prerouting rule paired with a filter rule on +// its own, for the paths that drop the filter rule's tracking without queueing +// a delete for it. +func (r *family) deleteMangleRule(pr *Rule, ruleID firewall.RuleID) error { + if pr.mangleRule == nil || pr.mangleRule.Handle == 0 { + return nil + } + + r.queueMangleDelete(pr) + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush mangle delete %s: %w", ruleID, err) + } + return nil +} + +// queueMangleDelete queues the delete of the rule's prerouting counterpart, if +// it has one. The caller commits it. +func (r *family) queueMangleDelete(pr *Rule) { + if pr.mangleRule == nil { + return + } + if err := r.conn.DelRule(pr.mangleRule); err != nil { + log.Errorf("queue mangle rule delete: %v", err) + } +} + +func (r *family) decrementSetCounter(rule *nftables.Rule) error { + if r.ipsetCounter == nil { + return nil + } + sets := findSets(rule) + + var merr *multierror.Error + for _, setName := range sets { + if _, err := r.ipsetCounter.Decrement(setName); err != nil { + merr = multierror.Append(merr, fmt.Errorf("decrement set counter: %w", err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +// dropNetworkMatch undoes whatever the source/destination match +// reserved. Safe to call when the spec is empty or holds only inline +// matchers. +func (r *family) dropNetworkMatch(exprs []expr.Any) { + if r.ipsetCounter == nil { + return + } + for _, e := range exprs { + lookup, ok := e.(*expr.Lookup) + if !ok { + continue + } + if _, err := r.ipsetCounter.Decrement(lookup.SetName); err != nil { + log.Errorf("rollback ipset decrement %s: %v", lookup.SetName, err) + } + } +} + +func (r *family) applyNetwork( + network firewall.Network, + setPrefixes []netip.Prefix, + isSource bool, +) ([]expr.Any, error) { + if network.IsSet() { + exprs, err := r.getIpSet(network.Set, setPrefixes, isSource) + if err != nil { + side := "destination" + if isSource { + side = "source" + } + return nil, fmt.Errorf("%s set: %w", side, err) + } + return exprs, nil + } + + if network.IsPrefix() { + return prefixMatchExprs(r.af, network.Prefix, isSource), nil + } + + return nil, nil +} + +// applyPort builds the transport-header port match. A single value +// compares directly, a range uses a range expression, and multiple +// values go through an anonymous constant set: consecutive cmp +// expressions AND together, so chained equality comparisons could +// never match more than one port. The set is queued on the +// connection and committed by the caller's flush together with the +// rule that binds it. +func (r *family) applyPort(port *firewall.Port, isSource bool) ([]expr.Any, error) { + if port == nil || len(port.Values) == 0 { + return nil, nil + } + + // dst port + offset := uint32(2) + if isSource { + // src port + offset = 0 + } + + exprs := []expr.Any{ + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: offset, + Len: 2, + }, + } + + switch { + case port.IsRange && len(port.Values) == 2: + exprs = append(exprs, &expr.Range{ + Op: expr.CmpOpEq, + Register: 1, + FromData: binaryutil.BigEndian.PutUint16(port.Values[0]), + ToData: binaryutil.BigEndian.PutUint16(port.Values[1]), + }) + case len(port.Values) == 1: + exprs = append(exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(port.Values[0]), + }) + default: + lookup, err := r.anonymousPortSet(port.Values) + if err != nil { + return nil, err + } + exprs = append(exprs, lookup) + } + + return exprs, nil +} + +// anonymousPortSet queues an anonymous constant set holding the given +// ports on the connection and returns a lookup against it. The set is +// committed by the caller's flush together with the rule that binds it. +func (r *family) anonymousPortSet(values []uint16) (*expr.Lookup, error) { + set := &nftables.Set{ + Anonymous: true, + Constant: true, + Table: r.workTable, + KeyType: nftables.TypeInetService, + } + elements := make([]nftables.SetElement, 0, len(values)) + for _, p := range values { + elements = append(elements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(p)}) + } + if err := r.conn.AddSet(set, elements); err != nil { + return nil, fmt.Errorf("add anonymous port set: %w", err) + } + return &expr.Lookup{ + SourceRegister: 1, + SetID: set.ID, + SetName: set.Name, + }, nil +} + +// applyPorts builds the source then destination port matches. +func (r *family) applyPorts(sPort, dPort *firewall.Port) ([]expr.Any, error) { + sPortExprs, err := r.applyPort(sPort, true) + if err != nil { + return nil, fmt.Errorf("apply source port: %w", err) + } + + dPortExprs, err := r.applyPort(dPort, false) + if err != nil { + return nil, fmt.Errorf("apply destination port: %w", err) + } + + return append(sPortExprs, dPortExprs...), nil +} + +// prefixMatchExprs is the family-aware match sequence for a CIDR +// prefix. /0 returns nil; a host prefix (full bit length for the +// family) skips the bitwise step since the mask is all-ones. Shared +// between family and aclManager so both treat single prefixes +// identically. +func prefixMatchExprs(af addrFamily, prefix netip.Prefix, isSource bool) []expr.Any { + offset := af.dstAddrOffset + if isSource { + offset = af.srcAddrOffset + } + + ones := prefix.Bits() + if ones == 0 { + return nil + } + + payload := &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: af.addrLen, + } + cmp := &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: prefix.Masked().Addr().AsSlice(), + } + + if ones == af.totalBits { + return []expr.Any{payload, cmp} + } + + mask := net.CIDRMask(ones, af.totalBits) + xor := make([]byte, af.addrLen) + return []expr.Any{ + payload, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: af.addrLen, + Mask: mask, + Xor: xor, + }, + cmp, + } +} + +func getCtNewExprs() []expr.Any { + return []expr.Any{ + &expr.Ct{ + Key: expr.CtKeySTATE, + Register: 1, + }, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW), + Xor: binaryutil.NativeEndian.PutUint32(0), + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0, 0, 0, 0}, + }, + } +} + +// sourceNetwork classifies a source-prefix list into the firewall.Network +// shape the rest of the spec-builder consumes: empty for match-any, a +// single prefix inline, or an ipset for multiple sources. +func sourceNetwork(sources []netip.Prefix) firewall.Network { + switch { + case len(sources) == 0: + return firewall.Network{} + case len(sources) == 1 && sources[0].Bits() == 0: + return firewall.Network{} + case len(sources) == 1: + return firewall.Network{Prefix: sources[0]} + default: + return firewall.Network{Set: firewall.NewPrefixSet(sources)} + } +} + +func ifname(n string) []byte { + b := make([]byte, 16) + copy(b, n+"\x00") + return b +} + +// findSets scans an nftables rule's expressions for expr.Lookup and +// returns the named sets in occurrence order. Used at delete time to +// drop ipsetCounter references; peer and route ACLs go through it. +func findSets(rule *nftables.Rule) []string { + var sets []string + for _, e := range rule.Exprs { + if lookup, ok := e.(*expr.Lookup); ok { + sets = append(sets, lookup.SetName) + } + } + return sets +} diff --git a/client/firewall/nftables/interface_allower_integration_linux_test.go b/client/firewall/nftables/interface_allower_integration_linux_test.go new file mode 100644 index 000000000..4d4bc6187 --- /dev/null +++ b/client/firewall/nftables/interface_allower_integration_linux_test.go @@ -0,0 +1,90 @@ +//go:build privileged + +package nftables + +import ( + "bytes" + "os" + "testing" + + "github.com/google/nftables" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/iface" +) + +// TestInterfaceAllowerInputOnly verifies the userspace-mode allower opens the +// interface on the INPUT hook of foreign chains only (not FORWARD, since the +// userspace router never forwards in the kernel), creates no netbird work +// table, and removes its rules on Close. +func TestInterfaceAllowerInputOnly(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("root required") + } + + require.False(t, ipTableExists(t, getTableName()), "precondition: no stale netbird table") + + conn := &nftables.Conn{} + extTable := conn.AddTable(&nftables.Table{Name: "nbtest_extchains", Family: nftables.TableFamilyINet}) + inputChain := conn.AddChain(&nftables.Chain{ + Name: "ext_input", Table: extTable, + Hooknum: nftables.ChainHookInput, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter, + }) + forwardChain := conn.AddChain(&nftables.Chain{ + Name: "ext_forward", Table: extTable, + Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter, + }) + require.NoError(t, conn.Flush(), "create external table and chains") + t.Cleanup(func() { + c := &nftables.Conn{} + c.DelTable(extTable) + _ = c.Flush() + }) + + allower, err := NewInterfaceAllower(ifaceMock, iface.DefaultMTU) + require.NoError(t, err, "create allower") + require.NoError(t, allower.Apply(), "apply") + + require.True(t, chainHasUserData(t, extTable, inputChain, userDataAcceptInputRule), + "external INPUT chain should get the accept rule") + require.Len(t, listRules(t, extTable, forwardChain), 0, + "external FORWARD chain must not be opened in userspace mode") + require.False(t, ipTableExists(t, getTableName()), + "allower must not create a netbird work table") + + require.NoError(t, allower.Close(), "close") + require.False(t, chainHasUserData(t, extTable, inputChain, userDataAcceptInputRule), + "accept rule should be removed on close") +} + +func listRules(t *testing.T, table *nftables.Table, chain *nftables.Chain) []*nftables.Rule { + t.Helper() + c := &nftables.Conn{} + rules, err := c.GetRules(table, chain) + require.NoError(t, err) + return rules +} + +func chainHasUserData(t *testing.T, table *nftables.Table, chain *nftables.Chain, ud string) bool { + for _, r := range listRules(t, table, chain) { + if bytes.Equal(r.UserData, []byte(ud)) { + return true + } + } + return false +} + +func ipTableExists(t *testing.T, name string) bool { + t.Helper() + c := &nftables.Conn{} + for _, fam := range []nftables.TableFamily{nftables.TableFamilyIPv4, nftables.TableFamilyIPv6} { + tbls, err := c.ListTablesOfFamily(fam) + require.NoError(t, err) + for _, tb := range tbls { + if tb.Name == name { + return true + } + } + } + return false +} diff --git a/client/firewall/nftables/interface_allower_linux.go b/client/firewall/nftables/interface_allower_linux.go new file mode 100644 index 000000000..e7232e2bf --- /dev/null +++ b/client/firewall/nftables/interface_allower_linux.go @@ -0,0 +1,107 @@ +package nftables + +import ( + "fmt" + + "github.com/google/nftables" + "github.com/hashicorp/go-multierror" + + nberrors "github.com/netbirdio/netbird/client/errors" +) + +// InterfaceAllower opens the NetBird interface in the kernel's filter table and +// external chains and keeps them reconciled via a netlink monitor, so the host +// firewall doesn't drop traffic the NetBird firewall handles. It is used by the +// userspace firewall, where routing happens in the forwarder, so only INPUT is +// opened (the userspace router never forwards in the kernel). +// +// It owns its own families/connection and never creates a netbird work table. +// firewalld trust is handled by the caller, not here. Its operations are serial +// (Apply before the monitor starts; reconciles run on the single monitor +// goroutine; Close stops the monitor before removing), so it needs no locking. +// +// TODO: this opens nftables and the iptables-nft filter table (detected via +// nft), but not a legacy-iptables ruleset running in parallel with nftables. +// Such a host would keep its legacy filter chains closed for the interface. +type InterfaceAllower struct { + family4 *family + family6 *family + extMonitor *externalChainMonitor +} + +// NewInterfaceAllower builds an allower for the given interface. It returns an +// error when nftables is unavailable (e.g. an iptables-legacy host), so the +// caller can fall back to firewalld trust. +func NewInterfaceAllower(wgIface iFaceMapper, mtu uint16) (*InterfaceAllower, error) { + tableName := getTableName() + + family4 := newFamily(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4}, wgIface, mtu) + + // Probe nftables availability before committing to this backend. + if _, err := family4.conn.ListChainsOfTableFamily(nftables.TableFamilyINet); err != nil { + return nil, fmt.Errorf("nftables not available: %w", err) + } + + a := &InterfaceAllower{family4: family4} + + if wgIface.Address().HasIPv6() { + a.family6 = newFamily(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6}, wgIface, mtu) + } + + a.extMonitor = newExternalChainMonitor(a) + return a, nil +} + +// Apply opens the interface (INPUT only) in the foreign filter chains and starts +// reconciling them on nftables changes. +func (a *InterfaceAllower) Apply() error { + var merr *multierror.Error + for _, f := range a.families() { + // Remove any stale accepts first so a prior unclean exit (e.g. SIGKILL, + // where Close never ran) is recovered deterministically rather than + // accumulating duplicate rules on the iptables filter table. + if err := f.removeAcceptFilterRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clean stale accept rules: %w", err)) + } + if err := f.openInterface(false); err != nil { + merr = multierror.Append(merr, err) + } + } + + a.extMonitor.start() + return nberrors.FormatErrorOrNil(merr) +} + +// families returns the configured address families (v4, and v6 when present). +func (a *InterfaceAllower) families() []*family { + families := []*family{a.family4} + if a.family6 != nil { + families = append(families, a.family6) + } + return families +} + +// reconcileExternalChains re-applies the INPUT accepts to external chains. It +// implements externalChainReconciler for the monitor. +func (a *InterfaceAllower) reconcileExternalChains() error { + var merr *multierror.Error + for _, f := range a.families() { + if err := f.acceptExternalChainsRules(false); err != nil { + merr = multierror.Append(merr, err) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +// Close stops the monitor and removes the accept rules. +func (a *InterfaceAllower) Close() error { + a.extMonitor.stop() + + var merr *multierror.Error + for _, f := range a.families() { + if err := f.removeAcceptFilterRules(); err != nil { + merr = multierror.Append(merr, err) + } + } + return nberrors.FormatErrorOrNil(merr) +} diff --git a/client/firewall/nftables/ipset_linux.go b/client/firewall/nftables/ipset_linux.go new file mode 100644 index 000000000..34783bbeb --- /dev/null +++ b/client/firewall/nftables/ipset_linux.go @@ -0,0 +1,210 @@ +//go:build !android + +package nftables + +import ( + "encoding/binary" + "fmt" + "net/netip" + + "github.com/google/nftables" + "github.com/google/nftables/expr" + log "github.com/sirupsen/logrus" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" +) + +func (r *family) getIpSet(set firewall.Set, prefixes []netip.Prefix, isSource bool) ([]expr.Any, error) { + ref, err := r.ipsetCounter.Increment(set.HashedName(), setInput{ + set: set, + prefixes: prefixes, + }) + if err != nil { + return nil, fmt.Errorf("create or get ipset: %w", err) + } + + return r.getIpSetExprs(ref, isSource) +} + +func (r *family) createIpSet(setName string, input setInput) (*nftables.Set, error) { + // overlapping prefixes will result in an error, so we need to merge them + prefixes := firewall.MergeIPRanges(input.prefixes) + + nfset := &nftables.Set{ + Name: setName, + Comment: input.set.Comment(), + Table: r.workTable, + // required for prefixes + Interval: true, + KeyType: r.af.setKeyType, + } + + elements := r.convertPrefixesToSet(prefixes) + nElements := len(elements) + + maxElements := maxPrefixesSet * 2 + initialElements := elements[:min(maxElements, nElements)] + + if err := r.conn.AddSet(nfset, initialElements); err != nil { + return nil, fmt.Errorf("error adding set %s: %w", setName, err) + } + if err := r.conn.Flush(); err != nil { + return nil, fmt.Errorf("flush error: %w", err) + } + log.Debugf("Created new ipset: %s with %d initial prefixes (total prefixes %d)", setName, len(initialElements)/2, len(prefixes)) + + // The set is committed now. If a later batch fails, destroy it: the + // refcounter records nothing on a create-callback error, so it would + // otherwise leak, and a partial source set fails-open for deny rules. + if err := r.addRemainingElements(nfset, elements, maxElements); err != nil { + if derr := r.deleteIpSet(setName, nfset); derr != nil { + log.Warnf("rollback ipset %s after add failure: %v", setName, derr) + } + return nil, err + } + + log.Infof("Created new ipset: %s with %d prefixes", setName, len(prefixes)) + return nfset, nil +} + +// addRemainingElements adds element batches beyond the initial one in +// maxElements-sized chunks, flushing each. Called after the set has been +// created with its first batch. +func (r *family) addRemainingElements(nfset *nftables.Set, elements []nftables.SetElement, maxElements int) error { + nElements := len(elements) + for subStart := maxElements; subStart < nElements; subStart += maxElements { + subEnd := min(subStart+maxElements, nElements) + subElement := elements[subStart:subEnd] + nSubPrefixes := len(subElement) / 2 + log.Tracef("Adding new prefixes (%d) in ipset: %s", nSubPrefixes, nfset.Name) + if err := r.conn.SetAddElements(nfset, subElement); err != nil { + return fmt.Errorf("error adding prefixes (%d) to set %s: %w", nSubPrefixes, nfset.Name, err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush error: %w", err) + } + log.Debugf("Added new prefixes (%d) in ipset: %s", nSubPrefixes, nfset.Name) + } + return nil +} + +func (r *family) convertPrefixesToSet(prefixes []netip.Prefix) []nftables.SetElement { + var elements []nftables.SetElement + for _, prefix := range prefixes { + // nftables needs half-open intervals [firstIP, lastIP) for prefixes + // e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc + firstIP := prefix.Addr() + + // For a /0 the last address is the broadcast and its Next() overflows + // to an invalid Addr with an empty key, so wrap to the zero address, + // which nftables reads as the open end of a full-range interval. + var lastKey []byte + if prefix.Bits() == 0 { + lastKey = make([]byte, r.af.addrLen) + } else { + lastKey = calculateLastIP(prefix).Next().AsSlice() + } + + // the nft tool also adds a zero-address IntervalEnd element, see https://github.com/google/nftables/issues/247 + // nftables.SetElement{Key: make([]byte, r.af.addrLen), IntervalEnd: true}, + elements = append(elements, + nftables.SetElement{Key: firstIP.AsSlice()}, + nftables.SetElement{Key: lastKey, IntervalEnd: true}, + ) + } + return elements +} + +// calculateLastIP determines the last IP in a given prefix. +func calculateLastIP(prefix netip.Prefix) netip.Addr { + masked := prefix.Masked() + if masked.Addr().Is4() { + hostMask := ^uint32(0) >> masked.Bits() + lastIP := uint32FromNetipAddr(masked.Addr()) | hostMask + return netip.AddrFrom4(uint32ToBytes(lastIP)) + } + + // IPv6: set host bits to all 1s + b := masked.Addr().As16() + bits := masked.Bits() + for i := bits; i < 128; i++ { + b[i/8] |= 1 << (7 - i%8) + } + return netip.AddrFrom16(b) +} + +// Utility function to convert netip.Addr to uint32. +func uint32FromNetipAddr(addr netip.Addr) uint32 { + b := addr.As4() + return binary.BigEndian.Uint32(b[:]) +} + +// Utility function to convert uint32 to a netip-compatible byte slice. +func uint32ToBytes(ip uint32) [4]byte { + var b [4]byte + binary.BigEndian.PutUint32(b[:], ip) + return b +} + +func (r *family) deleteIpSet(setName string, nfset *nftables.Set) error { + r.conn.DelSet(nfset) + if err := r.conn.Flush(); err != nil { + return fmt.Errorf(flushError, err) + } + + log.Debugf("Deleted unused ipset %s", setName) + return nil +} + +func (r *family) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { + nfset, err := r.conn.GetSetByName(r.workTable, set.HashedName()) + if err != nil { + return fmt.Errorf("get set %s: %w", set.HashedName(), err) + } + + // Overlapping prefixes (e.g. duplicate resolved addresses) make the + // interval set reject the batch, so merge them as createIpSet does. + prefixes = firewall.MergeIPRanges(prefixes) + elements := r.convertPrefixesToSet(prefixes) + + // Add in batches sized like createIpSet so a large update does not + // exceed the netlink message size limit. + maxElements := maxPrefixesSet * 2 + for start := 0; start < len(elements); start += maxElements { + end := min(start+maxElements, len(elements)) + if err := r.conn.SetAddElements(nfset, elements[start:end]); err != nil { + return fmt.Errorf("add elements to set %s: %w", set.HashedName(), err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf(flushError, err) + } + } + + log.Debugf("updated set %s with %d prefixes", set.HashedName(), len(prefixes)) + + return nil +} + +func (r *family) getIpSetExprs(ref refcounter.Ref[*nftables.Set], isSource bool) ([]expr.Any, error) { + // dst offset by default + offset := r.af.dstAddrOffset + if isSource { + // src offset + offset = r.af.srcAddrOffset + } + + return []expr.Any{ + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: r.af.addrLen, + }, + &expr.Lookup{ + SourceRegister: 1, + SetName: ref.Out.Name, + SetID: ref.Out.ID, + }, + }, nil +} diff --git a/client/firewall/nftables/ipset_linux_test.go b/client/firewall/nftables/ipset_linux_test.go new file mode 100644 index 000000000..7ab2f6c3f --- /dev/null +++ b/client/firewall/nftables/ipset_linux_test.go @@ -0,0 +1,36 @@ +package nftables + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestConvertPrefixesToSetWildcard verifies that a /0 prefix produces a +// usable interval. The last address of a /0 is the broadcast, whose Next() +// overflows to an invalid Addr with an empty key; the IntervalEnd must wrap +// to the zero address instead so nftables sees a full-range interval. +func TestConvertPrefixesToSetWildcard(t *testing.T) { + tests := []struct { + name string + af addrFamily + prefix string + }{ + {"IPv4 /0", afIPv4, "0.0.0.0/0"}, + {"IPv6 /0", afIPv6, "::/0"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &family{af: tt.af} + elements := r.convertPrefixesToSet([]netip.Prefix{netip.MustParsePrefix(tt.prefix)}) + + require.Len(t, elements, 2, "expected start and interval-end element") + assert.False(t, elements[0].IntervalEnd, "first element is the interval start") + assert.True(t, elements[1].IntervalEnd, "second element is the interval end") + assert.Len(t, elements[1].Key, int(tt.af.addrLen), "interval-end key must be a zero address, not empty") + }) + } +} diff --git a/client/firewall/nftables/ipsetstore_linux.go b/client/firewall/nftables/ipsetstore_linux.go deleted file mode 100644 index a6c2e9496..000000000 --- a/client/firewall/nftables/ipsetstore_linux.go +++ /dev/null @@ -1,85 +0,0 @@ -package nftables - -import ( - "net" -) - -type ipsetStore struct { - ipsetReference map[string]int - ipsets map[string]map[string]struct{} // ipsetName -> list of ips -} - -func newIpsetStore() *ipsetStore { - return &ipsetStore{ - ipsetReference: make(map[string]int), - ipsets: make(map[string]map[string]struct{}), - } -} - -func (s *ipsetStore) ips(ipsetName string) (map[string]struct{}, bool) { - r, ok := s.ipsets[ipsetName] - return r, ok -} - -func (s *ipsetStore) newIpset(ipsetName string) map[string]struct{} { - s.ipsetReference[ipsetName] = 0 - ipList := make(map[string]struct{}) - s.ipsets[ipsetName] = ipList - return ipList -} - -func (s *ipsetStore) deleteIpset(ipsetName string) { - delete(s.ipsetReference, ipsetName) - delete(s.ipsets, ipsetName) -} - -func (s *ipsetStore) DeleteIpFromSet(ipsetName string, ip net.IP) { - ipList, ok := s.ipsets[ipsetName] - if !ok { - return - } - delete(ipList, ip.String()) -} - -func (s *ipsetStore) AddIpToSet(ipsetName string, ip net.IP) { - ipList, ok := s.ipsets[ipsetName] - if !ok { - return - } - ipList[ip.String()] = struct{}{} -} - -func (s *ipsetStore) IsIpInSet(ipsetName string, ip net.IP) bool { - ipList, ok := s.ipsets[ipsetName] - if !ok { - return false - } - _, ok = ipList[ip.String()] - return ok -} - -func (s *ipsetStore) AddReferenceToIpset(ipsetName string) { - s.ipsetReference[ipsetName]++ -} - -func (s *ipsetStore) DeleteReferenceFromIpSet(ipsetName string) { - r, ok := s.ipsetReference[ipsetName] - if !ok { - return - } - if r == 0 { - return - } - s.ipsetReference[ipsetName]-- -} - -func (s *ipsetStore) HasReferenceToSet(ipsetName string) bool { - if _, ok := s.ipsetReference[ipsetName]; !ok { - return false - } - if s.ipsetReference[ipsetName] == 0 { - return false - } - - return true -} diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index 984b1c3ba..dbd5e4fa2 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -3,7 +3,6 @@ package nftables import ( "context" "fmt" - "net" "net/netip" "os" "sync" @@ -16,7 +15,6 @@ import ( "golang.org/x/sys/unix" nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/firewall/firewalld" firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/internal/statemanager" @@ -45,18 +43,17 @@ type iFaceMapper interface { Address() wgaddr.Address } -// Manager of iptables firewall +// Manager of nftables firewall. Per-family state (peer ACLs, route +// ACLs, NAT, DNAT, MSS clamping) lives on family; Manager dispatches +// by family and provides the public firewall.Manager surface. type Manager struct { mutex sync.Mutex rConn *nftables.Conn wgIface iFaceMapper - router *router - aclManager *AclManager - - // IPv6 counterparts, nil when no v6 overlay - router6 *router - aclManager6 *AclManager + family4 *family + // IPv6 counterpart, nil when no v6 overlay. + family6 *family notrackOutputChain *nftables.Chain notrackPreroutingChain *nftables.Chain @@ -74,21 +71,10 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) { tableName := getTableName() workTable := &nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4} - var err error - m.router, err = newRouter(workTable, wgIface, mtu) - if err != nil { - return nil, fmt.Errorf("create router: %w", err) - } - - m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw) - if err != nil { - return nil, fmt.Errorf("create acl manager: %w", err) - } + m.family4 = newFamily(workTable, wgIface, mtu) if wgIface.Address().HasIPv6() { - if err := m.createIPv6Components(tableName, wgIface, mtu); err != nil { - return nil, fmt.Errorf("create IPv6 firewall: %w", err) - } + m.createIPv6Components(tableName, wgIface, mtu) } m.extMonitor = newExternalChainMonitor(m) @@ -96,30 +82,19 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) { return m, nil } -func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mtu uint16) error { +func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mtu uint16) { workTable6 := &nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6} - var err error - m.router6, err = newRouter(workTable6, wgIface, mtu) - if err != nil { - return fmt.Errorf("create v6 router: %w", err) - } + m.family6 = newFamily(workTable6, wgIface, mtu) - // Share the per-family forwarding refcounter with the v4 router so a v4 + // Share the per-family forwarding refcounter with the v4 family so a v4 // rule and a v6 rule against the same state machine cooperate cleanly. - m.router6.ipFwdState = m.router.ipFwdState - - m.aclManager6, err = newAclManager(workTable6, wgIface, chainNameRoutingFw) - if err != nil { - return fmt.Errorf("create v6 acl manager: %w", err) - } - - return nil + m.family6.ipFwdState = m.family4.ipFwdState } // hasIPv6 reports whether the manager has IPv6 components initialized. func (m *Manager) hasIPv6() bool { - return m.router6 != nil + return m.family6 != nil } func (m *Manager) initIPv6() error { @@ -128,12 +103,8 @@ func (m *Manager) initIPv6() error { return fmt.Errorf("create v6 work table: %w", err) } - if err := m.router6.init(workTable6); err != nil { - return fmt.Errorf("v6 router init: %w", err) - } - - if err := m.aclManager6.init(workTable6); err != nil { - return fmt.Errorf("v6 acl manager init: %w", err) + if err := m.family6.init(workTable6); err != nil { + return fmt.Errorf("v6 family init: %w", err) } return nil @@ -156,19 +127,20 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { // reconcileExternalChains re-applies passthrough accept rules to external // filter chains for both IPv4 and IPv6 routers. Called by the monitor when -// tables or chains appear (e.g. after firewalld reloads). +// tables or chains appear (e.g. after firewalld reloads). Kernel routing opens +// both INPUT and FORWARD. func (m *Manager) reconcileExternalChains() error { m.mutex.Lock() defer m.mutex.Unlock() var merr *multierror.Error - if m.router != nil { - if err := m.router.acceptExternalChainsRules(); err != nil { + if m.family4 != nil { + if err := m.family4.acceptExternalChainsRules(true); err != nil { merr = multierror.Append(merr, fmt.Errorf("v4: %w", err)) } } if m.hasIPv6() { - if err := m.router6.acceptExternalChainsRules(); err != nil { + if err := m.family6.acceptExternalChainsRules(true); err != nil { merr = multierror.Append(merr, fmt.Errorf("v6: %w", err)) } } @@ -187,12 +159,8 @@ func (m *Manager) initFirewall() (err error) { } }() - if err := m.router.init(workTable); err != nil { - return fmt.Errorf("router init: %w", err) - } - - if err := m.aclManager.init(workTable); err != nil { - return fmt.Errorf("acl manager init: %w", err) + if err := m.family4.init(workTable); err != nil { + return fmt.Errorf("family init: %w", err) } if m.hasIPv6() { @@ -220,7 +188,7 @@ func (m *Manager) persistState(stateManager *statemanager.Manager) { InterfaceState: &InterfaceState{ NameStr: m.wgIface.Name(), WGAddress: m.wgIface.Address(), - MTU: m.router.mtu, + MTU: m.family4.mtu, }, }); err != nil { log.Errorf("failed to update state: %v", err) @@ -235,12 +203,12 @@ func (m *Manager) persistState(stateManager *statemanager.Manager) { // rollbackInit performs best-effort cleanup of already-initialized state when Init fails partway through. func (m *Manager) rollbackInit() { - if err := m.router.Reset(); err != nil { - log.Warnf("rollback router: %v", err) + if err := m.family4.Reset(); err != nil { + log.Warnf("rollback family: %v", err) } if m.hasIPv6() { - if err := m.router6.Reset(); err != nil { - log.Warnf("rollback v6 router: %v", err) + if err := m.family6.Reset(); err != nil { + log.Warnf("rollback v6 family: %v", err) } } if err := m.cleanupNetbirdTables(); err != nil { @@ -251,118 +219,82 @@ func (m *Manager) rollbackInit() { } } -// AddPeerFiltering rule to the firewall +// AddFilterRule installs a packet-filtering rule. // -// If comment argument is empty firewall manager should set -// rule ID as comment for the rule -func (m *Manager) AddPeerFiltering( - id []byte, - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if ip.To4() != nil { - return m.aclManager.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) - } - - if !m.hasIPv6() { - return nil, fmt.Errorf("add peer filtering for %s: %w", ip, firewall.ErrIPv6NotInitialized) - } - return m.aclManager6.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) -} - -func (m *Manager) AddRouteFiltering( +// Destination semantics: zero Network → input chain (peer ACL); +// set Network → forward chain (route ACL). +// +// Sources are a single address family; the rule is dispatched to the +// matching per-family backend. +func (m *Manager) AddFilterRule( id []byte, sources []netip.Prefix, destination firewall.Network, proto firewall.Protocol, - sPort, dPort *firewall.Port, + sPort *firewall.Port, + dPort *firewall.Port, action firewall.Action, ) (firewall.Rule, error) { + if len(sources) == 0 { + return nil, firewall.ErrNoSources + } + m.mutex.Lock() defer m.mutex.Unlock() - if isIPv6RouteRule(sources, destination) { + fam := m.family4 + if isIPv6Rule(sources, destination) { if !m.hasIPv6() { - return nil, fmt.Errorf("add route filtering: %w", firewall.ErrIPv6NotInitialized) + return nil, fmt.Errorf("add filtering: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + fam = m.family6 } - - return m.router.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + return fam.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) } -// DeletePeerRule from the firewall by rule definition -func (m *Manager) DeletePeerRule(rule firewall.Rule) error { +// DeleteFilterRule removes a filtering rule. The owning family is found +// by id in the in-memory filter maps, which are the only tracking for +// filter rules. family.DeleteFilterRule is idempotent when the id is +// absent. +func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - if m.hasIPv6() && isIPv6Rule(rule) { - return m.aclManager6.DeletePeerRule(rule) - } - return m.aclManager.DeletePeerRule(rule) -} - -func isIPv6Rule(rule firewall.Rule) bool { - r, ok := rule.(*Rule) - return ok && r.nftRule != nil && r.nftRule.Table != nil && r.nftRule.Table.Family == nftables.TableFamilyIPv6 -} - -// isIPv6RouteRule determines whether a route rule belongs to the v6 table. -// For static routes, the destination prefix determines the family. For dynamic -// routes (DomainSet), the sources determine the family since management -// duplicates dynamic rules per family. -func isIPv6RouteRule(sources []netip.Prefix, destination firewall.Network) bool { - if destination.IsPrefix() { - return destination.Prefix.Addr().Is6() - } - return len(sources) > 0 && sources[0].Addr().Is6() -} - -// DeleteRouteRule deletes a routing rule. Route rules live in exactly one -// router; the cached maps are normally authoritative, so the kernel is only -// consulted when neither map knows about the rule. -func (m *Manager) DeleteRouteRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - id := rule.ID() - r, err := m.routerForRuleID(id, (*router).hasRule) + fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule, false) if err != nil { return err } - return r.DeleteRouteRule(rule) + return fam.DeleteFilterRule(rule) } -// routerForRuleID picks the router holding the rule with the given id, using -// the supplied lookup. If the cached maps disagree (or both miss), it refreshes -// from the kernel once and re-checks before falling back to the v4 router. -func (m *Manager) routerForRuleID(id string, has func(*router, string) bool) (*router, error) { - if has(m.router, id) { - return m.router, nil - } - if m.hasIPv6() && has(m.router6, id) { - return m.router6, nil +// familyForRuleID picks the family holding the rule with the given id, using +// the supplied lookup. With refresh set, a miss in both cached maps reloads +// the NAT/DNAT rule maps from the kernel once and re-checks before falling +// back to the v4 family. Filter rules are tracked only in memory and have no +// kernel-backed reload, so their callers pass refresh as false. +func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool, refresh bool) (*family, error) { + if has(m.family4, id) { + return m.family4, nil } if !m.hasIPv6() { - return m.router, nil + return m.family4, nil } - if err := m.router.refreshRulesMap(); err != nil { + if has(m.family6, id) { + return m.family6, nil + } + if !refresh { + return m.family4, nil + } + if err := m.family4.refreshRulesMap(); err != nil { return nil, fmt.Errorf("refresh v4 rules: %w", err) } - if err := m.router6.refreshRulesMap(); err != nil { + if err := m.family6.refreshRulesMap(); err != nil { return nil, fmt.Errorf("refresh v6 rules: %w", err) } - if has(m.router6, id) && !has(m.router, id) { - return m.router6, nil + if has(m.family6, id) && !has(m.family4, id) { + return m.family6, nil } - return m.router, nil + return m.family4, nil } func (m *Manager) IsServerRouteSupported() bool { @@ -381,10 +313,10 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return fmt.Errorf("add NAT rule: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddNatRule(pair) + return m.family6.AddNatRule(pair) } - if err := m.router.AddNatRule(pair); err != nil { + if err := m.family4.AddNatRule(pair); err != nil { return err } @@ -396,7 +328,7 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { // so the eventual cleanup still works. if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.AddNatRule(v6Pair); err != nil { + if err := m.family6.AddNatRule(v6Pair); err != nil { return fmt.Errorf("add v6 NAT rule: %w", err) } } @@ -412,18 +344,18 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return nil } - return m.router6.RemoveNatRule(pair) + return m.family6.RemoveNatRule(pair) } var merr *multierror.Error - if err := m.router.RemoveNatRule(pair); err != nil { + if err := m.family4.RemoveNatRule(pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v4 NAT rule: %w", err)) } if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.RemoveNatRule(v6Pair); err != nil { + if err := m.family6.RemoveNatRule(v6Pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v6 NAT rule: %w", err)) } } @@ -431,46 +363,13 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { return nberrors.FormatErrorOrNil(merr) } -// AllowNetbird allows netbird interface traffic. -// This is called when USPFilter wraps the native firewall, adding blanket accept -// rules so that packet filtering is handled in userspace instead of by netfilter. -// -// TODO: In USP mode this only adds ACCEPT to the netbird table's own chains, -// which doesn't override DROP rules in external tables (e.g. firewalld). -// Should add passthrough rules to external chains (like the native mode router's -// addExternalChainsRules does) for both the netbird table family and inet tables. -// The netbird table itself is fine (routing chains already exist there), but -// non-netbird tables with INPUT/FORWARD hooks can still DROP our WG traffic. -func (m *Manager) AllowNetbird() error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if err := m.aclManager.createDefaultAllowRules(); err != nil { - return fmt.Errorf("create default allow rules: %w", err) - } - if m.hasIPv6() { - if err := m.aclManager6.createDefaultAllowRules(); err != nil { - return fmt.Errorf("create v6 default allow rules: %w", err) - } - } - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush allow input netbird rules: %w", err) - } - - if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - - return nil -} - // SetLegacyManagement sets the route manager to use legacy management func (m *Manager) SetLegacyManagement(isLegacy bool) error { - if err := firewall.SetLegacyManagement(m.router, isLegacy); err != nil { + if err := firewall.SetLegacyManagement(m.family4, isLegacy); err != nil { return err } if m.hasIPv6() { - return firewall.SetLegacyManagement(m.router6, isLegacy) + return firewall.SetLegacyManagement(m.family6, isLegacy) } return nil } @@ -484,13 +383,13 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { var merr *multierror.Error - if err := m.router.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset router: %v", err)) + if err := m.family4.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset family: %w", err)) } if m.hasIPv6() { - if err := m.router6.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset v6 router: %v", err)) + if err := m.family6.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) } } @@ -531,11 +430,11 @@ func (m *Manager) SetLogLevel(log.Level) { func (m *Manager) EnableRouting() error { // v6 only when the overlay actually has v6. - return m.router.ipFwdState.RequestRouting(m.router6 != nil) + return m.family4.ipFwdState.RequestRouting(m.hasIPv6()) } func (m *Manager) DisableRouting() error { - return m.router.ipFwdState.ReleaseRouting() + return m.family4.ipFwdState.ReleaseRouting() } // Flush rule/chain/set operations from the buffer @@ -546,13 +445,13 @@ func (m *Manager) Flush() error { m.mutex.Lock() defer m.mutex.Unlock() - if err := m.aclManager.Flush(); err != nil { + if err := m.family4.Flush(); err != nil { return err } if m.hasIPv6() { - if err := m.aclManager6.Flush(); err != nil { - return fmt.Errorf("flush v6 acl: %w", err) + if err := m.family6.Flush(); err != nil { + return fmt.Errorf("flush v6 family: %w", err) } } @@ -572,9 +471,9 @@ func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) if !m.hasIPv6() { return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddDNATRule(rule) + return m.family6.AddDNATRule(rule) } - return m.router.AddDNATRule(rule) + return m.family4.AddDNATRule(rule) } // DeleteDNATRule deletes a DNAT rule @@ -582,7 +481,7 @@ func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - r, err := m.routerForRuleID(rule.ID(), (*router).hasDNATRule) + r, err := m.familyForRuleID(rule.ID(), (*family).hasDNATRule, true) if err != nil { return err } @@ -603,12 +502,12 @@ func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { } } - if err := m.router.UpdateSet(set, v4Prefixes); err != nil { + if err := m.family4.UpdateSet(set, v4Prefixes); err != nil { return err } if m.hasIPv6() && len(v6Prefixes) > 0 { - if err := m.router6.UpdateSet(set, v6Prefixes); err != nil { + if err := m.family6.UpdateSet(set, v6Prefixes); err != nil { return fmt.Errorf("update v6 set: %w", err) } } @@ -625,9 +524,9 @@ func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protoco if !m.hasIPv6() { return fmt.Errorf("add inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveInboundDNAT removes an inbound DNAT rule. @@ -639,9 +538,9 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot if !m.hasIPv6() { return fmt.Errorf("remove inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. @@ -653,9 +552,9 @@ func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol if !m.hasIPv6() { return fmt.Errorf("add output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. @@ -667,9 +566,9 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto if !m.hasIPv6() { return fmt.Errorf("remove output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } const ( @@ -898,3 +797,14 @@ func getEstablishedExprs(register uint32) []expr.Any { }, } } + +// isIPv6Rule reports whether the rule belongs to the v6 table. For a +// prefix destination the destination family decides; otherwise the +// (single-family) sources do, since management duplicates rules per +// family. +func isIPv6Rule(sources []netip.Prefix, destination firewall.Network) bool { + if destination.IsPrefix() { + return destination.Prefix.Addr().Is6() + } + return len(sources) > 0 && sources[0].Addr().Is6() +} diff --git a/client/firewall/nftables/manager_linux_test.go b/client/firewall/nftables/manager_linux_test.go index 4eb466281..0ca56409e 100644 --- a/client/firewall/nftables/manager_linux_test.go +++ b/client/firewall/nftables/manager_linux_test.go @@ -72,13 +72,13 @@ func TestNftablesManager(t *testing.T) { testClient := &nftables.Conn{} - rule, err := manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop, "") + rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop) require.NoError(t, err, "failed to add rule") err = manager.Flush() require.NoError(t, err, "failed to flush") - rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) require.NoError(t, err, "failed to get rules") require.Len(t, rules, 2, "expected 2 rules") @@ -149,15 +149,12 @@ func TestNftablesManager(t *testing.T) { // Compare connection tracking rule at position 1 (pushed down by DROP rule insertion) compareExprsIgnoringCounters(t, rules[1].Exprs, expectedExprs1) - for _, r := range rule { - err = manager.DeletePeerRule(r) - require.NoError(t, err, "failed to delete rule") - } + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete rule") err = manager.Flush() require.NoError(t, err, "failed to flush") - rules, err = testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + rules, err = testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) require.NoError(t, err, "failed to get rules") // established rule remains require.Len(t, rules, 1, "expected 1 rules after deletion") @@ -182,47 +179,39 @@ func TestNftablesManagerRuleOrder(t *testing.T) { testClient := &nftables.Conn{} // Add accept rule first - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "accept-http") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err, "failed to add accept rule") // Add deny rule second for the same traffic - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop, "deny-http") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err, "failed to add deny rule") err = manager.Flush() require.NoError(t, err, "failed to flush") - rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) require.NoError(t, err, "failed to get rules") t.Logf("Found %d rules in nftables chain", len(rules)) - // Find the accept and deny rules and verify deny comes before accept + // Single-source rules emit a direct payload+cmp on the source IP + // (no set lookup). Match by source-IP + port + verdict instead of + // the legacy per-(action,port) set names ("deny-http"/"accept-http") + // that this test predates. + wantSrc := ip.AsSlice() var acceptRuleIndex, denyRuleIndex = -1, -1 for i, rule := range rules { - hasAcceptHTTPSet := false - hasDenyHTTPSet := false - hasPort80 := false + var hasSrc, hasPort80 bool var action string - for _, e := range rule.Exprs { - // Check for set lookup - if lookup, ok := e.(*expr.Lookup); ok { - switch lookup.SetName { - case "accept-http": - hasAcceptHTTPSet = true - case "deny-http": - hasDenyHTTPSet = true + if cmp, ok := e.(*expr.Cmp); ok && cmp.Op == expr.CmpOpEq { + if bytes.Equal(cmp.Data, wantSrc) { + hasSrc = true } - - } - // Check for port 80 - if cmp, ok := e.(*expr.Cmp); ok { - if cmp.Op == expr.CmpOpEq && len(cmp.Data) == 2 && binary.BigEndian.Uint16(cmp.Data) == 80 { + if len(cmp.Data) == 2 && binary.BigEndian.Uint16(cmp.Data) == 80 { hasPort80 = true } } - // Check for verdict if verdict, ok := e.(*expr.Verdict); ok { switch verdict.Kind { case expr.VerdictAccept: @@ -233,11 +222,15 @@ func TestNftablesManagerRuleOrder(t *testing.T) { } } - if hasAcceptHTTPSet && hasPort80 && action == "ACCEPT" { - t.Logf("Rule [%d]: accept-http set + Port 80 + ACCEPT", i) + if !hasSrc || !hasPort80 { + continue + } + switch action { + case "ACCEPT": + t.Logf("Rule [%d]: src=%s port=80 ACCEPT", i, ip) acceptRuleIndex = i - } else if hasDenyHTTPSet && hasPort80 && action == "DROP" { - t.Logf("Rule [%d]: deny-http set + Port 80 + DROP", i) + case "DROP": + t.Logf("Rule [%d]: src=%s port=80 DROP", i, ip) denyRuleIndex = i } } @@ -281,7 +274,7 @@ func TestNFtablesCreatePerformance(t *testing.T) { start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") if i%100 == 0 { @@ -363,10 +356,10 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) { }) ip := netip.MustParseAddr("100.96.0.1") - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err, "failed to add peer filtering rule") - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("192.168.2.0/24")}, fw.Network{Prefix: netip.MustParsePrefix("10.1.0.0/24")}, @@ -439,10 +432,10 @@ func TestNftablesManagerIPv6CompatibilityWithIp6tables(t *testing.T) { }) ip := netip.MustParseAddr("fd00::2") - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err, "add v6 peer filtering rule") - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("fd00:1::/64")}, fw.Network{Prefix: netip.MustParsePrefix("2001:db8::/48")}, @@ -552,7 +545,7 @@ func TestNftablesManagerCompatibilityWithIptablesFor6kPrefixes(t *testing.T) { prefixes = append(prefixes, netip.PrefixFrom(addr, 24)) } } - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, prefixes, fw.Network{Prefix: netip.MustParsePrefix("10.2.0.0/24")}, @@ -567,7 +560,7 @@ func TestNftablesManagerCompatibilityWithIptablesFor6kPrefixes(t *testing.T) { verifyIptablesOutput(t, stdout, stderr) } -func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) { +func TestNftablesManagerCompatibilityWithIptablesForWildcardSource(t *testing.T) { if check() != NFTABLES { t.Skip("nftables not supported on this system") } @@ -593,9 +586,9 @@ func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) verifyIptablesOutput(t, stdout, stderr) }) - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, - []netip.Prefix{}, + []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Prefix: netip.MustParsePrefix("10.2.0.0/24")}, fw.ProtocolTCP, nil, @@ -608,6 +601,73 @@ func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) verifyIptablesOutput(t, stdout, stderr) } +func TestNftablesManagerMultiPortFilter(t *testing.T) { + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + t.Cleanup(func() { + require.NoError(t, manager.Close(nil), "failed to reset manager state") + }) + + ip := netip.MustParseAddr("100.96.0.1") + + rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80, 443}}, fw.ActionAccept) + require.NoError(t, err, "failed to add multi-port rule") + + testClient := &nftables.Conn{} + rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) + require.NoError(t, err, "failed to get rules") + + var lookup *expr.Lookup + for _, kernelRule := range rules { + if string(kernelRule.UserData) != string(rule.ID()) { + continue + } + for _, e := range kernelRule.Exprs { + if l, ok := e.(*expr.Lookup); ok { + lookup = l + } + } + } + require.NotNil(t, lookup, "multi-port rule must match ports via a set lookup") + + sets, err := testClient.GetSets(manager.family4.workTable) + require.NoError(t, err, "failed to get sets") + + var portSet *nftables.Set + for _, s := range sets { + if s.Name == lookup.SetName { + portSet = s + } + } + require.NotNil(t, portSet, "anonymous port set not found in kernel") + + portSet.Table = manager.family4.workTable + elements, err := testClient.GetSetElements(portSet) + require.NoError(t, err, "failed to get set elements") + + ports := make(map[uint16]bool) + for _, e := range elements { + require.Len(t, e.Key, 2, "port set element key should be 2 bytes") + ports[binary.BigEndian.Uint16(e.Key)] = true + } + require.True(t, ports[80], "port set should contain port 80") + require.True(t, ports[443], "port set should contain port 443") + + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete rule") + + rules, err = testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) + require.NoError(t, err, "failed to get rules after delete") + for _, kernelRule := range rules { + require.NotEqual(t, string(rule.ID()), string(kernelRule.UserData), "rule should be removed from kernel") + } +} + func compareExprsIgnoringCounters(t *testing.T, got, want []expr.Any) { t.Helper() require.Equal(t, len(got), len(want), "expression count mismatch") diff --git a/client/firewall/nftables/router_linux.go b/client/firewall/nftables/router_linux.go deleted file mode 100644 index c79f9b8c2..000000000 --- a/client/firewall/nftables/router_linux.go +++ /dev/null @@ -1,2268 +0,0 @@ -package nftables - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "net" - "net/netip" - "strings" - - "github.com/coreos/go-iptables/iptables" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" - "github.com/google/nftables/xt" - "github.com/hashicorp/go-multierror" - log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - - nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/firewall/firewalld" - firewall "github.com/netbirdio/netbird/client/firewall/manager" - nbid "github.com/netbirdio/netbird/client/internal/acl/id" - "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" - "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - tableNat = "nat" - tableMangle = "mangle" - tableRaw = "raw" - tableSecurity = "security" - - chainNameNatPrerouting = "PREROUTING" - chainNameRoutingFw = "netbird-rt-fwd" - chainNameRoutingNat = "netbird-rt-postrouting" - chainNameRoutingRdr = "netbird-rt-redirect" - chainNameNATOutput = "netbird-nat-output" - chainNameForward = "FORWARD" - chainNameMangleForward = "netbird-mangle-forward" - - firewalldTableName = "firewalld" - - userDataAcceptForwardRuleIif = "frwacceptiif" - userDataAcceptForwardRuleOif = "frwacceptoif" - userDataAcceptInputRule = "inputaccept" - - dnatSuffix = "_dnat" - snatSuffix = "_snat" - - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. - ipv4TCPHeaderSize = 40 - // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. - ipv6TCPHeaderSize = 60 - - // maxPrefixesSet 1638 prefixes start to fail, taking some margin - maxPrefixesSet = 1500 - refreshRulesMapError = "refresh rules map: %w" -) - -var ( - errFilterTableNotFound = fmt.Errorf("'filter' table not found") -) - -type setInput struct { - set firewall.Set - prefixes []netip.Prefix -} - -type router struct { - conn *nftables.Conn - workTable *nftables.Table - filterTable *nftables.Table - chains map[string]*nftables.Chain - // rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules - rules map[string]*nftables.Rule - ipsetCounter *refcounter.Counter[string, setInput, *nftables.Set] - - af addrFamily - wgIface iFaceMapper - ipFwdState *ipfwdstate.IPForwardingState - legacyManagement bool - mtu uint16 -} - -func newRouter(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) (*router, error) { - r := &router{ - conn: &nftables.Conn{}, - workTable: workTable, - chains: make(map[string]*nftables.Chain), - rules: make(map[string]*nftables.Rule), - af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4), - wgIface: wgIface, - ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), - mtu: mtu, - } - - r.ipsetCounter = refcounter.New( - r.createIpSet, - r.deleteIpSet, - ) - - var err error - r.filterTable, err = r.loadFilterTable() - if err != nil { - log.Debugf("ip filter table not found: %v", err) - } - - return r, nil -} - -func (r *router) init(workTable *nftables.Table) error { - r.workTable = workTable - - if err := r.removeAcceptFilterRules(); err != nil { - log.Errorf("failed to clean up rules from filter table: %s", err) - } - - if err := r.createContainers(); err != nil { - return fmt.Errorf("create containers: %w", err) - } - - if err := r.setupDataPlaneMark(); err != nil { - log.Errorf("failed to set up data plane mark: %v", err) - } - - return nil -} - -// Reset cleans existing nftables filter table rules from the system -func (r *router) Reset() error { - // clear without deleting the ipsets, the nf table will be deleted by the caller - r.ipsetCounter.Clear() - - var merr *multierror.Error - - if err := r.removeAcceptFilterRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove accept filter rules: %w", err)) - } - - if err := firewalld.UntrustInterface(r.wgIface.Name()); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.removeNatPreroutingRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove filter prerouting rules: %w", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) removeNatPreroutingRules() error { - table := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - chain := &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: table, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - } - rules, err := r.conn.GetRules(table, chain) - if err != nil { - return fmt.Errorf("get rules from nat table: %w", err) - } - - var merr *multierror.Error - - // Delete rules that have our UserData suffix - for _, rule := range rules { - if len(rule.UserData) == 0 || !strings.HasSuffix(string(rule.UserData), dnatSuffix) { - continue - } - if err := r.conn.DelRule(rule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete rule %s: %w", rule.UserData, err)) - } - } - - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) loadFilterTable() (*nftables.Table, error) { - tables, err := r.conn.ListTablesOfFamily(r.af.tableFamily) - if err != nil { - return nil, fmt.Errorf("list tables: %w", err) - } - - for _, table := range tables { - if table.Name == "filter" { - return table, nil - } - } - - return nil, errFilterTableNotFound -} - -func hookName(hook *nftables.ChainHook) string { - if hook == nil { - return "unknown" - } - switch *hook { - case *nftables.ChainHookForward: - return chainNameForward - case *nftables.ChainHookInput: - return chainNameInput - default: - return fmt.Sprintf("hook(%d)", *hook) - } -} - -func familyName(family nftables.TableFamily) string { - switch family { - case nftables.TableFamilyIPv4: - return "ip" - case nftables.TableFamilyIPv6: - return "ip6" - case nftables.TableFamilyINet: - return "inet" - default: - return fmt.Sprintf("family(%d)", family) - } -} - -func (r *router) createContainers() error { - r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameRoutingFw, - Table: r.workTable, - }) - - prio := *nftables.ChainPriorityNATSource - 1 - r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameRoutingNat, - Table: r.workTable, - Hooknum: nftables.ChainHookPostrouting, - Priority: &prio, - Type: nftables.ChainTypeNAT, - }) - - r.chains[chainNameRoutingRdr] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameRoutingRdr, - Table: r.workTable, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - }) - - r.chains[chainNameManglePostrouting] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameManglePostrouting, - Table: r.workTable, - Hooknum: nftables.ChainHookPostrouting, - Priority: nftables.ChainPriorityMangle, - Type: nftables.ChainTypeFilter, - }) - - r.chains[chainNameManglePrerouting] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameManglePrerouting, - Table: r.workTable, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityMangle, - Type: nftables.ChainTypeFilter, - }) - - r.chains[chainNameMangleForward] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameMangleForward, - Table: r.workTable, - Hooknum: nftables.ChainHookForward, - Priority: nftables.ChainPriorityMangle, - Type: nftables.ChainTypeFilter, - }) - - insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw]) - - r.addPostroutingRules() - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("initialize tables: %v", err) - } - - if err := r.addMSSClampingRules(); err != nil { - log.Errorf("failed to add MSS clamping rules: %s", err) - } - - if err := r.acceptForwardRules(); err != nil { - log.Errorf("failed to add accept rules for the forward chain: %s", err) - } - - if err := firewalld.TrustInterface(r.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - - if err := r.refreshRulesMap(); err != nil { - log.Errorf("failed to refresh rules: %s", err) - } - - return nil -} - -// setupDataPlaneMark configures the fwmark for the data plane -func (r *router) setupDataPlaneMark() error { - if r.chains[chainNameManglePrerouting] == nil || r.chains[chainNameManglePostrouting] == nil { - return errors.New("no mangle chains found") - } - - ctNew := getCtNewExprs() - preExprs := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyIIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - } - preExprs = append(preExprs, ctNew...) - preExprs = append(preExprs, - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkIn), - }, - &expr.Ct{ - Key: expr.CtKeyMARK, - Register: 1, - SourceRegister: true, - }, - ) - - preNftRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameManglePrerouting], - Exprs: preExprs, - } - r.conn.AddRule(preNftRule) - - postExprs := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - } - postExprs = append(postExprs, ctNew...) - postExprs = append(postExprs, - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkOut), - }, - &expr.Ct{ - Key: expr.CtKeyMARK, - Register: 1, - SourceRegister: true, - }, - ) - - postNftRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameManglePostrouting], - Exprs: postExprs, - } - r.conn.AddRule(postNftRule) - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush: %w", err) - } - - return nil -} - -// AddRouteFiltering appends a nftables rule to the routing chain -func (r *router) AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination firewall.Network, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, -) (firewall.Rule, error) { - - ruleKey := nbid.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action) - if _, ok := r.rules[string(ruleKey)]; ok { - return ruleKey, nil - } - - chain := r.chains[chainNameRoutingFw] - var exprs []expr.Any - - var source firewall.Network - switch { - case len(sources) == 1 && sources[0].Bits() == 0: - // If it's 0.0.0.0/0, we don't need to add any source matching - case len(sources) == 1: - // If there's only one source, we can use it directly - source.Prefix = sources[0] - default: - // If there are multiple sources, use a set - source.Set = firewall.NewPrefixSet(sources) - } - - sourceExp, err := r.applyNetwork(source, sources, true) - if err != nil { - return nil, fmt.Errorf("apply source: %w", err) - } - exprs = append(exprs, sourceExp...) - - destExp, err := r.applyNetwork(destination, nil, false) - if err != nil { - return nil, fmt.Errorf("apply destination: %w", err) - } - exprs = append(exprs, destExp...) - - // Handle protocol - if proto != firewall.ProtocolALL { - protoNum, err := r.af.protoNum(proto) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %w", err) - } - exprs = append(exprs, &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}) - exprs = append(exprs, &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }) - - exprs = append(exprs, applyPort(sPort, true)...) - exprs = append(exprs, applyPort(dPort, false)...) - } - - exprs = append(exprs, &expr.Counter{}) - - var verdict expr.VerdictKind - if action == firewall.ActionAccept { - verdict = expr.VerdictAccept - } else { - verdict = expr.VerdictDrop - } - exprs = append(exprs, &expr.Verdict{Kind: verdict}) - - rule := &nftables.Rule{ - Table: r.workTable, - Chain: chain, - Exprs: exprs, - UserData: []byte(ruleKey), - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - if action == firewall.ActionDrop { - // TODO: Insert after the established rule - rule = r.conn.InsertRule(rule) - } else { - rule = r.conn.AddRule(rule) - } - - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf(flushError, err) - } - - r.rules[string(ruleKey)] = rule - - log.Debugf("added route rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", sources, destination, proto, sPort, dPort, action) - - return ruleKey, nil -} - -func (r *router) getIpSet(set firewall.Set, prefixes []netip.Prefix, isSource bool) ([]expr.Any, error) { - ref, err := r.ipsetCounter.Increment(set.HashedName(), setInput{ - set: set, - prefixes: prefixes, - }) - if err != nil { - return nil, fmt.Errorf("create or get ipset: %w", err) - } - - return r.getIpSetExprs(ref, isSource) -} - -func (r *router) iptablesProto() iptables.Protocol { - if r.af.tableFamily == nftables.TableFamilyIPv6 { - return iptables.ProtocolIPv6 - } - return iptables.ProtocolIPv4 -} - -func (r *router) hasRule(id string) bool { - _, ok := r.rules[id] - return ok -} - -func (r *router) hasDNATRule(id string) bool { - _, ok := r.rules[id+dnatSuffix] - return ok -} - -func (r *router) DeleteRouteRule(rule firewall.Rule) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - ruleKey := rule.ID() - nftRule, exists := r.rules[ruleKey] - if !exists { - log.Debugf("route rule %s not found", ruleKey) - return nil - } - - if nftRule.Handle == 0 { - log.Warnf("route rule %s has no handle, removing stale entry", ruleKey) - if err := r.decrementSetCounter(nftRule); err != nil { - log.Warnf("decrement set counter for stale rule %s: %v", ruleKey, err) - } - delete(r.rules, ruleKey) - return nil - } - - if err := r.deleteNftRule(nftRule, ruleKey); err != nil { - return fmt.Errorf("delete: %w", err) - } - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - if err := r.decrementSetCounter(nftRule); err != nil { - return fmt.Errorf("decrement set counter: %w", err) - } - - return nil -} - -func (r *router) createIpSet(setName string, input setInput) (*nftables.Set, error) { - // overlapping prefixes will result in an error, so we need to merge them - prefixes := firewall.MergeIPRanges(input.prefixes) - - nfset := &nftables.Set{ - Name: setName, - Comment: input.set.Comment(), - Table: r.workTable, - // required for prefixes - Interval: true, - KeyType: r.af.setKeyType, - } - - elements := r.convertPrefixesToSet(prefixes) - nElements := len(elements) - - maxElements := maxPrefixesSet * 2 - initialElements := elements[:min(maxElements, nElements)] - - if err := r.conn.AddSet(nfset, initialElements); err != nil { - return nil, fmt.Errorf("error adding set %s: %w", setName, err) - } - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf("flush error: %w", err) - } - log.Debugf("Created new ipset: %s with %d initial prefixes (total prefixes %d)", setName, len(initialElements)/2, len(prefixes)) - - var subEnd int - for subStart := maxElements; subStart < nElements; subStart += maxElements { - subEnd = min(subStart+maxElements, nElements) - subElement := elements[subStart:subEnd] - nSubPrefixes := len(subElement) / 2 - log.Tracef("Adding new prefixes (%d) in ipset: %s", nSubPrefixes, setName) - if err := r.conn.SetAddElements(nfset, subElement); err != nil { - return nil, fmt.Errorf("error adding prefixes (%d) to set %s: %w", nSubPrefixes, setName, err) - } - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf("flush error: %w", err) - } - log.Debugf("Added new prefixes (%d) in ipset: %s", nSubPrefixes, setName) - } - - log.Infof("Created new ipset: %s with %d prefixes", setName, len(prefixes)) - return nfset, nil -} - -func (r *router) convertPrefixesToSet(prefixes []netip.Prefix) []nftables.SetElement { - var elements []nftables.SetElement - for _, prefix := range prefixes { - // nftables needs half-open intervals [firstIP, lastIP) for prefixes - // e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc - firstIP := prefix.Addr() - lastIP := calculateLastIP(prefix).Next() - - elements = append(elements, - // the nft tool also adds a zero-address IntervalEnd element, see https://github.com/google/nftables/issues/247 - // nftables.SetElement{Key: make([]byte, r.af.addrLen), IntervalEnd: true}, - nftables.SetElement{Key: firstIP.AsSlice()}, - nftables.SetElement{Key: lastIP.AsSlice(), IntervalEnd: true}, - ) - } - return elements -} - -// calculateLastIP determines the last IP in a given prefix. -func calculateLastIP(prefix netip.Prefix) netip.Addr { - masked := prefix.Masked() - if masked.Addr().Is4() { - hostMask := ^uint32(0) >> masked.Bits() - lastIP := uint32FromNetipAddr(masked.Addr()) | hostMask - return netip.AddrFrom4(uint32ToBytes(lastIP)) - } - - // IPv6: set host bits to all 1s - b := masked.Addr().As16() - bits := masked.Bits() - for i := bits; i < 128; i++ { - b[i/8] |= 1 << (7 - i%8) - } - return netip.AddrFrom16(b) -} - -// Utility function to convert netip.Addr to uint32. -func uint32FromNetipAddr(addr netip.Addr) uint32 { - b := addr.As4() - return binary.BigEndian.Uint32(b[:]) -} - -// Utility function to convert uint32 to a netip-compatible byte slice. -func uint32ToBytes(ip uint32) [4]byte { - var b [4]byte - binary.BigEndian.PutUint32(b[:], ip) - return b -} - -func (r *router) deleteIpSet(setName string, nfset *nftables.Set) error { - r.conn.DelSet(nfset) - if err := r.conn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - log.Debugf("Deleted unused ipset %s", setName) - return nil -} - -func (r *router) decrementSetCounter(rule *nftables.Rule) error { - sets := r.findSets(rule) - - var merr *multierror.Error - for _, setName := range sets { - if _, err := r.ipsetCounter.Decrement(setName); err != nil { - merr = multierror.Append(merr, fmt.Errorf("decrement set counter: %w", err)) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) findSets(rule *nftables.Rule) []string { - var sets []string - for _, e := range rule.Exprs { - if lookup, ok := e.(*expr.Lookup); ok { - sets = append(sets, lookup.SetName) - } - } - return sets -} - -func (r *router) deleteNftRule(rule *nftables.Rule, ruleKey string) error { - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete rule %s: %w", ruleKey, err) - } - delete(r.rules, ruleKey) - - log.Debugf("removed route rule %s", ruleKey) - - return nil -} - -// AddNatRule appends a nftables rule pair to the nat chain -func (r *router) AddNatRule(pair firewall.RouterPair) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - if r.legacyManagement { - log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) - if err := r.addLegacyRouteRule(pair); err != nil { - return fmt.Errorf("add legacy routing rule: %w", err) - } - } - - if pair.Masquerade { - if err := r.addNatRule(pair); err != nil { - return fmt.Errorf("add nat rule: %w", err) - } - - if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil { - return fmt.Errorf("add inverse nat rule: %w", err) - } - } - - if err := r.conn.Flush(); err != nil { - r.rollbackRules(pair) - return fmt.Errorf("insert rules for %s: %w", pair.Destination, err) - } - - return nil -} - -// rollbackRules cleans up unflushed rules and their set counters after a flush failure. -func (r *router) rollbackRules(pair firewall.RouterPair) { - keys := []string{ - firewall.GenKey(firewall.ForwardingFormat, pair), - firewall.GenKey(firewall.PreroutingFormat, pair), - firewall.GenKey(firewall.PreroutingFormat, firewall.GetInversePair(pair)), - } - for _, key := range keys { - rule, ok := r.rules[key] - if !ok { - continue - } - if err := r.decrementSetCounter(rule); err != nil { - log.Warnf("rollback set counter for %s: %v", key, err) - } - delete(r.rules, key) - } -} - -// addNatRule inserts a nftables rule to the conn client flush queue -func (r *router) addNatRule(pair firewall.RouterPair) error { - sourceExp, err := r.applyNetwork(pair.Source, nil, true) - if err != nil { - return fmt.Errorf("apply source: %w", err) - } - - destExp, err := r.applyNetwork(pair.Destination, nil, false) - if err != nil { - return fmt.Errorf("apply destination: %w", err) - } - - op := expr.CmpOpEq - if pair.Inverse { - op = expr.CmpOpNeq - } - - exprs := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyIIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: op, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - } - // We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading. - // Masquerading will take care of the conntrack state, which means we won't need to mark established connections. - exprs = append(exprs, getCtNewExprs()...) - - exprs = append(exprs, sourceExp...) - exprs = append(exprs, destExp...) - - markValue := nbnet.PreroutingFwmarkMasquerade - if pair.Inverse { - markValue = nbnet.PreroutingFwmarkMasqueradeReturn - } - - exprs = append(exprs, - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(markValue), - }, - &expr.Meta{ - Key: expr.MetaKeyMARK, - SourceRegister: true, - Register: 1, - }, - ) - - ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair) - - if _, exists := r.rules[ruleKey]; exists { - if err := r.removeNatRule(pair); err != nil { - return fmt.Errorf("remove prerouting rule: %w", err) - } - } - - // Ensure nat rules come first, so the mark can be overwritten. - // Currently overwritten by the dst-type LOCAL rules for redirected traffic. - r.rules[ruleKey] = r.conn.InsertRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameManglePrerouting], - Exprs: exprs, - UserData: []byte(ruleKey), - }) - - return nil -} - -// addPostroutingRules adds the masquerade rules -func (r *router) addPostroutingRules() { - // First masquerade rule for traffic coming in from WireGuard interface - exprs := []expr.Any{ - // Match on the first fwmark - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), - }, - - // We need to exclude the loopback interface as this changes the ebpf proxy port - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: ifname("lo"), - }, - &expr.Counter{}, - &expr.Masq{}, - } - - r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: exprs, - }) - - // Second masquerade rule for traffic going out through WireGuard interface - exprs2 := []expr.Any{ - // Match on the second fwmark - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn), - }, - - // Match WireGuard interface - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Counter{}, - &expr.Masq{}, - } - - r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: exprs2, - }) -} - -// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. -func (r *router) addMSSClampingRules() error { - overhead := uint16(ipv4TCPHeaderSize) - if r.af.tableFamily == nftables.TableFamilyIPv6 { - overhead = ipv6TCPHeaderSize - } - if r.mtu <= overhead { - log.Debugf("MTU %d too small for MSS clamping (overhead %d), skipping", r.mtu, overhead) - return nil - } - mss := r.mtu - overhead - - exprsOut := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{ - Key: expr.MetaKeyL4PROTO, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{unix.IPPROTO_TCP}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 13, - Len: 1, - }, - &expr.Bitwise{ - DestRegister: 1, - SourceRegister: 1, - Len: 1, - Mask: []byte{0x02}, - Xor: []byte{0x00}, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: []byte{0x00}, - }, - &expr.Counter{}, - &expr.Exthdr{ - DestRegister: 1, - Type: 2, - Offset: 2, - Len: 2, - Op: expr.ExthdrOpTcpopt, - }, - &expr.Cmp{ - Op: expr.CmpOpGt, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(uint16(mss)), - }, - &expr.Immediate{ - Register: 1, - Data: binaryutil.BigEndian.PutUint16(uint16(mss)), - }, - &expr.Exthdr{ - SourceRegister: 1, - Type: 2, - Offset: 2, - Len: 2, - Op: expr.ExthdrOpTcpopt, - }, - } - - r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameMangleForward], - Exprs: exprsOut, - }) - - return r.conn.Flush() -} - -func buildLegacyRouteRuleExpressions(sourceExp, destExp []expr.Any) []expr.Any { - exprs := make([]expr.Any, 0, len(sourceExp)+len(destExp)+2) - exprs = append(exprs, sourceExp...) - exprs = append(exprs, destExp...) - exprs = append(exprs, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictAccept}, - ) - return exprs -} - -// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls -func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error { - sourceExp, err := r.applyNetwork(pair.Source, nil, true) - if err != nil { - return fmt.Errorf("apply source: %w", err) - } - - destExp, err := r.applyNetwork(pair.Destination, nil, false) - if err != nil { - return fmt.Errorf("apply destination: %w", err) - } - - exprs := buildLegacyRouteRuleExpressions(sourceExp, destExp) - - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - if _, exists := r.rules[ruleKey]; exists { - if err := r.removeLegacyRouteRule(pair); err != nil { - return fmt.Errorf("remove legacy routing rule: %w", err) - } - } - - r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingFw], - Exprs: exprs, - UserData: []byte(ruleKey), - }) - return nil -} - -// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls -func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - rule, exists := r.rules[ruleKey] - if !exists { - return nil - } - - if rule.Handle == 0 { - log.Warnf("legacy forwarding rule %s has no handle, removing stale entry", ruleKey) - if err := r.decrementSetCounter(rule); err != nil { - log.Warnf("decrement set counter for stale rule %s: %v", ruleKey, err) - } - delete(r.rules, ruleKey) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("remove legacy forwarding rule %s -> %s: %w", pair.Source, pair.Destination, err) - } - - log.Debugf("removed legacy forwarding rule %s -> %s", pair.Source, pair.Destination) - - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement set counter: %w", err) - } - - return nil -} - -// GetLegacyManagement returns the route manager's legacy management mode -func (r *router) GetLegacyManagement() bool { - return r.legacyManagement -} - -// SetLegacyManagement sets the route manager to use legacy management mode -func (r *router) SetLegacyManagement(isLegacy bool) { - r.legacyManagement = isLegacy -} - -// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls -func (r *router) RemoveAllLegacyRouteRules() error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - for k, rule := range r.rules { - if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) { - continue - } - if err := r.conn.DelRule(rule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err)) - } else { - delete(r.rules, k) - } - - } - return nberrors.FormatErrorOrNil(merr) -} - -// acceptForwardRules adds iif/oif rules in the filter table/forward chain to make sure -// that our traffic is not dropped by existing rules there. -// The existing FORWARD rules/policies decide outbound traffic towards our interface. -// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule. -// This method also adds INPUT chain rules to allow traffic to the local interface. -func (r *router) acceptForwardRules() error { - var merr *multierror.Error - - if err := r.acceptFilterTableRules(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.acceptExternalChainsRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add accept rules to external chains: %w", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) acceptFilterTableRules() error { - if r.filterTable == nil { - return nil - } - - fw := "iptables" - - defer func() { - log.Debugf("Used %s to add accept forward and input rules", fw) - }() - - // Try iptables first and fallback to nftables if iptables is not available. - // Use the correct protocol (iptables vs ip6tables) for the address family. - ipt, err := iptables.NewWithProtocol(r.iptablesProto()) - if err != nil { - log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err) - - fw = "nftables" - return r.acceptFilterRulesNftables(r.filterTable) - } - - if err := r.acceptFilterRulesIptables(ipt); err != nil { - log.Warnf("iptables failed (table may be incompatible), falling back to nftables: %v", err) - fw = "nftables" - return r.acceptFilterRulesNftables(r.filterTable) - } - return nil -} - -func (r *router) acceptFilterRulesIptables(ipt *iptables.IPTables) error { - var merr *multierror.Error - - for _, rule := range r.getAcceptForwardRules() { - if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add iptables forward rule: %v", err)) - } else { - log.Debugf("added iptables forward rule: %v", rule) - } - } - - inputRule := r.getAcceptInputRule() - if err := ipt.Insert("filter", chainNameInput, 1, inputRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add iptables input rule: %v", err)) - } else { - log.Debugf("added iptables input rule: %v", inputRule) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) getAcceptForwardRules() [][]string { - intf := r.wgIface.Name() - return [][]string{ - {"-i", intf, "-j", "ACCEPT"}, - {"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, - } -} - -func (r *router) getAcceptInputRule() []string { - return []string{"-i", r.wgIface.Name(), "-j", "ACCEPT"} -} - -// acceptFilterRulesNftables adds accept rules to the ip filter table using nftables. -// This is used when iptables is not available. -func (r *router) acceptFilterRulesNftables(table *nftables.Table) error { - intf := ifname(r.wgIface.Name()) - - forwardChain := &nftables.Chain{ - Name: chainNameForward, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookForward, - Priority: nftables.ChainPriorityFilter, - } - r.insertForwardAcceptRules(forwardChain, intf) - - inputChain := &nftables.Chain{ - Name: chainNameInput, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookInput, - Priority: nftables.ChainPriorityFilter, - } - r.insertInputAcceptRule(inputChain, intf) - - return r.conn.Flush() -} - -// acceptExternalChainsRules adds accept rules to external chains (non-netbird, non-iptables tables). -// It dynamically finds chains at call time to handle chains that may have been created after startup. -func (r *router) acceptExternalChainsRules() error { - chains := r.findExternalChains() - if len(chains) == 0 { - return nil - } - - intf := ifname(r.wgIface.Name()) - for _, chain := range chains { - r.applyExternalChainAccept(chain, intf) - } - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush external chain rules: %w", err) - } - return nil -} - -func (r *router) applyExternalChainAccept(chain *nftables.Chain, intf []byte) { - if chain.Hooknum == nil { - log.Debugf("skipping external chain %s/%s: hooknum is nil", chain.Table.Name, chain.Name) - return - } - - log.Debugf("adding accept rules to external %s chain: %s %s/%s", - hookName(chain.Hooknum), familyName(chain.Table.Family), chain.Table.Name, chain.Name) - - switch *chain.Hooknum { - case *nftables.ChainHookForward: - r.insertForwardAcceptRules(chain, intf) - case *nftables.ChainHookInput: - r.insertInputAcceptRule(chain, intf) - } -} - -func (r *router) insertForwardAcceptRules(chain *nftables.Chain, intf []byte) { - existing, err := r.existingNetbirdRulesInChain(chain) - if err != nil { - log.Warnf("skip forward accept rules in %s/%s: %v", chain.Table.Name, chain.Name, err) - return - } - r.insertForwardIifRule(chain, intf, existing) - r.insertForwardOifEstablishedRule(chain, intf, existing) -} - -func (r *router) insertForwardIifRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { - if existing[userDataAcceptForwardRuleIif] { - return - } - r.conn.InsertRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - UserData: []byte(userDataAcceptForwardRuleIif), - }) -} - -func (r *router) insertForwardOifEstablishedRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { - if existing[userDataAcceptForwardRuleOif] { - return - } - exprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, - } - r.conn.InsertRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: append(exprs, getEstablishedExprs(2)...), - UserData: []byte(userDataAcceptForwardRuleOif), - }) -} - -func (r *router) insertInputAcceptRule(chain *nftables.Chain, intf []byte) { - existing, err := r.existingNetbirdRulesInChain(chain) - if err != nil { - log.Warnf("skip input accept rule in %s/%s: %v", chain.Table.Name, chain.Name, err) - return - } - if existing[userDataAcceptInputRule] { - return - } - r.conn.InsertRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - UserData: []byte(userDataAcceptInputRule), - }) -} - -// existingNetbirdRulesInChain returns the set of netbird-owned UserData tags present in a chain; callers must bail on error since InsertRule is additive. -func (r *router) existingNetbirdRulesInChain(chain *nftables.Chain) (map[string]bool, error) { - rules, err := r.conn.GetRules(chain.Table, chain) - if err != nil { - return nil, fmt.Errorf("list rules: %w", err) - } - present := map[string]bool{} - for _, rule := range rules { - if !isNetbirdAcceptRuleTag(rule.UserData) { - continue - } - present[string(rule.UserData)] = true - } - return present, nil -} - -func isNetbirdAcceptRuleTag(userData []byte) bool { - switch string(userData) { - case userDataAcceptForwardRuleIif, - userDataAcceptForwardRuleOif, - userDataAcceptInputRule: - return true - } - return false -} - -func (r *router) removeAcceptFilterRules() error { - var merr *multierror.Error - - if err := r.removeFilterTableRules(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.removeExternalChainsRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove external chain rules: %w", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) removeFilterTableRules() error { - if r.filterTable == nil { - return nil - } - - ipt, err := iptables.NewWithProtocol(r.iptablesProto()) - if err != nil { - log.Debugf("iptables not available, using nftables to remove filter rules: %v", err) - return r.removeAcceptRulesFromTable(r.filterTable) - } - - if err := r.removeAcceptFilterRulesIptables(ipt); err != nil { - log.Debugf("iptables removal failed (table may be incompatible), falling back to nftables: %v", err) - return r.removeAcceptRulesFromTable(r.filterTable) - } - return nil -} - -func (r *router) removeAcceptRulesFromTable(table *nftables.Table) error { - chains, err := r.conn.ListChainsOfTableFamily(table.Family) - if err != nil { - return fmt.Errorf("list chains: %v", err) - } - - for _, chain := range chains { - if chain.Table.Name != table.Name { - continue - } - - if chain.Name != chainNameForward && chain.Name != chainNameInput { - continue - } - - if err := r.removeAcceptRulesFromChain(table, chain); err != nil { - return err - } - } - - return r.conn.Flush() -} - -func (r *router) removeAcceptRulesFromChain(table *nftables.Table, chain *nftables.Chain) error { - rules, err := r.conn.GetRules(table, chain) - if err != nil { - return fmt.Errorf("get rules from %s/%s: %v", table.Name, chain.Name, err) - } - - for _, rule := range rules { - if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) || - bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) || - bytes.Equal(rule.UserData, []byte(userDataAcceptInputRule)) { - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete rule from %s/%s: %v", table.Name, chain.Name, err) - } - } - } - return nil -} - -// removeExternalChainsRules removes our accept rules from all external chains. -// This is deterministic - it scans for chains at removal time rather than relying on saved state, -// ensuring cleanup works even after a crash or if chains changed. -func (r *router) removeExternalChainsRules() error { - chains := r.findExternalChains() - if len(chains) == 0 { - return nil - } - - for _, chain := range chains { - if err := r.removeAcceptRulesFromChain(chain.Table, chain); err != nil { - log.Warnf("remove rules from external chain %s/%s: %v", chain.Table.Name, chain.Name, err) - } - } - - return r.conn.Flush() -} - -// findExternalChains scans for chains from non-netbird tables that have FORWARD or INPUT hooks. -// This is used both at startup (to know where to add rules) and at cleanup (to ensure deterministic removal). -func (r *router) findExternalChains() []*nftables.Chain { - var chains []*nftables.Chain - - families := []nftables.TableFamily{r.af.tableFamily, nftables.TableFamilyINet} - - for _, family := range families { - allChains, err := r.conn.ListChainsOfTableFamily(family) - if err != nil { - log.Debugf("list chains for family %d: %v", family, err) - continue - } - - for _, chain := range allChains { - if r.isExternalChain(chain) { - chains = append(chains, chain) - } - } - } - - return chains -} - -func (r *router) isExternalChain(chain *nftables.Chain) bool { - if r.workTable != nil && chain.Table.Name == r.workTable.Name { - return false - } - - // Skip firewalld-owned chains. Firewalld creates its chains with the - // NFT_CHAIN_OWNER flag, so inserting rules into them returns EPERM. - // We delegate acceptance to firewalld by trusting the interface instead. - if chain.Table.Name == firewalldTableName { - return false - } - - // Skip iptables/ip6tables-managed tables (adding nft-native rules breaks iptables-save compat) - if (chain.Table.Family == nftables.TableFamilyIPv4 || chain.Table.Family == nftables.TableFamilyIPv6) && isIptablesTable(chain.Table.Name) { - return false - } - - if chain.Type != nftables.ChainTypeFilter { - return false - } - - if chain.Hooknum == nil { - return false - } - - return *chain.Hooknum == *nftables.ChainHookForward || *chain.Hooknum == *nftables.ChainHookInput -} - -func isIptablesTable(name string) bool { - switch name { - case tableNameFilter, tableNat, tableMangle, tableRaw, tableSecurity: - return true - } - return false -} - -func (r *router) removeAcceptFilterRulesIptables(ipt *iptables.IPTables) error { - var merr *multierror.Error - - for _, rule := range r.getAcceptForwardRules() { - if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove iptables forward rule: %v", err)) - } - } - - inputRule := r.getAcceptInputRule() - if err := ipt.DeleteIfExists("filter", chainNameInput, inputRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove iptables input rule: %v", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -// RemoveNatRule removes the prerouting mark rule -func (r *router) RemoveNatRule(pair firewall.RouterPair) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - - if pair.Masquerade { - if err := r.removeNatRule(pair); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove prerouting rule: %w", err)) - } - - if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove inverse prerouting rule: %w", err)) - } - } - - if err := r.removeLegacyRouteRule(pair); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove legacy routing rule: %w", err)) - } - - // Set counters are decremented in the sub-methods above before flush. If flush fails, - // counters will be off until the next successful removal or refresh cycle. - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("flush remove nat rules %s: %w", pair.Destination, err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) removeNatRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair) - - rule, exists := r.rules[ruleKey] - if !exists { - log.Debugf("prerouting rule %s not found", ruleKey) - return nil - } - - if rule.Handle == 0 { - log.Warnf("prerouting rule %s has no handle, removing stale entry", ruleKey) - if err := r.decrementSetCounter(rule); err != nil { - log.Warnf("decrement set counter for stale rule %s: %v", ruleKey, err) - } - delete(r.rules, ruleKey) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("remove prerouting rule %s -> %s: %w", pair.Source, pair.Destination, err) - } - - log.Debugf("removed prerouting rule %s -> %s", pair.Source, pair.Destination) - - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement set counter: %w", err) - } - - return nil -} - -// refreshRulesMap rebuilds the rule map from the kernel. This removes stale entries -// (e.g. from failed flushes) and updates handles for all existing rules. -func (r *router) refreshRulesMap() error { - var merr *multierror.Error - newRules := make(map[string]*nftables.Rule) - for _, chain := range r.chains { - rules, err := r.conn.GetRules(chain.Table, chain) - if err != nil { - merr = multierror.Append(merr, fmt.Errorf("list rules for chain %s: %w", chain.Name, err)) - // preserve existing entries for this chain since we can't verify their state - for k, v := range r.rules { - if v.Chain != nil && v.Chain.Name == chain.Name { - newRules[k] = v - } - } - continue - } - for _, rule := range rules { - if len(rule.UserData) > 0 { - newRules[string(rule.UserData)] = rule - } - } - } - r.rules = newRules - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - ruleKey := rule.ID() - if _, exists := r.rules[ruleKey+dnatSuffix]; exists { - return rule, nil - } - - protoNum, err := r.af.protoNum(rule.Protocol) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %w", err) - } - - // Request forwarding before queueing rules: addDnatRedirect/addDnatMasq - // buffer netlink messages on r.conn that the next caller's Flush would - // commit if we returned without flushing them ourselves. - v6 := r.af.tableFamily == nftables.TableFamilyIPv6 - if err := r.ipFwdState.RequestForwarding(v6); err != nil { - return nil, fmt.Errorf("enable forwarding: %w", err) - } - - if err := r.addDnatRedirect(rule, protoNum, ruleKey); err != nil { - if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil { - log.Warnf("rollback forwarding refcount: %v", rerr) - } - return nil, err - } - - r.addDnatMasq(rule, protoNum, ruleKey) - - // Unlike iptables, there's no point in adding "out" rules in the forward chain here as our policy is ACCEPT. - // To overcome DROP policies in other chains, we'd have to add rules to the chains there. - // We also cannot just add "oif accept" there and filter in our own table as we don't know what is supposed to be allowed. - // TODO: find chains with drop policies and add rules there - - if err := r.conn.Flush(); err != nil { - if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil { - log.Warnf("rollback forwarding refcount: %v", rerr) - } - delete(r.rules, ruleKey+dnatSuffix) - delete(r.rules, ruleKey+snatSuffix) - return nil, fmt.Errorf("flush rules: %w", err) - } - - return &rule, nil -} - -func (r *router) addDnatRedirect(rule firewall.ForwardRule, protoNum uint8, ruleKey string) error { - dnatExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - } - dnatExprs = append(dnatExprs, applyPort(&rule.DestinationPort, false)...) - - // shifted translated port is not supported in nftables, so we hand this over to xtables - if rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2 { - if rule.TranslatedPort.Values[0] != rule.DestinationPort.Values[0] || - rule.TranslatedPort.Values[1] != rule.DestinationPort.Values[1] { - return r.addXTablesRedirect(dnatExprs, ruleKey, rule) - } - } - - additionalExprs, regProtoMin, regProtoMax, err := r.handleTranslatedPort(rule) - if err != nil { - return err - } - dnatExprs = append(dnatExprs, additionalExprs...) - - dnatExprs = append(dnatExprs, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: regProtoMin, - RegProtoMax: regProtoMax, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingRdr], - Exprs: dnatExprs, - UserData: []byte(ruleKey + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleKey+dnatSuffix] = dnatRule - - return nil -} - -func (r *router) handleTranslatedPort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - switch { - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - return r.handlePortRange(rule) - case len(rule.TranslatedPort.Values) == 0: - return r.handleAddressOnly(rule) - case len(rule.TranslatedPort.Values) == 1: - return r.handleSinglePort(rule) - default: - return nil, 0, 0, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } -} - -func (r *router) handlePortRange(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - &expr.Immediate{ - Register: 3, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[1]), - }, - } - return exprs, 2, 3, nil -} - -func (r *router) handleAddressOnly(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - return exprs, 0, 0, nil -} - -func (r *router) handleSinglePort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - } - return exprs, 2, 0, nil -} - -func (r *router) addXTablesRedirect(dnatExprs []expr.Any, ruleKey string, rule firewall.ForwardRule) error { - dnatExprs = append(dnatExprs, - &expr.Counter{}, - &expr.Target{ - Name: "DNAT", - Rev: 2, - Info: &xt.NatRange2{ - NatRange: xt.NatRange{ - Flags: uint(xt.NatRangeMapIPs | xt.NatRangeProtoSpecified | xt.NatRangeProtoOffset), - MinIP: rule.TranslatedAddress.AsSlice(), - MaxIP: rule.TranslatedAddress.AsSlice(), - MinPort: rule.TranslatedPort.Values[0], - MaxPort: rule.TranslatedPort.Values[1], - }, - BasePort: rule.DestinationPort.Values[0], - }, - }, - ) - - natTable := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - dnatRule := &nftables.Rule{ - Table: natTable, - Chain: &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: natTable, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - }, - Exprs: dnatExprs, - UserData: []byte(ruleKey + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleKey+dnatSuffix] = dnatRule - - return nil -} - -func (r *router) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleKey string) { - masqExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: r.af.dstAddrOffset, - Len: r.af.addrLen, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - - masqExprs = append(masqExprs, applyPort(&rule.TranslatedPort, false)...) - masqExprs = append(masqExprs, &expr.Masq{}) - - masqRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: masqExprs, - UserData: []byte(ruleKey + snatSuffix), - } - r.conn.AddRule(masqRule) - r.rules[ruleKey+snatSuffix] = masqRule -} - -func (r *router) DeleteDNATRule(rule firewall.Rule) error { - ruleKey := rule.ID() - - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - _, hadDNAT := r.rules[ruleKey+dnatSuffix] - _, hadSNAT := r.rules[ruleKey+snatSuffix] - if !hadDNAT && !hadSNAT { - return nil - } - - var merr *multierror.Error - var needsFlush bool - - if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists { - if dnatRule.Handle == 0 { - log.Warnf("dnat rule %s has no handle, removing stale entry", ruleKey+dnatSuffix) - delete(r.rules, ruleKey+dnatSuffix) - } else if err := r.conn.DelRule(dnatRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete dnat rule: %w", err)) - } else { - needsFlush = true - } - } - - if masqRule, exists := r.rules[ruleKey+snatSuffix]; exists { - if masqRule.Handle == 0 { - log.Warnf("snat rule %s has no handle, removing stale entry", ruleKey+snatSuffix) - delete(r.rules, ruleKey+snatSuffix) - } else if err := r.conn.DelRule(masqRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete snat rule: %w", err)) - } else { - needsFlush = true - } - } - - if needsFlush { - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - } - - // Release the refcount only once the rules are gone from the kernel. On - // failure (including the refreshRulesMap error above) the rules and their - // map entries remain, keeping forwarding on until a retry removes them. - if merr == nil { - delete(r.rules, ruleKey+dnatSuffix) - delete(r.rules, ruleKey+snatSuffix) - - if err := r.ipFwdState.ReleaseForwarding(r.af.tableFamily == nftables.TableFamilyIPv6); err != nil { - log.Errorf("%v", err) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { - nfset, err := r.conn.GetSetByName(r.workTable, set.HashedName()) - if err != nil { - return fmt.Errorf("get set %s: %w", set.HashedName(), err) - } - - elements := r.convertPrefixesToSet(prefixes) - if err := r.conn.SetAddElements(nfset, elements); err != nil { - return fmt.Errorf("add elements to set %s: %w", set.HashedName(), err) - } - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - log.Debugf("updated set %s with prefixes %v", set.HashedName(), prefixes) - - return nil -} - -// AddInboundDNAT adds an inbound DNAT rule redirecting traffic from NetBird peers to local services. -func (r *router) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - protoNum, err := r.af.protoNum(protocol) - if err != nil { - return fmt.Errorf("convert protocol to number: %w", err) - } - - exprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 2}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 2, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 3, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 3, - Data: binaryutil.BigEndian.PutUint16(originalPort), - }, - } - - bits := 32 - if localAddr.Is6() { - bits = 128 - } - exprs = append(exprs, r.applyPrefix(netip.PrefixFrom(localAddr, bits), false)...) - - exprs = append(exprs, - &expr.Immediate{ - Register: 1, - Data: localAddr.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(translatedPort), - }, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: 2, - RegProtoMax: 0, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingRdr], - Exprs: exprs, - UserData: []byte(ruleID), - } - r.conn.AddRule(dnatRule) - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("add inbound DNAT rule: %w", err) - } - - r.rules[ruleID] = dnatRule - - return nil -} - -// RemoveInboundDNAT removes an inbound DNAT rule. -func (r *router) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - rule, exists := r.rules[ruleID] - if !exists { - return nil - } - - if rule.Handle == 0 { - log.Warnf("inbound DNAT rule %s has no handle, removing stale entry", ruleID) - delete(r.rules, ruleID) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete inbound DNAT rule %s: %w", ruleID, err) - } - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush delete inbound DNAT rule: %w", err) - } - delete(r.rules, ruleID) - - return nil -} - -// ensureNATOutputChain lazily creates the OUTPUT NAT chain on first use. -func (r *router) ensureNATOutputChain() error { - if _, exists := r.chains[chainNameNATOutput]; exists { - return nil - } - - r.chains[chainNameNATOutput] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameNATOutput, - Table: r.workTable, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - }) - - if err := r.conn.Flush(); err != nil { - delete(r.chains, chainNameNATOutput) - return fmt.Errorf("create NAT output chain: %w", err) - } - return nil -} - -// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. -func (r *router) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - if err := r.ensureNATOutputChain(); err != nil { - return err - } - - protoNum, err := r.af.protoNum(protocol) - if err != nil { - return fmt.Errorf("convert protocol to number: %w", err) - } - - exprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 2, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 2, - Data: binaryutil.BigEndian.PutUint16(originalPort), - }, - } - - bits := 32 - if localAddr.Is6() { - bits = 128 - } - exprs = append(exprs, r.applyPrefix(netip.PrefixFrom(localAddr, bits), false)...) - - exprs = append(exprs, - &expr.Immediate{ - Register: 1, - Data: localAddr.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(translatedPort), - }, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: 2, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameNATOutput], - Exprs: exprs, - UserData: []byte(ruleID), - } - r.conn.AddRule(dnatRule) - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("add output DNAT rule: %w", err) - } - - r.rules[ruleID] = dnatRule - - return nil -} - -// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. -func (r *router) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - rule, exists := r.rules[ruleID] - if !exists { - return nil - } - - if rule.Handle == 0 { - log.Warnf("output DNAT rule %s has no handle, removing stale entry", ruleID) - delete(r.rules, ruleID) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete output DNAT rule %s: %w", ruleID, err) - } - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush delete output DNAT rule: %w", err) - } - delete(r.rules, ruleID) - - return nil -} - -// applyNetwork generates nftables expressions for networks (CIDR) or sets -func (r *router) applyNetwork( - network firewall.Network, - setPrefixes []netip.Prefix, - isSource bool, -) ([]expr.Any, error) { - if network.IsSet() { - exprs, err := r.getIpSet(network.Set, setPrefixes, isSource) - if err != nil { - return nil, fmt.Errorf("source: %w", err) - } - return exprs, nil - } - - if network.IsPrefix() { - return r.applyPrefix(network.Prefix, isSource), nil - } - - return nil, nil -} - -// applyPrefix generates nftables expressions for a CIDR prefix -func (r *router) applyPrefix(prefix netip.Prefix, isSource bool) []expr.Any { - // dst offset by default - offset := r.af.dstAddrOffset - if isSource { - // src offset - offset = r.af.srcAddrOffset - } - - ones := prefix.Bits() - // unspecified address (/0) doesn't need extra expressions - if ones == 0 { - return nil - } - - mask := net.CIDRMask(ones, r.af.totalBits) - xor := make([]byte, r.af.addrLen) - - return []expr.Any{ - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: offset, - Len: r.af.addrLen, - }, - &expr.Bitwise{ - DestRegister: 1, - SourceRegister: 1, - Len: r.af.addrLen, - Mask: mask, - Xor: xor, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: prefix.Masked().Addr().AsSlice(), - }, - } -} - -func applyPort(port *firewall.Port, isSource bool) []expr.Any { - if port == nil { - return nil - } - - var exprs []expr.Any - - offset := uint32(2) // Default offset for destination port - if isSource { - offset = 0 // Offset for source port - } - - exprs = append(exprs, &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: offset, - Len: 2, - }) - - if port.IsRange && len(port.Values) == 2 { - // Handle port range - exprs = append(exprs, - &expr.Range{ - Op: expr.CmpOpEq, - Register: 1, - FromData: binaryutil.BigEndian.PutUint16(port.Values[0]), - ToData: binaryutil.BigEndian.PutUint16(port.Values[1]), - }, - ) - } else { - // Handle single port or multiple ports - for i, p := range port.Values { - if i > 0 { - // Add a bitwise OR operation between port checks - exprs = append(exprs, &expr.Bitwise{ - SourceRegister: 1, - DestRegister: 1, - Len: 4, - Mask: []byte{0x00, 0x00, 0xff, 0xff}, - Xor: []byte{0x00, 0x00, 0x00, 0x00}, - }) - } - exprs = append(exprs, &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(p), - }) - } - } - - return exprs -} - -func getCtNewExprs() []expr.Any { - return []expr.Any{ - &expr.Ct{ - Key: expr.CtKeySTATE, - Register: 1, - }, - &expr.Bitwise{ - SourceRegister: 1, - DestRegister: 1, - Len: 4, - Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW), - Xor: binaryutil.NativeEndian.PutUint32(0), - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: []byte{0, 0, 0, 0}, - }, - } -} - -func (r *router) getIpSetExprs(ref refcounter.Ref[*nftables.Set], isSource bool) ([]expr.Any, error) { - // dst offset by default - offset := r.af.dstAddrOffset - if isSource { - // src offset - offset = r.af.srcAddrOffset - } - - return []expr.Any{ - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: offset, - Len: r.af.addrLen, - }, - &expr.Lookup{ - SourceRegister: 1, - SetName: ref.Out.Name, - SetID: ref.Out.ID, - }, - }, nil -} diff --git a/client/firewall/nftables/router_linux_test.go b/client/firewall/nftables/router_linux_test.go index 2fc664d51..49dbfc8f2 100644 --- a/client/firewall/nftables/router_linux_test.go +++ b/client/firewall/nftables/router_linux_test.go @@ -37,7 +37,7 @@ func TestNftablesManager_AddNatRule(t *testing.T) { for _, testCase := range test.InsertRuleTestCases { t.Run(testCase.Name, func(t *testing.T) { - // need fw manager to init both acl mgr and router for all chains to be present + // need fw manager to init both acl mgr and family for all chains to be present manager, err := Create(ifaceMock, iface.DefaultMTU) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -47,7 +47,7 @@ func TestNftablesManager_AddNatRule(t *testing.T) { nftablesTestingClient := &nftables.Conn{} - rtr := manager.router + rtr := manager.family4 err = rtr.AddNatRule(testCase.InputPair) require.NoError(t, err, "pair should be inserted") @@ -90,9 +90,9 @@ func TestNftablesManager_AddNatRule(t *testing.T) { } // Build CIDR matching expressions - testRouter := &router{af: afIPv4} - sourceExp := testRouter.applyPrefix(testCase.InputPair.Source.Prefix, true) - destExp := testRouter.applyPrefix(testCase.InputPair.Destination.Prefix, false) + testRouter := &family{af: afIPv4} + sourceExp := prefixMatchExprs(testRouter.af, testCase.InputPair.Source.Prefix, true) + destExp := prefixMatchExprs(testRouter.af, testCase.InputPair.Destination.Prefix, false) // Combine all expressions in the correct order // nolint:gocritic @@ -100,14 +100,14 @@ func TestNftablesManager_AddNatRule(t *testing.T) { testingExpression = append(testingExpression, sourceExp...) testingExpression = append(testingExpression, destExp...) - natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.PreroutingFormat) found := 0 for _, chain := range rtr.chains { if chain.Name == chainNameManglePrerouting { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { // Compare expressions up to the mark setting expressions require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "prerouting nat rule elements should match") found = 1 @@ -135,19 +135,19 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) { require.NoError(t, err) require.NoError(t, manager.Init(nil)) - rtr := manager.router + rtr := manager.family4 - // First add the NAT rule using the router's method + // First add the NAT rule using the family's method err = rtr.AddNatRule(testCase.InputPair) require.NoError(t, err, "should add NAT rule") // Verify the rule was added - natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.PreroutingFormat) found := false rules, err := rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameManglePrerouting]) require.NoError(t, err, "should list rules") for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { found = true break } @@ -163,7 +163,7 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) { rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameManglePrerouting]) require.NoError(t, err, "should list rules after removal") for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { found = true break } @@ -200,11 +200,10 @@ func TestRouter_AddRouteFiltering(t *testing.T) { defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router") + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) - defer func(r *router) { + defer func(r *family) { require.NoError(t, r.Reset(), "Failed to reset rules") }(r) @@ -314,16 +313,16 @@ func TestRouter_AddRouteFiltering(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ruleKey, err := r.AddRouteFiltering(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) - require.NoError(t, err, "AddRouteFiltering failed") + ruleKey, err := r.AddFilterRule(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) + require.NoError(t, err, "AddFilterRule failed") t.Cleanup(func() { - require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule") + require.NoError(t, r.DeleteFilterRule(ruleKey), "Failed to delete rule") }) - // Check if the rule is in the internal map - rule, ok := r.rules[ruleKey.ID()] - assert.True(t, ok, "Rule not found in internal map") + stored, ok := r.filters[id.RuleID(ruleKey.ID())] + require.True(t, ok, "Rule not found in filters map") + rule := stored.nftRule t.Log("Internal rule expressions:") for i, expr := range rule.Exprs { @@ -339,7 +338,7 @@ func TestRouter_AddRouteFiltering(t *testing.T) { var nftRule *nftables.Rule for _, rule := range rules { - if string(rule.UserData) == ruleKey.ID() { + if firewall.RuleID(rule.UserData) == ruleKey.ID() { nftRule = rule break } @@ -367,12 +366,11 @@ func TestNftablesCreateIpSet(t *testing.T) { defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router") + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { - require.NoError(t, r.Reset(), "Failed to reset router") + require.NoError(t, r.Reset(), "Failed to reset family") }() tests := []struct { @@ -509,6 +507,58 @@ func TestNftablesCreateIpSet(t *testing.T) { } } +// TestNftablesUpdateSetMergesOverlapping verifies that UpdateSet merges +// overlapping prefixes before adding them. An interval set rejects +// overlapping elements, so without the merge a batch holding a /32 already +// covered by a /24, or a duplicate address as DNS resolution can produce, +// would fail. +func TestNftablesUpdateSetMergesOverlapping(t *testing.T) { + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + + workTable, err := createWorkTable() + require.NoError(t, err, "create work table") + defer deleteWorkTable() + + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) + require.NoError(t, r.init(workTable)) + defer func() { + require.NoError(t, r.Reset(), "reset family") + }() + + initial := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")} + set := firewall.NewPrefixSet(initial) + + created, err := r.createIpSet(set.HashedName(), setInput{prefixes: initial}) + require.NoError(t, err, "create ip set") + require.NotNil(t, created) + + overlapping := []netip.Prefix{ + netip.MustParsePrefix("192.168.1.0/24"), + netip.MustParsePrefix("192.168.1.1/32"), + netip.MustParsePrefix("192.168.1.1/32"), + } + require.NoError(t, r.UpdateSet(set, overlapping), "UpdateSet must merge overlapping prefixes") + + fetchedSet, err := r.conn.GetSetByName(r.workTable, set.HashedName()) + require.NoError(t, err, "fetch updated set") + elements, err := r.conn.GetSetElements(fetchedSet) + require.NoError(t, err, "get set elements") + + starts := make(map[string]bool) + for _, elem := range elements { + if elem.IntervalEnd { + continue + } + starts[netip.AddrFrom4(*(*[4]byte)(elem.Key)).String()] = true + } + // The /32s are covered by the /24, so the update adds one interval and + // leaves the one created earlier in place. + assert.Equal(t, map[string]bool{"10.0.0.0": true, "192.168.1.0": true}, starts, + "merged set must hold the original and the merged interval") +} + func TestNftablesCreateIpSet_IPv6(t *testing.T) { if check() != NFTABLES { t.Skip("nftables not supported on this system") @@ -518,11 +568,10 @@ func TestNftablesCreateIpSet_IPv6(t *testing.T) { require.NoError(t, err, "Failed to create v6 work table") defer deleteWorkTableIPv6() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router") + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { - require.NoError(t, r.Reset(), "Failed to reset router") + require.NoError(t, r.Reset(), "Failed to reset family") }() tests := []struct { @@ -748,6 +797,14 @@ func containsPort(exprs []expr.Any, port *firewall.Port, isSource bool) bool { } } } + case *expr.Lookup: + // Multiple discrete ports compile to an anonymous set lookup + // rather than a chain of comparisons. The set's id and name are + // assigned dynamically, so matching the lookup is enough here; + // the set elements are verified separately. + if !port.IsRange && len(port.Values) > 1 { + portMatchFound = true + } } if payloadFound && portMatchFound { return true @@ -861,13 +918,12 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) { require.NoError(t, err) defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err) + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { require.NoError(t, r.Reset()) }() // Add a real rule to the kernel - ruleKey, err := r.AddRouteFiltering( + ruleKey, err := r.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")}, firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")}, @@ -878,11 +934,11 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) { ) require.NoError(t, err) t.Cleanup(func() { - require.NoError(t, r.DeleteRouteRule(ruleKey)) + require.NoError(t, r.DeleteFilterRule(ruleKey)) }) // Inject a stale entry with Handle=0 (simulates store-before-flush failure) - staleKey := "stale-rule-that-does-not-exist" + staleKey := firewall.RuleID("stale-rule-that-does-not-exist") r.rules[staleKey] = &nftables.Rule{ Table: r.workTable, Chain: r.chains[chainNameRoutingFw], @@ -902,6 +958,54 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) { assert.NotZero(t, realRule.Handle, "real rule should have a valid handle") } +// TestRouter_DeleteRouteRule_RemovesKernelRule verifies a route filter +// rule is actually removed from the kernel on delete. The route chain is +// not refreshed by Flush, so the stored rule carries a zero handle; +// DeleteFilterRule must pull live handles itself before issuing the +// delete or the kernel rule leaks. Regression test for that path. +func TestRouter_DeleteRouteRule_RemovesKernelRule(t *testing.T) { + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + + workTable, err := createWorkTable() + require.NoError(t, err) + defer deleteWorkTable() + + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) + require.NoError(t, r.init(workTable)) + defer func() { require.NoError(t, r.Reset()) }() + + ruleKey, err := r.AddFilterRule( + nil, + []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")}, + firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")}, + firewall.ProtocolTCP, + nil, + &firewall.Port{Values: []uint16{80}}, + firewall.ActionAccept, + ) + require.NoError(t, err) + + countKernelRules := func() int { + list, err := r.conn.GetRules(r.workTable, r.chains[chainNameRoutingFw]) + require.NoError(t, err) + n := 0 + for _, rule := range list { + if string(rule.UserData) == string(ruleKey.ID()) { + n++ + } + } + return n + } + + require.Equal(t, 1, countKernelRules(), "rule should be present in the kernel after add") + + require.NoError(t, r.DeleteFilterRule(ruleKey)) + assert.Equal(t, 0, countKernelRules(), "rule must be removed from the kernel after delete") + assert.NotContains(t, r.filters, ruleKey.ID(), "filters map entry should be cleared") +} + func TestRouter_DeleteRouteRule_StaleHandle(t *testing.T) { if check() != NFTABLES { t.Skip("nftables not supported on this system") @@ -911,24 +1015,27 @@ func TestRouter_DeleteRouteRule_StaleHandle(t *testing.T) { require.NoError(t, err) defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err) + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { require.NoError(t, r.Reset()) }() // Inject a stale entry with Handle=0 - staleKey := "stale-route-rule" - r.rules[staleKey] = &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingFw], - Handle: 0, - UserData: []byte(staleKey), + staleKey := id.RuleID("stale-route-rule") + staleRule := &Rule{ + nftRule: &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingFw], + Handle: 0, + UserData: []byte(staleKey), + }, + id: staleKey, } + r.filters[staleKey] = staleRule - // DeleteRouteRule should not return an error for stale handles - err = r.DeleteRouteRule(id.RuleID(staleKey)) + // DeleteFilterRule should not return an error for stale handles + err = r.DeleteFilterRule(staleRule) assert.NoError(t, err, "deleting a stale rule should not error") - assert.NotContains(t, r.rules, staleKey, "stale entry should be cleaned up") + assert.NotContains(t, r.filters, staleKey, "stale entry should be cleaned up") } func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { @@ -950,7 +1057,7 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { Masquerade: true, } - rtr := manager.router + rtr := manager.family4 // First add succeeds err = rtr.AddNatRule(pair) @@ -960,11 +1067,11 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { }) // Corrupt the handle to simulate stale state - natRuleKey := firewall.GenKey(firewall.PreroutingFormat, pair) + natRuleKey := pair.GenKey(firewall.PreroutingFormat) if rule, exists := rtr.rules[natRuleKey]; exists { rule.Handle = 0 } - inverseKey := firewall.GenKey(firewall.PreroutingFormat, firewall.GetInversePair(pair)) + inverseKey := firewall.GetInversePair(pair).GenKey(firewall.PreroutingFormat) if rule, exists := rtr.rules[inverseKey]; exists { rule.Handle = 0 } @@ -979,7 +1086,7 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { found := 0 for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { found++ } } @@ -1010,7 +1117,7 @@ func TestCalculateLastIP(t *testing.T) { } func TestConvertPrefixesToSet_IPv6(t *testing.T) { - r := &router{af: afIPv6} + r := &family{af: afIPv6} prefixes := []netip.Prefix{ netip.MustParsePrefix("fd00::/64"), netip.MustParsePrefix("2001:db8::1/128"), diff --git a/client/firewall/nftables/routing_linux.go b/client/firewall/nftables/routing_linux.go new file mode 100644 index 000000000..4115c94bd --- /dev/null +++ b/client/firewall/nftables/routing_linux.go @@ -0,0 +1,558 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "strings" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) AddNatRule(pair firewall.RouterPair) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + // Resolve every rule's match expressions before queueing any of them: a + // message buffered on the shared connection cannot be un-queued, so + // returning an error after queueing would leave the next caller's Flush + // to commit a rule nothing tracks. + var legacyExprs []expr.Any + if r.legacyManagement { + log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) + + var err error + legacyExprs, err = r.legacyRouteRuleExprs(pair) + if err != nil { + return fmt.Errorf("build legacy routing rule: %w", err) + } + } + + inverse := firewall.GetInversePair(pair) + var natExprs, inverseExprs []expr.Any + if pair.Masquerade { + var err error + natExprs, err = r.natRuleExprs(pair) + if err != nil { + r.dropNetworkMatch(legacyExprs) + return fmt.Errorf("build nat rule: %w", err) + } + + inverseExprs, err = r.natRuleExprs(inverse) + if err != nil { + r.dropNetworkMatch(legacyExprs) + r.dropNetworkMatch(natExprs) + return fmt.Errorf("build inverse nat rule: %w", err) + } + } + + if legacyExprs != nil { + r.queueLegacyRouteRule(pair, legacyExprs) + } + if pair.Masquerade { + r.queueNatRule(pair, natExprs) + r.queueNatRule(inverse, inverseExprs) + } + + if err := r.conn.Flush(); err != nil { + r.rollbackRules(pair) + return fmt.Errorf("insert rules for %s: %w", pair.Destination, err) + } + + return nil +} + +// rollbackRules cleans up unflushed rules and their set counters after a flush failure. +func (r *family) rollbackRules(pair firewall.RouterPair) { + keys := []firewall.RuleID{ + pair.GenKey(firewall.ForwardingFormat), + pair.GenKey(firewall.PreroutingFormat), + firewall.GetInversePair(pair).GenKey(firewall.PreroutingFormat), + } + for _, key := range keys { + rule, ok := r.rules[key] + if !ok { + continue + } + if err := r.decrementSetCounter(rule); err != nil { + log.Warnf("rollback set counter for %s: %v", key, err) + } + delete(r.rules, key) + } +} + +// natRuleExprs resolves the match expressions of the pair's prerouting +// marking rule. It reserves the ipset references the matches need but queues +// nothing on the connection, so its error paths leave the connection clean. +func (r *family) natRuleExprs(pair firewall.RouterPair) ([]expr.Any, error) { + sourceExp, err := r.applyNetwork(pair.Source, nil, true) + if err != nil { + return nil, fmt.Errorf("apply source: %w", err) + } + + destExp, err := r.applyNetwork(pair.Destination, nil, false) + if err != nil { + r.dropNetworkMatch(sourceExp) + return nil, fmt.Errorf("apply destination: %w", err) + } + + op := expr.CmpOpEq + if pair.Inverse { + op = expr.CmpOpNeq + } + + exprs := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: op, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + } + // We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading. + // Masquerading will take care of the conntrack state, which means we won't need to mark established connections. + exprs = append(exprs, getCtNewExprs()...) + + exprs = append(exprs, sourceExp...) + exprs = append(exprs, destExp...) + + markValue := nbnet.PreroutingFwmarkMasquerade + if pair.Inverse { + markValue = nbnet.PreroutingFwmarkMasqueradeReturn + } + + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(markValue), + }, + &expr.Meta{ + Key: expr.MetaKeyMARK, + SourceRegister: true, + Register: 1, + }, + ) + + return exprs, nil +} + +// queueNatRule replaces any tracked rule for the pair and queues the new +// prerouting marking rule on the connection. Failures are logged rather than +// returned: the caller has already queued messages that only a Flush can +// commit, so it must not return early. +func (r *family) queueNatRule(pair firewall.RouterPair, exprs []expr.Any) { + ruleID := pair.GenKey(firewall.PreroutingFormat) + + if _, exists := r.rules[ruleID]; exists { + if err := r.removeNatRule(pair); err != nil { + // The rule this replaces may still be in the kernel. Keep tracking + // it and skip the new one: overwriting the entry would leave the old + // rule installed with nothing that can find it again, while keeping + // it lets the next update retry the whole replacement. + log.Errorf("replace prerouting rule %s: %v", ruleID, err) + r.dropNetworkMatch(exprs) + return + } + } + + // Ensure nat rules come first, so the mark can be overwritten. + // Currently overwritten by the dst-type LOCAL rules for redirected traffic. + r.rules[ruleID] = r.conn.InsertRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameManglePrerouting], + Exprs: exprs, + UserData: []byte(ruleID), + }) +} + +func (r *family) addPostroutingRules() { + // First masquerade rule for traffic coming in from WireGuard interface + exprs := []expr.Any{ + // Match on the first fwmark + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), + }, + + // We need to exclude the loopback interface as this changes the ebpf proxy port + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: ifname("lo"), + }, + &expr.Counter{}, + &expr.Masq{}, + } + + r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingNat], + Exprs: exprs, + }) + + // Second masquerade rule for traffic going out through WireGuard interface + exprs2 := []expr.Any{ + // Match on the second fwmark + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn), + }, + + // Match WireGuard interface + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Counter{}, + &expr.Masq{}, + } + + r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingNat], + Exprs: exprs2, + }) +} + +// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. +func (r *family) addMSSClampingRules() error { + overhead := uint16(ipv4TCPHeaderSize) + if r.af.tableFamily == nftables.TableFamilyIPv6 { + overhead = ipv6TCPHeaderSize + } + if r.mtu <= overhead { + log.Debugf("MTU %d too small for MSS clamping (overhead %d), skipping", r.mtu, overhead) + return nil + } + mss := r.mtu - overhead + + exprsOut := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_TCP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 13, + Len: 1, + }, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 1, + Mask: []byte{0x02}, + Xor: []byte{0x00}, + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0x00}, + }, + &expr.Counter{}, + &expr.Exthdr{ + DestRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + &expr.Cmp{ + Op: expr.CmpOpGt, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(uint16(mss)), + }, + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(uint16(mss)), + }, + &expr.Exthdr{ + SourceRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + } + + r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameMangleForward], + Exprs: exprsOut, + }) + + return r.conn.Flush() +} + +func buildLegacyRouteRuleExpressions(sourceExp, destExp []expr.Any) []expr.Any { + exprs := make([]expr.Any, 0, len(sourceExp)+len(destExp)+2) + exprs = append(exprs, sourceExp...) + exprs = append(exprs, destExp...) + exprs = append(exprs, + &expr.Counter{}, + &expr.Verdict{Kind: expr.VerdictAccept}, + ) + return exprs +} + +// legacyRouteRuleExprs resolves the match expressions of the pair's legacy +// forwarding rule, queueing nothing on the connection. +func (r *family) legacyRouteRuleExprs(pair firewall.RouterPair) ([]expr.Any, error) { + sourceExp, err := r.applyNetwork(pair.Source, nil, true) + if err != nil { + return nil, fmt.Errorf("apply source: %w", err) + } + + destExp, err := r.applyNetwork(pair.Destination, nil, false) + if err != nil { + r.dropNetworkMatch(sourceExp) + return nil, fmt.Errorf("apply destination: %w", err) + } + + return buildLegacyRouteRuleExpressions(sourceExp, destExp), nil +} + +// queueLegacyRouteRule replaces any tracked rule for the pair and queues the +// new legacy forwarding rule. Failures are logged for the same reason as in +// queueNatRule. +func (r *family) queueLegacyRouteRule(pair firewall.RouterPair, exprs []expr.Any) { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + if _, exists := r.rules[ruleID]; exists { + if err := r.removeLegacyRouteRule(pair); err != nil { + // Keep the old rule tracked instead of losing it, as in queueNatRule. + log.Errorf("replace legacy forwarding rule %s: %v", ruleID, err) + r.dropNetworkMatch(exprs) + return + } + } + + r.rules[ruleID] = r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingFw], + Exprs: exprs, + UserData: []byte(ruleID), + }) +} + +// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls +func (r *family) removeLegacyRouteRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + rule, exists := r.rules[ruleID] + if !exists { + return nil + } + + return r.deleteLegacyRuleEntry(ruleID, rule) +} + +// deleteLegacyRuleEntry removes one legacy forwarding rule and drops its +// ipset references. It also clears stale entries that never got a handle. +func (r *family) deleteLegacyRuleEntry(ruleID firewall.RuleID, rule *nftables.Rule) error { + if rule.Handle == 0 { + log.Warnf("legacy forwarding rule %s has no handle, removing stale entry", ruleID) + if err := r.decrementSetCounter(rule); err != nil { + log.Warnf("decrement set counter for stale rule %s: %v", ruleID, err) + } + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("remove legacy forwarding rule %s: %w", ruleID, err) + } + + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement set counter: %w", err) + } + + return nil +} + +// GetLegacyManagement returns the route manager's legacy management mode +func (r *family) GetLegacyManagement() bool { + return r.legacyManagement +} + +// SetLegacyManagement sets the route manager to use legacy management mode +func (r *family) SetLegacyManagement(isLegacy bool) { + r.legacyManagement = isLegacy +} + +// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls +func (r *family) RemoveAllLegacyRouteRules() error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + var merr *multierror.Error + var found bool + for k, rule := range r.rules { + if !strings.HasPrefix(string(k), firewall.ForwardingFormatPrefix) { + continue + } + found = true + if err := r.deleteLegacyRuleEntry(k, rule); err != nil { + merr = multierror.Append(merr, err) + } + } + + // Commit the queued deletes here instead of leaving them for whichever + // caller flushes next: the tracking entries are already gone, so an + // uncommitted delete would leave a rule in the kernel that nothing can + // find again. + if found { + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf(flushError, err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) removeNatPreroutingRules() error { + table := &nftables.Table{ + Name: tableNat, + Family: r.af.tableFamily, + } + chain := &nftables.Chain{ + Name: chainNameNatPrerouting, + Table: table, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + Type: nftables.ChainTypeNAT, + } + rules, err := r.conn.GetRules(table, chain) + if err != nil { + return fmt.Errorf("get rules from nat table: %w", err) + } + + var merr *multierror.Error + + // Delete rules that have our UserData suffix + for _, rule := range rules { + if len(rule.UserData) == 0 || !strings.HasSuffix(string(rule.UserData), string(dnatSuffix)) { + continue + } + if err := r.conn.DelRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete rule %s: %w", rule.UserData, err)) + } + } + + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf(flushError, err)) + } + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) RemoveNatRule(pair firewall.RouterPair) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + var merr *multierror.Error + + if pair.Masquerade { + if err := r.removeNatRule(pair); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove prerouting rule: %w", err)) + } + + if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove inverse prerouting rule: %w", err)) + } + } + + if err := r.removeLegacyRouteRule(pair); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove legacy routing rule: %w", err)) + } + + // Set counters are decremented in the sub-methods above before flush. If flush fails, + // counters will be off until the next successful removal or refresh cycle. + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("flush remove nat rules %s: %w", pair.Destination, err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) removeNatRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.PreroutingFormat) + + rule, exists := r.rules[ruleID] + if !exists { + log.Debugf("prerouting rule %s not found", ruleID) + return nil + } + + if rule.Handle == 0 { + log.Warnf("prerouting rule %s has no handle, removing stale entry", ruleID) + if err := r.decrementSetCounter(rule); err != nil { + log.Warnf("decrement set counter for stale rule %s: %v", ruleID, err) + } + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("remove prerouting rule %s -> %s: %w", pair.Source, pair.Destination, err) + } + + log.Debugf("removed prerouting rule %s -> %s", pair.Source, pair.Destination) + + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement set counter: %w", err) + } + + return nil +} diff --git a/client/firewall/nftables/rule_linux.go b/client/firewall/nftables/rule_linux.go index a90b74e36..8f3c0aebc 100644 --- a/client/firewall/nftables/rule_linux.go +++ b/client/firewall/nftables/rule_linux.go @@ -1,21 +1,26 @@ package nftables import ( - "net" + "net/netip" "github.com/google/nftables" + + "github.com/netbirdio/netbird/client/firewall/manager" ) -// Rule to handle management of rules +// Rule wraps an installed filter rule (peer or route). Source set +// membership is encoded in the rule's expressions; DeleteFilterRule +// recovers the set name via findSets so the refcounter can drop the +// right reference. mangleRule is set only for peer rules. type Rule struct { nftRule *nftables.Rule mangleRule *nftables.Rule - nftSet *nftables.Set - ruleID string - ip net.IP + // sources is the canonical source list this rule was created for. + sources []netip.Prefix + id manager.RuleID } -// GetRuleID returns the rule id -func (r *Rule) ID() string { - return r.ruleID +// ID returns the rule id +func (r *Rule) ID() manager.RuleID { + return r.id } diff --git a/client/firewall/nftables/testhelpers_linux_test.go b/client/firewall/nftables/testhelpers_linux_test.go new file mode 100644 index 000000000..72db3f7d2 --- /dev/null +++ b/client/firewall/nftables/testhelpers_linux_test.go @@ -0,0 +1,27 @@ +//go:build privileged + +package nftables + +import ( + "fmt" + "net" + "net/netip" +) + +func pfx(ip net.IP) []netip.Prefix { + if ip == nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + if ip.IsUnspecified() { + if ip.To4() != nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + } + a, ok := netip.AddrFromSlice(ip) + if !ok { + panic(fmt.Sprintf("invalid IP length: %d", len(ip))) + } + a = a.Unmap() + return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())} +} diff --git a/client/firewall/uspfilter/allow_netbird.go b/client/firewall/uspfilter/allow_netbird.go deleted file mode 100644 index b120cdf12..000000000 --- a/client/firewall/uspfilter/allow_netbird.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build !windows - -package uspfilter - -import ( - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/client/firewall/firewalld" - "github.com/netbirdio/netbird/client/internal/statemanager" -) - -// Close cleans up the firewall manager by removing all rules and closing trackers -func (m *Manager) Close(stateManager *statemanager.Manager) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - m.resetState() - - if m.nativeFirewall != nil { - return m.nativeFirewall.Close(stateManager) - } - if err := firewalld.UntrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to untrust interface in firewalld: %v", err) - } - return nil -} - -// AllowNetbird allows netbird interface traffic -func (m *Manager) AllowNetbird() error { - if m.nativeFirewall != nil { - return m.nativeFirewall.AllowNetbird() - } - if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - return nil -} diff --git a/client/firewall/uspfilter/common/iface.go b/client/firewall/uspfilter/common/iface.go deleted file mode 100644 index 9c06eb3f7..000000000 --- a/client/firewall/uspfilter/common/iface.go +++ /dev/null @@ -1,17 +0,0 @@ -package common - -import ( - wgdevice "golang.zx2c4.com/wireguard/device" - - "github.com/netbirdio/netbird/client/iface/device" - "github.com/netbirdio/netbird/client/iface/wgaddr" -) - -// IFaceMapper defines subset methods of interface required for manager -type IFaceMapper interface { - Name() string - SetFilter(device.PacketFilter) error - Address() wgaddr.Address - GetWGDevice() *wgdevice.Device - GetDevice() *device.FilteredDevice -} diff --git a/client/firewall/uspfilter/filter.go b/client/firewall/uspfilter/filter.go index 7376e59ca..5e1366c1f 100644 --- a/client/firewall/uspfilter/filter.go +++ b/client/firewall/uspfilter/filter.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "net" "net/netip" "os" "slices" @@ -20,14 +19,18 @@ import ( "github.com/google/uuid" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" + wgdevice "golang.zx2c4.com/wireguard/device" nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/firewall/firewalld" firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/firewall/uspfilter/common" "github.com/netbirdio/netbird/client/firewall/uspfilter/conntrack" "github.com/netbirdio/netbird/client/firewall/uspfilter/forwarder" nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" + "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/iface/netstack" + "github.com/netbirdio/netbird/client/iface/wgaddr" nbid "github.com/netbirdio/netbird/client/internal/acl/id" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" "github.com/netbirdio/netbird/client/internal/statemanager" @@ -58,7 +61,10 @@ const ( // EnvDisableMSSClamping disables TCP MSS clamping for forwarded traffic. EnvDisableMSSClamping = "NB_DISABLE_MSS_CLAMPING" - // EnvForceUserspaceRouter forces userspace routing even if native routing is available. + // EnvForceUserspaceRouter is a deprecated alias for + // NB_FORCE_USERSPACE_FIREWALL: the userspace firewall always routes in + // userspace, so forcing one forces the other. Kept for backward + // compatibility. EnvForceUserspaceRouter = "NB_FORCE_USERSPACE_ROUTER" // EnvEnableLocalForwarding enables forwarding of local traffic to the native stack for internal (non-NetBird) interfaces. @@ -70,14 +76,20 @@ const ( EnvEnableNetstackLocalForwarding = "NB_ENABLE_NETSTACK_LOCAL_FORWARDING" ) -var errNatNotSupported = errors.New("nat not supported with userspace firewall") +// errNotSupported is returned by firewall operations that only make sense with +// a kernel firewall (kernel NAT/DNAT, eBPF) and are not implemented in +// userspace mode, where they should not be called. +var errNotSupported = errors.New("not supported with userspace firewall") -// RuleSet is a set of rules grouped by a string key -type RuleSet map[string]PeerRule +// peerRules is the canonical list-based storage for peer ACL rules. +// Drop and accept rules live in separate slices; drop-before-accept +// ordering comes from consulting the deny slice (and its index) before +// the accept one. +type peerRules []*PeerRule -type RouteRules []*RouteRule +type routeRules []*RouteRule -func (r RouteRules) Sort() { +func (r routeRules) Sort() { slices.SortStableFunc(r, func(a, b *RouteRule) int { // Deny rules come first if a.action == firewall.ActionDrop && b.action != firewall.ActionDrop { @@ -86,22 +98,74 @@ func (r RouteRules) Sort() { if a.action != firewall.ActionDrop && b.action == firewall.ActionDrop { return 1 } - return strings.Compare(a.id, b.id) + return strings.Compare(string(a.id), string(b.id)) }) } +// peerRuleSpec carries the parameters that define a peer filter rule, +// threaded together through the build path so the builders take a single +// argument instead of a long parameter list. +type peerRuleSpec struct { + mgmtID []byte + sources []netip.Prefix + ipLayer gopacket.LayerType + proto firewall.Protocol + sPort *firewall.Port + dPort *firewall.Port + action firewall.Action +} + +// Iface is the network interface the userspace firewall attaches to: the +// methods of the WireGuard device it actually uses. +type Iface interface { + Name() string + Address() wgaddr.Address + SetFilter(device.PacketFilter) error + GetWGDevice() *wgdevice.Device +} + +// InterfaceAllower opens the NetBird interface in the host firewall so it +// doesn't drop traffic the userspace firewall handles, without taking over +// packet filtering. Implementations (nftables, iptables, firewalld, the windows +// netsh rule) are selected per platform and injected into Create; Apply runs at +// creation and Close on teardown. +type InterfaceAllower interface { + Apply() error + Close() error +} + +// Config holds the dependencies and options for the userspace firewall. +type Config struct { + // IFace is the overlay interface the filter attaches to. + IFace Iface + // InterfaceAllower opens the NetBird interface in foreign kernel filter + // chains so the kernel doesn't drop traffic the userspace firewall handles. + // Nil in netstack mode, on non-Linux platforms without a backend, or when + // neither nftables nor iptables is available. firewalld trust is applied by + // the manager regardless, since firewalld owns its own chains and we cannot + // insert into them. + InterfaceAllower InterfaceAllower + // DisableServerRoutes indicates whether server routes are disabled. + DisableServerRoutes bool + FlowLogger nftypes.FlowLogger + MTU uint16 +} + // Manager userspace firewall manager type Manager struct { - outgoingRules map[netip.Addr]RuleSet - incomingDenyRules map[netip.Addr]RuleSet - incomingRules map[netip.Addr]RuleSet - routeRules RouteRules - routeRulesMap map[nbid.RuleID]*RouteRule - decoders sync.Pool - wgIface common.IFaceMapper - nativeFirewall firewall.Manager + decoders sync.Pool + wgIface Iface + ifaceAllower InterfaceAllower + mutex sync.RWMutex - mutex sync.RWMutex + incomingDenyRules peerRules + incomingAcceptRules peerRules + incomingDenyIndex peerRuleIndex + incomingAcceptIndex peerRuleIndex + peerRulesMap map[nbid.RuleID]*PeerRule + + routeRules routeRules + routeRulesMap map[nbid.RuleID]*RouteRule // indicates whether server routes are disabled disableServerRoutes bool @@ -219,24 +283,6 @@ func (d *decoder) decodeTransport(proto layers.IPProtocol, payload []byte) bool return true } -// Create userspace firewall manager constructor -func Create(iface common.IFaceMapper, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (*Manager, error) { - return create(iface, nil, disableServerRoutes, flowLogger, mtu) -} - -func CreateWithNativeFirewall(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (*Manager, error) { - if nativeFirewall == nil { - return nil, errors.New("native firewall is nil") - } - - mgr, err := create(iface, nativeFirewall, disableServerRoutes, flowLogger, mtu) - if err != nil { - return nil, err - } - - return mgr, nil -} - func parseCreateEnv() (bool, bool, bool) { var disableConntrack, enableLocalForwarding, disableMSSClamping bool var err error @@ -267,7 +313,7 @@ func parseCreateEnv() (bool, bool, bool) { return disableConntrack, enableLocalForwarding, disableMSSClamping } -func create(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (*Manager, error) { +func Create(cfg Config) (_ *Manager, err error) { disableConntrack, enableLocalForwarding, disableMSSClamping := parseCreateEnv() m := &Manager{ @@ -290,65 +336,133 @@ func create(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableSe return d }, }, - nativeFirewall: nativeFirewall, - outgoingRules: make(map[netip.Addr]RuleSet), - incomingDenyRules: make(map[netip.Addr]RuleSet), - incomingRules: make(map[netip.Addr]RuleSet), - wgIface: iface, + wgIface: cfg.IFace, + ifaceAllower: cfg.InterfaceAllower, localipmanager: newLocalIPManager(), - disableServerRoutes: disableServerRoutes, + disableServerRoutes: cfg.DisableServerRoutes, stateful: !disableConntrack, logger: nblog.NewFromLogrus(log.StandardLogger()), - flowLogger: flowLogger, + flowLogger: cfg.FlowLogger, netstack: netstack.IsEnabled(), localForwarding: enableLocalForwarding, + peerRulesMap: make(map[nbid.RuleID]*PeerRule), routeRulesMap: make(map[nbid.RuleID]*RouteRule), dnatMappings: make(map[netip.Addr]netip.Addr), portDNATRules: []portDNATRule{}, netstackServices: make(map[serviceKey]struct{}), - mtu: mtu, + mtu: cfg.MTU, } m.routingEnabled.Store(false) + // Release the allower (and its monitor) if setup fails after it was wired in. + defer func() { + if err != nil { + m.closeAllowerOnError() + } + }() + if !disableMSSClamping { - m.mssClampEnabled = true - if mtu > ipv4TCPHeaderMinSize { - m.mssClampValueIPv4 = mtu - ipv4TCPHeaderMinSize - } - if mtu > ipv6TCPHeaderMinSize { - m.mssClampValueIPv6 = mtu - ipv6TCPHeaderMinSize - } + m.enableMSSClamping(cfg.MTU) } - if err := m.localipmanager.UpdateLocalIPs(iface); err != nil { + if err := m.localipmanager.UpdateLocalIPs(cfg.IFace); err != nil { return nil, fmt.Errorf("update local IPs: %w", err) } m.fragments = newFragmentTracker(m.logger) - - if disableConntrack { - log.Info("conntrack is disabled") - } else { - m.udpTracker = conntrack.NewUDPTracker(conntrack.DefaultUDPTimeout, m.logger, flowLogger) - m.icmpTracker = conntrack.NewICMPTracker(conntrack.DefaultICMPTimeout, m.logger, flowLogger) - m.tcpTracker = conntrack.NewTCPTracker(conntrack.DefaultTCPTimeout, m.logger, flowLogger) - } + m.setupConntrack(disableConntrack) if m.netstack && m.localForwarding { if err := m.initForwarder(); err != nil { log.Errorf("failed to initialize forwarder: %v", err) } } - if err := iface.SetFilter(m); err != nil { + if err := cfg.IFace.SetFilter(m); err != nil { m.fragments.Close() return nil, fmt.Errorf("set filter: %w", err) } + + m.openHostFirewall(cfg.IFace.Name()) + return m, nil } +// closeAllowerOnError releases the allower (and its monitor) when Create fails +// after the allower was wired in. +func (m *Manager) closeAllowerOnError() { + if m.ifaceAllower == nil { + return + } + if err := m.ifaceAllower.Close(); err != nil { + log.Warnf("close interface allower after failed firewall setup: %v", err) + } +} + +// enableMSSClamping enables MSS clamping and computes the per-family clamp values. +func (m *Manager) enableMSSClamping(mtu uint16) { + m.mssClampEnabled = true + if mtu > ipv4TCPHeaderMinSize { + m.mssClampValueIPv4 = mtu - ipv4TCPHeaderMinSize + } + if mtu > ipv6TCPHeaderMinSize { + m.mssClampValueIPv6 = mtu - ipv6TCPHeaderMinSize + } +} + +// setupConntrack initializes the stateful trackers unless conntrack is disabled. +func (m *Manager) setupConntrack(disabled bool) { + if disabled { + log.Info("conntrack is disabled") + return + } + m.udpTracker = conntrack.NewUDPTracker(conntrack.DefaultUDPTimeout, m.logger, m.flowLogger) + m.icmpTracker = conntrack.NewICMPTracker(conntrack.DefaultICMPTimeout, m.logger, m.flowLogger) + m.tcpTracker = conntrack.NewTCPTracker(conntrack.DefaultTCPTimeout, m.logger, m.flowLogger) +} + +// openHostFirewall opens the NetBird interface in the kernel firewall so it +// doesn't drop traffic the userspace firewall handles. Best-effort: failures +// here shouldn't prevent the firewall from coming up. +func (m *Manager) openHostFirewall(ifaceName string) { + if m.ifaceAllower != nil { + if err := m.ifaceAllower.Apply(); err != nil { + log.Errorf("failed to allow netbird interface traffic: %v", err) + } + } + // firewalld owns its own chains we can't insert into, so trust the interface + // there in addition to the allower. Netstack has no kernel interface. + if !m.netstack { + if err := firewalld.TrustInterface(ifaceName); err != nil { + log.Warnf("failed to trust interface in firewalld: %v", err) + } + } +} + +// Close cleans up the firewall manager: removes rules, closes trackers, and +// closes the interface allower. +func (m *Manager) Close(*statemanager.Manager) error { + m.mutex.Lock() + defer m.mutex.Unlock() + + m.resetState() + + var merr *multierror.Error + if m.ifaceAllower != nil { + if err := m.ifaceAllower.Close(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("close interface allower: %w", err)) + } + } + if !m.netstack { + if err := firewalld.UntrustInterface(m.wgIface.Name()); err != nil { + merr = multierror.Append(merr, fmt.Errorf("untrust interface in firewalld: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + // blockInvalidRouted installs drop rules for traffic to the wg overlay that // arrives via the routing path. v4 and v6 are independent: a v6 install // failure leaves v4 protection in place (and vice versa) so the returned // slice always contains whatever was successfully installed, even on error. // Callers must persist the slice so DisableRouting can clean partial state. -func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, error) { +func (m *Manager) blockInvalidRouted(iface Iface) ([]firewall.Rule, error) { wgPrefix := iface.Address().Network log.Debugf("blocking invalid routed traffic for %s", wgPrefix) @@ -359,7 +473,7 @@ func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, } var rules []firewall.Rule - v4Rule, err := m.addRouteFiltering( + v4Rule, err := m.addRouteRule( nil, sources, firewall.Network{Prefix: wgPrefix}, @@ -375,7 +489,7 @@ func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, if v6Net.IsValid() { log.Debugf("blocking invalid routed traffic for %s", v6Net) - v6Rule, err := m.addRouteFiltering( + v6Rule, err := m.addRouteRule( nil, sources, firewall.Network{Prefix: v6Net}, @@ -396,20 +510,14 @@ func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, } func (m *Manager) determineRouting() error { - var disableUspRouting, forceUserspaceRouter bool - var err error + var disableUspRouting bool if val := os.Getenv(EnvDisableUserspaceRouting); val != "" { + var err error disableUspRouting, err = strconv.ParseBool(val) if err != nil { log.Warnf("failed to parse %s: %v", EnvDisableUserspaceRouting, err) } } - if val := os.Getenv(EnvForceUserspaceRouter); val != "" { - forceUserspaceRouter, err = strconv.ParseBool(val) - if err != nil { - log.Warnf("failed to parse %s: %v", EnvForceUserspaceRouter, err) - } - } switch { case disableUspRouting: @@ -424,26 +532,11 @@ func (m *Manager) determineRouting() error { log.Info("server routes are disabled") - case forceUserspaceRouter: - m.routingEnabled.Store(true) - m.nativeRouter.Store(false) - - log.Info("userspace routing is forced") - - case !m.netstack && m.nativeFirewall != nil: - // if the OS supports routing natively, then we don't need to filter/route ourselves - // netstack mode won't support native routing as there is no interface - - m.routingEnabled.Store(true) - m.nativeRouter.Store(true) - - log.Info("native routing is enabled") - default: m.routingEnabled.Store(true) m.nativeRouter.Store(false) - log.Info("userspace routing enabled by default") + log.Info("userspace routing enabled") } if m.routingEnabled.Load() && !m.nativeRouter.Load() { @@ -509,96 +602,118 @@ func (m *Manager) IsStateful() bool { return m.stateful } -func (m *Manager) AddNatRule(pair firewall.RouterPair) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.AddNatRule(pair) - } - +func (m *Manager) AddNatRule(firewall.RouterPair) error { // userspace routed packets are always SNATed to the inbound direction // TODO: implement outbound SNAT return nil } // RemoveNatRule removes a routing firewall rule -func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.RemoveNatRule(pair) - } +func (m *Manager) RemoveNatRule(firewall.RouterPair) error { return nil } -// AddPeerFiltering rule to the firewall -// -// If comment argument is empty firewall manager should set -// rule ID as comment for the rule -func (m *Manager) AddPeerFiltering( +// addPeerRule installs an input-chain rule that matches packets +// by source only. Called from AddFilterRule when the caller doesn't +// specify a destination. Sources are expected to share one address +// family; the family selects the ipLayer so the ICMP variant matches +// what the decoder produces. +func (m *Manager) addPeerRule( id []byte, - ip net.IP, + sources []netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action, - _ string, -) ([]firewall.Rule, error) { - // TODO: fix in upper layers - i, ok := netip.AddrFromSlice(ip) - if !ok { - return nil, fmt.Errorf("invalid IP: %s", ip) - } - - i = i.Unmap() - r := PeerRule{ - id: uuid.New().String(), - mgmtId: id, - ip: i, - ipLayer: layers.LayerTypeIPv6, - matchByIP: true, - drop: action == firewall.ActionDrop, - } - if i.Is4() { - r.ipLayer = layers.LayerTypeIPv4 - } - - if s := r.ip.String(); s == "0.0.0.0" || s == "::" { - r.matchByIP = false - } - - r.sPort = sPort - r.dPort = dPort - - r.protoLayer = protoToLayer(proto, r.ipLayer) - - m.mutex.Lock() - var targetMap map[netip.Addr]RuleSet - if r.drop { - targetMap = m.incomingDenyRules - } else { - targetMap = m.incomingRules - } - - if _, ok := targetMap[r.ip]; !ok { - targetMap[r.ip] = make(RuleSet) - } - targetMap[r.ip][r.id] = r - m.mutex.Unlock() - return []firewall.Rule{&r}, nil -} - -func (m *Manager) AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination firewall.Network, - proto firewall.Protocol, - sPort, dPort *firewall.Port, - action firewall.Action, ) (firewall.Rule, error) { m.mutex.Lock() defer m.mutex.Unlock() - return m.addRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + // Sources are a single family; normalize v4-mapped prefixes to plain + // v4 and pick the matching IP layer. A /0 source matches any address + // of its own family only, mirroring the kernel backends. + normalized := make([]netip.Prefix, len(sources)) + ipLayer := layers.LayerTypeIPv4 + for i, p := range sources { + normalized[i] = firewall.UnmapPrefix(p) + if normalized[i].Addr().Is6() { + ipLayer = layers.LayerTypeIPv6 + } + } + spec := peerRuleSpec{ + mgmtID: id, + sources: normalized, + ipLayer: ipLayer, + proto: proto, + sPort: sPort, + dPort: dPort, + action: action, + } + return m.addOnePeerRule(spec), nil } -func (m *Manager) addRouteFiltering( +// addOnePeerRule builds and registers a single-family peer rule, or +// returns the existing rule when one with the same content key is +// already installed. The caller must hold m.mutex. The content key is +// the shared GenerateRuleID with an empty destination, so peer rules +// dedup the same way route rules and the kernel backends do; it is +// order-independent, so callers passing the same sources in any order +// dedup to one rule. +// +// There is no refcount: a content key is installed once and deleted on +// the first DeleteFilterRule for that key. The caller must therefore +// key its own tracking by the returned rule id so add and delete stay +// balanced per content key; the acl manager does this via +// peerRulesPairs. +func (m *Manager) addOnePeerRule(spec peerRuleSpec) *PeerRule { + ruleID := nbid.GenerateRuleID(spec.sources, firewall.Network{}, spec.proto, spec.sPort, spec.dPort, spec.action) + if existing, ok := m.peerRulesMap[ruleID]; ok { + return existing + } + + rule := m.buildPeerRule(ruleID, spec) + m.registerPeerRule(rule) + return rule +} + +func (m *Manager) buildPeerRule(ruleID nbid.RuleID, spec peerRuleSpec) *PeerRule { + r := &PeerRule{ + id: ruleID, + mgmtId: spec.mgmtID, + sources: spec.sources, + action: spec.action, + srcPort: spec.sPort, + dstPort: spec.dPort, + } + r.sourceAddrs = make(map[netip.Addr]struct{}, len(spec.sources)) + for _, p := range spec.sources { + if p.Bits() == p.Addr().BitLen() { + r.sourceAddrs[p.Addr()] = struct{}{} + } + } + r.protoLayer = protoToLayer(spec.proto, spec.ipLayer) + return r +} + +// registerPeerRule records a freshly built peer rule in the matching +// slice, index, and dedup map. The caller must hold m.mutex. +func (m *Manager) registerPeerRule(r *PeerRule) { + if r.action == firewall.ActionDrop { + m.incomingDenyRules = append(m.incomingDenyRules, r) + m.incomingDenyIndex.add(r) + } else { + m.incomingAcceptRules = append(m.incomingAcceptRules, r) + m.incomingAcceptIndex.add(r) + } + m.peerRulesMap[r.id] = r +} + +// AddFilterRule is the unified entry point for both peer (input chain) +// and route (forward chain) filtering rules. The destination +// distinguishes the two semantics: a zero Network installs an +// input-side rule that matches by source only; a set Network installs +// a forward-side rule that also matches the destination. +func (m *Manager) AddFilterRule( id []byte, sources []netip.Prefix, destination firewall.Network, @@ -606,19 +721,49 @@ func (m *Manager) addRouteFiltering( sPort, dPort *firewall.Port, action firewall.Action, ) (firewall.Rule, error) { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + if len(sources) == 0 { + return nil, firewall.ErrNoSources } - ruleKey := nbid.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action) + if destination.IsZero() { + return m.addPeerRule(id, sources, proto, sPort, dPort, action) + } - if existingRule, ok := m.routeRulesMap[ruleKey]; ok { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.addRouteRule(id, sources, destination, proto, sPort, dPort, action) +} + +// DeleteFilterRule deletes a filtering rule. The rule's underlying type +// is used to route to the correct internal path. +func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { + m.mutex.Lock() + defer m.mutex.Unlock() + + if r, ok := rule.(*PeerRule); ok { + return m.deletePeerRuleLocked(r) + } + + // Anything else is a route rule (matched on the forward path). + return m.deleteRouteRule(rule) +} + +func (m *Manager) addRouteRule( + id []byte, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort, dPort *firewall.Port, + action firewall.Action, +) (firewall.Rule, error) { + ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action) + + if existingRule, ok := m.routeRulesMap[ruleID]; ok { return existingRule, nil } rule := RouteRule{ - // TODO: consolidate these IDs - id: string(ruleKey), + id: ruleID, mgmtId: id, sources: sources, dstSet: destination.Set, @@ -633,78 +778,58 @@ func (m *Manager) addRouteFiltering( m.routeRules = append(m.routeRules, &rule) m.routeRules.Sort() - m.routeRulesMap[ruleKey] = &rule + m.routeRulesMap[ruleID] = &rule return &rule, nil } -func (m *Manager) DeleteRouteRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - return m.deleteRouteRule(rule) -} - func (m *Manager) deleteRouteRule(rule firewall.Rule) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.DeleteRouteRule(rule) + ruleID := rule.ID() + trimmed, _, ok := removeRuleByID(m.routeRules, ruleID) + if !ok { + return fmt.Errorf("route rule not found: %s", ruleID) } - - ruleKey := nbid.RuleID(rule.ID()) - if _, ok := m.routeRulesMap[ruleKey]; !ok { - return fmt.Errorf("route rule not found: %s", ruleKey) - } - - idx := slices.IndexFunc(m.routeRules, func(r *RouteRule) bool { - return r.id == string(ruleKey) - }) - if idx < 0 { - return fmt.Errorf("route rule not found in slice: %s", ruleKey) - } - - m.routeRules = slices.Delete(m.routeRules, idx, idx+1) - delete(m.routeRulesMap, ruleKey) + m.routeRules = trimmed + delete(m.routeRulesMap, ruleID) return nil } -// DeletePeerRule from the firewall by rule definition -func (m *Manager) DeletePeerRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() +// deletePeerRuleLocked removes a peer rule from the matching slice, +// index, and dedup map. The caller must hold m.mutex. +func (m *Manager) deletePeerRuleLocked(r *PeerRule) error { + target, index := &m.incomingAcceptRules, &m.incomingAcceptIndex + if r.action == firewall.ActionDrop { + target, index = &m.incomingDenyRules, &m.incomingDenyIndex + } - r, ok := rule.(*PeerRule) + trimmed, stored, ok := removeRuleByID(*target, r.id) if !ok { - return fmt.Errorf("delete rule: invalid rule type: %T", rule) - } - - var sourceMap map[netip.Addr]RuleSet - if r.drop { - sourceMap = m.incomingDenyRules - } else { - sourceMap = m.incomingRules - } - - if ruleset, ok := sourceMap[r.ip]; ok { - if _, exists := ruleset[r.id]; !exists { - return fmt.Errorf("delete rule: no rule with such id: %v", r.id) - } - delete(ruleset, r.id) - if len(ruleset) == 0 { - delete(sourceMap, r.ip) - } - } else { return fmt.Errorf("delete rule: no rule with such id: %v", r.id) } - + *target = trimmed + index.remove(stored) + delete(m.peerRulesMap, r.id) return nil } -// SetLegacyManagement doesn't need to be implemented for this manager -func (m *Manager) SetLegacyManagement(isLegacy bool) error { - if m.nativeFirewall == nil { - return nil +// removeRuleByID removes the first rule whose id matches ruleID from +// rules, preserving order. It returns the trimmed slice, the removed +// rule, and whether a match was found. +func removeRuleByID[S ~[]T, T firewall.Rule](rules S, ruleID firewall.RuleID) (S, T, bool) { + idx := slices.IndexFunc(rules, func(r T) bool { return r.ID() == ruleID }) + var removed T + if idx < 0 { + return rules, removed, false } - return m.nativeFirewall.SetLegacyManagement(isLegacy) + removed = rules[idx] + return slices.Delete(rules, idx, idx+1), removed, true +} + +// SetLegacyManagement is a no-op for the userspace firewall: it only matters +// when an old management server can't send route firewall rules, which the +// userspace router doesn't rely on. +func (m *Manager) SetLegacyManagement(bool) error { + return nil } // Flush doesn't need to be implemented for this manager @@ -713,11 +838,14 @@ func (m *Manager) Flush() error { return nil } // resetState clears all firewall rules and closes connection trackers. // Must be called with m.mutex held. func (m *Manager) resetState() { - clear(m.outgoingRules) - clear(m.incomingDenyRules) - clear(m.incomingRules) + m.incomingDenyRules = m.incomingDenyRules[:0] + m.incomingAcceptRules = m.incomingAcceptRules[:0] + m.incomingDenyIndex.reset() + m.incomingAcceptIndex.reset() + clear(m.peerRulesMap) clear(m.routeRulesMap) m.routeRules = m.routeRules[:0] + m.blockRules = nil m.udpHookOut.Store(nil) m.tcpHookOut.Store(nil) @@ -751,21 +879,15 @@ func (m *Manager) resetState() { } } -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - if m.nativeFirewall == nil { - return nil - } - return m.nativeFirewall.SetupEBPFProxyNoTrack(proxyPort, wgPort) +// SetupEBPFProxyNoTrack is not supported by the userspace firewall: eBPF isn't +// used in userspace mode, so this should never be called. +func (m *Manager) SetupEBPFProxyNoTrack(uint16, uint16) error { + return errNotSupported } // UpdateSet updates the rule destinations associated with the given set // by merging the existing prefixes with the new ones, then deduplicating. func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.UpdateSet(set, prefixes) - } - m.mutex.Lock() defer m.mutex.Unlock() @@ -863,11 +985,11 @@ func (m *Manager) extractIPs(d *decoder) (srcIP, dstIP netip.Addr) { case layers.LayerTypeIPv4: src, _ := netip.AddrFromSlice(d.ip4.SrcIP) dst, _ := netip.AddrFromSlice(d.ip4.DstIP) - return src, dst + return src.Unmap(), dst.Unmap() case layers.LayerTypeIPv6: src, _ := netip.AddrFromSlice(d.ip6.SrcIP) dst, _ := netip.AddrFromSlice(d.ip6.DstIP) - return src, dst + return src.Unmap(), dst.Unmap() default: return netip.Addr{}, netip.Addr{} } @@ -1622,20 +1744,12 @@ func (m *Manager) peerACLsBlock(srcIP netip.Addr, d *decoder, packetData []byte) return nil, false } - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingDenyRules[srcIP], d); ok { + if mgmtId, filter, ok := m.incomingDenyIndex.match(srcIP, d); ok { return mgmtId, filter } - - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingRules[srcIP], d); ok { + if mgmtId, filter, ok := m.incomingAcceptIndex.match(srcIP, d); ok { return mgmtId, filter } - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingRules[netip.IPv4Unspecified()], d); ok { - return mgmtId, filter - } - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingRules[netip.IPv6Unspecified()], d); ok { - return mgmtId, filter - } - return nil, true } @@ -1656,39 +1770,6 @@ func portsMatch(rulePort *firewall.Port, packetPort uint16) bool { return false } -func validateRule(ip netip.Addr, packetData []byte, rules map[string]PeerRule, d *decoder) ([]byte, bool, bool) { - payloadLayer := d.decoded[1] - - for _, rule := range rules { - if rule.matchByIP && ip.Compare(rule.ip) != 0 { - continue - } - - if rule.protoLayer == layerTypeAll { - return rule.mgmtId, rule.drop, true - } - - if !protoLayerMatches(rule.protoLayer, payloadLayer) { - continue - } - - switch payloadLayer { - case layers.LayerTypeTCP: - if portsMatch(rule.sPort, uint16(d.tcp.SrcPort)) && portsMatch(rule.dPort, uint16(d.tcp.DstPort)) { - return rule.mgmtId, rule.drop, true - } - case layers.LayerTypeUDP: - if portsMatch(rule.sPort, uint16(d.udp.SrcPort)) && portsMatch(rule.dPort, uint16(d.udp.DstPort)) { - return rule.mgmtId, rule.drop, true - } - case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6: - return rule.mgmtId, rule.drop, true - } - } - - return nil, false, false -} - // routeACLsPass returns true if the packet is allowed by the route ACLs func (m *Manager) routeACLsPass(srcIP, dstIP netip.Addr, protoLayer gopacket.LayerType, srcPort, dstPort uint16) ([]byte, bool) { m.mutex.RLock() @@ -1765,10 +1846,13 @@ func (m *Manager) EnableRouting() error { } rules, err := m.blockInvalidRouted(m.wgIface) - // Persist whatever was installed even on partial failure, so DisableRouting - // can clean it up later. m.blockRules = rules if err != nil { + // Roll back so forwarding can't stay active without the full set of + // block rules. + if derr := m.disableRouting(); derr != nil { + log.Warnf("roll back routing after block rule failure: %v", derr) + } return fmt.Errorf("block invalid routed: %w", err) } @@ -1779,6 +1863,10 @@ func (m *Manager) DisableRouting() error { m.mutex.Lock() defer m.mutex.Unlock() + return m.disableRouting() +} + +func (m *Manager) disableRouting() error { fwder := m.forwarder.Load() if fwder == nil { return nil diff --git a/client/firewall/uspfilter/filter_bench_test.go b/client/firewall/uspfilter/filter_bench_test.go index 4dccb0f65..72f3417f2 100644 --- a/client/firewall/uspfilter/filter_bench_test.go +++ b/client/firewall/uspfilter/filter_bench_test.go @@ -94,7 +94,7 @@ func BenchmarkCoreFiltering(b *testing.B) { stateful: false, setupFunc: func(m *Manager) { // Single rule allowing all traffic - _, err := m.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolALL, nil, nil, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) require.NoError(b, err) }, desc: "Baseline: Single 'allow all' rule without connection tracking", @@ -114,15 +114,13 @@ func BenchmarkCoreFiltering(b *testing.B) { // Add explicit rules matching return traffic pattern for i := 0; i < 1000; i++ { // Simulate realistic ruleset size ip := generateRandomIPs(1)[0] - _, err := m.AddPeerFiltering( + _, err := m.AddFilterRule( nil, - ip, + pfx(ip), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{uint16(1024 + i)}}, &fw.Port{Values: []uint16{80}}, - fw.ActionAccept, - "", - ) + fw.ActionAccept) require.NoError(b, err) } }, @@ -133,15 +131,13 @@ func BenchmarkCoreFiltering(b *testing.B) { stateful: true, setupFunc: func(m *Manager) { // Add some basic rules but rely on state for established connections - _, err := m.AddPeerFiltering( + _, err := m.AddFilterRule( nil, - net.ParseIP("0.0.0.0"), + pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, nil, nil, - fw.ActionDrop, - "", - ) + fw.ActionDrop) require.NoError(b, err) }, desc: "Connection tracking with established connections", @@ -168,9 +164,12 @@ func BenchmarkCoreFiltering(b *testing.B) { } // Create manager and basic setup - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -208,9 +207,12 @@ func BenchmarkStateScaling(b *testing.B) { for _, count := range connCounts { b.Run(fmt.Sprintf("conns_%d", count), func(b *testing.B) { - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -251,9 +253,12 @@ func BenchmarkEstablishmentOverhead(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -409,9 +414,12 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -536,9 +544,12 @@ func BenchmarkLongLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -546,7 +557,7 @@ func BenchmarkLongLivedConnections(b *testing.B) { // Setup initial state based on scenario if sc.rules { // Single rule to allow all return traffic from port 80 - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -619,9 +630,12 @@ func BenchmarkShortLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -629,7 +643,7 @@ func BenchmarkShortLivedConnections(b *testing.B) { // Setup initial state based on scenario if sc.rules { // Single rule to allow all return traffic from port 80 - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -730,16 +744,19 @@ func BenchmarkParallelLongLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) // Setup initial state based on scenario if sc.rules { - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -810,15 +827,18 @@ func BenchmarkParallelShortLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) if sc.rules { - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -931,7 +951,7 @@ func BenchmarkRouteACLs(b *testing.B) { for _, r := range rules { dst := fw.Network{Prefix: r.dest} - _, err := manager.AddRouteFiltering(nil, r.sources, dst, r.proto, nil, r.port, fw.ActionAccept) + _, err := manager.AddFilterRule(nil, r.sources, dst, r.proto, nil, r.port, fw.ActionAccept) if err != nil { b.Fatal(err) } @@ -1014,9 +1034,11 @@ func BenchmarkMSSClamping(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -1079,9 +1101,11 @@ func BenchmarkMSSClampingOverhead(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -1134,9 +1158,11 @@ func BenchmarkMSSClampingMemory(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) diff --git a/client/firewall/uspfilter/filter_filter_test.go b/client/firewall/uspfilter/filter_filter_test.go index 5ca8538be..57b59c9bc 100644 --- a/client/firewall/uspfilter/filter_filter_test.go +++ b/client/firewall/uspfilter/filter_filter_test.go @@ -32,7 +32,7 @@ func TestPeerACLFiltering(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) require.NotNil(t, manager) @@ -496,40 +496,32 @@ func TestPeerACLFiltering(t *testing.T) { t.Run(tc.name, func(t *testing.T) { if tc.ruleAction == fw.ActionDrop { // add general accept rule for the same IP to test drop rule precedence - rules, err := manager.AddPeerFiltering( + rules, err := manager.AddFilterRule( nil, - net.ParseIP(tc.ruleIP), + pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, fw.ProtocolALL, nil, nil, - fw.ActionAccept, - "", - ) + fw.ActionAccept) require.NoError(t, err) - require.NotEmpty(t, rules) + require.NotNil(t, rules) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) } - rules, err := manager.AddPeerFiltering( + rules, err := manager.AddFilterRule( nil, - net.ParseIP(tc.ruleIP), + pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, tc.ruleProto, tc.ruleSrcPort, tc.ruleDstPort, - tc.ruleAction, - "", - ) + tc.ruleAction) require.NoError(t, err) - require.NotEmpty(t, rules) + require.NotNil(t, rules) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) packet := createTestPacket(t, tc.srcIP, tc.dstIP, tc.proto, tc.srcPort, tc.dstPort) @@ -557,7 +549,7 @@ func TestPeerACLFilteringIPv6(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) }) @@ -652,14 +644,24 @@ func TestPeerACLFilteringIPv6(t *testing.T) { shouldBeBlocked: false, }, { - name: "IPv6: v4 wildcard ICMP rule matches ICMPv6 via protoLayerMatches", + name: "IPv6: v4 wildcard ICMP rule does not match ICMPv6", srcIP: "fd00::1", dstIP: "fd00::100", proto: fw.ProtocolICMP, ruleIP: "0.0.0.0", ruleProto: fw.ProtocolICMP, ruleAction: fw.ActionAccept, - shouldBeBlocked: false, + shouldBeBlocked: true, + }, + { + name: "IPv4: v6 wildcard ICMP rule does not match ICMPv4", + srcIP: "100.10.0.1", + dstIP: "100.10.0.100", + proto: fw.ProtocolICMP, + ruleIP: "::", + ruleProto: fw.ProtocolICMP, + ruleAction: fw.ActionAccept, + shouldBeBlocked: true, }, } @@ -672,22 +674,18 @@ func TestPeerACLFilteringIPv6(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.ruleAction == fw.ActionDrop { - rules, err := manager.AddPeerFiltering(nil, net.ParseIP(tc.ruleIP), fw.ProtocolALL, nil, nil, fw.ActionAccept, "") + rules, err := manager.AddFilterRule(nil, pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) require.NoError(t, err) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) } - rules, err := manager.AddPeerFiltering(nil, net.ParseIP(tc.ruleIP), tc.ruleProto, nil, tc.ruleDstPort, tc.ruleAction, "") + rules, err := manager.AddFilterRule(nil, pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, tc.ruleProto, nil, tc.ruleDstPort, tc.ruleAction) require.NoError(t, err) - require.NotEmpty(t, rules) + require.NotNil(t, rules) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) packet := createTestPacket(t, tc.srcIP, tc.dstIP, tc.proto, tc.srcPort, tc.dstPort) @@ -800,7 +798,7 @@ func setupRoutedManager(tb testing.TB, network string) *Manager { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(tb, err) require.NoError(tb, manager.EnableRouting()) require.NotNil(tb, manager) @@ -1405,7 +1403,7 @@ func TestRouteACLFiltering(t *testing.T) { t.Run(tc.name, func(t *testing.T) { if tc.rule.action == fw.ActionDrop { // add general accept rule to test drop rule - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Prefix: netip.MustParsePrefix("0.0.0.0/0")}, @@ -1415,13 +1413,13 @@ func TestRouteACLFiltering(t *testing.T) { fw.ActionAccept, ) require.NoError(t, err) - require.NotNil(t, rule) + require.NotEmpty(t, rule) t.Cleanup(func() { - require.NoError(t, manager.DeleteRouteRule(rule)) + require.NoError(t, manager.DeleteFilterRule(rule)) }) } - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, tc.rule.sources, tc.rule.dest, @@ -1431,10 +1429,10 @@ func TestRouteACLFiltering(t *testing.T) { tc.rule.action, ) require.NoError(t, err) - require.NotNil(t, rule) + require.NotEmpty(t, rule) t.Cleanup(func() { - require.NoError(t, manager.DeleteRouteRule(rule)) + require.NoError(t, manager.DeleteFilterRule(rule)) }) srcIP := netip.MustParseAddr(tc.srcIP) @@ -1602,9 +1600,9 @@ func TestRouteACLOrder(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - var rules []fw.Rule + var addedRules []fw.Rule for _, r := range tc.rules { - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, r.sources, r.dest, @@ -1615,12 +1613,12 @@ func TestRouteACLOrder(t *testing.T) { ) require.NoError(t, err) require.NotNil(t, rule) - rules = append(rules, rule) + addedRules = append(addedRules, rule) } t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeleteRouteRule(rule)) + for _, rule := range addedRules { + require.NoError(t, manager.DeleteFilterRule(rule)) } }) @@ -1646,7 +1644,7 @@ func TestRouteACLSet(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -1655,7 +1653,7 @@ func TestRouteACLSet(t *testing.T) { set := fw.NewDomainSet(domain.List{"example.org"}) // Add rule that uses the set (initially empty) - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Set: set}, @@ -1689,7 +1687,7 @@ func TestRouteACLFilteringIPv6(t *testing.T) { manager := setupRoutedManager(t, "10.10.0.100/16") v6Dst := netip.MustParsePrefix("fd00:dead:beef::/48") - _, err := manager.AddRouteFiltering( + _, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("fd00::/16")}, fw.Network{Prefix: v6Dst}, @@ -1700,7 +1698,7 @@ func TestRouteACLFilteringIPv6(t *testing.T) { ) require.NoError(t, err) - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("fd00::/16")}, fw.Network{Prefix: netip.MustParsePrefix("fd00:dead:beef:1::/64")}, diff --git a/client/firewall/uspfilter/filter_routeacl_test.go b/client/firewall/uspfilter/filter_routeacl_test.go index b6397d09b..2a0980e83 100644 --- a/client/firewall/uspfilter/filter_routeacl_test.go +++ b/client/firewall/uspfilter/filter_routeacl_test.go @@ -29,7 +29,7 @@ func TestAddRouteFilteringReturnsExistingRule(t *testing.T) { destination := fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")} // Add rule first time - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -42,7 +42,7 @@ func TestAddRouteFilteringReturnsExistingRule(t *testing.T) { require.NotNil(t, rule1) // Add the same rule again - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -74,7 +74,7 @@ func TestAddRouteFilteringDifferentRulesGetDifferentIDs(t *testing.T) { sources := []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")} // Add first rule - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")}, @@ -86,7 +86,7 @@ func TestAddRouteFilteringDifferentRulesGetDifferentIDs(t *testing.T) { require.NoError(t, err) // Add different rule (different destination) - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-2"), sources, fw.Network{Prefix: netip.MustParsePrefix("192.168.2.0/24")}, // Different! @@ -115,7 +115,7 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { sources := []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")} destination := fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")} - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -132,7 +132,7 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { require.True(t, pass, "Traffic should pass with rule in place") // Re-add same rule (simulates network map update) - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -147,7 +147,7 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { // won't delete rule1 during cleanup. If IDs differed, deleting rule1 // would remove the only matching rule and cause a traffic gap. if rule1.ID() != rule2.ID() { - err = manager.DeleteRouteRule(rule1) + err = manager.DeleteFilterRule(rule1) require.NoError(t, err) } @@ -156,6 +156,59 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { "Traffic should still pass after rule update - no gap should occur") } +// TestBlockInvalidRoutedDualStack verifies that when the interface has an +// IPv6 overlay address, blockInvalidRouted installs a block rule for both +// the v4 and v6 WG prefixes and that routed traffic to the v6 prefix is +// denied. The v4-only soft-skip path is covered by +// TestBlockInvalidRoutedIdempotent, whose mock leaves IPv6Net invalid. +func TestBlockInvalidRoutedDualStack(t *testing.T) { + ctrl := gomock.NewController(t) + dev := mocks.NewMockDevice(ctrl) + dev.EXPECT().MTU().Return(1500, nil).AnyTimes() + + wgNet := netip.MustParsePrefix("100.64.0.1/16") + wgNet6 := netip.MustParsePrefix("fd00:1234::1/64") + + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: wgNet.Addr(), + Network: wgNet, + IPv6: wgNet6.Addr(), + IPv6Net: wgNet6, + } + }, + GetDeviceFunc: func() *device.FilteredDevice { + return &device.FilteredDevice{Device: dev} + }, + GetWGDeviceFunc: func() *wgdevice.Device { + return &wgdevice.Device{} + }, + } + + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, manager.Close(nil)) + }) + + rules, err := manager.blockInvalidRouted(ifaceMock) + require.NoError(t, err) + require.Len(t, rules, 2, "dual-stack interface must produce a v4 and a v6 block rule") + + manager.mutex.RLock() + ruleCount := len(manager.routeRules) + manager.mutex.RUnlock() + assert.Equal(t, 2, ruleCount, "should have one block rule per family") + + // v6 routed traffic to the WG prefix must be denied. + srcIP := netip.MustParseAddr("2001:db8::1") + dstIP := netip.MustParseAddr("fd00:1234::50") + _, pass := manager.routeACLsPass(srcIP, dstIP, layers.LayerTypeTCP, 12345, 80) + assert.False(t, pass, "block rule should deny routed traffic to the v6 WG prefix") +} + // TestBlockInvalidRoutedIdempotent verifies that blockInvalidRouted creates // exactly one drop rule for the WireGuard network prefix, and calling it again // returns the same rule without duplicating. @@ -182,7 +235,7 @@ func TestBlockInvalidRoutedIdempotent(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -245,7 +298,7 @@ func TestBlockRuleNotAccumulatedOnRepeatedEnableRouting(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -274,7 +327,7 @@ func TestRouteRuleCountStableAcrossUpdates(t *testing.T) { // Simulate 5 network map updates with the same route rule for i := 0; i < 5; i++ { - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -304,7 +357,7 @@ func TestDeleteRouteRuleAfterIdempotentAdd(t *testing.T) { destination := fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")} // Add same rule twice - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -315,7 +368,7 @@ func TestDeleteRouteRuleAfterIdempotentAdd(t *testing.T) { ) require.NoError(t, err) - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -329,7 +382,7 @@ func TestDeleteRouteRuleAfterIdempotentAdd(t *testing.T) { require.Equal(t, rule1.ID(), rule2.ID(), "Should return same rule ID") // Delete using first reference - err = manager.DeleteRouteRule(rule1) + err = manager.DeleteFilterRule(rule1) require.NoError(t, err) // Verify traffic no longer passes @@ -364,7 +417,7 @@ func setupTestManager(t *testing.T) *Manager { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) require.NoError(t, manager.EnableRouting()) diff --git a/client/firewall/uspfilter/filter_test.go b/client/firewall/uspfilter/filter_test.go index f19c4bb56..280ce9311 100644 --- a/client/firewall/uspfilter/filter_test.go +++ b/client/firewall/uspfilter/filter_test.go @@ -78,18 +78,19 @@ func TestManagerCreate(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return } + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) if m == nil { t.Error("Manager is nil") } } -func TestManagerAddPeerFiltering(t *testing.T) { +func TestManagerAddFilterRule(t *testing.T) { isSetFilterCalled := false ifaceMock := &IFaceMock{ SetFilterFunc: func(device.PacketFilter) error { @@ -98,18 +99,19 @@ func TestManagerAddPeerFiltering(t *testing.T) { }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return } + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) ip := net.ParseIP("192.168.1.1") proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - rule, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + rule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return @@ -131,74 +133,47 @@ func TestManagerDeleteRule(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return } + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) ip := netip.MustParseAddr("192.168.1.1") proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - rule2, err := m.AddPeerFiltering(nil, ip.AsSlice(), proto, nil, port, action, "") + rule2, err := m.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, proto, nil, port, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return } - // Check rules exist in appropriate maps - for _, r := range rule2 { - peerRule, ok := r.(*PeerRule) - if !ok { - t.Errorf("rule should be a PeerRule") - continue - } - // Check if rule exists in deny or allow maps based on action - var found bool - if peerRule.drop { - _, found = m.incomingDenyRules[ip][r.ID()] - } else { - _, found = m.incomingRules[ip][r.ID()] - } - if !found { - t.Errorf("rule2 is not in the expected rules map") + peerRule, ok := rule2.(*PeerRule) + require.True(t, ok, "rule should be a peer rule") + + inMap := func() bool { + if peerRule.action == fw.ActionDrop { + return findRuleByID(m.incomingDenyRules, ip, rule2.ID()) } + return findRuleByID(m.incomingAcceptRules, ip, rule2.ID()) } - for _, r := range rule2 { - err = m.DeletePeerRule(r) - if err != nil { - t.Errorf("failed to delete rule: %v", err) - return - } - } + require.True(t, inMap(), "rule2 should be in the expected rules list") - // Check rules are removed from appropriate maps - for _, r := range rule2 { - peerRule, ok := r.(*PeerRule) - if !ok { - t.Errorf("rule should be a PeerRule") - continue - } - // Check if rule is removed from deny or allow maps based on action - var found bool - if peerRule.drop { - _, found = m.incomingDenyRules[ip][r.ID()] - } else { - _, found = m.incomingRules[ip][r.ID()] - } - if found { - t.Errorf("rule2 should be removed from the rules map") - } - } + require.NoError(t, m.DeleteFilterRule(rule2), "failed to delete rule") + + require.False(t, inMap(), "rule2 should be removed from the rules list") } func TestSetUDPPacketHook(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) }) @@ -220,9 +195,11 @@ func TestSetUDPPacketHook(t *testing.T) { } func TestSetTCPPacketHook(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) }) @@ -250,7 +227,7 @@ func TestPeerRuleLifecycleDenyRules(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, m.Close(nil)) @@ -260,36 +237,34 @@ func TestPeerRuleLifecycleDenyRules(t *testing.T) { addr := netip.MustParseAddr("192.168.1.1") // Add multiple deny rules for different ports - rule1, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{22}}, fw.ActionDrop, "") + rule1, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{22}}, fw.ActionDrop) require.NoError(t, err) - rule2, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{80}}, fw.ActionDrop, "") + rule2, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err) m.mutex.RLock() - denyCount := len(m.incomingDenyRules[addr]) + denyCount := countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 2, denyCount, "Should have exactly 2 deny rules") // Delete the first deny rule - err = m.DeletePeerRule(rule1[0]) + err = m.DeleteFilterRule(rule1) require.NoError(t, err) m.mutex.RLock() - denyCount = len(m.incomingDenyRules[addr]) + denyCount = countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 1, denyCount, "Should have 1 deny rule after deleting first") // Delete the second deny rule - err = m.DeletePeerRule(rule2[0]) + err = m.DeleteFilterRule(rule2) require.NoError(t, err) m.mutex.RLock() - _, exists := m.incomingDenyRules[addr] + exists := countRulesForAddr(m.incomingDenyRules, addr) > 0 m.mutex.RUnlock() - require.False(t, exists, "Deny rules IP entry should be cleaned up when empty") + require.False(t, exists, "Deny rules should be cleaned up when empty") } // TestPeerRuleAddAndDeleteDontLeak verifies that repeatedly adding and deleting @@ -299,7 +274,7 @@ func TestPeerRuleAddAndDeleteDontLeak(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, m.Close(nil)) @@ -311,27 +286,21 @@ func TestPeerRuleAddAndDeleteDontLeak(t *testing.T) { // Simulate 10 network map updates: add rule, delete old, add new for i := 0; i < 10; i++ { // Add a deny rule - rules, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{22}}, fw.ActionDrop, "") + rules, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{22}}, fw.ActionDrop) require.NoError(t, err) // Add an allow rule - allowRules, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + allowRules, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) // Delete them (simulating ACL manager cleanup) - for _, r := range rules { - require.NoError(t, m.DeletePeerRule(r)) - } - for _, r := range allowRules { - require.NoError(t, m.DeletePeerRule(r)) - } + require.NoError(t, m.DeleteFilterRule(rules)) + require.NoError(t, m.DeleteFilterRule(allowRules)) } m.mutex.RLock() - denyCount := len(m.incomingDenyRules[addr]) - allowCount := len(m.incomingRules[addr]) + denyCount := countRulesForAddr(m.incomingDenyRules, addr) + allowCount := countRulesForAddr(m.incomingAcceptRules, addr) m.mutex.RUnlock() require.Equal(t, 0, denyCount, "No deny rules should remain after cleanup") @@ -345,7 +314,7 @@ func TestMixedAllowDenyRulesSameIP(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, m.Close(nil)) @@ -354,41 +323,39 @@ func TestMixedAllowDenyRulesSameIP(t *testing.T) { ip := net.ParseIP("192.168.1.1") // Add allow rule for port 80 - allowRule, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + allowRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) // Add deny rule for port 22 - denyRule, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{22}}, fw.ActionDrop, "") + denyRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{22}}, fw.ActionDrop) require.NoError(t, err) addr := netip.MustParseAddr("192.168.1.1") m.mutex.RLock() - allowCount := len(m.incomingRules[addr]) - denyCount := len(m.incomingDenyRules[addr]) + allowCount := countRulesForAddr(m.incomingAcceptRules, addr) + denyCount := countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 1, allowCount, "Should have 1 allow rule") require.Equal(t, 1, denyCount, "Should have 1 deny rule") // Delete allow rule should not affect deny rule - err = m.DeletePeerRule(allowRule[0]) + err = m.DeleteFilterRule(allowRule) require.NoError(t, err) m.mutex.RLock() - denyCountAfter := len(m.incomingDenyRules[addr]) + denyCountAfter := countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 1, denyCountAfter, "Deny rule should still exist after deleting allow rule") // Delete deny rule - err = m.DeletePeerRule(denyRule[0]) + err = m.DeleteFilterRule(denyRule) require.NoError(t, err) m.mutex.RLock() - _, denyExists := m.incomingDenyRules[addr] - _, allowExists := m.incomingRules[addr] + denyExists := countRulesForAddr(m.incomingDenyRules, addr) > 0 + allowExists := countRulesForAddr(m.incomingAcceptRules, addr) > 0 m.mutex.RUnlock() require.False(t, denyExists, "Deny rules should be empty") @@ -400,7 +367,7 @@ func TestManagerReset(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return @@ -411,7 +378,7 @@ func TestManagerReset(t *testing.T) { port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - _, err = m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err = m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return @@ -423,7 +390,7 @@ func TestManagerReset(t *testing.T) { return } - if len(m.outgoingRules) != 0 || len(m.incomingRules) != 0 || len(m.incomingDenyRules) != 0 { + if len(m.incomingAcceptRules) != 0 || len(m.incomingDenyRules) != 0 { t.Errorf("rules are not empty") } } @@ -439,7 +406,7 @@ func TestNotMatchByIP(t *testing.T) { }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return @@ -449,7 +416,7 @@ func TestNotMatchByIP(t *testing.T) { proto := fw.ProtocolUDP action := fw.ActionAccept - _, err = m.AddPeerFiltering(nil, ip, proto, nil, nil, action, "") + _, err = m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, nil, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return @@ -502,7 +469,7 @@ func TestRemovePacketHook(t *testing.T) { } // creating manager instance - manager, err := Create(iface, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: iface, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Fatalf("Failed to create Manager: %s", err) } @@ -519,9 +486,11 @@ func TestRemovePacketHook(t *testing.T) { } func TestProcessOutgoingHooks(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) manager.udpTracker.Close() @@ -606,7 +575,7 @@ func TestUSPFilterCreatePerformance(t *testing.T) { ifaceMock := &IFaceMock{ SetFilterFunc: func(device.PacketFilter) error { return nil }, } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) time.Sleep(time.Second) @@ -621,7 +590,7 @@ func TestUSPFilterCreatePerformance(t *testing.T) { start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip, "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") } @@ -631,9 +600,11 @@ func TestUSPFilterCreatePerformance(t *testing.T) { } func TestStatefulFirewall_UDPTracking(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) manager.udpTracker.Close() // Close the existing tracker @@ -845,7 +816,7 @@ func TestUpdateSetMerge(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -858,7 +829,7 @@ func TestUpdateSetMerge(t *testing.T) { netip.MustParsePrefix("192.168.1.0/24"), } - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Set: set}, @@ -931,7 +902,7 @@ func TestUpdateSetDeduplication(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -939,7 +910,7 @@ func TestUpdateSetDeduplication(t *testing.T) { set := fw.NewDomainSet(domain.List{"example.org"}) - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Set: set}, @@ -1051,7 +1022,7 @@ func TestMSSClamping(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, 1280) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: 1280}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -1243,7 +1214,7 @@ func TestShouldForward(t *testing.T) { return wgaddr.Address{IP: wgIP, Network: netip.PrefixFrom(wgIP, 24)} } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -1358,7 +1329,7 @@ func TestShouldForward(t *testing.T) { // Re-create manager to pick up the new address with IPv6 require.NoError(t, manager.Close(nil)) - manager, err = Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err = Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) v6Cases := []struct { diff --git a/client/firewall/uspfilter/forwarder/forwarder.go b/client/firewall/uspfilter/forwarder/forwarder.go index 28320ad88..7308b38bd 100644 --- a/client/firewall/uspfilter/forwarder/forwarder.go +++ b/client/firewall/uspfilter/forwarder/forwarder.go @@ -12,6 +12,7 @@ import ( "time" log "github.com/sirupsen/logrus" + wgdevice "golang.zx2c4.com/wireguard/device" "gvisor.dev/gvisor/pkg/buffer" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/header" @@ -22,9 +23,9 @@ import ( "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" "gvisor.dev/gvisor/pkg/tcpip/transport/udp" - "github.com/netbirdio/netbird/client/firewall/uspfilter/common" "github.com/netbirdio/netbird/client/firewall/uspfilter/conntrack" nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" + "github.com/netbirdio/netbird/client/iface/wgaddr" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" ) @@ -40,6 +41,12 @@ const ( envForceTCPRACK = "NB_FORCE_TCP_RACK" ) +// IFace provides the WireGuard device and overlay addresses the forwarder needs. +type IFace interface { + GetWGDevice() *wgdevice.Device + Address() wgaddr.Address +} + type Forwarder struct { logger *nblog.Logger flowLogger nftypes.FlowLogger @@ -58,7 +65,7 @@ type Forwarder struct { pingSemaphore chan struct{} } -func New(iface common.IFaceMapper, logger *nblog.Logger, flowLogger nftypes.FlowLogger, netstack bool, mtu uint16) (*Forwarder, error) { +func New(iface IFace, logger *nblog.Logger, flowLogger nftypes.FlowLogger, netstack bool, mtu uint16) (*Forwarder, error) { s := stack.New(stack.Options{ NetworkProtocols: []stack.NetworkProtocolFactory{ ipv4.NewProtocol, diff --git a/client/firewall/uspfilter/fragment_test.go b/client/firewall/uspfilter/fragment_test.go index 6960e4dda..8f99864be 100644 --- a/client/firewall/uspfilter/fragment_test.go +++ b/client/firewall/uspfilter/fragment_test.go @@ -39,7 +39,7 @@ func newFragmentTestManager(tb testing.TB) *Manager { }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(tb, err) require.NoError(tb, m.UpdateLocalIPs()) tb.Cleanup(func() { require.NoError(tb, m.Close(nil)) }) @@ -175,8 +175,8 @@ func normalUDPPacket(tb testing.TB, dstPort uint16, payloadLen int) []byte { func allowUDP(tb testing.TB, m *Manager, dstPort uint16) { tb.Helper() - _, err := m.AddPeerFiltering(nil, net.ParseIP(fragTestSrc), fw.ProtocolUDP, nil, - &fw.Port{Values: []uint16{dstPort}}, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolUDP, nil, + &fw.Port{Values: []uint16{dstPort}}, fw.ActionAccept) require.NoError(tb, err) } @@ -228,8 +228,8 @@ func TestFragment_DeniedFirstDropsTrailing(t *testing.T) { // the overlap lands on real header bytes (the flags at byte 13). func TestFragment_OverlappingHeaderDropped(t *testing.T) { m := newFragmentTestManager(t) - _, err := m.AddPeerFiltering(nil, net.ParseIP(fragTestSrc), fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{8080}}, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) require.NoError(t, err) // First fragment: TCP header (20) + 12 data = 32 bytes -> headerEnd = 4 octets. @@ -290,8 +290,8 @@ func TestFragment_TinyFirstDropped(t *testing.T) { // trailing fragments inherit the verdict. func TestFragment_TCPFirstFragment(t *testing.T) { m := newFragmentTestManager(t) - _, err := m.AddPeerFiltering(nil, net.ParseIP(fragTestSrc), fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{8080}}, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) require.NoError(t, err) // TCP header (20) + 12 data = 32 bytes -> headerEnd = 4 octets. @@ -308,8 +308,8 @@ func TestFragment_TCPFirstFragment(t *testing.T) { // bytes would satisfy a UDP header but falls short of the 20-byte TCP header. func TestFragment_TCPTinyFirstDropped(t *testing.T) { m := newFragmentTestManager(t) - _, err := m.AddPeerFiltering(nil, net.ParseIP(fragTestSrc), fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{8080}}, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) require.NoError(t, err) tiny := trailingFragmentTo(t, fragTestDst, layers.IPProtocolTCP, 0x7777, 0, true, 12) @@ -353,7 +353,7 @@ func TestFragment_RouteACL(t *testing.T) { m.routingEnabled.Store(true) m.nativeRouter.Store(false) - _, err := m.AddRouteFiltering( + _, err := m.AddFilterRule( []byte("rt-1"), []netip.Prefix{netip.MustParsePrefix("100.10.0.0/16")}, fw.Network{Prefix: netip.MustParsePrefix("198.51.100.0/24")}, @@ -511,8 +511,8 @@ func TestFragmentV6_TrailingWithoutFirstDropped(t *testing.T) { // through. func TestFragmentV6_AllowedFirstPassesTrailing(t *testing.T) { m := newFragmentTestManager(t) - _, err := m.AddPeerFiltering(nil, net.ParseIP(fragTestSrcV6), fw.ProtocolUDP, nil, - &fw.Port{Values: []uint16{8080}}, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrcV6)), fw.Network{}, fw.ProtocolUDP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) require.NoError(t, err) // First fragment: UDP header (8) + 32 data = 40 octets -> headerEnd = 5. @@ -531,8 +531,8 @@ func TestFragmentV6_AllowedFirstPassesTrailing(t *testing.T) { // exhaust the verdict table. func TestFragmentV6_AtomicNotCached(t *testing.T) { m := newFragmentTestManager(t) - _, err := m.AddPeerFiltering(nil, net.ParseIP(fragTestSrcV6), fw.ProtocolUDP, nil, - &fw.Port{Values: []uint16{8080}}, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrcV6)), fw.Network{}, fw.ProtocolUDP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) require.NoError(t, err) atomic := fragmentUDPv6(t, 0xA70301C, 8080, 16, false) diff --git a/client/firewall/uspfilter/allow_netbird_windows.go b/client/firewall/uspfilter/interface_allower_windows.go similarity index 79% rename from client/firewall/uspfilter/allow_netbird_windows.go rename to client/firewall/uspfilter/interface_allower_windows.go index 10a2b9116..7f525e28c 100644 --- a/client/firewall/uspfilter/allow_netbird_windows.go +++ b/client/firewall/uspfilter/interface_allower_windows.go @@ -9,7 +9,6 @@ import ( log "github.com/sirupsen/logrus" nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/internal/statemanager" ) type action string @@ -20,35 +19,20 @@ const ( firewallRuleName = "Netbird" ) -// Close cleans up the firewall manager by removing all rules and closing trackers -func (m *Manager) Close(*statemanager.Manager) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - m.resetState() - - if !isWindowsFirewallReachable() { - return nil - } - - var merr *multierror.Error - if isFirewallRuleActive(firewallRuleName) { - if err := manageFirewallRule(firewallRuleName, deleteRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove windows firewall rule: %w", err)) - } - } - - if isFirewallRuleActive(firewallRuleName + "-v6") { - if err := manageFirewallRule(firewallRuleName+"-v6", deleteRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove windows v6 firewall rule: %w", err)) - } - } - - return nberrors.FormatErrorOrNil(merr) +// WindowsInterfaceAllower opens the NetBird interface in the Windows firewall +// via netsh advfirewall rules. It implements InterfaceAllower for the userspace +// firewall on Windows. +type WindowsInterfaceAllower struct { + iface Iface } -// AllowNetbird allows netbird interface traffic -func (m *Manager) AllowNetbird() error { +// NewWindowsInterfaceAllower builds the Windows netsh-based interface allower. +func NewWindowsInterfaceAllower(iface Iface) *WindowsInterfaceAllower { + return &WindowsInterfaceAllower{iface: iface} +} + +// Apply adds inbound-allow netsh rules for the interface's addresses. +func (a *WindowsInterfaceAllower) Apply() error { if !isWindowsFirewallReachable() { return nil } @@ -60,13 +44,13 @@ func (m *Manager) AllowNetbird() error { "enable=yes", "action=allow", "profile=any", - "localip="+m.wgIface.Address().IP.String(), + "localip="+a.iface.Address().IP.String(), ); err != nil { return err } } - if v6 := m.wgIface.Address().IPv6; v6.IsValid() && !isFirewallRuleActive(firewallRuleName+"-v6") { + if v6 := a.iface.Address().IPv6; v6.IsValid() && !isFirewallRuleActive(firewallRuleName+"-v6") { if err := manageFirewallRule(firewallRuleName+"-v6", addRule, "dir=in", @@ -82,8 +66,27 @@ func (m *Manager) AllowNetbird() error { return nil } -func manageFirewallRule(ruleName string, action action, extraArgs ...string) error { +// Close removes the netsh rules added by Apply. +func (a *WindowsInterfaceAllower) Close() error { + if !isWindowsFirewallReachable() { + return nil + } + var merr *multierror.Error + if isFirewallRuleActive(firewallRuleName) { + if err := manageFirewallRule(firewallRuleName, deleteRule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove windows firewall rule: %w", err)) + } + } + if isFirewallRuleActive(firewallRuleName + "-v6") { + if err := manageFirewallRule(firewallRuleName+"-v6", deleteRule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove windows v6 firewall rule: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +func manageFirewallRule(ruleName string, action action, extraArgs ...string) error { args := []string{"advfirewall", "firewall", string(action), "rule", "name=" + ruleName} if action == addRule { args = append(args, extraArgs...) diff --git a/client/firewall/uspfilter/localip.go b/client/firewall/uspfilter/localip.go index b35be56c6..869832732 100644 --- a/client/firewall/uspfilter/localip.go +++ b/client/firewall/uspfilter/localip.go @@ -7,8 +7,6 @@ import ( "sync/atomic" log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/client/firewall/uspfilter/common" ) // localIPSnapshot is an immutable snapshot of local IP addresses, swapped @@ -60,7 +58,7 @@ func processInterface(iface net.Interface, ips map[netip.Addr]struct{}, addresse } // UpdateLocalIPs rebuilds the local IP snapshot and swaps it in atomically. -func (m *localIPManager) UpdateLocalIPs(iface common.IFaceMapper) (err error) { +func (m *localIPManager) UpdateLocalIPs(iface Iface) (err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: %v", r) diff --git a/client/firewall/uspfilter/nat.go b/client/firewall/uspfilter/nat.go index 5d51c1538..06312aabf 100644 --- a/client/firewall/uspfilter/nat.go +++ b/client/firewall/uspfilter/nat.go @@ -487,19 +487,13 @@ func incrementalUpdate(oldChecksum uint16, oldBytes, newBytes []byte) uint16 { } // AddDNATRule adds outbound DNAT rule for forwarding external traffic to NetBird network. -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - if m.nativeFirewall == nil { - return nil, errNatNotSupported - } - return m.nativeFirewall.AddDNATRule(rule) +func (m *Manager) AddDNATRule(firewall.ForwardRule) (firewall.Rule, error) { + return nil, errNotSupported } // DeleteDNATRule deletes outbound DNAT rule. -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - if m.nativeFirewall == nil { - return errNatNotSupported - } - return m.nativeFirewall.DeleteDNATRule(rule) +func (m *Manager) DeleteDNATRule(firewall.Rule) error { + return errNotSupported } // addPortRedirection adds a port redirection rule. @@ -521,7 +515,6 @@ func (m *Manager) addPortRedirection(targetIP netip.Addr, protocol gopacket.Laye } // AddInboundDNAT adds an inbound DNAT rule redirecting traffic from NetBird peers to local services. -// TODO: also delegate to nativeFirewall when available for kernel WG mode func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { var layerType gopacket.LayerType switch protocol { @@ -567,20 +560,16 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot return m.removePortRedirection(localAddr, layerType, originalPort, translatedPort) } -// AddOutputDNAT delegates to the native firewall if available. -func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if m.nativeFirewall == nil { - return fmt.Errorf("output DNAT not supported without native firewall") - } - return m.nativeFirewall.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) +// AddOutputDNAT is not supported by the userspace firewall: it backs kernel DNS +// redirection, but userspace DNS is served in-process on the gVisor netstack, so +// this should never be called. +func (m *Manager) AddOutputDNAT(netip.Addr, firewall.Protocol, uint16, uint16) error { + return errNotSupported } -// RemoveOutputDNAT delegates to the native firewall if available. -func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if m.nativeFirewall == nil { - return nil - } - return m.nativeFirewall.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) +// RemoveOutputDNAT is a no-op for the userspace firewall (see AddOutputDNAT). +func (m *Manager) RemoveOutputDNAT(netip.Addr, firewall.Protocol, uint16, uint16) error { + return nil } // translateInboundPortDNAT applies port-specific DNAT translation to inbound packets. diff --git a/client/firewall/uspfilter/nat_bench_test.go b/client/firewall/uspfilter/nat_bench_test.go index 1e15c8c0c..422c6b849 100644 --- a/client/firewall/uspfilter/nat_bench_test.go +++ b/client/firewall/uspfilter/nat_bench_test.go @@ -64,9 +64,11 @@ func BenchmarkDNATTranslation(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -124,9 +126,11 @@ func BenchmarkDNATTranslation(b *testing.B) { // BenchmarkDNATConcurrency tests DNAT performance under concurrent load func BenchmarkDNATConcurrency(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -196,9 +200,11 @@ func BenchmarkDNATScaling(b *testing.B) { for _, count := range mappingCounts { b.Run(fmt.Sprintf("mappings_%d", count), func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -308,9 +314,11 @@ func BenchmarkChecksumUpdate(b *testing.B) { // BenchmarkDNATMemoryAllocations checks for memory allocations in DNAT operations func BenchmarkDNATMemoryAllocations(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -481,9 +489,11 @@ func BenchmarkPortDNAT(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) diff --git a/client/firewall/uspfilter/nat_stateful_test.go b/client/firewall/uspfilter/nat_stateful_test.go index 21c6da06e..5fa5da027 100644 --- a/client/firewall/uspfilter/nat_stateful_test.go +++ b/client/firewall/uspfilter/nat_stateful_test.go @@ -13,9 +13,11 @@ import ( // TestPortDNATBasic tests basic port DNAT functionality func TestPortDNATBasic(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -49,9 +51,11 @@ func TestPortDNATBasic(t *testing.T) { // TestPortDNATMultipleRules tests multiple port DNAT rules func TestPortDNATMultipleRules(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) diff --git a/client/firewall/uspfilter/nat_test.go b/client/firewall/uspfilter/nat_test.go index 4598c3901..5b5840383 100644 --- a/client/firewall/uspfilter/nat_test.go +++ b/client/firewall/uspfilter/nat_test.go @@ -15,9 +15,11 @@ import ( // TestDNATTranslationCorrectness verifies DNAT translation works correctly func TestDNATTranslationCorrectness(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -104,9 +106,11 @@ func parsePacket(t testing.TB, packetData []byte) *decoder { // TestDNATMappingManagement tests adding/removing DNAT mappings func TestDNATMappingManagement(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -152,9 +156,11 @@ func TestDNATMappingManagement(t *testing.T) { } func TestInboundPortDNAT(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -202,9 +208,11 @@ func TestInboundPortDNAT(t *testing.T) { } func TestInboundPortDNATNegative(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) diff --git a/client/firewall/uspfilter/peer_acl_bench_test.go b/client/firewall/uspfilter/peer_acl_bench_test.go new file mode 100644 index 000000000..bcb0ca5c2 --- /dev/null +++ b/client/firewall/uspfilter/peer_acl_bench_test.go @@ -0,0 +1,333 @@ +//go:build uspbench + +package uspfilter + +import ( + "fmt" + "io" + "math/rand" + "net" + "net/netip" + "runtime" + "testing" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +// BenchmarkPeerACLMatch measures the per-packet cost of the peer ACL +// matcher (peerACLsBlock) across realistic shapes: M distinct policy +// rules, each with K source peers in its set. +// +// With the reverse-source index, miss cost is independent of M and +// hit cost grows only with the number of rules touching a single +// srcIP, not with total rule count. +func BenchmarkPeerACLMatch(b *testing.B) { + shapes := []struct{ M, K int }{ + {1, 100}, {10, 100}, {50, 100}, {100, 100}, {100, 1000}, + } + families := []struct { + name string + v6 bool + }{{"v4", false}, {"v6", true}} + + for _, fam := range families { + for _, s := range shapes { + b.Run(fmt.Sprintf("%s/M=%d/K=%d/hit", fam.name, s.M, s.K), func(b *testing.B) { + runPeerACLBench(b, s.M, s.K, true, fam.v6) + }) + b.Run(fmt.Sprintf("%s/M=%d/K=%d/miss", fam.name, s.M, s.K), func(b *testing.B) { + runPeerACLBench(b, s.M, s.K, false, fam.v6) + }) + } + } +} + +func runPeerACLBench(b *testing.B, m, k int, hit, v6 bool) { + log.SetOutput(io.Discard) // keep manager logs out of the benchmark output + + // Miss packets are dropped, so they always traverse the full peer + // ACL matcher (every bucket) without short-circuiting and without + // touching conntrack. Disable conntrack for the miss case so it + // measures the matcher, not established-state lookups. The hit case + // keeps conntrack on: an accepted packet reaches trackInbound, which + // needs the trackers conntrack creates. + if !hit { + b.Setenv("NB_DISABLE_CONNTRACK", "1") + } + + bits := 32 + genPkt := generatePacket + addrs := uniqueAddrs + if v6 { + bits = 128 + genPkt = generatePacket6 + addrs = uniqueAddrs6 + } + + // dstIP must be a local IP so filterInbound takes the local-traffic + // path (handleLocalTraffic → peerACLsBlock) we are measuring; an + // address the manager doesn't own would be treated as routed and + // short-circuit before the peer matcher. + dstIP := addrs(1, 2)[0] + mockAddr := wgaddr.Address{IP: dstIP, Network: netip.PrefixFrom(dstIP, bits)} + if v6 { + // The local-IP manager needs a valid v4 address too; expose the v6 + // dst as the interface's IPv6 so IsLocalIP recognizes it. + mockAddr = wgaddr.Address{ + IP: netip.MustParseAddr("100.64.0.1"), + Network: netip.MustParsePrefix("100.64.0.0/16"), + IPv6: dstIP, + IPv6Net: netip.PrefixFrom(dstIP, bits), + } + } + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { return mockAddr }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) + b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) + + // Generate M policies × K source peers, all distinct. + all := addrs(m*k, 1) + for i := 0; i < m; i++ { + sources := make([]netip.Prefix, k) + for j, a := range all[i*k : (i+1)*k] { + sources[j] = netip.PrefixFrom(a, bits) + } + _, err := manager.AddFilterRule( + nil, sources, fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{uint16(80 + i)}}, + fw.ActionAccept) + require.NoError(b, err) + } + + // Hit: cycle through real sources, picking the matching policy's port. + // Miss: a source from a disjoint range, port 80 (matches no policy). + var pktFn func(i int) []byte + if hit { + pktFn = func(i int) []byte { + policy := i % m + src := all[policy*k+(i%k)] + return genPkt(b, src.AsSlice(), dstIP.AsSlice(), + uint16(1024+i%60000), uint16(80+policy), layers.IPProtocolTCP) + } + } else { + miss := addrs(4096, 99) + pktFn = func(i int) []byte { + return genPkt(b, miss[i%len(miss)].AsSlice(), dstIP.AsSlice(), + uint16(1024+i%60000), 80, layers.IPProtocolTCP) + } + } + + // Pre-build a pool to avoid allocations dominating the measurement. + pool := make([][]byte, 1024) + for i := range pool { + pool[i] = pktFn(i) + } + + // Confirm the matcher is actually exercised: a hit packet must be + // allowed and a miss packet dropped. Without this the benchmark + // could silently time the routed early-return instead. + require.Equal(b, !hit, manager.filterInbound(pool[0], 0), + "benchmark must reach the peer ACL matcher") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + manager.filterInbound(pool[i%len(pool)], 0) + } +} + +// BenchmarkPeerACLIndexMemory reports the resident memory cost of +// the source-keyed index across realistic deployment shapes. Two +// dimensions matter: (M, K), the number of policies × peers-per-policy, +// and overlap, the fraction of peers shared between policies. +// +// The output uses ReportMetric("bytes/rule") so the cost can be +// compared across shapes directly. Total bytes = bytes/rule * M. +func BenchmarkPeerACLIndexMemory(b *testing.B) { + cases := []struct { + name string + M, K int + overlapFrac float64 // 0 = disjoint per-policy sources, 1 = all share the same pool + }{ + {"M=10/K=100/disjoint", 10, 100, 0}, + {"M=100/K=100/disjoint", 100, 100, 0}, + {"M=100/K=1000/disjoint", 100, 1000, 0}, + {"M=100/K=1000/overlap=0.5", 100, 1000, 0.5}, + {"M=100/K=1000/overlap=1.0", 100, 1000, 1.0}, + {"M=1000/K=100/overlap=1.0", 1000, 100, 1.0}, + } + + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + mgr, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) + + populateIndexedRules(b, mgr, c.M, c.K, c.overlapFrac) + + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + before := ms.HeapAlloc + + // Drop the manager's external roots so we can isolate + // the index cost. We hold the manager itself live; the + // index is what we measure on the second pass. + mgr.incomingAcceptIndex.reset() + mgr.incomingDenyIndex.reset() + mgr.incomingAcceptRules = mgr.incomingAcceptRules[:0] + mgr.incomingDenyRules = mgr.incomingDenyRules[:0] + runtime.GC() + runtime.ReadMemStats(&ms) + after := ms.HeapAlloc + + delta := int64(before) - int64(after) + if delta < 0 { + delta = 0 + } + b.ReportMetric(float64(delta)/float64(c.M), "bytes/rule") + b.ReportMetric(float64(delta), "bytes/total") + + require.NoError(b, mgr.Close(nil)) + } + }) + } +} + +func populateIndexedRules(b *testing.B, mgr *Manager, m, k int, overlapFrac float64) { + b.Helper() + pool := uniqueAddrs(k+m*k, 1) // big enough universe + sharedLen := int(float64(k) * overlapFrac) + if sharedLen > k { + sharedLen = k + } + shared := pool[:sharedLen] + uniquePool := pool[sharedLen:] + + for i := 0; i < m; i++ { + sources := make([]netip.Prefix, 0, k) + for _, a := range shared { + sources = append(sources, netip.PrefixFrom(a, 32)) + } + // each policy gets (k-sharedLen) addresses unique to it from the unique pool + unique := uniquePool[i*(k-sharedLen) : (i+1)*(k-sharedLen)] + for _, a := range unique { + sources = append(sources, netip.PrefixFrom(a, 32)) + } + _, err := mgr.AddFilterRule( + nil, sources, fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{uint16(80 + i)}}, + fw.ActionAccept) + require.NoError(b, err) + } +} + +// uniqueAddrs returns n distinct addrs. Seeds 1, 2 are used for +// policy sources / dst; seed 99 puts misses in 10/8. +func uniqueAddrs(n int, seed int64) []netip.Addr { + out := make([]netip.Addr, 0, n) + seen := make(map[netip.Addr]struct{}, n) + r := rand.New(rand.NewSource(seed)) + miss := seed == 99 + for len(out) < n { + var b [4]byte + if miss { + b[0] = 10 + b[1] = byte(r.Intn(256)) + } else { + b[0] = 100 + b[1] = byte(64 + r.Intn(63)) + } + b[2] = byte(r.Intn(256)) + b[3] = byte(1 + r.Intn(254)) + a := netip.AddrFrom4(b) + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + out = append(out, a) + } + return out +} + +// uniqueAddrs6 mirrors uniqueAddrs for IPv6: sources come from the ULA +// range fd00::/8, the miss set (seed 99) from 2001:db8::/32 so it is +// disjoint from any source. +func uniqueAddrs6(n int, seed int64) []netip.Addr { + out := make([]netip.Addr, 0, n) + seen := make(map[netip.Addr]struct{}, n) + r := rand.New(rand.NewSource(seed)) + miss := seed == 99 + for len(out) < n { + var b [16]byte + if miss { + b[0], b[1], b[2], b[3] = 0x20, 0x01, 0x0d, 0xb8 + } else { + b[0] = 0xfd + } + for x := 8; x < 16; x++ { + b[x] = byte(r.Intn(256)) + } + a := netip.AddrFrom16(b) + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + out = append(out, a) + } + return out +} + +// generatePacket6 builds an IPv6 TCP/UDP packet, mirroring +// generatePacket for the v4 case. +func generatePacket6(b *testing.B, srcIP, dstIP net.IP, srcPort, dstPort uint16, protocol layers.IPProtocol) []byte { + b.Helper() + + ipv6 := &layers.IPv6{ + Version: 6, + HopLimit: 64, + NextHeader: protocol, + SrcIP: srcIP, + DstIP: dstIP, + } + + var transportLayer gopacket.SerializableLayer + switch protocol { + case layers.IPProtocolTCP: + tcp := &layers.TCP{ + SrcPort: layers.TCPPort(srcPort), + DstPort: layers.TCPPort(dstPort), + SYN: true, + } + require.NoError(b, tcp.SetNetworkLayerForChecksum(ipv6)) + transportLayer = tcp + case layers.IPProtocolUDP: + udp := &layers.UDP{ + SrcPort: layers.UDPPort(srcPort), + DstPort: layers.UDPPort(dstPort), + } + require.NoError(b, udp.SetNetworkLayerForChecksum(ipv6)) + transportLayer = udp + } + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(b, gopacket.SerializeLayers(buf, opts, ipv6, transportLayer, gopacket.Payload("test"))) + return buf.Bytes() +} diff --git a/client/firewall/uspfilter/peer_acl_dedup_test.go b/client/firewall/uspfilter/peer_acl_dedup_test.go new file mode 100644 index 000000000..696766d80 --- /dev/null +++ b/client/firewall/uspfilter/peer_acl_dedup_test.go @@ -0,0 +1,150 @@ +package uspfilter + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + nbiface "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" +) + +func newTestManager(t *testing.T) *Manager { + t.Helper() + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + } + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) + require.NoError(t, err, "create manager") + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) + return m +} + +// TestAddPeerFiltering_DeduplicatesIdenticalRules verifies that adding +// the same peer rule twice does not create two backing rules. The acl +// manager keys its own cache, but the firewall backend must be +// idempotent on its own so a double-apply cannot leak rules, matching +// the route path and the kernel backends. +func TestAddPeerFiltering_DeduplicatesIdenticalRules(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + action := fw.ActionDrop + + first, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "first add") + + second, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "second add") + + assert.Equal(t, first.ID(), second.ID(), "duplicate add should return the same rule id") + assert.Len(t, m.incomingDenyRules, 1, "duplicate add must not create a second backing rule") +} + +// TestDeletePeerFiltering_NoRefcountSingleDeleteRemoves locks the +// backend's owner accounting for the same-owner case: a content key +// installed twice by the same owner registers one owner claim, so the +// first DeleteFilterRule removes the rule. Owner counting only kicks +// in for distinct management rule IDs (see the peer owner tests); the +// acl manager keys its tracking per (policy, content) and deletes once +// per key, so adds and deletes stay balanced. +func TestDeletePeerFiltering_NoRefcountSingleDeleteRemoves(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + action := fw.ActionDrop + + first, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "first add") + + second, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "second add") + require.Equal(t, first.ID(), second.ID(), "dedup to one rule") + require.Len(t, m.incomingDenyRules, 1, "still one backing rule after duplicate add") + + require.NoError(t, m.DeleteFilterRule(first), "delete once") + assert.Empty(t, m.incomingDenyRules, "single delete removes the backing rule (no refcount)") + assert.NotContains(t, m.peerRulesMap, first.ID(), "dedup map entry cleared") +} + +// TestAddPeerFiltering_DeterministicID verifies the peer rule id is a +// content hash, not a random UUID: identical inputs produce the same id +// across independent managers. A random id breaks caller-side dedup. +func TestAddPeerFiltering_DeterministicID(t *testing.T) { + ip := net.ParseIP("10.0.0.5") + proto := fw.ProtocolUDP + port := &fw.Port{Values: []uint16{53}} + action := fw.ActionAccept + + m1 := newTestManager(t) + r1, err := m1.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err) + + m2 := newTestManager(t) + r2, err := m2.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err) + + assert.Equal(t, r1.ID(), r2.ID(), "same inputs must produce the same rule id") +} + +// TestAddPeerFiltering_DistinctRulesNotDeduped verifies that rules +// differing only by port are kept separate. +func TestAddPeerFiltering_DistinctRulesNotDeduped(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + action := fw.ActionAccept + + r80, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, &fw.Port{Values: []uint16{80}}, action) + require.NoError(t, err) + + r443, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, &fw.Port{Values: []uint16{443}}, action) + require.NoError(t, err) + + assert.NotEqual(t, r80.ID(), r443.ID(), "different ports must produce different rule ids") + assert.Len(t, m.incomingAcceptRules, 2, "distinct rules must both be stored") +} + +// TestAddPeerFiltering_SourceVsDestPortNotDeduped verifies that a rule +// matching on source port and one matching on destination port for the +// same selector do not collide: the port lands in a different slot, so +// the content key must differ. +func TestAddPeerFiltering_SourceVsDestPortNotDeduped(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + action := fw.ActionAccept + + dPortRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err) + + sPortRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, port, nil, action) + require.NoError(t, err) + + assert.NotEqual(t, dPortRule.ID(), sPortRule.ID(), "source-port and dest-port matches must produce different rule ids") +} + +// TestAddFilterRule_EmptySourcesRejected verifies that an empty source +// list is rejected rather than treated as "match any". "Match any" must +// be an explicit /0, so a zeroed list can never silently widen a rule to +// every source. +func TestAddFilterRule_EmptySourcesRejected(t *testing.T) { + m := newTestManager(t) + + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + + _, err := m.AddFilterRule(nil, nil, fw.Network{}, proto, nil, port, fw.ActionAccept) + require.ErrorIs(t, err, fw.ErrNoSources, "empty sources must be rejected") + assert.Empty(t, m.incomingAcceptRules, "no rule should be stored for empty sources") +} diff --git a/client/firewall/uspfilter/peer_acl_ipv6_test.go b/client/firewall/uspfilter/peer_acl_ipv6_test.go new file mode 100644 index 000000000..282b44ebb --- /dev/null +++ b/client/firewall/uspfilter/peer_acl_ipv6_test.go @@ -0,0 +1,105 @@ +package uspfilter + +import ( + "net" + "net/netip" + "testing" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + nbiface "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +func newV6TestManager(t *testing.T, localV6 string) *Manager { + t.Helper() + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("100.10.0.100"), + Network: netip.MustParsePrefix("100.10.0.0/16"), + IPv6: netip.MustParseAddr(localV6), + IPv6Net: netip.MustParsePrefix("fd00::/64"), + } + }, + } + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) + require.NoError(t, err, "create manager") + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) + return m +} + +func v6UDPPacket(t *testing.T, src, dst string, dstPort uint16) []byte { + t.Helper() + ip6 := &layers.IPv6{ + Version: 6, + HopLimit: 64, + NextHeader: layers.IPProtocolUDP, + SrcIP: net.ParseIP(src), + DstIP: net.ParseIP(dst), + } + udp := &layers.UDP{SrcPort: 51334, DstPort: layers.UDPPort(dstPort)} + require.NoError(t, udp.SetNetworkLayerForChecksum(ip6)) + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(t, gopacket.SerializeLayers(buf, opts, ip6, udp, gopacket.Payload("test"))) + return buf.Bytes() +} + +// TestPeerACL_IPv6HostRule verifies the source index resolves /128 v6 +// rules: a matching v6 source is accepted, a non-matching one is +// denied by the default. This is the end-to-end proof that the index +// is not v4-only. +func TestPeerACL_IPv6HostRule(t *testing.T) { + m := newV6TestManager(t, "fd00::100") + + src := net.ParseIP("fd00::1") + _, err := m.AddFilterRule(nil, pfx(src), fw.Network{}, fw.ProtocolUDP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionAccept) + require.NoError(t, err, "add v6 accept rule") + + require.False(t, m.filterInbound(v6UDPPacket(t, "fd00::1", "fd00::100", 53), 0), + "v6 packet from the allowed /128 source must be accepted") + require.True(t, m.filterInbound(v6UDPPacket(t, "fd00::2", "fd00::100", 53), 0), + "v6 packet from an unlisted source must be denied by default") +} + +// TestPeerACL_IPv6IndexBuckets verifies that v6 sources land in the +// right index bucket: a /128 in bySource keyed by its address, and +// coarser prefixes (including ::/0) in the nonHost slice. +func TestPeerACL_IPv6IndexBuckets(t *testing.T) { + m := newV6TestManager(t, "fd00::100") + port := &fw.Port{Values: []uint16{53}} + + host := netip.MustParseAddr("fd00::1") + _, err := m.AddFilterRule(nil, []netip.Prefix{netip.PrefixFrom(host, 128)}, fw.Network{}, fw.ProtocolUDP, nil, port, fw.ActionAccept) + require.NoError(t, err) + assert.Contains(t, m.incomingAcceptIndex.bySource, host, "/128 v6 source must be indexed by address") + + _, err = m.AddFilterRule(nil, []netip.Prefix{netip.MustParsePrefix("fd00:dead::/64")}, fw.Network{}, fw.ProtocolUDP, nil, port, fw.ActionAccept) + require.NoError(t, err) + require.Len(t, m.incomingAcceptIndex.nonHost, 1, "coarser v6 prefix must land in nonHost") + + _, err = m.AddFilterRule(nil, []netip.Prefix{netip.MustParsePrefix("::/0")}, fw.Network{}, fw.ProtocolUDP, nil, port, fw.ActionAccept) + require.NoError(t, err) + require.Len(t, m.incomingAcceptIndex.nonHost, 2, "::/0 source must also land in nonHost") +} + +// TestPeerACL_IPv4MappedSourceNormalized verifies a v4-mapped v6 +// source prefix is normalized to v4 so a plain v4 packet matches it. +func TestPeerACL_IPv4MappedSourceNormalized(t *testing.T) { + m := newTestManager(t) + + mapped := netip.MustParseAddr("::ffff:192.168.1.1") + _, err := m.AddFilterRule(nil, []netip.Prefix{netip.PrefixFrom(mapped, mapped.BitLen())}, fw.Network{}, fw.ProtocolUDP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionAccept) + require.NoError(t, err) + + v4 := netip.MustParseAddr("192.168.1.1") + assert.Contains(t, m.incomingAcceptIndex.bySource, v4, "v4-mapped v6 source must be indexed as plain v4") +} diff --git a/client/firewall/uspfilter/peer_family_scope_test.go b/client/firewall/uspfilter/peer_family_scope_test.go new file mode 100644 index 000000000..1cf3498fb --- /dev/null +++ b/client/firewall/uspfilter/peer_family_scope_test.go @@ -0,0 +1,104 @@ +package uspfilter + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" +) + +// peerACLCheck decodes the packet and runs it through the peer ACLs, +// returning the attributed management rule id and the drop verdict. +func peerACLCheck(t *testing.T, m *Manager, packet []byte) ([]byte, bool) { + t.Helper() + d := m.decoders.Get().(*decoder) + defer m.decoders.Put(d) + require.NoError(t, d.decodePacket(packet)) + src, _ := m.extractIPs(d) + return m.peerACLsBlock(src, d, packet) +} + +// TestPeerACL_MultiValuePortMatchesEachListedPort guards the multi-value +// port path: a rule listing several discrete destination ports must +// match a packet to each listed port and drop one that is not listed. +// Management currently splits a multi-port policy into one rule per port +// (and the wire format carries a single port), so this list shape is not +// emitted today; the test locks correct matching in case that changes. +func TestPeerACL_MultiValuePortMatchesEachListedPort(t *testing.T) { + m := newTestManager(t) + + src := net.ParseIP("192.168.1.1") + ports := &fw.Port{Values: []uint16{80, 443}} + _, err := m.AddFilterRule(nil, pfx(src), fw.Network{}, fw.ProtocolTCP, nil, ports, fw.ActionAccept) + require.NoError(t, err, "add multi-value port rule") + + for _, p := range []uint16{80, 443} { + _, blocked := peerACLCheck(t, m, createTestPacket(t, "192.168.1.1", "10.0.0.2", fw.ProtocolTCP, 12345, p)) + assert.False(t, blocked, "packet to listed port %d must match the rule", p) + } + + _, blocked := peerACLCheck(t, m, createTestPacket(t, "192.168.1.1", "10.0.0.2", fw.ProtocolTCP, 12345, 8080)) + assert.True(t, blocked, "packet to a port not in the list must not match the rule") +} + +// TestPeerACL_MatchAnyIsFamilyScoped verifies that a /0 source matches +// only packets of its own family: 0.0.0.0/0 must not match IPv6 packets +// and ::/0 must not match IPv4 packets, matching kernel backend +// semantics. +func TestPeerACL_MatchAnyIsFamilyScoped(t *testing.T) { + m := newTestManager(t) + + v4Packet := createTestPacket(t, "10.0.0.1", "10.0.0.2", fw.ProtocolUDP, 12345, 53) + v6Packet := v6UDPPacket(t, "fd00::1", "fd00::100", 53) + + v4Any := []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + rule, err := m.AddFilterRule(nil, v4Any, fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err, "add v4 /0 rule") + + _, blocked := peerACLCheck(t, m, v4Packet) + assert.False(t, blocked, "0.0.0.0/0 must match IPv4 packets") + _, blocked = peerACLCheck(t, m, v6Packet) + assert.True(t, blocked, "0.0.0.0/0 must not match IPv6 packets") + + require.NoError(t, m.DeleteFilterRule(rule)) + + v6Any := []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + _, err = m.AddFilterRule(nil, v6Any, fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err, "add v6 /0 rule") + + _, blocked = peerACLCheck(t, m, v6Packet) + assert.False(t, blocked, "::/0 must match IPv6 packets") + _, blocked = peerACLCheck(t, m, v4Packet) + assert.True(t, blocked, "::/0 must not match IPv4 packets") +} + +// TestRouteACL_MixedFamilyZeroSourcesStayFamilySafe verifies the route +// path keeps per-prefix family matching when a single rule carries both +// 0.0.0.0/0 and ::/0 sources, as blockInvalidRouted does. +func TestRouteACL_MixedFamilyZeroSourcesStayFamilySafe(t *testing.T) { + m := newTestManager(t) + + sources := []netip.Prefix{ + netip.PrefixFrom(netip.IPv4Unspecified(), 0), + netip.PrefixFrom(netip.IPv6Unspecified(), 0), + } + + _, err := m.AddFilterRule(nil, sources, fw.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")}, + fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err) + _, err = m.AddFilterRule(nil, sources, fw.Network{Prefix: netip.MustParsePrefix("fd00:1::/64")}, + fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err) + + v4Src := netip.MustParseAddr("192.168.1.1") + v6Src := netip.MustParseAddr("fd00::1") + + _, pass := m.routeACLsPass(v4Src, netip.MustParseAddr("10.0.0.5"), 255, 0, 0) + assert.True(t, pass, "v4 source must match the v4 destination rule via 0.0.0.0/0") + _, pass = m.routeACLsPass(v6Src, netip.MustParseAddr("fd00:1::5"), 255, 0, 0) + assert.True(t, pass, "v6 source must match the v6 destination rule via ::/0") +} diff --git a/client/firewall/uspfilter/peer_index.go b/client/firewall/uspfilter/peer_index.go new file mode 100644 index 000000000..552ffd2d7 --- /dev/null +++ b/client/firewall/uspfilter/peer_index.go @@ -0,0 +1,140 @@ +package uspfilter + +import ( + "net/netip" + "slices" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +// peerRuleIndex is the source-side dispatcher consulted on the packet +// hot path. It splits rules into two buckets by the shape of their +// source list: +// +// - bySource: every source is a host prefix (/32 for v4, /128 for +// v6). Keyed by the concrete source address, so a hit guarantees +// the source filter passes and the matcher goes straight to +// proto/port checks. This is the common case for peer ACLs. +// - nonHost: any source list with a prefix coarser than a host, +// including a /0 "match any". Walked linearly with a per-rule +// Contains() check. Expected small or empty for typical peer ACLs. +// +// Maintained incrementally by add/remove, never rebuilt. +type peerRuleIndex struct { + bySource map[netip.Addr][]*PeerRule + nonHost []*PeerRule +} + +func (i *peerRuleIndex) add(r *PeerRule) { + if hasNonHostSource(r) { + i.nonHost = append(i.nonHost, r) + return + } + if i.bySource == nil { + i.bySource = make(map[netip.Addr][]*PeerRule) + } + for a := range r.sourceAddrs { + i.bySource[a] = append(i.bySource[a], r) + } +} + +func (i *peerRuleIndex) remove(r *PeerRule) { + if hasNonHostSource(r) { + i.nonHost = slices.DeleteFunc(i.nonHost, eqRule(r)) + return + } + if i.bySource == nil { + return + } + for a := range r.sourceAddrs { + entries := slices.DeleteFunc(i.bySource[a], eqRule(r)) + if len(entries) == 0 { + delete(i.bySource, a) + } else { + i.bySource[a] = entries + } + } +} + +func (i *peerRuleIndex) reset() { + i.bySource = nil + i.nonHost = i.nonHost[:0] +} + +// match returns the first rule matching src and the decoded packet. +// Host rules are found by direct map lookup; nonHost rules run a +// per-rule source Contains() check. Containment is family-scoped, so +// a /0 source matches every address of its own family only (0.0.0.0/0 +// never matches v6 sources and ::/0 never matches v4). Within either +// bucket the matcher runs the proto/port filter. +func (i *peerRuleIndex) match(src netip.Addr, d *decoder) ([]byte, bool, bool) { + payloadLayer := d.decoded[1] + + for _, rule := range i.bySource[src] { + if id, drop, ok := matchProto(rule, d, payloadLayer); ok { + return id, drop, true + } + } + for _, rule := range i.nonHost { + if !prefixesContain(rule.sources, src) { + continue + } + if id, drop, ok := matchProto(rule, d, payloadLayer); ok { + return id, drop, true + } + } + return nil, false, false +} + +func eqRule(target *PeerRule) func(*PeerRule) bool { + return func(p *PeerRule) bool { return p == target } +} + +// hasNonHostSource reports whether the rule has any source prefix +// that is not a single host address. Called only at add/remove time, +// not on the packet path. +func hasNonHostSource(r *PeerRule) bool { + for _, p := range r.sources { + if p.Bits() != p.Addr().BitLen() { + return true + } + } + return false +} + +// matchProto applies the proto/port half of a rule against the +// decoded packet. Source matching is the caller's responsibility. +func matchProto(rule *PeerRule, d *decoder, payloadLayer gopacket.LayerType) ([]byte, bool, bool) { + drop := rule.action == firewall.ActionDrop + if rule.protoLayer == layerTypeAll { + return rule.mgmtId, drop, true + } + if !protoLayerMatches(rule.protoLayer, payloadLayer) { + return nil, false, false + } + switch payloadLayer { + case layers.LayerTypeTCP: + if portsMatch(rule.srcPort, uint16(d.tcp.SrcPort)) && portsMatch(rule.dstPort, uint16(d.tcp.DstPort)) { + return rule.mgmtId, drop, true + } + case layers.LayerTypeUDP: + if portsMatch(rule.srcPort, uint16(d.udp.SrcPort)) && portsMatch(rule.dstPort, uint16(d.udp.DstPort)) { + return rule.mgmtId, drop, true + } + case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6: + return rule.mgmtId, drop, true + } + return nil, false, false +} + +func prefixesContain(sources []netip.Prefix, src netip.Addr) bool { + for _, p := range sources { + if p.Contains(src) { + return true + } + } + return false +} diff --git a/client/firewall/uspfilter/rule.go b/client/firewall/uspfilter/rule.go index 08d68a78e..6d73b19ac 100644 --- a/client/firewall/uspfilter/rule.go +++ b/client/firewall/uspfilter/rule.go @@ -10,24 +10,43 @@ import ( // PeerRule to handle management of rules type PeerRule struct { - id string - mgmtId []byte - ip netip.Addr - ipLayer gopacket.LayerType - matchByIP bool - protoLayer gopacket.LayerType - sPort *firewall.Port - dPort *firewall.Port - drop bool + id firewall.RuleID + mgmtId []byte + // sources is the canonical list of source prefixes this rule + // matches against. + sources []netip.Prefix + // sourceAddrs is a fast-path membership set for host-prefix + // sources (/32 v4, /128 v6). Populated alongside sources; + // consulted before falling back to prefix scan. + sourceAddrs map[netip.Addr]struct{} + protoLayer gopacket.LayerType + srcPort *firewall.Port + dstPort *firewall.Port + action firewall.Action +} + +// matchesSource reports whether the given source address is covered +// by this rule's source list. Prefix containment is family-scoped, so +// a /0 source matches every address of its own family only. +func (r *PeerRule) matchesSource(src netip.Addr) bool { + if _, ok := r.sourceAddrs[src]; ok { + return true + } + for _, p := range r.sources { + if p.Contains(src) { + return true + } + } + return false } // ID returns the rule id -func (r *PeerRule) ID() string { +func (r *PeerRule) ID() firewall.RuleID { return r.id } type RouteRule struct { - id string + id firewall.RuleID mgmtId []byte sources []netip.Prefix dstSet firewall.Set @@ -39,6 +58,6 @@ type RouteRule struct { } // ID returns the rule id -func (r *RouteRule) ID() string { +func (r *RouteRule) ID() firewall.RuleID { return r.id } diff --git a/client/firewall/uspfilter/testhelpers_test.go b/client/firewall/uspfilter/testhelpers_test.go new file mode 100644 index 000000000..a0760f5ba --- /dev/null +++ b/client/firewall/uspfilter/testhelpers_test.go @@ -0,0 +1,50 @@ +package uspfilter + +import ( + "net" + "net/netip" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +// countRulesForAddr reports how many rules in the given slice match +// the supplied source address. +func countRulesForAddr(rules peerRules, src netip.Addr) int { + n := 0 + for _, r := range rules { + if r.matchesSource(src) { + n++ + } + } + return n +} + +// findRuleByID returns true if the rules slice contains a rule with +// the given id whose source set covers src. +func findRuleByID(rules peerRules, src netip.Addr, id firewall.RuleID) bool { + for _, r := range rules { + if r.id == id && r.matchesSource(src) { + return true + } + } + return false +} + +// pfx converts a single net.IP into the []netip.Prefix form +// AddFilterRule expects. A nil or unspecified address becomes a /0 +// ("match any") prefix in the matching family; any other address +// becomes its /32 (or /128) host prefix. +func pfx(ip net.IP) []netip.Prefix { + if ip == nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + if ip.IsUnspecified() { + if ip.To4() != nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + } + a, _ := netip.AddrFromSlice(ip) + a = a.Unmap() + return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())} +} diff --git a/client/firewall/uspfilter/tracer.go b/client/firewall/uspfilter/tracer.go index 696489e95..3c081314c 100644 --- a/client/firewall/uspfilter/tracer.go +++ b/client/firewall/uspfilter/tracer.go @@ -285,6 +285,14 @@ func (m *Manager) TracePacket(packetData []byte, direction fw.RuleDirection) *Pa trace.SourceIP = srcIP trace.DestinationIP = dstIP + // A fragment or otherwise truncated packet has no transport layer. + // The inbound datapath drops these via isValidPacket; the tracer must + // guard explicitly since every downstream stage reads d.decoded[1]. + if len(d.decoded) < 2 { + trace.AddResult(StageReceived, "Packet has no transport layer (fragment or unsupported protocol)", false) + return trace + } + // Determine protocol and ports switch d.decoded[1] { case layers.LayerTypeTCP: diff --git a/client/firewall/uspfilter/tracer_test.go b/client/firewall/uspfilter/tracer_test.go index 657f96fc0..27b5e3f9e 100644 --- a/client/firewall/uspfilter/tracer_test.go +++ b/client/firewall/uspfilter/tracer_test.go @@ -45,7 +45,7 @@ func TestTracePacket(t *testing.T) { }, } - m, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) if !statefulMode { @@ -97,7 +97,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -121,7 +121,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -150,7 +150,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -178,7 +178,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -205,7 +205,7 @@ func TestTracePacket(t *testing.T) { src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32) dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32) - _, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) + _, err := m.AddFilterRule(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -231,7 +231,7 @@ func TestTracePacket(t *testing.T) { src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32) dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32) - _, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) + _, err := m.AddFilterRule(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -332,7 +332,7 @@ func TestTracePacket(t *testing.T) { ip := net.ParseIP("1.1.1.1") proto := fw.ProtocolICMP action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, nil, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, nil, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -355,7 +355,7 @@ func TestTracePacket(t *testing.T) { ip := net.ParseIP("1.1.1.1") proto := fw.ProtocolICMP action := fw.ActionDrop - _, err := m.AddPeerFiltering(nil, ip, proto, nil, nil, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, nil, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -379,7 +379,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolUDP port := &fw.Port{Values: []uint16{53}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -423,7 +423,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { diff --git a/client/iface/device/device_android.go b/client/iface/device/device_android.go index cbe88c10c..0ed1299ae 100644 --- a/client/iface/device/device_android.go +++ b/client/iface/device/device_android.go @@ -63,7 +63,12 @@ func (t *WGTunDevice) Create(routes []string, dns string, searchDomains []string searchDomainsToString = "" } - fd, err := t.tunAdapter.ConfigureInterface(t.address.String(), t.address.IPv6String(), int(t.mtu), dns, searchDomainsToString, routesString) + ipv6Host := "" + if t.address.HasIPv6() { + ipv6Host = t.address.IPv6HostPrefix().String() + } + + fd, err := t.tunAdapter.ConfigureInterface(t.address.HostPrefix().String(), ipv6Host, int(t.mtu), dns, searchDomainsToString, routesString) if err != nil { log.Errorf("failed to create Android interface: %s", err) return nil, err diff --git a/client/iface/wgaddr/address.go b/client/iface/wgaddr/address.go index 43d1ec9aa..148e724f4 100644 --- a/client/iface/wgaddr/address.go +++ b/client/iface/wgaddr/address.go @@ -59,6 +59,19 @@ func (addr Address) IPv6Prefix() netip.Prefix { return netip.PrefixFrom(addr.IPv6, addr.IPv6Net.Bits()) } +// HostPrefix returns the v4 address as a single-host prefix. +func (addr Address) HostPrefix() netip.Prefix { + return netip.PrefixFrom(addr.IP, addr.IP.BitLen()) +} + +// IPv6HostPrefix returns the v6 address as a single-host prefix, or an invalid prefix when no v6 overlay address is assigned. +func (addr Address) IPv6HostPrefix() netip.Prefix { + if !addr.HasIPv6() { + return netip.Prefix{} + } + return netip.PrefixFrom(addr.IPv6, addr.IPv6.BitLen()) +} + // SetIPv6FromCompact decodes a compact prefix (5 or 17 bytes) and sets the IPv6 fields. // Returns an error if the bytes are invalid. A nil or empty input is a no-op. // diff --git a/client/iface/wgaddr/address_test.go b/client/iface/wgaddr/address_test.go new file mode 100644 index 000000000..61478b24c --- /dev/null +++ b/client/iface/wgaddr/address_test.go @@ -0,0 +1,24 @@ +package wgaddr + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAddress_HostPrefix(t *testing.T) { + addr := MustParseWGAddress("100.91.96.107/16") + + assert.Equal(t, netip.MustParsePrefix("100.91.96.107/32"), addr.HostPrefix(), "v4 host prefix must be a single host") + assert.Equal(t, netip.MustParsePrefix("100.91.0.0/16"), addr.Network, "network must keep the overlay prefix length") + assert.False(t, addr.IPv6HostPrefix().IsValid(), "no v6 overlay means no v6 host prefix") +} + +func TestAddress_IPv6HostPrefix(t *testing.T) { + addr := MustParseWGAddress("100.91.96.107/16") + addr.IPv6 = netip.MustParseAddr("fd00:1234::1") + addr.IPv6Net = netip.MustParsePrefix("fd00:1234::/64") + + assert.Equal(t, netip.MustParsePrefix("fd00:1234::1/128"), addr.IPv6HostPrefix(), "v6 host prefix must be a single host") +} diff --git a/client/internal/acl/dispatch_test.go b/client/internal/acl/dispatch_test.go new file mode 100644 index 000000000..be82e414f --- /dev/null +++ b/client/internal/acl/dispatch_test.go @@ -0,0 +1,190 @@ +package acl + +import ( + "net/netip" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/netbirdio/netbird/client/firewall" + fwmgr "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/acl/mocks" + mgmProto "github.com/netbirdio/netbird/shared/management/proto" +) + +// TestNetworkZeroPrefixIsRoute guards the route-vs-peer dispatch +// invariant: the backends classify a rule as a peer rule purely by the +// absence of a destination (neither prefix nor set). A default route +// (0.0.0.0/0 or ::/0) is a valid prefix and must therefore classify as +// a route, not collapse into the peer path. +func TestNetworkZeroPrefixIsRoute(t *testing.T) { + for _, p := range []string{"0.0.0.0/0", "::/0", "10.0.0.0/8"} { + n := fwmgr.Network{Prefix: netip.MustParsePrefix(p)} + assert.True(t, n.IsPrefix(), "%s must report IsPrefix", p) + assert.True(t, n.IsPrefix() || n.IsSet(), "%s must classify as a route", p) + } + + // A zero-value Network is the only peer-rule shape. + var empty fwmgr.Network + assert.False(t, empty.IsPrefix(), "zero Network must not be a prefix") + assert.False(t, empty.IsSet(), "zero Network must not be a set") +} + +// TestDetermineDestinationAlwaysRoute verifies determineDestination +// never yields an empty Network for a valid route rule: every branch +// (static prefix, default route, dynamic with/without domains, with and +// without a local resolver) produces a destination that classifies as a +// route. If this regresses, a route rule would be dispatched down the +// peer path, which matches on source only. +func TestDetermineDestinationAlwaysRoute(t *testing.T) { + v4 := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")} + v6 := []netip.Prefix{netip.MustParsePrefix("2001:db8::/48")} + + cases := []struct { + name string + rule *mgmProto.RouteFirewallRule + resolver bool + sources []netip.Prefix + }{ + {"static prefix", &mgmProto.RouteFirewallRule{Destination: "192.168.0.0/16"}, false, v4}, + {"static default route", &mgmProto.RouteFirewallRule{Destination: "0.0.0.0/0"}, false, v4}, + {"dynamic with domains + resolver", &mgmProto.RouteFirewallRule{IsDynamic: true, Domains: []string{"example.com"}}, true, v4}, + {"dynamic no domains + resolver (v4)", &mgmProto.RouteFirewallRule{IsDynamic: true}, true, v4}, + {"dynamic no domains + resolver (v6)", &mgmProto.RouteFirewallRule{IsDynamic: true}, true, v6}, + {"dynamic + no local resolver (v4)", &mgmProto.RouteFirewallRule{IsDynamic: true}, false, v4}, + {"dynamic + no local resolver (v6)", &mgmProto.RouteFirewallRule{IsDynamic: true}, false, v6}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dest, err := determineDestination(tc.rule, tc.resolver, tc.sources) + require.NoError(t, err) + assert.True(t, dest.IsPrefix() || dest.IsSet(), + "destination must classify as a route, got empty Network") + }) + } +} + +// countingFirewall wraps a real firewall.Manager and counts filter-rule +// add/delete calls so a test can assert how many backing rules the acl +// manager actually creates and tears down. +type countingFirewall struct { + fwmgr.Manager + mu sync.Mutex + addCalls int + dels int + ruleIDs map[fwmgr.RuleID]struct{} +} + +// distinctRules returns the number of distinct backing rules the +// backend produced. Because the backend dedups identical content, +// repeated AddFilterRule calls for the same rule resolve to one id. +func (f *countingFirewall) distinctRules() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.ruleIDs) +} + +func (f *countingFirewall) AddFilterRule(id []byte, sources []netip.Prefix, destination fwmgr.Network, proto fwmgr.Protocol, sPort, dPort *fwmgr.Port, action fwmgr.Action) (fwmgr.Rule, error) { + rule, err := f.Manager.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) + if err == nil { + f.mu.Lock() + f.addCalls++ + if f.ruleIDs == nil { + f.ruleIDs = make(map[fwmgr.RuleID]struct{}) + } + if rule != nil { + f.ruleIDs[rule.ID()] = struct{}{} + } + f.mu.Unlock() + } + return rule, err +} + +func (f *countingFirewall) DeleteFilterRule(r fwmgr.Rule) error { + err := f.Manager.DeleteFilterRule(r) + if err == nil { + f.mu.Lock() + f.dels++ + delete(f.ruleIDs, r.ID()) + f.mu.Unlock() + } + return err +} + +func newCountingACL(t *testing.T) (*DefaultManager, *countingFirewall, func()) { + t.Helper() + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv(firewall.EnvForceUserspaceFirewall, "true") + + ctrl := gomock.NewController(t) + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + network := netip.MustParsePrefix("172.0.0.1/32") + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU) + require.NoError(t, err) + + fw := &countingFirewall{Manager: realFW} + cleanup := func() { + require.NoError(t, realFW.Close(nil)) + ctrl.Finish() + } + return NewDefaultManager(fw), fw, cleanup +} + +// TestDuplicateContentPoliciesShareOneRule verifies the dedup contract +// the backends rely on: two policies that authorize an identical flow +// (same selector and sources) collapse to a single backing firewall +// rule, and that rule survives until BOTH policies are gone. This is +// why the backend can dedup on add without refcounting on delete: the +// acl manager's pair key matches the backend's content key, so add and +// delete stay balanced per content key across full-state reapplies. +func TestDuplicateContentPoliciesShareOneRule(t *testing.T) { + acl, fw, cleanup := newCountingACL(t) + defer cleanup() + + ruleA := &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + } + ruleB := &mgmProto.FirewallRule{ + PolicyID: []byte("policy-B"), + PeerIP: "10.0.0.1", //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + } + + // Both policies present: identical content collapses to one rule. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: []*mgmProto.FirewallRule{ruleA, ruleB}, FirewallRulesIsEmpty: false}, false) + assert.Equal(t, 1, fw.distinctRules(), "identical-content policies must produce one backing rule") + assert.Equal(t, 1, len(acl.peerRulesPairs), "one content key, one pair") + + // Drop policy A only: the shared rule is still authorized by B, so + // nothing is deleted. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: []*mgmProto.FirewallRule{ruleB}, FirewallRulesIsEmpty: false}, false) + assert.Equal(t, 1, fw.distinctRules(), "no new backing rule on reapply") + assert.Equal(t, 0, fw.dels, "rule must survive while any policy still authorizes it") + assert.Equal(t, 1, len(acl.peerRulesPairs)) + + // Drop policy B too: now the content key has no authorizer and the + // single backing rule is removed exactly once. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: true}, false) + assert.Equal(t, 1, fw.dels, "rule removed once when last policy is gone") + assert.Equal(t, 0, len(acl.peerRulesPairs)) +} diff --git a/client/internal/acl/grouping_test.go b/client/internal/acl/grouping_test.go new file mode 100644 index 000000000..d6cf29b59 --- /dev/null +++ b/client/internal/acl/grouping_test.go @@ -0,0 +1,318 @@ +package acl + +import ( + "errors" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/netbirdio/netbird/client/firewall" + fwmgr "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/acl/mocks" + mgmProto "github.com/netbirdio/netbird/shared/management/proto" + "github.com/netbirdio/netbird/shared/netiputil" +) + +// TestGroupPeerRulesPolicyIDSeparates verifies that two FirewallRules +// with identical selectors but different PolicyIDs do NOT get merged +// into one group, so each policy's sources merge under its own +// attribution id. (Identical-content groups may still dedup to one +// backing rule at the backend; see TestDuplicateContentPoliciesShareOneRule.) +func TestGroupPeerRulesPolicyIDSeparates(t *testing.T) { + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + { + PolicyID: []byte("policy-B"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "rules with different PolicyIDs must produce separate groups") +} + +// TestGroupPeerRulesFamilySeparates verifies that v4 and v6 rules +// belonging to the same policy don't merge. +func TestGroupPeerRulesFamilySeparates(t *testing.T) { + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + { + PolicyID: []byte("policy-A"), + PeerIP: "2001:db8::1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "rules of different families must produce separate groups") + + var sawV4, sawV6 bool + for _, g := range groups { + require.Len(t, g.sources, 1) + if g.sources[0].Addr().Is4() { + sawV4 = true + } + if g.sources[0].Addr().Is6() { + sawV6 = true + } + } + assert.True(t, sawV4 && sawV6) +} + +// TestGroupPeerRulesSplitsMixedFamilySingleRule verifies that a single +// FirewallRule carrying both v4 and v6 source prefixes is split into one +// group per family. Each backend keys a rule to a single family, so a +// group whose sources span families would mismatch the other family's +// sources. mgmt normally emits one rule per family; this guards against +// a mixed-family rule slipping through. +func TestGroupPeerRulesSplitsMixedFamilySingleRule(t *testing.T) { + srcs := [][]byte{ + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.1")), + netiputil.EncodeAddr(netip.MustParseAddr("2001:db8::1")), + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.2")), + netiputil.EncodeAddr(netip.MustParseAddr("2001:db8::2")), + } + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + SourcePrefixes: srcs, + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "mixed-family sources in one rule must split into two groups") + + for _, g := range groups { + require.Len(t, g.sources, 2) + v6 := prefixIsV6(g.sources[0]) + for _, s := range g.sources { + assert.Equal(t, v6, prefixIsV6(s), "every source in a group must share one family") + } + } +} + +// TestGroupPeerRulesMergesSameSelector verifies that rules sharing +// every distinguishing field (policy, family, direction, action, +// proto, port) collapse into a single multi-source group. +func TestGroupPeerRulesMergesSameSelector(t *testing.T) { + mk := func(peerIP string) *mgmProto.FirewallRule { + return &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: peerIP, //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + } + } + rules := []*mgmProto.FirewallRule{mk("10.0.0.1"), mk("10.0.0.2"), mk("10.0.0.3")} + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 1) + require.Len(t, groups[0].sources, 3) +} + +// TestGroupPeerRulesPortSeparates verifies that PortInfo is part of the +// selector key: rules differing only in port must not merge, and a +// single port must not merge with a range. A regression dropping the +// port from the key would collapse rules for different ports into one. +func TestGroupPeerRulesPortSeparates(t *testing.T) { + mkPort := func(peerIP string, port uint32) *mgmProto.FirewallRule { + return &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: peerIP, //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + PortInfo: &mgmProto.PortInfo{PortSelection: &mgmProto.PortInfo_Port{Port: port}}, + } + } + + groups, denyErr, err := groupPeerRules([]*mgmProto.FirewallRule{ + mkPort("10.0.0.1", 80), mkPort("10.0.0.2", 80), mkPort("10.0.0.3", 443), + }) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "rules on different ports must not merge") + + rangeRule := &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.4", //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + PortInfo: &mgmProto.PortInfo{PortSelection: &mgmProto.PortInfo_Range_{Range: &mgmProto.PortInfo_Range{Start: 80, End: 90}}}, + } + groups, denyErr, err = groupPeerRules([]*mgmProto.FirewallRule{mkPort("10.0.0.1", 80), rangeRule}) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "a single port and a range must not merge") +} + +// TestGroupPeerRulesUsesSourcePrefixesWhenPresent verifies that the +// new sourcePrefixes wire field is consumed and produces a +// multi-source group in one shot (no client-side merging needed). +func TestGroupPeerRulesUsesSourcePrefixesWhenPresent(t *testing.T) { + srcs := [][]byte{ + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.1")), + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.2")), + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.3")), + } + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + SourcePrefixes: srcs, + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 1) + require.Len(t, groups[0].sources, 3) +} + +// TestGroupPeerRulesActionSeparates verifies the obvious: accept +// and drop rules with the same selector don't merge. +func TestGroupPeerRulesActionSeparates(t *testing.T) { + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_DROP, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2) +} + +// failingDeleteFirewall wraps a real firewall.Manager and forces the +// next N DeleteFilterRule calls to fail. Used to verify that the acl +// manager retains rules whose deletion was rejected by the backend, +// so they get retried on the next ApplyFiltering pass instead of +// becoming orphans. +type failingDeleteFirewall struct { + fwmgr.Manager + failCount int +} + +func (f *failingDeleteFirewall) DeleteFilterRule(r fwmgr.Rule) error { + if f.failCount > 0 { + f.failCount-- + return errors.New("simulated delete failure") + } + return f.Manager.DeleteFilterRule(r) +} + +// TestApplyFilteringRetainsRulesOnDeleteFailure verifies that a +// transient DeleteFilterRule error doesn't make the acl manager +// forget about a rule. The rule must remain in peerRulesPairs so the +// next ApplyFiltering pass attempts the delete again. +func TestApplyFilteringRetainsRulesOnDeleteFailure(t *testing.T) { + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv(firewall.EnvForceUserspaceFirewall, "true") + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + network := netip.MustParsePrefix("172.0.0.1/32") + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU) + require.NoError(t, err) + defer func() { require.NoError(t, realFW.Close(nil)) }() + + fw := &failingDeleteFirewall{Manager: realFW} + acl := NewDefaultManager(fw) + + // First pass: install a rule. + netmap1 := &mgmProto.NetworkMap{ + FirewallRules: []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_DROP, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "22", + }, + }, + FirewallRulesIsEmpty: false, + } + acl.ApplyFiltering(netmap1, false) + require.Equal(t, 1, len(acl.peerRulesPairs), "rule should be installed") + + // Second pass: remove the rule from the map. The backend will + // fail the delete; the acl manager must retain the rule. + fw.failCount = 1 + netmap2 := &mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: true} + acl.ApplyFiltering(netmap2, false) + require.Equal(t, 1, len(acl.peerRulesPairs), + "rule must be retained when DeleteFilterRule fails so it gets retried") + + // Third pass: same map, backend no longer fails. The rule + // should now succeed in being removed. + acl.ApplyFiltering(netmap2, false) + require.Equal(t, 0, len(acl.peerRulesPairs), "retry should succeed") +} diff --git a/client/internal/acl/id/id.go b/client/internal/acl/id/id.go index 23451453e..952403bbd 100644 --- a/client/internal/acl/id/id.go +++ b/client/internal/acl/id/id.go @@ -5,18 +5,18 @@ import ( "encoding/hex" "fmt" "net/netip" + "slices" "strconv" "github.com/netbirdio/netbird/client/firewall/manager" ) -type RuleID string +// RuleID aliases manager.RuleID so existing nbid.RuleID references +// keep working while the canonical type lives in the firewall package. +type RuleID = manager.RuleID -func (r RuleID) ID() string { - return string(r) -} - -func GenerateRouteRuleKey( +// GenerateRuleID returns a deterministic content hash identifying a filter rule. +func GenerateRuleID( sources []netip.Prefix, destination manager.Network, proto manager.Protocol, @@ -24,6 +24,7 @@ func GenerateRouteRuleKey( dPort *manager.Port, action manager.Action, ) RuleID { + sources = slices.Clone(sources) manager.SortPrefixes(sources) h := sha256.New() diff --git a/client/internal/acl/legacy_fallback_test.go b/client/internal/acl/legacy_fallback_test.go new file mode 100644 index 000000000..00ca013f3 --- /dev/null +++ b/client/internal/acl/legacy_fallback_test.go @@ -0,0 +1,75 @@ +package acl + +import ( + "net/netip" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/netbirdio/netbird/client/firewall" + fwmgr "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/acl/mocks" + mgmProto "github.com/netbirdio/netbird/shared/management/proto" +) + +// sourcesRecordingFirewall wraps a real firewall.Manager and records +// the source prefixes of every AddFilterRule call. +type sourcesRecordingFirewall struct { + fwmgr.Manager + mu sync.Mutex + sources [][]netip.Prefix +} + +func (f *sourcesRecordingFirewall) AddFilterRule(id []byte, sources []netip.Prefix, destination fwmgr.Network, proto fwmgr.Protocol, sPort, dPort *fwmgr.Port, action fwmgr.Action) (fwmgr.Rule, error) { + f.mu.Lock() + f.sources = append(f.sources, sources) + f.mu.Unlock() + return f.Manager.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) +} + +// TestLegacyManagementFallbackUsesMatchAnySources verifies the +// allow-all fallback for old management servers (empty FirewallRules +// without the FirewallRulesIsEmpty flag) reaches the firewall as /0 +// match-any sources. The fallback rule carries PeerIP 0.0.0.0; if that +// were converted to a host prefix (0.0.0.0/32) it would match nothing +// and all peer traffic would be dropped. +func TestLegacyManagementFallbackUsesMatchAnySources(t *testing.T) { + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv(firewall.EnvForceUserspaceFirewall, "true") + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + network := netip.MustParsePrefix("172.0.0.1/32") + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU) + require.NoError(t, err) + defer func() { require.NoError(t, realFW.Close(nil)) }() + + fw := &sourcesRecordingFirewall{Manager: realFW} + acl := NewDefaultManager(fw) + + // Old management: no rules and no FirewallRulesIsEmpty flag. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: false}, false) + + fw.mu.Lock() + defer fw.mu.Unlock() + require.NotEmpty(t, fw.sources, "legacy fallback must install at least one allow-all rule") + for _, sources := range fw.sources { + require.NotEmpty(t, sources) + for _, p := range sources { + assert.Equal(t, 0, p.Bits(), "legacy fallback source %s must be a /0 match-any prefix", p) + } + } +} diff --git a/client/internal/acl/manager.go b/client/internal/acl/manager.go index cbd9c5ab1..6c544b345 100644 --- a/client/internal/acl/manager.go +++ b/client/internal/acl/manager.go @@ -1,8 +1,6 @@ package acl import ( - "crypto/md5" - "encoding/hex" "errors" "fmt" "net/netip" @@ -24,6 +22,10 @@ import ( var ErrSourceRangesEmpty = errors.New("sources range is empty") +// ErrNoRuleReturned is returned when the firewall backend reports success +// from AddFilterRule but yields no rule to track. +var ErrNoRuleReturned = errors.New("backend returned no rule") + // Manager is a ACL rules manager type Manager interface { ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRouteFeatureFlag bool) @@ -32,19 +34,48 @@ type Manager interface { // DefaultManager uses firewall manager to handle type DefaultManager struct { firewall firewall.Manager - ipsetCounter int peerRulesPairs map[id.RuleID][]firewall.Rule - routeRules map[id.RuleID]struct{} + routeRules map[id.RuleID]firewall.Rule previousConfigHash uint64 hasAppliedConfig bool mutex sync.Mutex } +// peerRuleGroup collapses a set of single-source FirewallRules sharing +// the same selector into one multi-source rule to push to the backend. +type peerRuleGroup struct { + direction mgmProto.RuleDirection + action mgmProto.RuleAction + protocol mgmProto.RuleProtocol + port *mgmProto.PortInfo + // legacyPort is used only when PortInfo is empty (old management). + legacyPort string + policyID []byte + sources []netip.Prefix +} + +// peerRuleKey is the comparable selector that decides which single-source +// rules merge into one group. Rules with an equal key collapse into one +// multi-source backend rule. PortInfo is flattened into its scalar fields +// so the key compares by value; policyID keeps policies separate so two +// policies authorizing different peers don't merge under one attribution. +type peerRuleKey struct { + v6 bool + policyID string + direction mgmProto.RuleDirection + action mgmProto.RuleAction + protocol mgmProto.RuleProtocol + legacyPort string + port uint16 + rangeStart uint16 + rangeEnd uint16 +} + func NewDefaultManager(fm firewall.Manager) *DefaultManager { return &DefaultManager{ firewall: fm, peerRulesPairs: make(map[id.RuleID][]firewall.Rule), - routeRules: make(map[id.RuleID]struct{}), + routeRules: make(map[id.RuleID]firewall.Rule), } } @@ -88,11 +119,14 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout time.Since(start), total) }() - d.applyPeerACLs(networkMap) + peerErr := d.applyPeerACLs(networkMap) + if peerErr != nil { + log.Errorf("apply peer ACLs: %v", peerErr) + } routeErr := d.applyRouteACLs(networkMap.RoutesFirewallRules, dnsRouteFeatureFlag) if routeErr != nil { - log.Errorf("Failed to apply route ACLs: %v", routeErr) + log.Errorf("apply route ACLs: %v", routeErr) } flushErr := d.firewall.Flush() @@ -104,7 +138,7 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout // If applying or flushing failed, leave the previous hash untouched so the // next (possibly identical) update is not skipped and gets a chance to // reconcile the firewall state. - if err == nil && routeErr == nil && flushErr == nil { + if err == nil && peerErr == nil && routeErr == nil && flushErr == nil { d.previousConfigHash = hash d.hasAppliedConfig = true } else { @@ -135,7 +169,7 @@ func (d *DefaultManager) firewallConfigHash(networkMap *mgmProto.NetworkMap, dns }) } -func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) { +func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) error { rules := networkMap.FirewallRules // if we got empty rules list but management not set networkMap.FirewallRulesIsEmpty flag @@ -158,59 +192,167 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) { ) } - newRulePairs := make(map[id.RuleID][]firewall.Rule) - ipsetByRuleSelectors := make(map[string]string) + // Group incoming single-source rules from management by their + // (direction, action, proto, port) selector and merge sources. + // One call to the firewall backend per merged rule. + // A deny we cannot decode would leave its traffic unblocked, so skip + // the whole pass and keep existing rules until the next sync. + groups, denyErr, err := groupPeerRules(rules) + if denyErr != nil { + return fmt.Errorf("decode deny rule sources: %w", denyErr) + } - // TODO: deny rules should be fatal: if a deny rule fails to apply, we must - // roll back all allow rules to avoid a fail-open where allowed traffic bypasses - // the missing deny. Currently we accumulate errors and continue. + newRulePairs := make(map[id.RuleID][]firewall.Rule) var merr *multierror.Error - for _, r := range rules { - // if this rule is member of rule selection with more than DefaultIPsCountForSet - // it's IP address can be used in the ipset for firewall manager which supports it - selector := d.getRuleGroupingSelector(r) - ipsetName, ok := ipsetByRuleSelectors[selector] - if !ok { - d.ipsetCounter++ - ipsetName = fmt.Sprintf("nb%07d", d.ipsetCounter) - ipsetByRuleSelectors[selector] = ipsetName - } - pairID, rulePair, err := d.protoRuleToFirewallRule(r, ipsetName) - if err != nil { - merr = multierror.Append(merr, fmt.Errorf("apply firewall rule: %w", err)) + if err != nil { + merr = multierror.Append(merr, err) + } + + // Apply denies first. A deny that fails to install is a security + // failure (fail-open), so if any deny errors we roll back the + // denies we already installed in this pass and bail out without + // installing any accept. Pre-existing rules stay untouched until + // the next successful pass clears them. + denies, accepts := splitDenyAccept(groups) + if err := d.installPeerGroups(denies, newRulePairs, true); err != nil { + return fmt.Errorf("install deny rules: %w", err) + } + + if err := d.installPeerGroups(accepts, newRulePairs, false); err != nil { + merr = multierror.Append(merr, err) + } + + // Tear down rules that disappeared from the networkmap. Any rule + // the backend refuses to delete stays in our tracking so it gets + // retried on the next ApplyFiltering. Otherwise a transient + // delete failure would leak the rule in the firewall until the + // process exits. + for pairID, rules := range d.peerRulesPairs { + if _, ok := newRulePairs[pairID]; ok { continue } - if len(rulePair) > 0 { - d.peerRulesPairs[pairID] = rulePair - newRulePairs[pairID] = rulePair - } - } - - if merr != nil { - log.Errorf("failed to apply %d peer ACL rule(s): %v", merr.Len(), nberrors.FormatErrorOrNil(merr)) - } - - for pairID, rules := range d.peerRulesPairs { - if _, ok := newRulePairs[pairID]; !ok { - for _, rule := range rules { - if err := d.firewall.DeletePeerRule(rule); err != nil { - log.Errorf("failed to delete peer firewall rule: %v", err) - continue - } + var remaining []firewall.Rule + for _, rule := range rules { + if err := d.firewall.DeleteFilterRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete peer rule, will retry: %w", err)) + remaining = append(remaining, rule) } - delete(d.peerRulesPairs, pairID) + } + if len(remaining) > 0 { + newRulePairs[pairID] = remaining } } d.peerRulesPairs = newRulePairs + + return nberrors.FormatErrorOrNil(merr) +} + +// installPeerGroups applies each group and records the resulting rule +// pairs in newRulePairs. With atomic set (deny rules), a single failure +// rolls back every rule installed in this call and returns, leaving the +// firewall exactly as before: denies are fail-closed and must be applied +// all-or-nothing. With atomic unset (accept rules), failures are +// accumulated and the remaining groups still install, so one malformed +// allow cannot drop every other legitimate allow in the pass. +func (d *DefaultManager) installPeerGroups(groups []*peerRuleGroup, newRulePairs map[id.RuleID][]firewall.Rule, atomic bool) error { + var freshlyInstalled []id.RuleID + var merr *multierror.Error + for _, g := range groups { + pairID, rulePair, err := d.applyPeerGroup(g) + if err != nil { + if atomic { + d.rollbackInstalled(freshlyInstalled) + return fmt.Errorf("apply firewall rule: %w", err) + } + merr = multierror.Append(merr, fmt.Errorf("apply firewall rule: %w", err)) + continue + } + if len(rulePair) == 0 { + continue + } + if _, existed := d.peerRulesPairs[pairID]; !existed { + freshlyInstalled = append(freshlyInstalled, pairID) + } + d.peerRulesPairs[pairID] = rulePair + newRulePairs[pairID] = rulePair + } + return nberrors.FormatErrorOrNil(merr) +} + +func (d *DefaultManager) rollbackInstalled(pairIDs []id.RuleID) { + var merr *multierror.Error + for _, pairID := range pairIDs { + // Keep any rule the backend refuses to delete tracked so it is + // retried on the next ApplyFiltering instead of leaking in the + // firewall with no tracking left to remove it. + var remaining []firewall.Rule + for _, rule := range d.peerRulesPairs[pairID] { + if err := d.firewall.DeleteFilterRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("rule %s: %w", pairID, err)) + remaining = append(remaining, rule) + } + } + if len(remaining) > 0 { + d.peerRulesPairs[pairID] = remaining + } else { + delete(d.peerRulesPairs, pairID) + } + } + if err := nberrors.FormatErrorOrNil(merr); err != nil { + log.Errorf("rollback peer rules: %v", err) + } +} + +func (d *DefaultManager) applyPeerGroup(g *peerRuleGroup) (id.RuleID, []firewall.Rule, error) { + protocol, err := ConvertToFirewallProtocol(g.protocol) + if err != nil { + return "", nil, fmt.Errorf("skipping firewall rule: %w", err) + } + action, err := convertFirewallAction(g.action) + if err != nil { + return "", nil, fmt.Errorf("skipping firewall rule: %w", err) + } + port, err := resolveGroupPort(g) + if err != nil { + return "", nil, err + } + + var fwRule firewall.Rule + switch g.direction { + case mgmProto.RuleDirection_IN: + fwRule, err = d.firewall.AddFilterRule(g.policyID, g.sources, firewall.Network{}, protocol, nil, port, action) + case mgmProto.RuleDirection_OUT: + if d.firewall.IsStateful() { + return "", nil, nil + } + if shouldSkipInvertedRule(protocol, port) { + return "", nil, nil + } + fwRule, err = d.firewall.AddFilterRule(g.policyID, g.sources, firewall.Network{}, protocol, port, nil, action) + default: + return "", nil, errors.New("invalid direction") + } + + if err != nil { + return "", nil, fmt.Errorf("add firewall rule: %w", err) + } + if fwRule == nil { + return "", nil, fmt.Errorf("add firewall rule: %w", ErrNoRuleReturned) + } + + // Derive the pair id from the backend rule, like the route path: + // the backend dedups identical content, so two policies authorizing + // the same flow resolve to the same id and a single backing rule. + return fwRule.ID(), []firewall.Rule{fwRule}, nil } func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dynamicResolver bool) error { - newRouteRules := make(map[id.RuleID]struct{}, len(rules)) + newRouteRules := make(map[id.RuleID]firewall.Rule, len(rules)) var merr *multierror.Error - // Apply new rules - firewall manager will return existing rule ID if already present + // Apply new rules - firewall manager will return the existing rule if already present for _, rule := range rules { - id, err := d.applyRouteACL(rule, dynamicResolver) + addedRule, err := d.applyRouteACL(rule, dynamicResolver) if err != nil { if errors.Is(err, ErrSourceRangesEmpty) { log.Debugf("skipping empty sources rule with destination %s: %v", rule.Destination, err) @@ -219,16 +361,18 @@ func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dyn } continue } - newRouteRules[id] = struct{}{} + newRouteRules[addedRule.ID()] = addedRule } - // Clean up old firewall rules - for id := range d.routeRules { - if _, exists := newRouteRules[id]; !exists { - if err := d.firewall.DeleteRouteRule(id); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete route rule: %w", err)) - } - // implicitly deleted from the map + // Tear down old route rules; retain ones the backend refused so a + // transient failure doesn't leave orphaned rules in the firewall. + for ruleID, rule := range d.routeRules { + if _, exists := newRouteRules[ruleID]; exists { + continue + } + if err := d.firewall.DeleteFilterRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete route rule, will retry: %w", err)) + newRouteRules[ruleID] = rule } } @@ -236,102 +380,202 @@ func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dyn return nberrors.FormatErrorOrNil(merr) } -func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule, dynamicResolver bool) (id.RuleID, error) { +func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule, dynamicResolver bool) (firewall.Rule, error) { if len(rule.SourceRanges) == 0 { - return "", ErrSourceRangesEmpty + return nil, ErrSourceRangesEmpty } var sources []netip.Prefix for _, sourceRange := range rule.SourceRanges { source, err := netip.ParsePrefix(sourceRange) if err != nil { - return "", fmt.Errorf("parse source range: %w", err) + return nil, fmt.Errorf("parse source range: %w", err) } - sources = append(sources, source) + sources = append(sources, firewall.UnmapPrefix(source)) } destination, err := determineDestination(rule, dynamicResolver, sources) if err != nil { - return "", fmt.Errorf("determine destination: %w", err) + return nil, fmt.Errorf("determine destination: %w", err) } - protocol, err := convertToFirewallProtocol(rule.Protocol) + protocol, err := ConvertToFirewallProtocol(rule.Protocol) if err != nil { - return "", fmt.Errorf("invalid protocol: %w", err) + return nil, fmt.Errorf("invalid protocol: %w", err) } action, err := convertFirewallAction(rule.Action) if err != nil { - return "", fmt.Errorf("invalid action: %w", err) + return nil, fmt.Errorf("invalid action: %w", err) } dPorts := convertPortInfo(rule.PortInfo) - addedRule, err := d.firewall.AddRouteFiltering(rule.PolicyID, sources, destination, protocol, nil, dPorts, action) + addedRule, err := d.firewall.AddFilterRule(rule.PolicyID, sources, destination, protocol, nil, dPorts, action) if err != nil { - return "", fmt.Errorf("add route rule: %w", err) + return nil, fmt.Errorf("add route rule: %w", err) + } + if addedRule == nil { + return nil, fmt.Errorf("add route rule: %w", ErrNoRuleReturned) } - return id.RuleID(addedRule.ID()), nil + return addedRule, nil } -func (d *DefaultManager) protoRuleToFirewallRule( - r *mgmProto.FirewallRule, - ipsetName string, -) (id.RuleID, []firewall.Rule, error) { - ip, err := extractRuleIP(r) - if err != nil { - return "", nil, err +// splitDenyAccept partitions groups by action so denies can be +// applied before accepts. Order within each bucket is preserved. +func splitDenyAccept(groups []*peerRuleGroup) (denies, accepts []*peerRuleGroup) { + for _, g := range groups { + if g.action == mgmProto.RuleAction_DROP { + denies = append(denies, g) + } else { + accepts = append(accepts, g) + } + } + return denies, accepts +} + +// groupPeerRules merges single-source rules sharing a selector into +// multi-source groups. It splits source-decode failures by action: +// denyErr is non-nil when a deny rule could not be decoded, which is a +// fail-open risk the caller must treat as fatal for the pass; err +// carries the tolerable accept-rule failures the caller can log and +// continue past. +func groupPeerRules(rules []*mgmProto.FirewallRule) (groups []*peerRuleGroup, denyErr error, err error) { + var denyMerr, acceptMerr *multierror.Error + byKey := make(map[peerRuleKey]*peerRuleGroup) + order := make([]peerRuleKey, 0) + + for _, r := range rules { + srcs, decErr := extractRuleSources(r) + if decErr != nil { + if r.Action == mgmProto.RuleAction_DROP { + denyMerr = multierror.Append(denyMerr, decErr) + } else { + acceptMerr = multierror.Append(acceptMerr, decErr) + } + continue + } + // A single FirewallRule normally carries one address family, but + // split by family defensively: each backend keys a rule to one + // family and would mismatch sources of the other, so a group's + // sources must never span families. + v4, v6 := splitPrefixesByFamily(srcs) + for _, sub := range []struct { + isV6 bool + sources []netip.Prefix + }{{false, v4}, {true, v6}} { + if len(sub.sources) == 0 { + continue + } + key := ruleGroupKey(r, sub.isV6) + g, ok := byKey[key] + if !ok { + g = &peerRuleGroup{ + direction: r.Direction, + action: r.Action, + protocol: r.Protocol, + port: r.PortInfo, + legacyPort: r.Port, + policyID: r.PolicyID, + } + byKey[key] = g + order = append(order, key) + } + g.sources = append(g.sources, sub.sources...) + } } - protocol, err := convertToFirewallProtocol(r.Protocol) - if err != nil { - return "", nil, fmt.Errorf("skipping firewall rule: %s", err) + out := make([]*peerRuleGroup, 0, len(order)) + for _, k := range order { + out = append(out, byKey[k]) + } + return out, nberrors.FormatErrorOrNil(denyMerr), nberrors.FormatErrorOrNil(acceptMerr) +} + +func prefixIsV6(p netip.Prefix) bool { + return p.Addr().Is6() && !p.Addr().Is4In6() +} + +// splitPrefixesByFamily partitions prefixes into IPv4 and IPv6 groups. +func splitPrefixesByFamily(prefixes []netip.Prefix) (v4, v6 []netip.Prefix) { + for _, p := range prefixes { + if prefixIsV6(p) { + v6 = append(v6, p) + } else { + v4 = append(v4, p) + } + } + return v4, v6 +} + +// ruleGroupKey builds the selector key for a rule. v6 must reflect the +// rule's source family: mgmt emits one rule per family and mixing them +// would break ICMP-variant selection in uspfilter. +func ruleGroupKey(r *mgmProto.FirewallRule, v6 bool) peerRuleKey { + k := peerRuleKey{ + v6: v6, + policyID: string(r.PolicyID), + direction: r.Direction, + action: r.Action, + protocol: r.Protocol, + legacyPort: r.Port, + } + if pi := r.PortInfo; pi != nil { + k.port = uint16(pi.GetPort()) + if rng := pi.GetRange(); rng != nil { + k.rangeStart = uint16(rng.GetStart()) + k.rangeEnd = uint16(rng.GetEnd()) + } + } + return k +} + +// extractRuleSources returns all source prefixes the rule applies to. +// New management populates sourcePrefixes; older management sets PeerIP. +func extractRuleSources(r *mgmProto.FirewallRule) ([]netip.Prefix, error) { + if len(r.SourcePrefixes) > 0 { + out := make([]netip.Prefix, 0, len(r.SourcePrefixes)) + for _, raw := range r.SourcePrefixes { + addr, err := netiputil.DecodeAddr(raw) + if err != nil { + return nil, fmt.Errorf("decode source prefix: %w", err) + } + out = append(out, netip.PrefixFrom(addr.Unmap(), addr.Unmap().BitLen())) + } + return out, nil } - action, err := convertFirewallAction(r.Action) + peerIP := r.PeerIP //nolint:staticcheck // PeerIP is the legacy source field for old management servers + addr, err := netip.ParseAddr(peerIP) if err != nil { - return "", nil, fmt.Errorf("skipping firewall rule: %s", err) + return nil, fmt.Errorf("parse peer IP %q: %w", peerIP, err) } + addr = addr.Unmap() + // An unspecified PeerIP means "any peer" (legacy management + // allow-all fallback); only a /0 prefix matches any source in the + // backends, a full-length prefix would match nothing. + if addr.IsUnspecified() { + return []netip.Prefix{netip.PrefixFrom(addr, 0)}, nil + } + return []netip.Prefix{netip.PrefixFrom(addr, addr.BitLen())}, nil +} - var port *firewall.Port - if !portInfoEmpty(r.PortInfo) { - port = convertPortInfo(r.PortInfo) - } else if r.Port != "" { - // old version of management, single port - value, err := strconv.Atoi(r.Port) +func resolveGroupPort(g *peerRuleGroup) (*firewall.Port, error) { + if !portInfoEmpty(g.port) { + return convertPortInfo(g.port), nil + } + if g.legacyPort != "" { + value, err := strconv.ParseUint(g.legacyPort, 10, 16) if err != nil { - return "", nil, fmt.Errorf("invalid port: %w", err) + return nil, fmt.Errorf("invalid port: %w", err) } - port = &firewall.Port{ + return &firewall.Port{ Values: []uint16{uint16(value)}, - } + }, nil } - - ruleID := d.getPeerRuleID(ip, protocol, int(r.Direction), port, action) - if rulesPair, ok := d.peerRulesPairs[ruleID]; ok { - return ruleID, rulesPair, nil - } - - var rules []firewall.Rule - switch r.Direction { - case mgmProto.RuleDirection_IN: - rules, err = d.addInRules(r.PolicyID, ip, protocol, port, action, ipsetName) - case mgmProto.RuleDirection_OUT: - if d.firewall.IsStateful() { - return "", nil, nil - } - // return traffic for outbound connections if firewall is stateless - rules, err = d.addOutRules(r.PolicyID, ip, protocol, port, action, ipsetName) - default: - return "", nil, fmt.Errorf("invalid direction, skipping firewall rule") - } - - if err != nil { - return "", nil, err - } - - return ruleID, rules, nil + // nolint:nilnil // a nil port legitimately means "no port restriction" + return nil, nil } func portInfoEmpty(portInfo *mgmProto.PortInfo) bool { @@ -350,84 +594,9 @@ func portInfoEmpty(portInfo *mgmProto.PortInfo) bool { } } -func (d *DefaultManager) addInRules( - id []byte, - ip netip.Addr, - protocol firewall.Protocol, - port *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - rule, err := d.firewall.AddPeerFiltering(id, ip.AsSlice(), protocol, nil, port, action, ipsetName) - if err != nil { - return nil, fmt.Errorf("add firewall rule: %w", err) - } - - return rule, nil -} - -func (d *DefaultManager) addOutRules( - id []byte, - ip netip.Addr, - protocol firewall.Protocol, - port *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - if shouldSkipInvertedRule(protocol, port) { - return nil, nil - } - - rule, err := d.firewall.AddPeerFiltering(id, ip.AsSlice(), protocol, port, nil, action, ipsetName) - if err != nil { - return nil, fmt.Errorf("add firewall rule: %w", err) - } - - return rule, nil -} - -// getPeerRuleID returns unique ID for the rule based on its parameters. -func (d *DefaultManager) getPeerRuleID( - ip netip.Addr, - proto firewall.Protocol, - direction int, - port *firewall.Port, - action firewall.Action, -) id.RuleID { - idStr := ip.String() + string(proto) + strconv.Itoa(direction) + strconv.Itoa(int(action)) - if port != nil { - idStr += port.String() - } - - return id.RuleID(hex.EncodeToString(md5.New().Sum([]byte(idStr)))) -} - -// getRuleGroupingSelector takes all rule properties except IP address to build selector -func (d *DefaultManager) getRuleGroupingSelector(rule *mgmProto.FirewallRule) string { - return fmt.Sprintf("%v:%v:%v:%s:%v", strconv.Itoa(int(rule.Direction)), rule.Action, rule.Protocol, rule.Port, rule.PortInfo) -} - -// extractRuleIP extracts the peer IP from a firewall rule. -// If sourcePrefixes is populated (new management), decode the first entry and use its address. -// Otherwise fall back to the deprecated PeerIP string field (old management). -func extractRuleIP(r *mgmProto.FirewallRule) (netip.Addr, error) { - if len(r.SourcePrefixes) > 0 { - addr, err := netiputil.DecodeAddr(r.SourcePrefixes[0]) - if err != nil { - return netip.Addr{}, fmt.Errorf("decode source prefix: %w", err) - } - return addr.Unmap(), nil - } - - //nolint:staticcheck // PeerIP used for backward compatibility with old management - addr, err := netip.ParseAddr(r.PeerIP) - if err != nil { - return netip.Addr{}, fmt.Errorf("invalid IP address, skipping firewall rule") - } - return addr.Unmap(), nil -} - -func convertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewall.Protocol, error) { +// ConvertToFirewallProtocol maps a management rule protocol to the +// firewall protocol type. +func ConvertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewall.Protocol, error) { switch protocol { case mgmProto.RuleProtocol_TCP: return firewall.ProtocolTCP, nil diff --git a/client/internal/acl/manager_test.go b/client/internal/acl/manager_test.go index 8f737706e..19bef9058 100644 --- a/client/internal/acl/manager_test.go +++ b/client/internal/acl/manager_test.go @@ -10,6 +10,7 @@ import ( "go.uber.org/mock/gomock" "github.com/netbirdio/netbird/client/firewall" + fwmanager "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/internal/acl/mocks" @@ -77,9 +78,9 @@ func TestDefaultManager(t *testing.T) { }) t.Run("add extra rules", func(t *testing.T) { - existedPairs := map[string]struct{}{} + existedPairs := map[fwmanager.RuleID]struct{}{} for id := range acl.peerRulesPairs { - existedPairs[id.ID()] = struct{}{} + existedPairs[id] = struct{}{} } // remove first rule @@ -106,7 +107,7 @@ func TestDefaultManager(t *testing.T) { // check that old rule was removed previousCount := 0 for id := range acl.peerRulesPairs { - if _, ok := existedPairs[id.ID()]; ok { + if _, ok := existedPairs[id]; ok { previousCount++ } } diff --git a/client/internal/connect_test.go b/client/internal/connect_test.go index c317c88d8..3212e6abf 100644 --- a/client/internal/connect_test.go +++ b/client/internal/connect_test.go @@ -5,65 +5,78 @@ import ( "testing" ) -func Test_freePort(t *testing.T) { - tests := []struct { - name string - port int - want int - shouldMatch bool - }{ - { - name: "when port is 0 use random port", - port: 0, - want: 0, - shouldMatch: false, - }, - { - name: "provided and available", - port: 51821, - want: 51821, - shouldMatch: true, - }, - { - name: "provided and not available", - port: 51830, - want: 51830, - shouldMatch: false, - }, - } - c1, err := net.ListenUDP("udp", &net.UDPAddr{Port: 0}) +// probeFreePort asks the OS for a free UDP port and immediately releases it. +// The returned number is only a hint: nothing stops another process from +// grabbing the same port before the caller gets a chance to bind it. +// +// A hardcoded port number is not an option here: any fixed number can fall +// inside the ephemeral range and be held by an unrelated process on the test +// runner. +func probeFreePort(t *testing.T) int { + t.Helper() + + conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: 0}) if err != nil { - t.Errorf("freePort error = %v", err) + t.Fatalf("failed to bind probe port: %v", err) } - defer func(c1 *net.UDPConn) { - _ = c1.Close() - }(c1) - - if tests[1].port == c1.LocalAddr().(*net.UDPAddr).Port { - tests[1].port++ - tests[1].want++ - } - - tests[2].port = c1.LocalAddr().(*net.UDPAddr).Port - tests[2].want = c1.LocalAddr().(*net.UDPAddr).Port - - for _, tt := range tests { - - t.Run(tt.name, func(t *testing.T) { - got, err := freePort(tt.port) - - if err != nil { - t.Errorf("got an error while getting free port: %v", err) - } - - if tt.shouldMatch && got != tt.want { - t.Errorf("got a different port %v, want %v", got, tt.want) - } - - if !tt.shouldMatch && got == tt.want { - t.Errorf("got the same port %v, want a different port", tt.want) - } - }) - + port := conn.LocalAddr().(*net.UDPAddr).Port + if err := conn.Close(); err != nil { + t.Fatalf("failed to close probe port: %v", err) } + return port +} + +func Test_freePort(t *testing.T) { + t.Run("when port is 0 use random port", func(t *testing.T) { + got, err := freePort(0) + if err != nil { + t.Fatalf("got an error while getting free port: %v", err) + } + if got == 0 { + t.Errorf("got port 0, want a non-zero random port") + } + }) + + t.Run("provided and available", func(t *testing.T) { + const maxAttempts = 5 + + // The probed port is released before freePort binds it, so an + // unrelated process on the test runner can grab it in between, + // making freePort fall back to a different port. Retry with a + // freshly probed port instead of failing on a lost race. + for attempt := 1; attempt <= maxAttempts; attempt++ { + candidate := probeFreePort(t) + + got, err := freePort(candidate) + if err != nil { + t.Fatalf("got an error while getting free port: %v", err) + } + + if got == candidate { + return + } + t.Logf("attempt %d: freePort returned %d instead of the requested %d, retrying", attempt, got, candidate) + } + + t.Fatalf("freePort did not return the requested free port after %d attempts", maxAttempts) + }) + + t.Run("provided and not available", func(t *testing.T) { + busy, err := net.ListenUDP("udp", &net.UDPAddr{Port: 0}) + if err != nil { + t.Fatalf("failed to bind busy port: %v", err) + } + t.Cleanup(func() { + _ = busy.Close() + }) + busyPort := busy.LocalAddr().(*net.UDPAddr).Port + + got, err := freePort(busyPort) + if err != nil { + t.Fatalf("got an error while getting free port: %v", err) + } + if got == busyPort { + t.Errorf("got the same port %v, want a different port", busyPort) + } + }) } diff --git a/client/internal/dns/server_test.go b/client/internal/dns/server_test.go index 96e55a354..0144a4a8b 100644 --- a/client/internal/dns/server_test.go +++ b/client/internal/dns/server_test.go @@ -423,7 +423,7 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) { return nil, err } - pf, err := uspfilter.Create(wgIface, false, flowLogger, iface.DefaultMTU) + pf, err := uspfilter.Create(uspfilter.Config{IFace: wgIface, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) if err != nil { t.Fatalf("failed to create uspfilter: %v", err) return nil, err diff --git a/client/internal/dnsfwd/forwarder.go b/client/internal/dnsfwd/forwarder.go index b7e5a10e3..e3cb597be 100644 --- a/client/internal/dnsfwd/forwarder.go +++ b/client/internal/dnsfwd/forwarder.go @@ -54,12 +54,20 @@ type DNSForwarder struct { ttl uint32 statusRecorder *peer.Status - dnsServer *dns.Server - mux *dns.ServeMux - tcpServer *dns.Server - tcpMux *dns.ServeMux + mux *dns.ServeMux + tcpMux *dns.ServeMux - mutex sync.RWMutex + mutex sync.RWMutex + // closed records that Close has run, so a Listen still in flight does not + // go on to serve sockets nobody will shut down. + closed bool + // The sockets are kept alongside the servers because closing them is the + // only stop that always works: a server whose ActivateAndServe has not run + // yet refuses to shut down, and would otherwise start serving afterwards. + udpConn net.PacketConn + tcpLn net.Listener + dnsServer *dns.Server + tcpServer *dns.Server fwdEntries []*ForwarderEntry firewall firewaller resolver resolver @@ -106,7 +114,7 @@ func (f *DNSForwarder) Listen(entries []*ForwarderEntry) error { mux := dns.NewServeMux() f.mux = mux mux.HandleFunc(".", f.handleDNSQueryUDP) - f.dnsServer = &dns.Server{ + dnsServer := &dns.Server{ PacketConn: udpLn, Handler: mux, } @@ -114,22 +122,32 @@ func (f *DNSForwarder) Listen(entries []*ForwarderEntry) error { tcpMux := dns.NewServeMux() f.tcpMux = tcpMux tcpMux.HandleFunc(".", f.handleDNSQueryTCP) - f.tcpServer = &dns.Server{ + tcpServer := &dns.Server{ Listener: tcpLn, Handler: tcpMux, } - f.UpdateDomains(entries) + if !f.publish(udpLn, tcpLn, dnsServer, tcpServer, entries) { + log.Infof("DNS forwarder on %s was closed before it started serving", addrDesc) + if err := udpLn.Close(); err != nil { + log.Debugf("close UDP listener of a closed forwarder: %v", err) + } + if err := tcpLn.Close(); err != nil { + log.Debugf("close TCP listener of a closed forwarder: %v", err) + } + return nil + } + log.Debugf("DNS forwarder serving %d domains", len(entries)) errCh := make(chan error, 2) go func() { log.Infof("DNS UDP listener running on %s", addrDesc) - errCh <- f.dnsServer.ActivateAndServe() + errCh <- dnsServer.ActivateAndServe() }() go func() { log.Infof("DNS TCP listener running on %s", addrDesc) - errCh <- f.tcpServer.ActivateAndServe() + errCh <- tcpServer.ActivateAndServe() }() return <-errCh @@ -151,6 +169,46 @@ func (f *DNSForwarder) createTCPListener(netstackNet *netstack.Net) (net.Listene return net.ListenTCP("tcp", net.TCPAddrFromAddrPort(f.listenAddress)) } +// publish hands the sockets, servers and entries to the forwarder so Close can +// reach them and Domains can report them, and says whether serving may begin. +// Listen runs on its own goroutine, so a Close can arrive before it gets this +// far; false means the caller must close what it created instead of serving on +// it. +// +// The entries go in under the same lock rather than afterwards. Anything that +// reads them in between would otherwise see a forwarder that is listening and +// serves no domain, which for a caller rebuilding one means it comes back +// refusing every routed query. +func (f *DNSForwarder) publish( + udpConn net.PacketConn, + tcpLn net.Listener, + dnsServer, tcpServer *dns.Server, + entries []*ForwarderEntry, +) bool { + f.mutex.Lock() + defer f.mutex.Unlock() + + if f.closed { + return false + } + + f.udpConn = udpConn + f.tcpLn = tcpLn + f.dnsServer = dnsServer + f.tcpServer = tcpServer + f.fwdEntries = entries + return true +} + +// Domains returns the entries currently being served. The slice is replaced +// wholesale by UpdateDomains rather than mutated, so the caller may read it but +// must not write to it. +func (f *DNSForwarder) Domains() []*ForwarderEntry { + f.mutex.RLock() + defer f.mutex.RUnlock() + return f.fwdEntries +} + func (f *DNSForwarder) UpdateDomains(entries []*ForwarderEntry) { f.mutex.Lock() defer f.mutex.Unlock() @@ -189,19 +247,45 @@ func (f *DNSForwarder) removeStaleCacheEntries(oldEntries, newEntries []*Forward } func (f *DNSForwarder) Close(ctx context.Context) error { + // Marked closed under the lock so a Listen that has not published its + // servers yet gives up instead of racing this shutdown. The shutdowns + // themselves block, so they run outside it. + f.mutex.Lock() + f.closed = true + dnsServer, tcpServer := f.dnsServer, f.tcpServer + udpConn, tcpLn := f.udpConn, f.tcpLn + f.mutex.Unlock() + var result *multierror.Error - if f.dnsServer != nil { - if err := f.dnsServer.ShutdownContext(ctx); err != nil { + if dnsServer != nil { + if err := shutdownServer(ctx, dnsServer); err != nil { result = multierror.Append(result, fmt.Errorf("UDP shutdown: %w", err)) } } - if f.tcpServer != nil { - if err := f.tcpServer.ShutdownContext(ctx); err != nil { + if tcpServer != nil { + if err := shutdownServer(ctx, tcpServer); err != nil { result = multierror.Append(result, fmt.Errorf("TCP shutdown: %w", err)) } } + // The sockets are closed even when the shutdowns above reported nothing to + // do. A server that has been published but has not reached + // ActivateAndServe refuses to shut down, and closing what it was about to + // serve on is what stops it: the alternative is a listener still answering + // on an interface that has gone away. A shutdown that did run has already + // closed these, so the second close is expected to fail. + if udpConn != nil { + if err := udpConn.Close(); err != nil { + log.Debugf("close UDP socket of the DNS forwarder: %v", err) + } + } + if tcpLn != nil { + if err := tcpLn.Close(); err != nil { + log.Debugf("close TCP socket of the DNS forwarder: %v", err) + } + } + return nberrors.FormatErrorOrNil(result) } @@ -514,3 +598,16 @@ func attachEDE(resp *dns.Msg, code uint16, text string) { } opt.Option = append(opt.Option, &dns.EDNS0_EDE{InfoCode: code, ExtraText: text}) } + +// shutdownServer shuts a server down gracefully, treating "never started" as +// success. A server that was published but has not reached ActivateAndServe +// has nothing to wind down, and the caller closes its socket regardless, which +// is what actually stops it. dns exports no sentinel for this, so the message +// is all there is to match on. +func shutdownServer(ctx context.Context, server *dns.Server) error { + err := server.ShutdownContext(ctx) + if err == nil || strings.Contains(err.Error(), "server not started") { + return nil + } + return err +} diff --git a/client/internal/dnsfwd/forwarder_test.go b/client/internal/dnsfwd/forwarder_test.go index c69a9166e..a64ba80e7 100644 --- a/client/internal/dnsfwd/forwarder_test.go +++ b/client/internal/dnsfwd/forwarder_test.go @@ -1238,3 +1238,55 @@ func TestDNSForwarder_EmptyQuery(t *testing.T) { assert.Nil(t, mockWriter.GetLastResponse(), "Should not write response for empty query") } + +// TestDNSForwarder_ClosedBeforeItServes covers Listen reaching the point of +// serving after the forwarder has already been closed. Listen runs on its own +// goroutine, so it can get there late, and a socket it starts serving then is +// one nothing will ever close: on Android it keeps answering on an interface +// that has been replaced. The close is sequenced first here rather than raced, +// which pins the same state deterministically. +func TestDNSForwarder_ClosedBeforeItServes(t *testing.T) { + f := NewDNSForwarder(netip.MustParseAddrPort("127.0.0.1:0"), 60, nil, nil, nil) + + require.NoError(t, f.Close(context.Background()), "closing a forwarder that never started") + + done := make(chan error, 1) + go func() { done <- f.Listen(nil) }() + + select { + case err := <-done: + assert.NoError(t, err, "a closed forwarder should give up quietly, not serve") + case <-time.After(5 * time.Second): + t.Fatal("Listen went on to serve after the forwarder was closed") + } +} + +// TestDNSForwarder_CloseStopsUnactivatedServers covers the window between +// Listen publishing its servers and reaching ActivateAndServe. A server that +// has not been activated refuses to shut down, so Close has to close the +// sockets itself or they are left serving. +func TestDNSForwarder_CloseStopsUnactivatedServers(t *testing.T) { + f := NewDNSForwarder(netip.MustParseAddrPort("127.0.0.1:0"), 60, nil, nil, nil) + + udpConn, err := f.createUDPListener(nil) + require.NoError(t, err, "create UDP listener") + tcpLn, err := f.createTCPListener(nil) + require.NoError(t, err, "create TCP listener") + + // Published but deliberately never activated, which is the state Listen is + // in for the moment before it starts serving. + require.True(t, f.publish(udpConn, tcpLn, &dns.Server{PacketConn: udpConn}, &dns.Server{Listener: tcpLn}, nil), + "publishing to an open forwarder") + + tcpAddr := tcpLn.Addr().String() + require.NoError(t, f.Close(context.Background()), "close should report no error for servers it could not shut down") + + _, err = tcpLn.Accept() + assert.Error(t, err, "the TCP socket should be closed after Close") + + conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) + if err == nil { + _ = conn.Close() + t.Fatal("the forwarder is still accepting connections after Close") + } +} diff --git a/client/internal/dnsfwd/manager.go b/client/internal/dnsfwd/manager.go index 29ca0d247..1c62e908d 100644 --- a/client/internal/dnsfwd/manager.go +++ b/client/internal/dnsfwd/manager.go @@ -3,7 +3,6 @@ package dnsfwd import ( "context" "fmt" - "net" "net/netip" "os" "strconv" @@ -118,6 +117,16 @@ func (m *Manager) UpdateDomains(entries []*ForwarderEntry) { m.dnsForwarder.UpdateDomains(entries) } +// Domains returns the entries currently being served, or nil when the +// forwarder is not running. +func (m *Manager) Domains() []*ForwarderEntry { + if m.dnsForwarder == nil { + return nil + } + + return m.dnsForwarder.Domains() +} + func (m *Manager) Stop(ctx context.Context) error { if m.dnsForwarder == nil { return nil @@ -160,12 +169,13 @@ func (m *Manager) allowDNSFirewall() error { return nil } - dnsRules, err := m.firewall.AddPeerFiltering(nil, net.IP{0, 0, 0, 0}, firewall.ProtocolUDP, nil, dport, firewall.ActionAccept, "") + anyV4 := []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + dnsRule, err := m.firewall.AddFilterRule(nil, anyV4, firewall.Network{}, firewall.ProtocolUDP, nil, dport, firewall.ActionAccept) if err != nil { return fmt.Errorf("add udp firewall rule: %w", err) } - tcpRules, err := m.firewall.AddPeerFiltering(nil, net.IP{0, 0, 0, 0}, firewall.ProtocolTCP, nil, dport, firewall.ActionAccept, "") + tcpRule, err := m.firewall.AddFilterRule(nil, anyV4, firewall.Network{}, firewall.ProtocolTCP, nil, dport, firewall.ActionAccept) if err != nil { return fmt.Errorf("add tcp firewall rule: %w", err) } @@ -174,8 +184,12 @@ func (m *Manager) allowDNSFirewall() error { return fmt.Errorf("flush: %w", err) } - m.fwRules = dnsRules - m.tcpRules = tcpRules + if dnsRule != nil { + m.fwRules = []firewall.Rule{dnsRule} + } + if tcpRule != nil { + m.tcpRules = []firewall.Rule{tcpRule} + } m.registerNetstackServices() @@ -209,12 +223,12 @@ func (m *Manager) unregisterNetstackServices() { func (m *Manager) dropDNSFirewall() error { var mErr *multierror.Error for _, rule := range m.fwRules { - if err := m.firewall.DeletePeerRule(rule); err != nil { + if err := m.firewall.DeleteFilterRule(rule); err != nil { mErr = multierror.Append(mErr, fmt.Errorf("failed to delete DNS router rules, err: %v", err)) } } for _, rule := range m.tcpRules { - if err := m.firewall.DeletePeerRule(rule); err != nil { + if err := m.firewall.DeleteFilterRule(rule); err != nil { mErr = multierror.Append(mErr, fmt.Errorf("failed to delete DNS router rules, err: %v", err)) } } diff --git a/client/internal/engine.go b/client/internal/engine.go index 2cfd19a81..f8b65f7d8 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -94,6 +94,13 @@ const ( // exec, os.Stat); without this bound a single stuck call freezes handleSync, and // thus syncMsgMux, for as long as the call hangs (observed multi-minute freezes). systemInfoTimeout = 15 * time.Second + + // dnsForwarderStopTimeout bounds how long stopping the DNS forwarder waits + // for the queries still in flight. One waiting on an unresponsive upstream + // would otherwise hold the stop for the whole upstream timeout, and the + // stop runs with syncMsgMux held. The sockets are closed either way, so + // giving up costs a query that was already failing. + dnsForwarderStopTimeout = 2 * time.Second ) var ErrResetConnection = fmt.Errorf("reset connection") @@ -258,6 +265,8 @@ type Engine struct { // checks are the client-applied posture checks that need to be evaluated on the client checks []*mgmProto.Checks + infoSource system.InfoSource + relayManager *relayClient.Manager stateManager *statemanager.Manager portForwardManager *portforward.Manager @@ -321,6 +330,10 @@ type localIpUpdater interface { UpdateLocalIPs() error } +// overlayRebind rebuilds one subsystem's sockets on the current interface. The +// error it returns names its own subsystem, since the caller can only log it. +type overlayRebind func() error + // NewEngine creates a new Connection Engine with probes attached func NewEngine( clientCtx context.Context, @@ -745,6 +758,11 @@ func (e *Engine) initFirewall() error { return fmt.Errorf("set firewall: %w", err) } + // TODO: the firewall backends dedup filter rules by content, so a + // management route ACL with identical content would collapse onto the + // untracked drop rules installed here, and a later management delete + // could remove them. Needs backend refcounting or per-consumer key + // namespacing. if e.config.BlockLANAccess { e.blockLanAccess() } @@ -757,14 +775,14 @@ func (e *Engine) initFirewall() error { port := firewallManager.Port{Values: []uint16{uint16(rosenpassPort)}} // IPv4-only: rosenpass peers connect via AllowedIps[0] which is always v4. - if _, err := e.firewall.AddPeerFiltering( + if _, err := e.firewall.AddFilterRule( nil, - net.IP{0, 0, 0, 0}, + []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)}, + firewallManager.Network{}, firewallManager.ProtocolUDP, nil, &port, firewallManager.ActionAccept, - "", ); err != nil { log.Errorf("failed to allow rosenpass interface traffic: %v", err) return nil @@ -814,7 +832,7 @@ func (e *Engine) blockLanAccess() { if network.Addr().Is6() { source = v6 } - if _, err := e.firewall.AddRouteFiltering( + if _, err := e.firewall.AddFilterRule( nil, []netip.Prefix{source}, firewallManager.Network{Prefix: network}, @@ -1225,9 +1243,7 @@ func (e *Engine) updateChecksIfNew(checks []*mgmProto.Checks) error { if isChecksEqual(e.checks, checks) { return nil } - e.checks = checks - - info, ok := system.GetInfoWithChecksTimeout(e.ctx, systemInfoTimeout, checks, e.overlayAddresses()...) + info, ok := e.infoSource.Refresh(e.ctx, systemInfoTimeout, checks, e.overlayAddresses()...) if !ok { // Gathering timed out; skip the meta sync this cycle rather than blocking the // sync loop (and syncMsgMux) on a stuck system call. A later sync will retry. @@ -1238,6 +1254,7 @@ func (e *Engine) updateChecksIfNew(checks []*mgmProto.Checks) error { if err := e.mgmClient.SyncMeta(info); err != nil { return fmt.Errorf("could not sync meta: error %s", err) } + e.checks = checks return nil } @@ -1264,6 +1281,28 @@ func (e *Engine) applyInfoFlags(info *system.Info) { ) } +func (e *Engine) currentSystemInfo(ctx context.Context) *system.Info { + info := e.infoSource.Current(ctx, e.overlayAddresses()...) + e.applyInfoFlags(info) + return info +} + +// syncInfoFunc returns the info callback for the management sync stream. The +// first connect sends the info refreshed right before it instead of gathering +// again; every reconnect gathers a fresh one. The stream retry loop calls the +// callback sequentially, so the handoff needs no synchronization. +func (e *Engine) syncInfoFunc(refreshed *system.Info) func(ctx context.Context) *system.Info { + return func(ctx context.Context) *system.Info { + if refreshed == nil { + return e.currentSystemInfo(ctx) + } + info := refreshed + refreshed = nil + e.applyInfoFlags(info) + return info + } +} + // overlayAddresses returns our own WireGuard overlay address (v4 and v6) so it // can be excluded from the reported network addresses; the interface coming and // going otherwise churns the peer meta on the management server. @@ -1457,15 +1496,11 @@ func (e *Engine) receiveManagementEvents() { e.shutdownWg.Add(1) go func() { defer e.shutdownWg.Done() - info, ok := system.GetInfoWithChecksTimeout(e.ctx, systemInfoTimeout, e.checks, e.overlayAddresses()...) + info, ok := e.infoSource.Refresh(e.ctx, systemInfoTimeout, e.checks, e.overlayAddresses()...) if !ok { - // Gathering timed out; connect the stream with base info so management - // connectivity still comes up rather than blocking here. - info = system.GetInfo(e.ctx) + log.Warnf("posture checks not refreshed before the sync connect, sending the previous results") } - e.applyInfoFlags(info) - - err := e.mgmClient.Sync(e.ctx, info, e.handleSync) + err := e.mgmClient.Sync(e.ctx, e.syncInfoFunc(info), e.handleSync) if err != nil { // happens if management is unavailable for a long time. // We want to cancel the operation of the whole client @@ -2497,7 +2532,72 @@ func (e *Engine) RenewTun(fd int) error { return fmt.Errorf("wireguard interface not initialized") } - return wgInterface.RenewTun(fd) + if err := wgInterface.RenewTun(fd); err != nil { + return err + } + + e.rebindOverlayListeners() + return nil +} + +// rebindOverlayListeners gives the servers that listen on an overlay address +// sockets on the interface as it is now. +// +// A socket belongs to the interface generation it was created on. Renewing the +// TUN builds a new interface and moves the overlay addresses to it, which +// leaves the old sockets in LISTEN with the uspfilter still logging packets +// arriving for them, while every accept fails with EINVAL for the life of the +// socket: from the outside the server looks alive and answers nothing. On +// Android this happens during a normal startup, where the first TUN is +// established before the routes are known and replaced once they arrive. +// +// Rebinding costs whatever those sockets were carrying, which the renewal has +// already broken. Errors are logged rather than returned: the renewal itself +// succeeded, and failing it would hand the caller a working interface and an +// error. +func (e *Engine) rebindOverlayListeners() { + e.syncMsgMux.Lock() + defer e.syncMsgMux.Unlock() + + for _, rebind := range e.overlayRebinds() { + if err := rebind(); err != nil { + log.Errorf("after TUN renewal: %v", err) + } + } +} + +// overlayRebinds is every subsystem of this engine that holds sockets bound to +// an overlay address, and how to rebuild each one's. +// +// A subsystem that starts listening on an overlay address belongs in this list. +// Leaving it out costs nothing that review would notice and produces a listener +// that stays in LISTEN, is logged as receiving packets, and refuses every +// connection for the life of the process. +func (e *Engine) overlayRebinds() []overlayRebind { + return []overlayRebind{ + e.restartSSHListeners, + e.restartDNSForwarder, + } +} + +// restartDNSForwarder rebuilds the DNS forwarder serving the same domains. +// No-op when it is not running. See Engine.rebindOverlayListeners. +func (e *Engine) restartDNSForwarder() error { + if e.dnsForwardMgr == nil { + return nil + } + // Read from the forwarder before it goes away, so the replacement serves + // the domains in force now rather than a copy kept somewhere else. + entries := e.dnsForwardMgr.Domains() + e.stopDNSForwarder() + // Both halves log their own failures, so the only thing left to report is + // the outcome: a start that failed left the manager nil, and the forwarder + // is now down rather than merely rebound. + e.startDNSForwarder(entries) + if e.dnsForwardMgr == nil { + return errors.New("rebind DNS forwarder: it did not come back up") + } + return nil } // updateDNSForwarder start or stop the DNS forwarder based on the domains and the feature flag @@ -2543,7 +2643,14 @@ func (e *Engine) stopDNSForwarder() { return } - if err := e.dnsForwardMgr.Stop(context.Background()); err != nil { + // Bounded because the shutdown waits for queries still in flight, and one + // waiting on an unresponsive upstream holds it for as long as that lookup + // is allowed to take. This runs with syncMsgMux held, so that wait is one + // the whole engine spends. + ctx, cancel := context.WithTimeout(context.Background(), dnsForwarderStopTimeout) + defer cancel() + + if err := e.dnsForwardMgr.Stop(ctx); err != nil { log.Errorf("failed to stop DNS forward: %v", err) } @@ -2658,7 +2765,7 @@ func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewal var merr *multierror.Error forwardingRules := make([]firewallManager.ForwardRule, 0, len(rules)) for _, rule := range rules { - proto, err := convertToFirewallProtocol(rule.GetProtocol()) + proto, err := acl.ConvertToFirewallProtocol(rule.GetProtocol()) if err != nil { merr = multierror.Append(merr, fmt.Errorf("failed to convert protocol '%s': %w", rule.GetProtocol(), err)) continue diff --git a/client/internal/engine_privileged_test.go b/client/internal/engine_privileged_test.go index 1428b742c..1b047e017 100644 --- a/client/internal/engine_privileged_test.go +++ b/client/internal/engine_privileged_test.go @@ -193,7 +193,7 @@ func TestEngine_Sync(t *testing.T) { // feed updates to Engine via mocked Management client updates := make(chan *mgmtProto.SyncResponse) defer close(updates) - syncFunc := func(ctx context.Context, info *system.Info, msgHandler func(msg *mgmtProto.SyncResponse) error) error { + syncFunc := func(ctx context.Context, _ func(context.Context) *system.Info, msgHandler func(msg *mgmtProto.SyncResponse) error) error { for msg := range updates { err := msgHandler(msg) if err != nil { diff --git a/client/internal/engine_ssh.go b/client/internal/engine_ssh.go index 53d2c1122..60bdfd806 100644 --- a/client/internal/engine_ssh.go +++ b/client/internal/engine_ssh.go @@ -24,6 +24,8 @@ type sshServer interface { Stop() error GetStatus() (bool, []sshserver.SessionInfo) UpdateSSHAuth(config *sshauth.Config) + JWTConfig() *sshserver.JWTConfig + AuthConfig() *sshauth.Config } func (e *Engine) setupSSHPortRedirection() error { @@ -77,7 +79,7 @@ func (e *Engine) updateSSH(sshConf *mgmProto.SSHConfig) error { if e.config.DisableSSHAuth != nil && *e.config.DisableSSHAuth { log.Info("starting SSH server without JWT authentication (authentication disabled by config)") - return e.startSSHServer(nil) + return e.startSSHServer(nil, nil) } if protoJWT := sshConf.GetJwtConfig(); protoJWT != nil { @@ -95,7 +97,7 @@ func (e *Engine) updateSSH(sshConf *mgmProto.SSHConfig) error { MaxTokenAge: protoJWT.GetMaxTokenAge(), } - return e.startSSHServer(jwtConfig) + return e.startSSHServer(jwtConfig, nil) } return errors.New("SSH server requires valid JWT configuration") @@ -231,8 +233,33 @@ func (e *Engine) cleanupSSHConfig() { } } -// startSSHServer initializes and starts the SSH server with proper configuration. -func (e *Engine) startSSHServer(jwtConfig *sshserver.JWTConfig) error { +// restartSSHListeners rebuilds the SSH server so it listens on new sockets, on +// the same terms it was started with. No-op when it is not running. See +// Engine.rebindOverlayListeners for why this is needed. +func (e *Engine) restartSSHListeners() error { + if e.sshServer == nil { + return nil + } + // Read from the server before it goes away. A rebuilt one starts with an + // empty authorizer, which fails closed, so without carrying the + // authorization over every JWT login is refused until the next network map + // happens to bring one. + jwtConfig, authConfig := e.sshServer.JWTConfig(), e.sshServer.AuthConfig() + if err := e.stopSSHServer(); err != nil { + return fmt.Errorf("rebind SSH listeners: %w", err) + } + if err := e.startSSHServer(jwtConfig, authConfig); err != nil { + return fmt.Errorf("rebind SSH listeners: %w", err) + } + return nil +} + +// startSSHServer initializes and starts the SSH server with proper +// configuration. authConfig is the fine-grained authorization to open with, and +// is applied before the server accepts anything: a server that starts listening +// with an empty authorizer refuses the logins that arrive in the meantime. +// Nil leaves it as management has not sent one yet. +func (e *Engine) startSSHServer(jwtConfig *sshserver.JWTConfig, authConfig *sshauth.Config) error { if e.wgInterface == nil { return errors.New("wg interface not initialized") } @@ -240,6 +267,7 @@ func (e *Engine) startSSHServer(jwtConfig *sshserver.JWTConfig) error { serverConfig := &sshserver.Config{ HostKeyPEM: e.config.SSHKey, JWT: jwtConfig, + Auth: authConfig, } server := sshserver.New(serverConfig) diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 4e9faa437..ec388ac94 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -2,6 +2,7 @@ package internal import ( "context" + "errors" "fmt" "net" "net/netip" @@ -31,6 +32,7 @@ import ( icemaker "github.com/netbirdio/netbird/client/internal/peer/ice" "github.com/netbirdio/netbird/client/internal/profilemanager" "github.com/netbirdio/netbird/client/internal/routemanager" + "github.com/netbirdio/netbird/client/system" nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/monotime" "github.com/netbirdio/netbird/route" @@ -253,6 +255,118 @@ func TestEngine_SSHServerConsistency(t *testing.T) { }) } +func TestEngine_FirstSyncInfoCarriesLoginChecks(t *testing.T) { + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + exe, err := os.Executable() + require.NoError(t, err) + + ctx, cancel := context.WithCancel(CtxInitState(context.Background())) + defer cancel() + + infos := make(chan *system.Info, 1) + mgmClient := &mgmt.MockClient{ + SyncFunc: func(ctx context.Context, getInfo func(context.Context) *system.Info, _ func(*mgmtProto.SyncResponse) error) error { + infos <- getInfo(ctx) + return nil + }, + } + + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String(), iface.DefaultMTU) + engine := NewEngine(ctx, cancel, &EngineConfig{ + WgIfaceName: "utun104", + WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"), + WgPrivateKey: key, + WgPort: 33100, + MTU: iface.DefaultMTU, + }, EngineServices{ + SignalClient: &signal.MockClient{}, + MgmClient: mgmClient, + RelayManager: relayMgr, + StatusRecorder: peer.NewRecorder("https://mgm"), + Checks: []*mgmtProto.Checks{{Files: []string{exe}}}, + }, MobileDependency{}) + + engine.receiveManagementEvents() + + select { + case info := <-infos: + require.Len(t, info.Files, 1) + assert.Equal(t, exe, info.Files[0].Path) + assert.True(t, info.Files[0].Exist) + case <-time.After(20 * time.Second): + t.Fatal("timeout waiting for the first sync info") + } + engine.shutdownWg.Wait() +} + +func TestEngine_SyncInfoFuncReusesRefreshedInfoOnce(t *testing.T) { + engine := &Engine{config: &EngineConfig{}} + + refreshed := &system.Info{Hostname: "from-refresh"} + getInfo := engine.syncInfoFunc(refreshed) + + first := getInfo(context.Background()) + assert.Same(t, refreshed, first, "the first connect should send the refreshed info instead of gathering again") + + second := getInfo(context.Background()) + assert.NotSame(t, refreshed, second, "the reconnect should gather a fresh info") + assert.NotEqual(t, "from-refresh", second.Hostname, "the fresh info should not carry the refreshed hostname") +} + +func TestEngine_SyncInfoFuncGathersWhenRefreshFailed(t *testing.T) { + engine := &Engine{config: &EngineConfig{}} + + info := engine.syncInfoFunc(nil)(context.Background()) + require.NotNil(t, info, "a failed refresh should fall back to gathering the info") + assert.NotEmpty(t, info.Hostname, "the gathered info should carry the hostname") +} + +func TestEngine_UpdateChecksIfNewRetriesAfterFailedSyncMeta(t *testing.T) { + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + + exe, err := os.Executable() + require.NoError(t, err) + + ctx, cancel := context.WithCancel(CtxInitState(context.Background())) + defer cancel() + + syncMetaCalls := 0 + mgmClient := &mgmt.MockClient{ + SyncMetaFunc: func(*system.Info) error { + syncMetaCalls++ + if syncMetaCalls == 1 { + return errors.New("management unavailable") + } + return nil + }, + } + + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String(), iface.DefaultMTU) + engine := NewEngine(ctx, cancel, &EngineConfig{ + WgIfaceName: "utun105", + WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"), + WgPrivateKey: key, + WgPort: 33100, + MTU: iface.DefaultMTU, + }, EngineServices{ + SignalClient: &signal.MockClient{}, + MgmClient: mgmClient, + RelayManager: relayMgr, + StatusRecorder: peer.NewRecorder("https://mgm"), + }, MobileDependency{}) + + checks := []*mgmtProto.Checks{{Files: []string{exe}}} + + require.Error(t, engine.updateChecksIfNew(checks)) + require.NoError(t, engine.updateChecksIfNew(checks)) + require.NoError(t, engine.updateChecksIfNew(checks)) + + assert.Equal(t, 2, syncMetaCalls) +} + func TestEngine_UpdateNetworkMap(t *testing.T) { // test setup key, err := wgtypes.GeneratePrivateKey() diff --git a/client/internal/ingressgw/manager.go b/client/internal/ingressgw/manager.go index b8952e5c0..605543d1c 100644 --- a/client/internal/ingressgw/manager.go +++ b/client/internal/ingressgw/manager.go @@ -24,14 +24,14 @@ type RulePair struct { type Manager struct { dnatFirewall DNATFirewall - rules map[string]RulePair // keys is the ID of the ForwardRule + rules map[firewall.RuleID]RulePair rulesMu sync.Mutex } func NewManager(dnatFirewall DNATFirewall) *Manager { return &Manager{ dnatFirewall: dnatFirewall, - rules: make(map[string]RulePair), + rules: make(map[firewall.RuleID]RulePair), } } @@ -41,7 +41,7 @@ func (h *Manager) Update(forwardRules []firewall.ForwardRule) error { var mErr *multierror.Error - toDelete := make(map[string]RulePair, len(h.rules)) + toDelete := make(map[firewall.RuleID]RulePair, len(h.rules)) for id, r := range h.rules { toDelete[id] = r } @@ -59,6 +59,10 @@ func (h *Manager) Update(forwardRules []firewall.ForwardRule) error { mErr = multierror.Append(mErr, fmt.Errorf("add forward rule '%s': %v", fwdRule.String(), err)) continue } + if rule == nil { + mErr = multierror.Append(mErr, fmt.Errorf("add forward rule '%s': backend returned no rule", fwdRule.String())) + continue + } log.Infof("forward rule has been added '%s'", fwdRule) h.rules[id] = RulePair{ ForwardRule: fwdRule, @@ -90,7 +94,7 @@ func (h *Manager) Close() error { } } - h.rules = make(map[string]RulePair) + h.rules = make(map[firewall.RuleID]RulePair) return nberrors.FormatErrorOrNil(mErr) } diff --git a/client/internal/ingressgw/manager_test.go b/client/internal/ingressgw/manager_test.go index 591ea0dd8..0cd40fcc4 100644 --- a/client/internal/ingressgw/manager_test.go +++ b/client/internal/ingressgw/manager_test.go @@ -14,11 +14,11 @@ var ( ) type MocFwRule struct { - id string + id firewall.RuleID } -func (m *MocFwRule) ID() string { - return string(m.id) +func (m *MocFwRule) ID() firewall.RuleID { + return m.id } type MockDNATFirewall struct { diff --git a/client/internal/ipcauth/identity.go b/client/internal/ipcauth/identity.go index ff70c209a..d7d10f57d 100644 --- a/client/internal/ipcauth/identity.go +++ b/client/internal/ipcauth/identity.go @@ -91,6 +91,19 @@ func (i Identity) IsPrivileged() bool { return slices.Contains(i.Groups, sidAdministrators) } +// SameUser reports whether two identities are the same local principal. Only +// the account is compared: the group set and the elevation flag describe what a +// token may do, not who it belongs to. A SID on either side decides the +// comparison, so a Windows principal never matches a Unix one on the UID both +// happen to leave at zero. The zero Identity carries uid 0, so callers must +// establish that both identities are real before the answer means anything. +func (i Identity) SameUser(other Identity) bool { + if i.SID != "" || other.SID != "" { + return i.SID == other.SID + } + return i.UID == other.UID +} + // String renders the identity for audit logs and denial messages. func (i Identity) String() string { if i.IsWindows() { diff --git a/client/internal/ipcauth/identity_sameuser_test.go b/client/internal/ipcauth/identity_sameuser_test.go new file mode 100644 index 000000000..c98f583db --- /dev/null +++ b/client/internal/ipcauth/identity_sameuser_test.go @@ -0,0 +1,66 @@ +package ipcauth + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIdentitySameUser(t *testing.T) { + tests := []struct { + name string + a Identity + b Identity + want bool + }{ + { + name: "same uid", + a: Identity{UID: 1000, GID: 1000}, + b: Identity{UID: 1000, GID: 1000}, + want: true, + }, + { + name: "same uid, different gid and pid still the same user", + a: Identity{UID: 1000, GID: 1000, PID: 11}, + b: Identity{UID: 1000, GID: 27, PID: 22}, + want: true, + }, + { + name: "different uid", + a: Identity{UID: 1000}, + b: Identity{UID: 1001}, + want: false, + }, + { + name: "same sid", + a: Identity{SID: "S-1-5-21-1-2-3-1001"}, + b: Identity{SID: "S-1-5-21-1-2-3-1001"}, + want: true, + }, + { + name: "same sid, elevation and groups differ", + a: Identity{SID: "S-1-5-21-1-2-3-1001", Elevated: true, Groups: []string{sidAdministrators}}, + b: Identity{SID: "S-1-5-21-1-2-3-1001"}, + want: true, + }, + { + name: "different sid", + a: Identity{SID: "S-1-5-21-1-2-3-1001"}, + b: Identity{SID: "S-1-5-21-1-2-3-1002"}, + want: false, + }, + { + name: "a windows principal is never a unix one", + a: Identity{SID: "S-1-5-18"}, + b: Identity{UID: 0}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.a.SameUser(tt.b)) + assert.Equal(t, tt.want, tt.b.SameUser(tt.a), "SameUser must be symmetric") + }) + } +} diff --git a/client/internal/message_convert.go b/client/internal/message_convert.go index 97da32c06..60f19e228 100644 --- a/client/internal/message_convert.go +++ b/client/internal/message_convert.go @@ -10,21 +10,6 @@ import ( mgmProto "github.com/netbirdio/netbird/shared/management/proto" ) -func convertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewallManager.Protocol, error) { - switch protocol { - case mgmProto.RuleProtocol_TCP: - return firewallManager.ProtocolTCP, nil - case mgmProto.RuleProtocol_UDP: - return firewallManager.ProtocolUDP, nil - case mgmProto.RuleProtocol_ICMP: - return firewallManager.ProtocolICMP, nil - case mgmProto.RuleProtocol_ALL: - return firewallManager.ProtocolALL, nil - default: - return "", fmt.Errorf("invalid protocol type: %s", protocol.String()) - } -} - func convertPortInfo(portInfo *mgmProto.PortInfo) (*firewallManager.Port, error) { if portInfo == nil { return nil, errors.New("portInfo cannot be nil") diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 0ccfa83ac..981b0c987 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -8,6 +8,7 @@ import ( "net/netip" "net/url" "runtime" + "slices" "sort" "strings" "sync" @@ -472,27 +473,13 @@ func (m *DefaultManager) CurrentRouteRange() []string { m.mux.Lock() defer m.mux.Unlock() - if m.disableClientRoutes { - return nil - } - - filtered := m.routeSelector.FilterSelectedExitNodes(m.clientRoutes) - var nets []string - for _, routes := range filtered { - for _, r := range routes { - if r.IsDynamic() { - continue - } - nets = append(nets, r.NetString()) - } - } - - if m.fakeIPManager != nil { - nets = append(nets, m.fakeIPManager.GetFakeIPBlock().String(), m.fakeIPManager.GetFakeIPv6Block().String()) + nets := m.overlayNetworks() + if !m.disableClientRoutes { + nets = append(nets, m.clientRouteRange()...) } sort.Strings(nets) - return nets + return slices.Compact(nets) } // GetRouteSelector returns the route selector @@ -856,6 +843,42 @@ func (m *DefaultManager) logExitNodeUpdate(info exitNodeInfo, preferred route.Ne len(info.allIDs), preferred, len(info.userSelected), len(info.userDeselected), len(info.selectedByManagement)) } +// overlayNetworks returns the v4 and v6 overlay networks of the WireGuard interface, each only when it is set. +func (m *DefaultManager) overlayNetworks() []string { + if m.wgInterface == nil { + return nil + } + + addr := m.wgInterface.Address() + var nets []string + if addr.Network.IsValid() { + nets = append(nets, addr.Network.String()) + } + if addr.IPv6Net.IsValid() { + nets = append(nets, addr.IPv6Net.String()) + } + return nets +} + +// clientRouteRange returns the static client route networks of the selected exit nodes together with the fake IP blocks. +func (m *DefaultManager) clientRouteRange() []string { + filtered := m.routeSelector.FilterSelectedExitNodes(m.clientRoutes) + var nets []string + for _, routes := range filtered { + for _, r := range routes { + if r.IsDynamic() { + continue + } + nets = append(nets, r.NetString()) + } + } + + if m.fakeIPManager != nil { + nets = append(nets, m.fakeIPManager.GetFakeIPBlock().String(), m.fakeIPManager.GetFakeIPv6Block().String()) + } + return nets +} + // minNetID returns the lexicographically smallest NetID, for a deterministic // default pick that stays stable across restarts. func minNetID(ids []route.NetID) route.NetID { diff --git a/client/internal/routemanager/notifier/notifier_android.go b/client/internal/routemanager/notifier/notifier_android.go index 5fa329310..24cbb94db 100644 --- a/client/internal/routemanager/notifier/notifier_android.go +++ b/client/internal/routemanager/notifier/notifier_android.go @@ -4,8 +4,6 @@ package notifier import ( "net/netip" - "slices" - "sort" "sync" "github.com/netbirdio/netbird/client/internal/listener" @@ -75,19 +73,3 @@ func (n *Notifier) notifyLocked() { func (n *Notifier) Close() { // unused } - -func routesToStrings(routes []*route.Route) []string { - nets := make([]string, 0, len(routes)) - for _, r := range routes { - nets = append(nets, r.NetString()) - } - return nets -} - -func hasRouteDiff(a []*route.Route, b []*route.Route) bool { - as := routesToStrings(a) - bs := routesToStrings(b) - sort.Strings(as) - sort.Strings(bs) - return !slices.Equal(as, bs) -} diff --git a/client/internal/routemanager/notifier/route_diff.go b/client/internal/routemanager/notifier/route_diff.go new file mode 100644 index 000000000..52abddf36 --- /dev/null +++ b/client/internal/routemanager/notifier/route_diff.go @@ -0,0 +1,27 @@ +package notifier + +import ( + "slices" + "sort" + + "github.com/netbirdio/netbird/route" +) + +// routePrefixes returns the distinct prefixes a route set covers, sorted. +// Duplicates are dropped deliberately: an HA group hands us one route per +// peer serving the same prefix, and the platform is given the prefix, not the +// candidates. Counting them would report a change every time a peer joins or +// leaves a group, and on Android each report renews the TUN. +func routePrefixes(routes []*route.Route) []string { + nets := make([]string, 0, len(routes)) + for _, r := range routes { + nets = append(nets, r.NetString()) + } + sort.Strings(nets) + return slices.Compact(nets) +} + +// hasRouteDiff reports whether the prefixes the two route sets cover differ. +func hasRouteDiff(a []*route.Route, b []*route.Route) bool { + return !slices.Equal(routePrefixes(a), routePrefixes(b)) +} diff --git a/client/internal/routemanager/notifier/route_diff_test.go b/client/internal/routemanager/notifier/route_diff_test.go new file mode 100644 index 000000000..80df69d9d --- /dev/null +++ b/client/internal/routemanager/notifier/route_diff_test.go @@ -0,0 +1,88 @@ +package notifier + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/route" +) + +func routeFor(id route.ID, prefix string) *route.Route { + return &route.Route{ + ID: id, + NetID: "net", + Network: netip.MustParsePrefix(prefix), + } +} + +// TestHasRouteDiff_IgnoresHACandidateCount is the reason the comparison +// deduplicates. Every notification renews the TUN, and a renewed TUN +// invalidates the sockets the embedded servers are listening on, so a peer +// joining or leaving an HA group must not count as a route change when the +// prefixes the TUN carries are identical. +func TestHasRouteDiff_IgnoresHACandidateCount(t *testing.T) { + onePeer := []*route.Route{routeFor("a", "10.0.0.0/24")} + twoPeers := []*route.Route{ + routeFor("a", "10.0.0.0/24"), + routeFor("b", "10.0.0.0/24"), + } + + assert.False(t, hasRouteDiff(onePeer, twoPeers), + "a second peer serving the same prefix is not a route change") + assert.False(t, hasRouteDiff(twoPeers, onePeer), + "losing one of two peers serving the same prefix is not a route change") +} + +func TestHasRouteDiff_ReportsRealChanges(t *testing.T) { + tests := []struct { + name string + a []*route.Route + b []*route.Route + want bool + }{ + { + name: "added prefix", + a: []*route.Route{routeFor("a", "10.0.0.0/24")}, + b: []*route.Route{routeFor("a", "10.0.0.0/24"), routeFor("b", "10.0.1.0/24")}, + want: true, + }, + { + name: "removed prefix", + a: []*route.Route{routeFor("a", "10.0.0.0/24"), routeFor("b", "10.0.1.0/24")}, + b: []*route.Route{routeFor("a", "10.0.0.0/24")}, + want: true, + }, + { + name: "replaced prefix", + a: []*route.Route{routeFor("a", "10.0.0.0/24")}, + b: []*route.Route{routeFor("a", "10.0.1.0/24")}, + want: true, + }, + { + name: "same prefix, different order", + a: []*route.Route{routeFor("a", "10.0.1.0/24"), routeFor("b", "10.0.0.0/24")}, + b: []*route.Route{routeFor("b", "10.0.0.0/24"), routeFor("a", "10.0.1.0/24")}, + want: false, + }, + { + name: "all routes gone", + a: []*route.Route{routeFor("a", "10.0.0.0/24")}, + b: nil, + want: true, + }, + { + name: "both empty", + a: nil, + b: nil, + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, hasRouteDiff(tc.a, tc.b), + "route diff for %s", tc.name) + }) + } +} diff --git a/client/internal/routemanager/reconcile_test.go b/client/internal/routemanager/reconcile_test.go index c6806a6cd..bc9693229 100644 --- a/client/internal/routemanager/reconcile_test.go +++ b/client/internal/routemanager/reconcile_test.go @@ -17,11 +17,12 @@ import ( "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" ) -// reconcileWGMock is a minimal iface.WGIface that only records AddAllowedIP calls; every other -// method is an inert stub because ReconcilePeerAllowedIPs exercises none of them. +// reconcileWGMock is a minimal iface.WGIface that records AddAllowedIP calls and reports the +// configured address; every other method is an inert stub because the tests exercise none of them. type reconcileWGMock struct { mu sync.Mutex adds map[string][]netip.Prefix + addr wgaddr.Address } func (m *reconcileWGMock) AddAllowedIP(peerKey string, allowedIP netip.Prefix) error { @@ -42,7 +43,7 @@ func (m *reconcileWGMock) added(peerKey string) []netip.Prefix { func (m *reconcileWGMock) RemoveAllowedIP(string, netip.Prefix) error { return nil } func (m *reconcileWGMock) Name() string { return "utun-test" } -func (m *reconcileWGMock) Address() wgaddr.Address { return wgaddr.Address{} } +func (m *reconcileWGMock) Address() wgaddr.Address { return m.addr } func (m *reconcileWGMock) ToInterface() *net.Interface { return nil } func (m *reconcileWGMock) IsUserspaceBind() bool { return false } func (m *reconcileWGMock) GetFilter() device.PacketFilter { return nil } diff --git a/client/internal/routemanager/route_range_test.go b/client/internal/routemanager/route_range_test.go new file mode 100644 index 000000000..b51b5747a --- /dev/null +++ b/client/internal/routemanager/route_range_test.go @@ -0,0 +1,95 @@ +//go:build !windows + +package routemanager + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/routeselector" + "github.com/netbirdio/netbird/route" +) + +func TestCurrentRouteRange_OverlayNetworkWithClientRoutesDisabled(t *testing.T) { + m := &DefaultManager{ + wgInterface: &reconcileWGMock{addr: wgaddr.MustParseWGAddress("100.91.96.107/16")}, + disableClientRoutes: true, + } + + assert.Equal(t, []string{"100.91.0.0/16"}, m.CurrentRouteRange(), "overlay network must be routed even when client routes are disabled") +} + +func TestCurrentRouteRange_OverlayNetworksAndClientRoutes(t *testing.T) { + addr := wgaddr.MustParseWGAddress("100.91.96.107/16") + addr.IPv6 = netip.MustParseAddr("fd00:1234::1") + addr.IPv6Net = netip.MustParsePrefix("fd00:1234::/64") + + static := &route.Route{ID: "static", NetID: "lan", Network: netip.MustParsePrefix("192.168.50.0/24"), NetworkType: route.IPv4Network} + dynamic := &route.Route{ID: "dynamic", NetID: "dyn", NetworkType: route.DomainNetwork} + + m := &DefaultManager{ + wgInterface: &reconcileWGMock{addr: addr}, + routeSelector: routeselector.NewRouteSelector(), + clientRoutes: route.HAMap{ + static.GetHAUniqueID(): {static}, + dynamic.GetHAUniqueID(): {dynamic}, + }, + } + + assert.Equal(t, []string{"100.91.0.0/16", "192.168.50.0/24", "fd00:1234::/64"}, m.CurrentRouteRange(), "overlay networks and static client routes must be listed, dynamic routes skipped") +} + +func TestCurrentRouteRange_NoInterfaceAddress(t *testing.T) { + m := &DefaultManager{ + wgInterface: &reconcileWGMock{}, + disableClientRoutes: true, + } + + assert.Empty(t, m.CurrentRouteRange(), "an unset interface address must not produce a route entry") +} + +func TestCurrentRouteRange_IPv6WithoutIPv4Network(t *testing.T) { + addr := wgaddr.Address{ + IPv6: netip.MustParseAddr("fd00:1234::1"), + IPv6Net: netip.MustParsePrefix("fd00:1234::/64"), + } + m := &DefaultManager{ + wgInterface: &reconcileWGMock{addr: addr}, + disableClientRoutes: true, + } + + assert.Equal(t, []string{"fd00:1234::/64"}, m.CurrentRouteRange(), "a v6 overlay network must not depend on a v4 network being set") +} + +func TestCurrentRouteRange_IPv6AddressWithoutNetwork(t *testing.T) { + addr := wgaddr.MustParseWGAddress("100.91.96.107/16") + addr.IPv6 = netip.MustParseAddr("fd00:1234::1") + + m := &DefaultManager{ + wgInterface: &reconcileWGMock{addr: addr}, + disableClientRoutes: true, + } + + assert.Equal(t, []string{"100.91.0.0/16"}, m.CurrentRouteRange(), "a v6 address without a network must not produce a route entry") +} + +func TestCurrentRouteRange_DeduplicatesPrefixes(t *testing.T) { + // Two HA peers serve the same prefix, and a client route announces the overlay network itself. + haPeerA := &route.Route{ID: "ha-a", NetID: "lan", Peer: "peer-a", Network: netip.MustParsePrefix("192.168.50.0/24"), NetworkType: route.IPv4Network} + haPeerB := &route.Route{ID: "ha-b", NetID: "lan", Peer: "peer-b", Network: netip.MustParsePrefix("192.168.50.0/24"), NetworkType: route.IPv4Network} + overlay := &route.Route{ID: "overlay", NetID: "overlay", Network: netip.MustParsePrefix("100.91.0.0/16"), NetworkType: route.IPv4Network} + + m := &DefaultManager{ + wgInterface: &reconcileWGMock{addr: wgaddr.MustParseWGAddress("100.91.96.107/16")}, + routeSelector: routeselector.NewRouteSelector(), + clientRoutes: route.HAMap{ + haPeerA.GetHAUniqueID(): {haPeerA, haPeerB}, + overlay.GetHAUniqueID(): {overlay}, + }, + } + + assert.Equal(t, []string{"100.91.0.0/16", "192.168.50.0/24"}, m.CurrentRouteRange(), "every prefix must be listed once regardless of how many routes carry it") +} diff --git a/client/internal/routemanager/server/server.go b/client/internal/routemanager/server/server.go index f569c0cac..38d7f0db4 100644 --- a/client/internal/routemanager/server/server.go +++ b/client/internal/routemanager/server/server.go @@ -135,6 +135,14 @@ func (r *Router) CleanUp() { } } + // Give back the routing reference taken in UpdateRoutes, after the routes + // are gone as above. Without this the sysctls enabling it changed (IPv6 + // forwarding and the accept_ra values that keep RA handling alive next to + // it) stay applied once the client stops. + if err := r.firewall.DisableRouting(); err != nil { + log.Errorf("Failed to disable routing: %v", err) + } + r.statusRecorder.CleanLocalPeerStateRoutes() } diff --git a/client/internal/routemanager/server/server_test.go b/client/internal/routemanager/server/server_test.go new file mode 100644 index 000000000..1b42115e2 --- /dev/null +++ b/client/internal/routemanager/server/server_test.go @@ -0,0 +1,66 @@ +package server + +import ( + "context" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/route" +) + +// routingFirewall records the routing lifecycle calls the router makes. The +// embedded interface covers the methods this test never reaches. +type routingFirewall struct { + firewall.Manager + + removed []firewall.RouterPair + enabled int + disabled int +} + +func (f *routingFirewall) RemoveNatRule(pair firewall.RouterPair) error { + f.removed = append(f.removed, pair) + return nil +} + +func (f *routingFirewall) EnableRouting() error { + f.enabled++ + return nil +} + +func (f *routingFirewall) DisableRouting() error { + f.disabled++ + return nil +} + +// TestRouterCleanUpReleasesRouting covers the shutdown path: the router holds a +// routing reference for as long as it serves routes, and CleanUp has to give it +// back. Without that the sysctls the reference enabled (IPv6 forwarding and the +// accept_ra values that keep RA handling working alongside it) stay applied +// after the client stops, leaving the host configured as a router. +func TestRouterCleanUpReleasesRouting(t *testing.T) { + fw := &routingFirewall{} + r := &Router{ + ctx: context.Background(), + firewall: fw, + statusRecorder: peer.NewRecorder("https://mgm"), + routes: map[route.ID]*route.Route{ + "route-1": { + ID: "route-1", + Network: netip.MustParsePrefix("192.168.55.0/24"), + NetworkType: route.IPv4Network, + Masquerade: true, + }, + }, + } + + r.CleanUp() + + require.Len(t, fw.removed, 1, "the route's NAT rule must be removed") + assert.Equal(t, 1, fw.disabled, "CleanUp must release the routing reference") +} diff --git a/client/ios/NetBirdSDK/preferences.go b/client/ios/NetBirdSDK/preferences.go index 8b40aa2bb..d57f01c55 100644 --- a/client/ios/NetBirdSDK/preferences.go +++ b/client/ios/NetBirdSDK/preferences.go @@ -128,6 +128,27 @@ func (p *Preferences) SetDisableIPv6(disable bool) { p.configInput.DisableIPv6 = &disable } +// GetRemoteJobsAllowed reads the remote jobs opt-in from config file +func (p *Preferences) GetRemoteJobsAllowed() (bool, error) { + if p.configInput.RemoteJobsAllowed != nil { + return *p.configInput.RemoteJobsAllowed, nil + } + + cfg, err := profilemanager.ReadOrGenerateConfig(p.configInput.ConfigPath) + if err != nil { + return false, err + } + if cfg.RemoteJobsAllowed == nil { + return false, nil + } + return *cfg.RemoteJobsAllowed, err +} + +// SetRemoteJobsAllowed stores the given value and waits for commit +func (p *Preferences) SetRemoteJobsAllowed(allowed bool) { + p.configInput.RemoteJobsAllowed = &allowed +} + // Commit write out the changes into config file func (p *Preferences) Commit() error { // Use DirectUpdateOrCreateConfig to avoid atomic file operations (temp file + rename) diff --git a/client/proto/generate.sh b/client/proto/generate.sh index cea8ae912..d73367d12 100755 --- a/client/proto/generate.sh +++ b/client/proto/generate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if ! which realpath >/dev/null 2>&1; then diff --git a/client/server/jwt_cache.go b/client/server/jwt_cache.go index 21e170517..73cec046d 100644 --- a/client/server/jwt_cache.go +++ b/client/server/jwt_cache.go @@ -6,11 +6,21 @@ import ( "github.com/awnumar/memguard" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/ipcauth" ) type jwtCache struct { - mu sync.RWMutex - enclave *memguard.Enclave + mu sync.RWMutex + enclave *memguard.Enclave + owner *ipcauth.Identity + + // generation counts the invalidations. A caller that starts an + // authentication takes the generation first and hands it back to store, so + // a token obtained under a session that ended while the IdP was being + // polled cannot land in the cache the new session is using. + generation uint64 + expiresAt time.Time timer *time.Timer maxTokenSize int @@ -22,10 +32,23 @@ func newJWTCache() *jwtCache { } } -func (c *jwtCache) store(token string, maxAge time.Duration) { +func (c *jwtCache) currentGeneration() uint64 { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.generation +} + +// store keeps the token only while generation is still the current one, and +// reports whether it did. See the generation field. +func (c *jwtCache) store(token string, owner ipcauth.Identity, maxAge time.Duration, generation uint64) bool { c.mu.Lock() defer c.mu.Unlock() + if c.generation != generation { + return false + } + c.cleanup() if c.timer != nil { @@ -35,6 +58,7 @@ func (c *jwtCache) store(token string, maxAge time.Duration) { tokenBytes := []byte(token) c.enclave = memguard.NewEnclave(tokenBytes) + c.owner = &owner c.expiresAt = time.Now().Add(maxAge) var timer *time.Timer @@ -49,9 +73,12 @@ func (c *jwtCache) store(token string, maxAge time.Duration) { log.Debugf("JWT token cache expired after %v, securely wiped from memory", maxAge) }) c.timer = timer + + return true } -func (c *jwtCache) get() (string, bool) { +// get returns the cached token to the identity that stored it. +func (c *jwtCache) get(caller ipcauth.Identity) (string, bool) { c.mu.RLock() defer c.mu.RUnlock() @@ -59,6 +86,11 @@ func (c *jwtCache) get() (string, bool) { return "", false } + if c.owner == nil || !c.owner.SameUser(caller) { + log.Warnf("refusing the cached SSH JWT: caller %s is not the identity that obtained it", caller) + return "", false + } + buffer, err := c.enclave.Open() if err != nil { log.Debugf("Failed to open JWT token enclave: %v", err) @@ -70,10 +102,23 @@ func (c *jwtCache) get() (string, bool) { return token, true } +func (c *jwtCache) clear() { + c.mu.Lock() + defer c.mu.Unlock() + + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + c.cleanup() + c.generation++ +} + // cleanup destroys the secure enclave, must be called with lock held func (c *jwtCache) cleanup() { if c.enclave != nil { c.enclave = nil } + c.owner = nil c.expiresAt = time.Time{} } diff --git a/client/server/jwt_cache_test.go b/client/server/jwt_cache_test.go new file mode 100644 index 000000000..11d208ade --- /dev/null +++ b/client/server/jwt_cache_test.go @@ -0,0 +1,176 @@ +package server + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal/ipcauth" +) + +const testTTL = time.Minute + +func unixCaller(uid uint32) ipcauth.Identity { + return ipcauth.Identity{UID: uid, GID: uid} +} + +func windowsCaller(sid string) ipcauth.Identity { + return ipcauth.Identity{SID: sid} +} + +func TestJWTCache_ServesTheOwner(t *testing.T) { + c := newJWTCache() + owner := unixCaller(1000) + c.store("token-for-1000", owner, testTTL, c.currentGeneration()) + + got, found := c.get(owner) + + require.True(t, found, "the identity that stored the token must get it back") + assert.Equal(t, "token-for-1000", got) +} + +// The disclosure this cache guards against: one local account collecting the +// SSH JWT another account's authentication put in the daemon-wide cache. +func TestJWTCache_RefusesAnotherLocalUser(t *testing.T) { + tests := []struct { + name string + owner ipcauth.Identity + caller ipcauth.Identity + }{ + {"different uid", unixCaller(1000), unixCaller(65534)}, + {"root is not the owner either", unixCaller(1000), unixCaller(0)}, + {"different sid", windowsCaller("S-1-5-21-1-2-3-1001"), windowsCaller("S-1-5-21-1-2-3-1002")}, + {"windows caller against a unix owner", unixCaller(0), windowsCaller("S-1-5-18")}, + {"unix caller against a windows owner", windowsCaller("S-1-5-18"), unixCaller(0)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newJWTCache() + c.store("victim-token", tt.owner, testTTL, c.currentGeneration()) + + got, found := c.get(tt.caller) + + assert.False(t, found, "a caller that is not the owner must get a miss") + assert.Empty(t, got) + }) + } +} + +func TestJWTCache_EmptyCacheMatchesNobody(t *testing.T) { + c := newJWTCache() + + got, found := c.get(unixCaller(0)) + + assert.False(t, found) + assert.Empty(t, got) +} + +// An entry with no recorded owner must match nobody, root included: an +// unidentified caller arrives as the zero Identity, which carries uid 0. This +// pins the nil-owner guard rather than the comparison, so it sets up an entry +// that exists and then drops its owner. +func TestJWTCache_UnownedEntryMatchesNobody(t *testing.T) { + c := newJWTCache() + c.store("token", unixCaller(1000), testTTL, c.currentGeneration()) + c.owner = nil + + got, found := c.get(unixCaller(0)) + + assert.False(t, found) + assert.Empty(t, got) +} + +// The same user calling once elevated and once not is still the same user, so +// hiding their own token from them would be wrong. +func TestJWTCache_ElevationDoesNotChangeTheOwner(t *testing.T) { + c := newJWTCache() + sid := "S-1-5-21-1-2-3-1001" + owner := windowsCaller(sid) + owner.Elevated = true + c.store("token", owner, testTTL, c.currentGeneration()) + + got, found := c.get(windowsCaller(sid)) + + require.True(t, found) + assert.Equal(t, "token", got) +} + +func TestJWTCache_Expiry(t *testing.T) { + c := newJWTCache() + owner := unixCaller(1000) + c.store("token", owner, testTTL, c.currentGeneration()) + c.expiresAt = time.Now().Add(-time.Second) + + _, found := c.get(owner) + + assert.False(t, found) +} + +// Logout and SwitchProfile call clear — Down deliberately does not: the NetBird +// session the token speaks for is over, so not even its owner may have it back. +func TestJWTCache_ClearDropsTheEntry(t *testing.T) { + c := newJWTCache() + owner := unixCaller(1000) + c.store("token", owner, testTTL, c.currentGeneration()) + + c.clear() + + _, found := c.get(owner) + assert.False(t, found) + assert.Nil(t, c.owner, "clear must forget the owner too") + assert.Nil(t, c.timer, "clear must stop the expiry timer") +} + +// WaitJWTToken polls the IdP unlocked, so a logout or a profile switch can +// clear the cache while a flow is still in the air. The token that flow returns +// belongs to the session that ended, so it must not land in the cache the new +// session is using. +func TestJWTCache_StoreFromAnEndedSessionIsDropped(t *testing.T) { + c := newJWTCache() + owner := unixCaller(1000) + + // The generation a caller takes when its authentication starts. + generation := c.currentGeneration() + + c.clear() // logout or profile switch, while the IdP is still being polled + + stored := c.store("stale-token", owner, testTTL, generation) + + assert.False(t, stored, "a token from an ended session must not be cached") + _, found := c.get(owner) + assert.False(t, found, "the cache must stay empty after the session ended") +} + +// The same caller must still be able to store once it re-reads the generation, so +// the guard does not wedge the cache after any invalidation. +func TestJWTCache_StoreWorksAgainAfterClear(t *testing.T) { + c := newJWTCache() + owner := unixCaller(1000) + + c.clear() + + require.True(t, c.store("token", owner, testTTL, c.currentGeneration())) + + got, found := c.get(owner) + require.True(t, found) + assert.Equal(t, "token", got) +} + +func TestJWTCache_StoreReplacesThePreviousOwner(t *testing.T) { + c := newJWTCache() + first := unixCaller(1000) + second := unixCaller(1001) + + c.store("first-token", first, testTTL, c.currentGeneration()) + c.store("second-token", second, testTTL, c.currentGeneration()) + + _, found := c.get(first) + assert.False(t, found, "the previous owner must not reach the new token") + + got, found := c.get(second) + require.True(t, found) + assert.Equal(t, "second-token", got) +} diff --git a/client/server/server.go b/client/server/server.go index 61e234fb9..12cd3f865 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -23,6 +23,7 @@ import ( "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/expose" + "github.com/netbirdio/netbird/client/internal/ipcauth" "github.com/prometheus/client_golang/prometheus" "github.com/netbirdio/netbird/client/internal/localmetrics" @@ -155,9 +156,17 @@ type Server struct { } type oauthAuthFlow struct { - expiresAt time.Time - flow auth.OAuthFlow - info auth.AuthFlowInfo + expiresAt time.Time + flow auth.OAuthFlow + info auth.AuthFlowInfo + + // cacheGeneration is the SSH JWT cache's generation as of the start of the + // request that created this flow. The flow outlives a profile switch, so + // reading the generation any later — when the IdP has answered, or when the + // token finally arrives — would read the new session's one and let the old + // session's token into the new session's cache. + cacheGeneration uint64 + waitCancel context.CancelFunc } @@ -1271,6 +1280,8 @@ func (s *Server) SwitchProfile(callerCtx context.Context, msg *proto.SwitchProfi s.config = config s.localMetrics.Reconcile(config.LocalMetricsEnabled, config.LocalMetricsAddress) + s.jwtCache.clear() + if msg != nil && msg.ProfileName != nil { s.publishProfileListChanged(*msg.ProfileName) } @@ -1441,6 +1452,7 @@ func (s *Server) cleanupAfterProfileLogout(id profilemanager.ID, username string if err := s.cleanupConnection(); err != nil && !errors.Is(err, ErrServiceNotUp) { log.Errorf("failed to cleanup connection: %v", err) } + s.jwtCache.clear() state := internal.CtxGetState(s.rootCtx) state.Set(internal.StatusNeedsLogin) } @@ -1469,6 +1481,7 @@ func (s *Server) handleActiveProfileLogout(ctx context.Context) (*proto.LogoutRe log.Errorf("failed to cleanup connection: %v", err) return nil, err } + s.jwtCache.clear() state := internal.CtxGetState(s.rootCtx) state.Set(internal.StatusNeedsLogin) @@ -1830,6 +1843,20 @@ func (s *Server) getJWTCacheTTL() time.Duration { return ttl } +// cachedJWT returns the cached SSH JWT to the identity that obtained it, and a +// miss on a control channel that carries no caller identity. +func (s *Server) cachedJWT(ctx context.Context) (string, bool) { + caller, ok := ipcauth.CallerIdentity(ctx) + if !ok { + // Expected and handled on a control channel with no peer identity: the + // caller re-authenticates. daemonServerOptions warns about it once at + // startup, so this stays out of the per-request log. + log.Debug("not serving the cached SSH JWT: the caller's identity cannot be verified on this control channel") + return "", false + } + return s.jwtCache.get(caller) +} + // RequestJWTAuth initiates JWT authentication flow for SSH func (s *Server) RequestJWTAuth( ctx context.Context, @@ -1839,8 +1866,14 @@ func (s *Server) RequestJWTAuth( return nil, ctx.Err() } + // The generation is read here, with the config and under the same lock, not + // where the flow is stored below: RequestAuthInfo talks to the IdP in + // between, and a switch or a logout during that call would otherwise be + // read as the generation this flow belongs to. SwitchProfile holds + // s.mutex across its own clear(), so the pair cannot be torn. s.mutex.Lock() config := s.config + cacheGeneration := s.jwtCache.currentGeneration() s.mutex.Unlock() if config == nil { @@ -1849,7 +1882,7 @@ func (s *Server) RequestJWTAuth( jwtCacheTTL := s.getJWTCacheTTL() if jwtCacheTTL > 0 { - if cachedToken, found := s.jwtCache.get(); found { + if cachedToken, found := s.cachedJWT(ctx); found { log.Debugf("JWT token found in cache, returning cached token for SSH authentication") return &proto.RequestJWTAuthResponse{ @@ -1883,6 +1916,7 @@ func (s *Server) RequestJWTAuth( s.oauthAuthFlow.flow = oAuthFlow s.oauthAuthFlow.info = authInfo s.oauthAuthFlow.expiresAt = time.Now().Add(time.Duration(authInfo.ExpiresIn) * time.Second) + s.oauthAuthFlow.cacheGeneration = cacheGeneration s.mutex.Unlock() return &proto.RequestJWTAuthResponse{ @@ -1907,6 +1941,10 @@ func (s *Server) WaitJWTToken( s.mutex.Lock() oAuthFlow := s.oauthAuthFlow.flow authInfo := s.oauthAuthFlow.info + // Recorded when the flow was created, not read here: the flow survives a + // profile switch, and everything from RequestJWTAuth to the IdP answering + // has to count as the same session for the cache. + generation := s.oauthAuthFlow.cacheGeneration s.mutex.Unlock() if oAuthFlow == nil || authInfo.DeviceCode != req.DeviceCode { @@ -1921,11 +1959,17 @@ func (s *Server) WaitJWTToken( token := tokenInfo.GetTokenToUse() jwtCacheTTL := s.getJWTCacheTTL() - if jwtCacheTTL > 0 { - s.jwtCache.store(token, jwtCacheTTL) - log.Debugf("JWT token cached for SSH authentication, TTL: %v", jwtCacheTTL) - } else { + switch caller, ok := ipcauth.CallerIdentity(ctx); { + case jwtCacheTTL <= 0: log.Debug("JWT caching disabled, not storing token") + case !ok: + log.Debug("not caching the SSH JWT: the caller's identity cannot be verified on this control channel") + default: + if s.jwtCache.store(token, caller, jwtCacheTTL, generation) { + log.Debugf("JWT token cached for SSH authentication, TTL: %v", jwtCacheTTL) + } else { + log.Debug("not caching the SSH JWT: the session it was obtained under ended while the IdP was polled") + } } s.mutex.Lock() diff --git a/client/server/server_connect_test.go b/client/server/server_connect_test.go index 0c6e03a4a..dc191a44f 100644 --- a/client/server/server_connect_test.go +++ b/client/server/server_connect_test.go @@ -18,6 +18,10 @@ func newTestServer() *Server { return &Server{ rootCtx: context.Background(), statusRecorder: peer.NewRecorder(""), + // New always populates the SSH JWT cache and the logout and + // profile-switch paths call into it unconditionally, so a Server + // assembled field by field has to populate it too. + jwtCache: newJWTCache(), } } diff --git a/client/server/server_jwt_test.go b/client/server/server_jwt_test.go new file mode 100644 index 000000000..3fec5598a --- /dev/null +++ b/client/server/server_jwt_test.go @@ -0,0 +1,188 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal/auth" + "github.com/netbirdio/netbird/client/internal/localmetrics" + "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/proto" +) + +// These cover the RPC side of the cache: the cache itself is exercised in +// jwt_cache_test.go, but a correct cache buys nothing if the handlers around it +// consult the wrong identity or forget to clear it. + +func TestCachedJWT_ServesTheOwner(t *testing.T) { + s := newTestServer() + owner := unprivilegedIdentity() + s.jwtCache.store("token", owner, testTTL, s.jwtCache.currentGeneration()) + + got, found := s.cachedJWT(ctxWithIdentity(owner)) + + require.True(t, found, "the identity that obtained the token must get it back") + assert.Equal(t, "token", got) +} + +func TestCachedJWT_RefusesAnotherCaller(t *testing.T) { + s := newTestServer() + s.jwtCache.store("token", unprivilegedIdentity(), testTTL, s.jwtCache.currentGeneration()) + + got, found := s.cachedJWT(ctxWithIdentity(privilegedIdentity())) + + assert.False(t, found, "a caller that did not obtain the token must get a miss") + assert.Empty(t, got) +} + +// A control channel that carries no caller identity — a TCP daemon socket, or a +// platform with no peer-credential primitive — cannot tell one local user from +// another, so cachedJWT must fail closed there. +func TestCachedJWT_WithoutCallerIdentity(t *testing.T) { + s := newTestServer() + s.jwtCache.store("token", unprivilegedIdentity(), testTTL, s.jwtCache.currentGeneration()) + + got, found := s.cachedJWT(context.Background()) + + assert.False(t, found) + assert.Empty(t, got) +} + +// profileFixture points the profile globals at a temp dir holding a single +// default profile, which is the one ActiveProfileState.FilePath resolves +// without consulting the current OS user. +func profileFixture(t *testing.T) string { + t.Helper() + + dir := t.TempDir() + defaultConfig := filepath.Join(dir, "default.json") + require.NoError(t, os.WriteFile(defaultConfig, []byte("{}"), 0o600)) + + origDir := profilemanager.DefaultConfigPathDir + origDefault := profilemanager.DefaultConfigPath + origState := profilemanager.ActiveProfileStatePath + origOverride := profilemanager.ConfigDirOverride + + profilemanager.DefaultConfigPathDir = dir + profilemanager.DefaultConfigPath = defaultConfig + profilemanager.ActiveProfileStatePath = filepath.Join(dir, "active_profile.json") + profilemanager.ConfigDirOverride = dir + + t.Cleanup(func() { + profilemanager.DefaultConfigPathDir = origDir + profilemanager.DefaultConfigPath = origDefault + profilemanager.ActiveProfileStatePath = origState + profilemanager.ConfigDirOverride = origOverride + }) + + return defaultConfig +} + +// A profile carries its own NetBird account, so a token obtained under the +// previous one must not survive the switch even for the local user who +// obtained it. +func TestSwitchProfile_ClearsJWTCache(t *testing.T) { + defaultConfig := profileFixture(t) + + // localmetrics.NewManager runs until its context is done, so the manager + // must not outlive the test. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + s := newTestServer() + s.profileManager = profilemanager.NewServiceManager(defaultConfig) + s.localMetrics = localmetrics.NewManager(ctx, s.statusRecorder, nil) + + // A second profile to move to, so the request goes through + // switchProfileIfNeeded rather than the no-op path a nil request takes. + const target = "second" + username := "tester" + _, err := profilemanager.UpdateOrCreateConfig(profilemanager.ConfigInput{ + ConfigPath: filepath.Join(profilemanager.DefaultConfigPathDir, target+".json"), + ManagementURL: "https://api.netbird.io:443", + }) + require.NoError(t, err) + + owner := unprivilegedIdentity() + s.jwtCache.store("token", owner, testTTL, s.jwtCache.currentGeneration()) + + name := target + _, err = s.SwitchProfile(ctx, &proto.SwitchProfileRequest{ProfileName: &name, Username: &username}) + require.NoError(t, err) + + active, err := s.profileManager.GetActiveProfileState() + require.NoError(t, err) + require.Equal(t, profilemanager.ID(target), active.ID, "the profile must actually have changed") + + _, found := s.jwtCache.get(owner) + assert.False(t, found, "switching profile must drop the cached SSH JWT") +} + +// Down ends the connection, not the session: the peer stays enrolled and the +// token still belongs to the same NetBird identity, so `down` followed by `up` +// must not cost the owner a fresh device-code flow. +// +// The logout handlers do call cleanupConnection, and SwitchProfile does not; +// what they have in common is that each clears the cache itself, right after, +// so tearing the connection down is no longer what decides the token's fate. +func TestCleanupConnection_KeepsJWTCache(t *testing.T) { + s := newTestServer() + _, cancel := context.WithCancel(context.Background()) + s.actCancel = cancel + + owner := unprivilegedIdentity() + s.jwtCache.store("token", owner, testTTL, s.jwtCache.currentGeneration()) + + require.NoError(t, s.cleanupConnection()) + + got, found := s.jwtCache.get(owner) + require.True(t, found, "going down must not drop the cached SSH JWT") + assert.Equal(t, "token", got) +} + +// fakeOAuthFlow stands in for the IdP round trip so a test can drive +// WaitJWTToken without a real device-code flow. +type fakeOAuthFlow struct { + token string +} + +func (f *fakeOAuthFlow) RequestAuthInfo(context.Context) (auth.AuthFlowInfo, error) { + return auth.AuthFlowInfo{DeviceCode: "device-code"}, nil +} + +func (f *fakeOAuthFlow) WaitToken(context.Context, auth.AuthFlowInfo) (auth.TokenInfo, error) { + return auth.TokenInfo{AccessToken: f.token}, nil +} + +func (f *fakeOAuthFlow) GetClientID(context.Context) string { return "client-id" } + +// The flow outlives a profile switch, because SwitchProfile does not reset +// s.oauthAuthFlow. A switch between RequestJWTAuth and the IdP answering must +// still keep the token out of the cache the new profile uses, and the +// generation the flow carries is what decides it: reading the cache's own +// generation at store time would already be the new one. +func TestWaitJWTToken_DropsTokenFromASessionThatEndedBeforeTheWait(t *testing.T) { + s := newTestServer() + owner := unprivilegedIdentity() + ttl := int(testTTL.Seconds()) + s.config = &profilemanager.Config{SSHJWTCacheTTL: &ttl} + + // RequestJWTAuth ran under the previous session and recorded its generation. + s.oauthAuthFlow.flow = &fakeOAuthFlow{token: "token-from-the-old-session"} + s.oauthAuthFlow.info = auth.AuthFlowInfo{DeviceCode: "device-code"} + s.oauthAuthFlow.cacheGeneration = s.jwtCache.currentGeneration() + + // A profile switch or a logout lands before the caller reaches WaitJWTToken. + s.jwtCache.clear() + + _, err := s.WaitJWTToken(ctxWithIdentity(owner), &proto.WaitJWTTokenRequest{DeviceCode: "device-code"}) + require.NoError(t, err) + + _, found := s.jwtCache.get(owner) + assert.False(t, found, "a token whose flow started under the previous session must not be cached") +} diff --git a/client/ssh/auth/auth.go b/client/ssh/auth/auth.go index 079282fdc..92f517fac 100644 --- a/client/ssh/auth/auth.go +++ b/client/ssh/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "errors" "fmt" + "slices" "sync" log "github.com/sirupsen/logrus" @@ -155,6 +156,24 @@ func (a *Authorizer) GetUserIDClaim() string { return a.userIDClaim } +// Config returns the authorization currently in force. The user list and the +// machine-user map are copies; the originals stay in use here. +func (a *Authorizer) Config() *Config { + a.mu.RLock() + defer a.mu.RUnlock() + + machineUsers := make(map[string][]uint32, len(a.machineUsers)) + for osUser, indexes := range a.machineUsers { + machineUsers[osUser] = slices.Clone(indexes) + } + + return &Config{ + UserIDClaim: a.userIDClaim, + AuthorizedUsers: slices.Clone(a.authorizedUsers), + MachineUsers: machineUsers, + } +} + // findUserIndex finds the index of a hashed user ID in the authorized users list // Returns the index and true if found, 0 and false if not found func (a *Authorizer) findUserIndex(hashedUserID sshuserhash.UserIDHash) (int, bool) { diff --git a/client/ssh/handshake.go b/client/ssh/handshake.go index e78a806be..a718748df 100644 --- a/client/ssh/handshake.go +++ b/client/ssh/handshake.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net" - "time" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" @@ -13,26 +12,23 @@ import ( // Handshake runs the SSH client handshake on an already dialed conn and // returns the resulting client. Dialing bounds only the TCP establishment; -// without a deadline on the socket a peer that accepts and then goes silent -// blocks the handshake forever, so the context deadline is applied to conn -// for the duration of the handshake. conn is closed on any error. +// a peer that accepts and then goes silent would block the handshake forever, +// so conn is closed as soon as ctx is done, which unblocks the handshake and +// surfaces the context error. conn is closed on any error. func Handshake(ctx context.Context, conn net.Conn, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { - if deadline, ok := ctx.Deadline(); ok { - if err := conn.SetDeadline(deadline); err != nil { - closeHandshake(conn, "conn after deadline error") - return nil, fmt.Errorf("set handshake deadline: %w", err) - } - } + stop := context.AfterFunc(ctx, func() { closeHandshake(conn, "conn on context done") }) sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) if err != nil { - closeHandshake(conn, "conn after handshake error") - return nil, fmt.Errorf("ssh handshake: %w", err) + if stop() { + closeHandshake(conn, "conn after handshake error") + } + return nil, handshakeError(ctx, err) } - if err := conn.SetDeadline(time.Time{}); err != nil { - closeHandshake(sshConn, "ssh conn after deadline clear error") - return nil, fmt.Errorf("clear handshake deadline: %w", err) + if !stop() { + closeHandshake(sshConn, "ssh conn after context done") + return nil, fmt.Errorf("ssh handshake: %w", ctx.Err()) } return ssh.NewClient(sshConn, chans, reqs), nil @@ -43,3 +39,10 @@ func closeHandshake(c io.Closer, label string) { log.Debugf("ssh: close %s: %v", label, err) } } + +func handshakeError(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("ssh handshake: %w: %w", ctxErr, err) + } + return fmt.Errorf("ssh handshake: %w", err) +} diff --git a/client/ssh/handshake_test.go b/client/ssh/handshake_test.go new file mode 100644 index 000000000..77a6f916b --- /dev/null +++ b/client/ssh/handshake_test.go @@ -0,0 +1,90 @@ +package ssh + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" +) + +func TestHandshake_ContextDeadlineWrapped(t *testing.T) { + conn := dialSilentServer(t) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err := Handshake(ctx, conn, conn.RemoteAddr().String(), testClientConfig()) + require.Error(t, err) + require.True(t, errors.Is(err, context.DeadlineExceeded), "expected context.DeadlineExceeded, got: %v", err) +} + +func TestHandshake_ContextCancelUnblocks(t *testing.T) { + conn := dialSilentServer(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + time.AfterFunc(50*time.Millisecond, cancel) + + errCh := make(chan error, 1) + go func() { + _, err := Handshake(ctx, conn, conn.RemoteAddr().String(), testClientConfig()) + errCh <- err + }() + + select { + case err := <-errCh: + require.Error(t, err) + require.True(t, errors.Is(err, context.Canceled), "expected context.Canceled, got: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("handshake did not return after context cancellation") + } +} + +func TestHandshake_NonContextErrorNotWrapped(t *testing.T) { + conn := dialSilentServer(t) + require.NoError(t, conn.Close()) + + _, err := Handshake(context.Background(), conn, conn.RemoteAddr().String(), testClientConfig()) + require.Error(t, err) + require.False(t, errors.Is(err, context.Canceled)) + require.False(t, errors.Is(err, context.DeadlineExceeded)) +} + +func testClientConfig() *ssh.ClientConfig { + return &ssh.ClientConfig{ + User: "test", + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } +} + +// dialSilentServer returns a client conn to a server that accepts and never +// sends anything, so the SSH handshake blocks until the context is done. +func dialSilentServer(t *testing.T) net.Conn { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + + go func() { + c, err := listener.Accept() + if err != nil { + return + } + defer func() { _ = c.Close() }() + <-done + }() + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + return conn +} diff --git a/client/ssh/server/server.go b/client/ssh/server/server.go index 6735e0f3b..b32da796e 100644 --- a/client/ssh/server/server.go +++ b/client/ssh/server/server.go @@ -197,6 +197,12 @@ type Config struct { // HostKey is the SSH server host key in PEM format HostKeyPEM []byte + + // Auth is the fine-grained authorization to open with. Nil starts with an + // empty authorizer, which authorizes nobody until UpdateSSHAuth is called. + // Setting it here rather than afterwards means the server never accepts a + // login before it knows who is allowed. + Auth *sshauth.Config } // SessionInfo contains information about an active SSH session @@ -220,7 +226,11 @@ func New(config *Config) *Server { connections: make(map[connKey]*connState), jwtEnabled: config.JWT != nil, jwtConfig: config.JWT, - authorizer: sshauth.NewAuthorizer(), // Initialize with empty config + authorizer: sshauth.NewAuthorizer(), + } + + if config.Auth != nil { + s.authorizer.Update(config.Auth) } return s @@ -461,6 +471,27 @@ func (s *Server) UpdateSSHAuth(config *sshauth.Config) { s.authorizer.Update(config) } +// JWTConfig returns the JWT authentication this server was built with, or nil +// when JWT authentication is disabled. +func (s *Server) JWTConfig() *JWTConfig { + s.mu.RLock() + defer s.mu.RUnlock() + return s.jwtConfig +} + +// AuthConfig returns the fine-grained authorization currently in force, or nil +// when the server has no authorizer. +func (s *Server) AuthConfig() *sshauth.Config { + s.mu.RLock() + authorizer := s.authorizer + s.mu.RUnlock() + + if authorizer == nil { + return nil + } + return authorizer.Config() +} + // ensureJWTValidator initializes the JWT validator and extractor if not already initialized func (s *Server) ensureJWTValidator() error { s.mu.RLock() diff --git a/client/ssh/server/test.go b/client/ssh/server/test.go index e2be0551c..7ca28f034 100644 --- a/client/ssh/server/test.go +++ b/client/ssh/server/test.go @@ -1,9 +1,9 @@ // This file is intentionally named test.go (not test_test.go) so the exported // StartTestServer helper is visible to the ssh/proxy and ssh/client external // test packages, not just this package's own tests. The //go:build !js tag -// keeps its "testing" import — and the whole testing/flag/regexp transitive -// chain it drags in — out of the wasm client, which links ssh/server through -// the engine but never runs Go tests under GOOS=js. +// keeps its "testing" import, along with the whole testing/flag/regexp +// transitive chain it drags in, out of the wasm client, which links +// ssh/server through the engine but never runs Go tests under GOOS=js. //go:build !js package server diff --git a/client/system/info_source.go b/client/system/info_source.go new file mode 100644 index 000000000..050e094e4 --- /dev/null +++ b/client/system/info_source.go @@ -0,0 +1,38 @@ +package system + +import ( + "context" + "net/netip" + "slices" + "sync/atomic" + "time" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// InfoSource gathers the system info sent to management, keeping the posture +// check results from the last Refresh for the cheap Current snapshots. +type InfoSource struct { + files atomic.Pointer[[]File] +} + +// Refresh gathers the info with the posture checks evaluated, bounded by timeout. +func (s *InfoSource) Refresh(ctx context.Context, timeout time.Duration, checks []*proto.Checks, excludeIPs ...netip.Addr) (*Info, bool) { + info, ok := GetInfoWithChecksTimeout(ctx, timeout, checks, excludeIPs...) + if !ok { + return nil, false + } + files := slices.Clone(info.Files) + s.files.Store(&files) + return info, true +} + +// Current gathers the info without evaluating the checks, reusing the last Refresh results. +func (s *InfoSource) Current(ctx context.Context, excludeIPs ...netip.Addr) *Info { + info := GetInfo(ctx) + info.removeAddresses(excludeIPs...) + if files := s.files.Load(); files != nil { + info.Files = *files + } + return info +} diff --git a/client/system/info_source_test.go b/client/system/info_source_test.go new file mode 100644 index 000000000..1c86806af --- /dev/null +++ b/client/system/info_source_test.go @@ -0,0 +1,59 @@ +package system + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +func TestInfoSource_CurrentBeforeRefresh(t *testing.T) { + var src InfoSource + + info := src.Current(context.Background()) + + assert.Empty(t, info.Files) +} + +func TestInfoSource_CurrentReusesRefreshedFiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "agent") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + checks := []*proto.Checks{{Files: []string{path}}} + + var src InfoSource + refreshed, ok := src.Refresh(context.Background(), 15*time.Second, checks) + require.True(t, ok) + require.Equal(t, []File{{Path: path, Exist: true}}, refreshed.Files) + + info := src.Current(context.Background()) + + assert.Equal(t, refreshed.Files, info.Files) +} + +func TestInfoSource_CurrentExcludesAddresses(t *testing.T) { + addrs := GetInfo(context.Background()).NetworkAddresses + if len(addrs) == 0 { + t.Skip("no network addresses on this host") + } + excluded := addrs[0].NetIP.Addr() + matching := 0 + for _, addr := range addrs { + if addr.NetIP.Addr() == excluded { + matching++ + } + } + + var src InfoSource + info := src.Current(context.Background(), excluded) + + assert.Len(t, info.NetworkAddresses, len(addrs)-matching) + for _, addr := range info.NetworkAddresses { + assert.NotEqual(t, excluded, addr.NetIP.Addr()) + } +} diff --git a/combined/cmd/root.go b/combined/cmd/root.go index 7eac84ce5..3e583ef20 100644 --- a/combined/cmd/root.go +++ b/combined/cmd/root.go @@ -365,7 +365,6 @@ func setupServerHooks(servers *serverInstances, cfg *CombinedConfig) { }) } } - } func startServers(wg *sync.WaitGroup, srv *relayServer.Server, httpHealthcheck *healthcheck.Server, stunServer *stun.Server, metricsServer *sharedMetrics.Metrics) { @@ -539,7 +538,7 @@ func createManagementServer(cfg *CombinedConfig, mgmtConfig *nbconfig.Config) (m &mgmtServer.Config{ NbConfig: mgmtConfig, DNSDomain: "", - MgmtSingleAccModeDomain: "", + MgmtSingleAccModeDomain: mgmtServer.DefaultSelfHostedDomain, AutoResolveDomains: true, MgmtPort: mgmtPort, MgmtMetricsPort: cfg.Server.MetricsPort, @@ -554,7 +553,7 @@ func createManagementServer(cfg *CombinedConfig, mgmtConfig *nbconfig.Config) (m } // createCombinedHandler creates an HTTP handler that multiplexes Management, Signal (via wsproxy), and Relay WebSocket traffic -func createCombinedHandler(grpcServer *grpc.Server, httpHandler http.Handler, idpHandler http.Handler, relaySrv *relayServer.Server, meter metric.Meter, cfg *CombinedConfig) http.Handler { +func createCombinedHandler(grpcServer *grpc.Server, httpHandler, idpHandler http.Handler, relaySrv *relayServer.Server, meter metric.Meter, cfg *CombinedConfig) http.Handler { wsProxy := wsproxyserver.New(grpcServer, wsproxyserver.WithOTelMeter(meter)) var relayAcceptFn func(conn listener.Conn) diff --git a/e2e/agentnetwork/agent_config_test.go b/e2e/agentnetwork/agent_config_test.go new file mode 100644 index 000000000..58bfddab3 --- /dev/null +++ b/e2e/agentnetwork/agent_config_test.go @@ -0,0 +1,143 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// joinGroup places the PAT's own user into the group so caller-scoped answers +// (GET /api/agent-network/agent-config) see the policies sourced from it, and +// restores the previous auto-groups on cleanup. Self-service updates of one's +// own auto_groups are permitted for every role, so this needs no second user. +func joinGroup(t *testing.T, ctx context.Context, groupID string) { + t.Helper() + me, err := srv.API().Users.Current(ctx) + require.NoError(t, err, "read current user") + before := append([]string(nil), me.AutoGroups...) + _, err = srv.API().Users.Update(ctx, me.Id, api.PutApiUsersUserIdJSONRequestBody{ + Role: me.Role, + IsBlocked: me.IsBlocked, + AutoGroups: append(append([]string(nil), before...), groupID), + }) + require.NoError(t, err, "add the caller to the policy source group") + t.Cleanup(func() { + _, _ = srv.API().Users.Update(context.Background(), me.Id, api.PutApiUsersUserIdJSONRequestBody{ + Role: me.Role, + IsBlocked: me.IsBlocked, + AutoGroups: before, + }) + }) +} + +// configProvider returns the agent-config entry for the named provider, nil +// when the answer does not offer it. The suite shares one account, so other +// tests' fixtures may add unrelated providers to the caller's answer. +func configProvider(cfg api.AgentNetworkAgentConfig, name string) *api.AgentNetworkAgentConfigProvider { + for i := range cfg.Providers { + if cfg.Providers[i].Name == name { + return &cfg.Providers[i] + } + } + return nil +} + +// TestAgentConfigAllowlistOfDeclaredModels reproduces the post-#7221 field +// report: a provider carrying a declared model set plus a policy guardrail +// whose allowlist holds those same declared ids must advertise the models on +// GET /api/agent-network/agent-config — the guardrail was built FROM the +// provider's model list (the dashboard's allowlist picker persists the +// declared ids verbatim), so nothing about the setup excludes them. +// +// The plain case passes today. The path-style case (Bedrock; Vertex has the +// same shape) fails: the declared id is compared through the proxy parser's +// canonical form (region prefix and version suffix stripped) while the +// allowlist entry is not, so the raw-vs-raw pair never intersects and the +// caller sees an empty model list. The same one-sided normalization sits in +// policyPermitsModel, so the proxy also denies the model at request time — +// the guardrail meant to allow exactly this model turns it off end to end. +func TestAgentConfigAllowlistOfDeclaredModels(t *testing.T) { + ctx := context.Background() + + cases := []struct { + name string + catalogID string + upstream string + declared string + }{ + { + name: "plain-declared-id", + catalogID: "openai_api", + upstream: "https://api.openai.com", + declared: "gpt-4o-mini", + }, + { + // The operator declares the id AWS issues — region-prefixed + // inference profile with a version suffix — and the allowlist + // picker copies it as-is. + name: "bedrock-declared-id", + catalogID: "bedrock_api", + upstream: "https://bedrock-runtime.eu-central-1.amazonaws.com", + declared: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-agentcfg-" + tc.name}) + require.NoError(t, err, "create source group") + t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) }) + + joinGroup(t, ctx, grp.Id) + + providerName := "e2e-agentcfg-" + tc.name + prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: providerName, + ProviderId: tc.catalogID, + UpstreamUrl: tc.upstream, + ApiKey: ptr("sk-dummy-e2e-key"), + Enabled: ptr(true), + Models: &[]api.AgentNetworkProviderModel{{Id: tc.declared, InputPer1k: 0.001, OutputPer1k: 0.002}}, + }) + require.NoError(t, err, "create provider") + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + + // Allowlist exactly the declared model, the way the dashboard + // builds a guardrail from the provider's model list. + var gr api.AgentNetworkGuardrailRequest + gr.Name = "e2e-agentcfg-" + tc.name + gr.Checks.ModelAllowlist.Enabled = true + gr.Checks.ModelAllowlist.Models = []string{tc.declared} + guard, err := srv.CreateGuardrail(ctx, gr) + require.NoError(t, err, "create guardrail") + t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) }) + + pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-agentcfg-" + tc.name, + Enabled: ptr(true), + SourceGroups: []string{grp.Id}, + DestinationProviderIds: []string{prov.Id}, + GuardrailIds: &[]string{guard.Id}, + }) + require.NoError(t, err, "create policy") + t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) + + cfg, err := srv.GetAgentConfig(ctx) + require.NoError(t, err, "read the caller-scoped agent config") + require.True(t, cfg.Configured, "the account endpoint is bootstrapped by TestMain") + + entry := configProvider(cfg, providerName) + require.NotNil(t, entry, "the policy authorizes the caller for the provider, so it must be offered") + assert.False(t, entry.AllModelsAllowed, "an allowlist guardrail restricts the provider") + assert.Equal(t, []string{tc.declared}, entry.Models, + "the allowlist holds the provider's own declared id, so that model must be advertised") + }) + } +} diff --git a/e2e/agentnetwork/credential_check_live_test.go b/e2e/agentnetwork/credential_check_live_test.go new file mode 100644 index 000000000..f83a18129 --- /dev/null +++ b/e2e/agentnetwork/credential_check_live_test.go @@ -0,0 +1,235 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/client/rest" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// credentialCase is one vendor to try the save-time check against. The key is +// the real one the suite already sources; corrupting it is what produces the +// refusal, so the pair of cases differ only in the credential. +type credentialCase struct { + name string + catalogID string + upstream string + apiKey string +} + +// liveCredentialCases mirrors the discovery matrix's env gating so a partial +// key set still yields partial coverage. Vertex is left out: its credential is +// a service-account keyfile, and mangling one produces a client-side parse +// failure rather than the vendor refusal this is about. +func liveCredentialCases() []credentialCase { + var cases []credentialCase + + if k := os.Getenv("OPENAI_TOKEN"); k != "" { + cases = append(cases, credentialCase{ + name: "openai", catalogID: "openai_api", + upstream: "https://api.openai.com", apiKey: k, + }) + } + if k := os.Getenv("ANTHROPIC_TOKEN"); k != "" { + cases = append(cases, credentialCase{ + name: "anthropic", catalogID: "anthropic_api", + upstream: "https://api.anthropic.com", apiKey: k, + }) + } + if k := os.Getenv("AWS_BEARER_TOKEN_BEDROCK"); k != "" { + region := os.Getenv("AWS_REGION") + if region == "" { + region = "eu-central-1" + } + cases = append(cases, credentialCase{ + name: "bedrock", catalogID: "bedrock_api", + upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k, + }) + } + + return cases +} + +// TestLiveProviderCredentialCheck drives the save-time check against the real +// vendors. A unit test can only assert that a mocked refusal is classified; +// what it cannot show is that these vendors refuse a bad key on their listing +// endpoint at all, which is the assumption the whole feature rests on. +// +// The good-key case matters just as much as the bad one: a check that refused +// everything would pass a test asserting only the refusal, and would make the +// product unusable. +// +// The suite asserts on the vendors themselves, so it inherits their +// availability: the check blocks on 5xx and 429 by design, and a vendor outage +// or a rate limit during a run fails "a good credential saves" with a +// perfectly valid key. There is no retry here on purpose — a retry loop would +// also mask the outage classification these tests exist to prove. Re-run the +// job. +func TestLiveProviderCredentialCheck(t *testing.T) { + cases := liveCredentialCases() + if len(cases) == 0 { + t.Skip("no live provider credentials in the environment; source ~/.llm-keys to run") + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Run("a good credential saves", func(t *testing.T) { + prov, err := srv.CreateProvider(ctx, credentialProviderRequest(tc, "e2e-cred-ok-"+tc.name, tc.apiKey)) + require.NoError(t, err, "the suite's own credential must pass its check") + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + require.NotEmpty(t, prov.Id) + }) + + t.Run("a rejected credential is refused", func(t *testing.T) { + _, err := srv.CreateProvider(ctx, credentialProviderRequest(tc, "e2e-cred-bad-"+tc.name, corrupt(tc.apiKey))) + require.Error(t, err, "a key the vendor rejects must not save") + + var apiErr *rest.APIError + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusUnprocessableEntity, apiErr.StatusCode, + "a refused credential is the caller's problem to fix, not a server fault") + require.Contains(t, strings.ToLower(apiErr.Message), "rejected the credential", + "the message must name the credential rather than the url") + + // The record must be absent, not merely unusable: a provider + // saved despite its check is the state this prevents. + all, listErr := srv.ListProviders(ctx) + require.NoError(t, listErr) + for _, p := range all { + require.NotEqual(t, "e2e-cred-bad-"+tc.name, p.Name, "a refused provider must not be stored") + } + }) + }) + } +} + +// TestLiveProviderUrlCheck points a real credential at a host that is not the +// vendor's API. It is the half of the split a wrong key cannot exercise: the +// operator has to be told the URL is at fault while their key is fine. +// +// One vendor, deliberately. The transport classification under test happens +// before any vendor is reached, so running it per configured vendor would +// repeat the same code path and multiply the wall-clock of a suite that +// already creates real records. cases[0] is whichever vendor the environment +// supplies first. +func TestLiveProviderUrlCheck(t *testing.T) { + cases := liveCredentialCases() + if len(cases) == 0 { + t.Skip("no live provider credentials in the environment; source ~/.llm-keys to run") + } + tc := cases[0] + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // A name that resolves nowhere. The check has to reach a verdict without + // the vendor's help, which is the transport half of the classification. + req := credentialProviderRequest(tc, "e2e-cred-badurl", tc.apiKey) + req.UpstreamUrl = "https://not-a-real-vendor-host.netbird-e2e.invalid" + + _, err := srv.CreateProvider(ctx, req) + require.Error(t, err, "an upstream that does not resolve must not save") + + var apiErr *rest.APIError + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusUnprocessableEntity, apiErr.StatusCode) + require.Contains(t, strings.ToLower(apiErr.Message), "could not be reached", + "the message must name the url rather than the credential") + + // An error is not the same fact as an absent record: a handler that saved + // first and reported afterwards would satisfy everything above. + all, listErr := srv.ListProviders(ctx) + require.NoError(t, listErr) + for _, p := range all { + require.NotEqual(t, "e2e-cred-badurl", p.Name, "a refused provider must not be stored") + } +} + +// TestLiveProviderUpdateKeepsTheWorkingKey is the state the check exists to +// prevent on the update path: a rejected rotation that has already replaced +// the credential would take a working provider down. +// +// Also one vendor: the behaviour is in the manager's merge, not in any +// vendor's response, and each run creates and mutates a real provider record. +func TestLiveProviderUpdateKeepsTheWorkingKey(t *testing.T) { + cases := liveCredentialCases() + if len(cases) == 0 { + t.Skip("no live provider credentials in the environment; source ~/.llm-keys to run") + } + tc := cases[0] + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + prov, err := srv.CreateProvider(ctx, credentialProviderRequest(tc, "e2e-cred-rotate", tc.apiKey)) + require.NoError(t, err) + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + + rotation := credentialProviderRequest(tc, "e2e-cred-rotate", corrupt(tc.apiKey)) + _, err = srv.UpdateProvider(ctx, prov.Id, rotation) + require.Error(t, err, "a rotation the vendor rejects must not be stored") + + var apiErr *rest.APIError + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusUnprocessableEntity, apiErr.StatusCode) + + // The stored key is never returned by the API, so the proof that it + // survived is that an edit which reuses it still passes its check. A + // replaced key would fail here exactly as the rotation just did. + // + // The trailing slash is what makes that an actual check: an edit touching + // neither the url, the key nor the catalog entry is stored without asking + // the vendor anything, so a rename alone would pass whatever is on the + // record. Only the host is read out of the upstream, so the same vendor is + // reached — but the string differs, and the check runs. + recheck := credentialProviderRequest(tc, "e2e-cred-rotate-renamed", "") + recheck.UpstreamUrl = tc.upstream + "/" + updated, err := srv.UpdateProvider(ctx, prov.Id, recheck) + require.NoError(t, err, "the working key must still be the stored one") + require.Equal(t, "e2e-cred-rotate-renamed", updated.Name) +} + +// credentialProviderRequest builds a create/update body for a case. An empty apiKey is +// omitted rather than sent blank, which is how the form asks to keep whatever +// is already stored. +func credentialProviderRequest(tc credentialCase, name, apiKey string) api.AgentNetworkProviderRequest { + req := api.AgentNetworkProviderRequest{ + Name: name, + ProviderId: tc.catalogID, + UpstreamUrl: tc.upstream, + Enabled: ptr(true), + } + if apiKey != "" { + req.ApiKey = &apiKey + } + return req +} + +// corrupt returns a key the vendor will reject while keeping the shape of the +// original. Replacing the last character rather than appending keeps any +// length or prefix validation satisfied, so the refusal comes from the vendor +// checking the secret rather than from it rejecting an obviously malformed +// one. +func corrupt(key string) string { + if key == "" { + return key + } + last := key[len(key)-1] + replacement := byte('A') + if last == 'A' { + replacement = 'B' + } + return key[:len(key)-1] + string(replacement) +} diff --git a/e2e/agentnetwork/guardrail_declared_ids_test.go b/e2e/agentnetwork/guardrail_declared_ids_test.go new file mode 100644 index 000000000..a34d545be --- /dev/null +++ b/e2e/agentnetwork/guardrail_declared_ids_test.go @@ -0,0 +1,132 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// TestModelAllowlistOfDeclaredIDsServed drives the setup an operator actually +// builds for a path-routed provider: the models are declared in the form the +// vendor issues (Bedrock's region-prefixed, versioned inference-profile id; +// Vertex's model@version), and the guardrail allowlist is built from that +// declared list — the dashboard's allowlist picker persists the declared ids +// verbatim. A request for the declared model must be served end to end, and a +// model outside the allowlist must still be denied. +// +// TestModelAllowlistEnforced never caught this because it registers and +// allowlists the pre-normalized catalog form (see the catalogModel comment +// there and the one in providerRequest: "register the normalized form here or +// routing fails as model_not_routable") — the harness encoded the +// canonicalization workaround instead of the shape operators configure. +func TestModelAllowlistOfDeclaredIDsServed(t *testing.T) { + var providers []providerCase + for _, pc := range availableProviders() { + if pc.kind == harness.WireBedrock || pc.kind == harness.WireVertex { + providers = append(providers, pc) + } + } + if len(providers) == 0 { + t.Skip("no path-routed provider keys set (AWS_BEARER_TOKEN_BEDROCK / GOOGLE_VERTEX_*); source ~/.llm-keys") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-declared-allowlist"}) + require.NoError(t, err, "create group") + t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) }) + + ephemeral := false + sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{ + Name: "e2e-declared-allowlist-client", + Type: "reusable", + ExpiresIn: 86400, + UsageLimit: 0, + AutoGroups: []string{grp.Id}, + Ephemeral: &ephemeral, + }) + require.NoError(t, err, "mint setup key") + t.Cleanup(func() { _ = srv.API().SetupKeys.Delete(context.Background(), sk.Id) }) + + // Providers declaring the raw vendor-issued model id — NOT the normalized + // catalog form providerRequest would register. + ids := make([]string, 0, len(providers)) + declared := make([]string, 0, len(providers)) + for _, pc := range providers { + req := providerRequest(pc) + req.Models = &[]api.AgentNetworkProviderModel{{Id: pc.model, InputPer1k: 0.001, OutputPer1k: 0.002}} + prov, perr := srv.CreateProvider(ctx, req) + require.NoError(t, perr, "create provider %s", pc.name) + id := prov.Id + ids = append(ids, id) + declared = append(declared, pc.model) + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) }) + } + + // Guardrail allowlisting the declared ids verbatim, the way the dashboard + // builds an allowlist from the providers' model lists. + var gr api.AgentNetworkGuardrailRequest + gr.Name = "e2e-declared-allowlist" + gr.Checks.ModelAllowlist.Enabled = true + gr.Checks.ModelAllowlist.Models = declared + guard, err := srv.CreateGuardrail(ctx, gr) + require.NoError(t, err, "create guardrail") + t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) }) + + enabled := true + pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-declared-allowlist", + Enabled: &enabled, + SourceGroups: []string{grp.Id}, + DestinationProviderIds: ids, + GuardrailIds: &[]string{guard.Id}, + }) + require.NoError(t, err, "create policy") + t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) + + settings, err := srv.GetSettings(ctx) + require.NoError(t, err, "read settings for endpoint") + require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned") + + proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy-declared-allowlist") + require.NoError(t, err, "mint proxy token via CLI") + px, err := harness.StartProxy(ctx, srv, proxyToken) + require.NoError(t, err, "start proxy") + t.Cleanup(func() { _ = px.Terminate(context.Background()) }) + + cl, err := harness.StartClient(ctx, srv, sk.Key) + require.NoError(t, err, "start client") + t.Cleanup(func() { _ = cl.Terminate(context.Background()) }) + + require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management") + // Probe first: the GET resolves the endpoint (DNS error fails) and its first packet wakes the lazy proxy peer, so WaitProxyPeer sees it connected; any HTTP status counts. + proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint) + require.NoError(t, err, "resolve agent-network endpoint to proxy IP") + if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil { + t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background())) + } + + for _, pc := range providers { + pc := pc + t.Run(pc.name, func(t *testing.T) { + // The model the operator declared and allowlisted is served end to + // end: the route must claim it and the guardrail must permit it, + // both through the canonicalization the parser applies at request + // time — whatever id form the operator configured. + assert.Equal(t, 200, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, pc.model), + "the declared and allowlisted model must be served for %s", pc.name) + // A model outside the allowlist stays denied. + assert.Equal(t, 403, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, disallowedModel(pc)), + "model outside the allowlist must be denied for %s", pc.name) + }) + } +} diff --git a/e2e/agentnetwork/management_test.go b/e2e/agentnetwork/management_test.go index 9e3176d68..6868a9d4d 100644 --- a/e2e/agentnetwork/management_test.go +++ b/e2e/agentnetwork/management_test.go @@ -16,14 +16,20 @@ import ( func ptr[T any](v T) *T { return &v } -// newProvider creates an OpenAI-catalog provider with a dummy key (these tests -// never call the upstream) and registers cleanup. +// newProvider creates an OpenAI-catalog provider these tests can hang a policy +// off, and registers cleanup. Nothing here calls the upstream. func newProvider(t *testing.T, ctx context.Context, name string) api.AgentNetworkProvider { t.Helper() + // A provider save is credential-checked against the vendor, and every + // caller here wants a provider row to hang a policy off rather than a + // working upstream. A private address is left unchecked — the proxy would + // reach it through the tunnel, management cannot reach it at all — which + // keeps this fixture independent of whether the run has vendor keys, and + // covers the unchecked-provider-still-saves path while it is at it. prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ Name: name, ProviderId: "openai_api", - UpstreamUrl: "https://api.openai.com", + UpstreamUrl: "https://10.255.255.1", ApiKey: ptr("sk-dummy-e2e-key"), }) require.NoError(t, err, "create provider %q", name) diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go index e85475dff..e51f2dd7a 100644 --- a/e2e/harness/agentnetwork.go +++ b/e2e/harness/agentnetwork.go @@ -100,6 +100,15 @@ func (c *Combined) SetProviderEnabled(ctx context.Context, id string, enabled bo return err } +// GetAgentConfig returns the caller-scoped self-service connection config — +// the answer the dashboard's "Connect Agent" view renders for the PAT's user. +// Providers appear only when the caller's own groups intersect an enabled +// policy's source groups, so tests must place the PAT user into the policy's +// source group first (via the Users API auto-groups). +func (c *Combined) GetAgentConfig(ctx context.Context) (api.AgentNetworkAgentConfig, error) { + return anRequest[api.AgentNetworkAgentConfig](ctx, c, http.MethodGet, "/api/agent-network/agent-config", nil) +} + // CreatePolicy creates an agent-network policy. func (c *Combined) CreatePolicy(ctx context.Context, req api.AgentNetworkPolicyRequest) (api.AgentNetworkPolicy, error) { return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPost, "/api/agent-network/policies", req) diff --git a/encryption/testprotos/generate.sh b/encryption/testprotos/generate.sh index 0ce6ebdea..ffbc481d6 100755 --- a/encryption/testprotos/generate.sh +++ b/encryption/testprotos/generate.sh @@ -1,2 +1,2 @@ -#!/bin/bash -protoc -I testprotos/ testprotos/testproto.proto --go_out=. \ No newline at end of file +#!/usr/bin/env bash +protoc -I testprotos/ testprotos/testproto.proto --go_out=. diff --git a/flow/proto/generate.sh b/flow/proto/generate.sh index 6bbf78e61..a031245fd 100755 --- a/flow/proto/generate.sh +++ b/flow/proto/generate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if ! which realpath > /dev/null 2>&1 diff --git a/integration_tests/management/network_map_db/network_router_test.go b/integration_tests/management/network_map_db/network_router_test.go index fa7ea2a04..2baf146a7 100644 --- a/integration_tests/management/network_map_db/network_router_test.go +++ b/integration_tests/management/network_map_db/network_router_test.go @@ -19,6 +19,14 @@ func TestGetNetworkRouters(t *testing.T) { execQuery(t, ctx, `insert into network_routers (id, account_id, public_id, peer, network_id, masquerade, metric, enabled, peer_groups) VALUES('test-nr-id-2','account-1','public-id-2','','network-id-2',TRUE,333,TRUE,'["group-two-resources-id","group-no-resources-id"]')`) + // empty peer_groups + execQuery(t, ctx, + `insert into network_routers (id, account_id, public_id, peer, network_id, masquerade, metric, enabled, peer_groups) + VALUES('test-nr-id-3','account-1','public-id-3','peer-id-3','network-id-3',TRUE,999,TRUE,'[]')`) + // nil peer_groups + execQuery(t, ctx, + `insert into network_routers (id, account_id, public_id, peer, network_id, masquerade, metric, enabled, peer_groups) + VALUES('test-nr-id-4','account-1','public-id-4','peer-id-4','network-id-4',TRUE,999,TRUE,null)`) routers, err := conn(t, ctx).GetNetworkRouters(ctx, "account-1") assert.NoError(t, err) @@ -30,4 +38,8 @@ func TestGetNetworkRouters(t *testing.T) { map[string]*nmdata.NetworkRouter{ "peer-id-2": {PublicID: "public-id-2", Masquerade: true, Metric: 333, Enabled: true, PeerGroups: []string{"group-two-resources-id", "group-no-resources-id"}}, "peer-id-3": {PublicID: "public-id-2", Masquerade: true, Metric: 333, Enabled: true, PeerGroups: []string{"group-two-resources-id", "group-no-resources-id"}}}) + assert.Equal(t, routers["network-id-3"], + map[string]*nmdata.NetworkRouter{"peer-id-3": {PublicID: "public-id-3", Masquerade: true, Metric: 999, Enabled: true, PeerGroups: []string{}}}) + assert.Equal(t, routers["network-id-4"], + map[string]*nmdata.NetworkRouter{"peer-id-4": {PublicID: "public-id-4", Masquerade: true, Metric: 999, Enabled: true, PeerGroups: nil}}) } diff --git a/integration_tests/management/network_map_db/user_test.go b/integration_tests/management/network_map_db/user_test.go index 132f749e2..fce1833d3 100644 --- a/integration_tests/management/network_map_db/user_test.go +++ b/integration_tests/management/network_map_db/user_test.go @@ -21,6 +21,14 @@ func TestGetAllowedUsers(t *testing.T) { execQuery(t, ctx, `insert into users (id, name, account_id, auto_groups, blocked, is_service_user) VALUES('user-3','user-3','account-1','["group-two-resources-id"]',false,false)`) + // empty auto_groups; shouldn't error out + execQuery(t, ctx, + `insert into users (id, name, account_id, auto_groups, blocked, is_service_user) + VALUES('user-31','user-31','account-1','[]',false,false)`) + // null auto_groups; shouldn't error out + execQuery(t, ctx, + `insert into users (id, name, account_id, auto_groups, blocked, is_service_user) + VALUES('user-32','user-32','account-1',null,false,false)`) // shouldn't be included as it's blocked execQuery(t, ctx, `insert into users (id, name, account_id, auto_groups, blocked, is_service_user) @@ -43,15 +51,17 @@ func TestGetAllowedUsers(t *testing.T) { assert.NoError(t, err) assert.Equal(t, userIdx, map[string]struct{}{ - "user-1": {}, - "user-2": {}, - "user-3": {}, + "user-1": {}, + "user-2": {}, + "user-3": {}, + "user-31": {}, + "user-32": {}, }) assert.Equal(t, groupIdToUserIds, map[string][]string{ "group-one-resource-id": {"user-1", "user-2"}, "group-two-resources-id": {"user-2", "user-3"}, - "all-group-1": {"user-1", "user-2", "user-3"}, - "all-group-2": {"user-1", "user-2", "user-3"}, - "all-group-3": {"user-1", "user-2", "user-3"}, + "all-group-1": {"user-1", "user-2", "user-3", "user-31", "user-32"}, + "all-group-2": {"user-1", "user-2", "user-3", "user-31", "user-32"}, + "all-group-3": {"user-1", "user-2", "user-3", "user-31", "user-32"}, }) } diff --git a/management/cmd/management.go b/management/cmd/management.go index 147985314..fc6bd0a46 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -236,6 +236,9 @@ func ApplyEmbeddedIdPConfig(ctx context.Context, cfg *nbconfig.Config) error { // Embedded IdP requires single account mode - multiple account mode is not supported return fmt.Errorf("embedded IdP requires single account mode; multiple account mode is not supported with embedded IdP. Please remove --disable-single-account-mode flag") } + if mgmtSingleAccModeDomain == "" { + return fmt.Errorf("embedded IdP requires single account mode; --single-account-mode-domain must not be empty") + } // Enable user deletion from IDP by default if EmbeddedIdP is enabled userDeleteFromIDPEnabled = true diff --git a/management/cmd/management_test.go b/management/cmd/management_test.go index 2c3481213..e34e1975e 100644 --- a/management/cmd/management_test.go +++ b/management/cmd/management_test.go @@ -5,8 +5,12 @@ import ( "os" "testing" - "github.com/netbirdio/netbird/shared/management/grpc" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + nbconfig "github.com/netbirdio/netbird/management/internals/server/config" + "github.com/netbirdio/netbird/management/server/idp" + "github.com/netbirdio/netbird/shared/management/grpc" ) const ( @@ -60,6 +64,22 @@ func Test_LoadMgmtConfig_Empty(t *testing.T) { assert.Nil(t, cfg.PerAccountHighestSupportedSyncMessageVersion) } +func TestApplyEmbeddedIdPConfigRequiresSingleAccountDomain(t *testing.T) { + previousDomain := mgmtSingleAccModeDomain + previousDisabled := disableSingleAccMode + t.Cleanup(func() { + mgmtSingleAccModeDomain = previousDomain + disableSingleAccMode = previousDisabled + }) + + mgmtSingleAccModeDomain = "" + disableSingleAccMode = false + cfg := &nbconfig.Config{ + EmbeddedIdP: &idp.EmbeddedIdPConfig{Enabled: true}, + } + require.ErrorContains(t, ApplyEmbeddedIdPConfig(context.Background(), cfg), "embedded IdP requires single account mode") +} + func createConfig(config string) (string, error) { tmpfile, err := os.CreateTemp("", "config.json") if err != nil { diff --git a/management/internals/modules/agentnetwork/agent_config.go b/management/internals/modules/agentnetwork/agent_config.go index 5571fd159..902652479 100644 --- a/management/internals/modules/agentnetwork/agent_config.go +++ b/management/internals/modules/agentnetwork/agent_config.go @@ -179,6 +179,10 @@ func policiesForProvider(policies []*types.Policy, providerID string) []*types.P // only claims declared models, so an allowlisted-but-undeclared model is // unreachable and must not be advertised. With no declared models the // router claims every model, so the allowlist union stands alone. +// Allowlist entries and declared ids both compare through the canonical +// id the proxy's parser emits, so an allowlist may hold either form: the +// raw declared id the dashboard's picker copies from the provider, or +// the stripped id the parser matches at request time. func effectiveModelsForProvider(provider *types.Provider, policies []*types.Policy, guardrailsByID map[string]*types.Guardrail) (bool, []string) { restricted := true union := make([]string, 0) @@ -192,7 +196,7 @@ func effectiveModelsForProvider(provider *types.Provider, policies []*types.Poli } policyRestricted = true for _, model := range g.Checks.ModelAllowlist.Models { - key := normaliseModelID(model) + key := canonicalModelKey(provider.ProviderID, model) if key == "" { continue } @@ -221,17 +225,26 @@ func effectiveModelsForProvider(provider *types.Provider, policies []*types.Poli for _, id := range declared { // Compare through the canonical id the proxy's parser emits — a // Bedrock declaration may carry the region/version form - // ("eu.anthropic.claude-...-v1:0") while the allowlist holds the - // stripped id the parser matches at request time, and the raw - // forms would never intersect. The declared id itself is what - // gets advertised, matching the router's route claim. - if _, ok := seen[normaliseModelID(normalizePricingModelID(provider.ProviderID, id))]; ok { + // ("eu.anthropic.claude-...-v1:0") that the parser strips at + // request time, and the raw forms would never intersect. The + // declared id itself is what gets advertised, matching the + // router's route claim. + if _, ok := seen[canonicalModelKey(provider.ProviderID, id)]; ok { out = append(out, id) } } return false, out } +// canonicalModelKey builds the compare key for a model id: lowercased, +// trimmed, and canonicalized through the provider-aware normalization the +// proxy's parser applies. Lowercase/trim comes FIRST — the path-style +// strippers anchor on a lowercase id's tail, so a trailing space or a +// case-variant geography/version would otherwise survive into the key. +func canonicalModelKey(catalogProviderID, id string) string { + return normaliseModelID(normalizePricingModelID(catalogProviderID, normaliseModelID(id))) +} + // providerModelsByID maps effective model ids (as effectiveModelsForProvider // returns them) back onto the operator's declared entries, keeping the // declared casing and prices. With no operator declaration the ids are the diff --git a/management/internals/modules/agentnetwork/agent_config_realstore_test.go b/management/internals/modules/agentnetwork/agent_config_realstore_test.go index 9a66e1190..2fd541fad 100644 --- a/management/internals/modules/agentnetwork/agent_config_realstore_test.go +++ b/management/internals/modules/agentnetwork/agent_config_realstore_test.go @@ -144,6 +144,54 @@ func TestAgentConfig_RealStore_AllowlistMatchesBedrockDeclaredIDsCanonically(t * "the allowlisted canonical id must admit the declared region/version form, and only it") } +func TestAgentConfig_RealStore_AllowlistHoldsRawDeclaredIDs(t *testing.T) { + // The dashboard's allowlist picker copies the provider's declared ids + // verbatim, so for path-style providers the allowlist carries the + // region/version form rather than the canonical id the parser emits. + // Both forms must admit the declared model. + cases := []struct { + name string + catalogID string + declared string + allowlist string + }{ + {"bedrock", "bedrock_api", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", ""}, + {"vertex", "vertex_ai_api", "claude-sonnet-4-5@20250929", ""}, + // The geography/version strippers anchor on a lowercase tail, so a + // case-variant entry must be lowercased before canonicalization or + // the prefix and suffix survive into the compare key. + {"bedrock-case-variant", "bedrock_api", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + " EU.Anthropic.Claude-Sonnet-4-5-20250929-V1:0 "}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mgr, s := newAgentConfigTestMgr(t) + ctx := context.Background() + + allowlisted := tc.allowlist + if allowlisted == "" { + allowlisted = tc.declared + } + require.NoError(t, s.SaveAgentNetworkSettings(ctx, newSynthTestSettings())) + provider := newSynthTestProvider() + provider.ProviderID = tc.catalogID + provider.Name = tc.name + provider.Models = []types.ProviderModel{{ID: tc.declared}} + require.NoError(t, s.SaveAgentNetworkProvider(ctx, provider)) + require.NoError(t, s.SaveAgentNetworkGuardrail(ctx, newSetupTestGuardrail("guard-1", allowlisted))) + require.NoError(t, s.SaveAgentNetworkPolicy(ctx, newSynthTestPolicy(provider.ID, "grp-eng", "guard-1"))) + + setup, err := mgr.agentConfigForGroups(ctx, testAccountID, []string{"grp-eng"}) + require.NoError(t, err) + require.Len(t, setup.Providers, 1) + p := setup.Providers[0] + assert.False(t, p.AllModelsAllowed) + assert.Equal(t, []string{tc.declared}, p.Models, + "an allowlist holding the raw declared id must admit that declared model") + }) + } +} + func TestAgentConfig_RealStore_UnrestrictedPolicyWinsOverRestricted(t *testing.T) { mgr, s := newAgentConfigTestMgr(t) ctx := context.Background() diff --git a/management/internals/modules/agentnetwork/credentialcheck.go b/management/internals/modules/agentnetwork/credentialcheck.go new file mode 100644 index 000000000..a4787d019 --- /dev/null +++ b/management/internals/modules/agentnetwork/credentialcheck.go @@ -0,0 +1,135 @@ +package agentnetwork + +import ( + "context" + "errors" + "net/http" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/modeldiscovery" + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" + "github.com/netbirdio/netbird/shared/management/status" +) + +// ModelLister is the vendor-facing half of the credential check. +// modeldiscovery.Client is the only production implementation; it is an +// interface because the check runs on a write path, so without a seam every +// test that saves a provider would reach a vendor to do it. +type ModelLister interface { + Fetch(ctx context.Context, req modeldiscovery.Request) ([]modeldiscovery.Model, error) +} + +// checkProviderCredential refuses a record whose upstream or credential the +// vendor will not accept. +// +// It reuses the discovery Fetch rather than a lighter status probe so it +// exercises the path the model picker takes: a URL answering 200 with a login +// page fails here instead of producing an empty picker later. +func (m *managerImpl) checkProviderCredential(ctx context.Context, provider *types.Provider) error { + // A record that asks the proxy to skip certificate verification is one this + // check cannot speak for. Discovery verifies certificates, so a self-hosted + // endpoint behind a self-signed one would be refused for a reason the + // operator already told us to ignore — a lockout of exactly the setup the + // flag exists for. Sending the credential over a connection management + // declines to verify is the other way out, and a worse one. + if provider.SkipTLSVerification { + log.WithContext(ctx).Debugf("agent network provider %s not credential-checked: tls verification is disabled for it", provider.ProviderID) + return nil + } + + _, err := m.modelDiscovery.Fetch(ctx, modeldiscovery.Request{ + CatalogID: provider.ProviderID, + UpstreamURL: provider.UpstreamURL, + APIKey: provider.APIKey, + }) + if err == nil { + return nil + } + + message, blocking := credentialCheckFailure(err) + if !blocking { + log.WithContext(ctx).Debugf("agent network provider %s not credential-checked: %v", provider.ProviderID, err) + return nil + } + + // WriteError logs only what we return, and that carries no status code, + // so the vendor's number is recorded here or nowhere. + log.WithContext(ctx).Infof("agent network provider %s failed its credential check: %v", provider.ProviderID, err) + + return status.Errorf(status.InvalidArgument, "%s", message) +} + +// discoveryFailure renders a failed model listing for the operator who pressed +// the button. Every outcome here is something they did or configured — a key +// the vendor refused, an upstream that does not answer — so it owes them the +// same sentence a refused save gives, not the generic 500 an unclassified +// error turns into. +// +// ErrNoDiscovery and ErrInvalidRequest pass through untouched: the handler +// already maps them, and "this provider has no listing endpoint" is a fact +// about the catalog rather than a failure to report as one. +func discoveryFailure(ctx context.Context, catalogID string, err error) error { + if errors.Is(err, modeldiscovery.ErrNoDiscovery) || errors.Is(err, modeldiscovery.ErrInvalidRequest) { + return err + } + + message, _ := credentialCheckFailure(err) + if message == "" { + return err + } + + // The operator's message carries no status code, so the vendor's number is + // recorded here or nowhere. + log.WithContext(ctx).Infof("agent network model discovery for %s failed: %v", catalogID, err) + + return status.Errorf(status.InvalidArgument, "%s", message) +} + +// credentialCheckFailure renders a discovery failure as the sentence the +// provider form shows, and reports whether it should block the write. +// +// The strings survive WriteError lowercasing them, and never echo the +// operator's URL: paths are case-sensitive, so an echoed URL comes back +// altered and describes something they did not type. +func credentialCheckFailure(err error) (message string, blocking bool) { + // Not checkable. The record may be perfectly good and we have no way to + // ask, so reporting a failure would be a guess. + switch { + case errors.Is(err, modeldiscovery.ErrNoDiscovery), + errors.Is(err, modeldiscovery.ErrNoDiscoveryHost), + errors.Is(err, modeldiscovery.ErrPrivateHost): + return "", false + } + + var vendor *modeldiscovery.VendorStatusError + if errors.As(err, &vendor) { + switch vendor.Status { + case http.StatusUnauthorized, http.StatusForbidden: + return "the provider rejected the credential", true + case http.StatusNotFound, http.StatusMethodNotAllowed: + return "the upstream url did not answer a model listing", true + default: + // 5xx and 429 included: an outage still leaves the record + // unverified, which is what this refuses to save. + return "the provider returned an error", true + } + } + + var unreachable *modeldiscovery.UnreachableError + if errors.As(err, &unreachable) { + if reason := unreachable.Reason(); reason != "" { + return "the upstream url could not be reached: " + reason, true + } + return "the upstream url could not be reached", true + } + + if errors.Is(err, modeldiscovery.ErrUnparseableListing) { + return "the upstream url answered, but not with a model listing", true + } + + // Ours rather than the vendor's — a request this code built badly, or a + // catalog entry that does not match its parser. Still unverified, so it + // still blocks. + return "the provider could not be checked", true +} diff --git a/management/internals/modules/agentnetwork/credentialcheck_test.go b/management/internals/modules/agentnetwork/credentialcheck_test.go new file mode 100644 index 000000000..8bc8f0b58 --- /dev/null +++ b/management/internals/modules/agentnetwork/credentialcheck_test.go @@ -0,0 +1,605 @@ +package agentnetwork + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/modeldiscovery" + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" + "github.com/netbirdio/netbird/management/server/permissions/modules" + "github.com/netbirdio/netbird/management/server/permissions/operations" + "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/shared/management/status" +) + +// stubLister stands in for the vendor on the write path. It records what it +// was asked so a test can assert not only that the check ran, but that it ran +// against the right upstream and the right credential — and, for an edit that +// touches neither, that it did not run at all. +type stubLister struct { + err error + requests []modeldiscovery.Request +} + +func (s *stubLister) Fetch(_ context.Context, req modeldiscovery.Request) ([]modeldiscovery.Model, error) { + s.requests = append(s.requests, req) + if s.err != nil { + return nil, s.err + } + return []modeldiscovery.Model{{ID: "a-model", PricingKnown: true}}, nil +} + +func (s *stubLister) calls() int { return len(s.requests) } + +func (s *stubLister) only(t *testing.T) modeldiscovery.Request { + t.Helper() + require.Len(t, s.requests, 1, "the vendor must be asked exactly once") + return s.requests[0] +} + +// TestCredentialCheckFailure_SeparatesTheUrlFromTheCredential is the contract +// the provider form is written against: an operator gets told which of the two +// fields they have to look at, and the message says so without a status code +// and without echoing the URL back at them. +func TestCredentialCheckFailure_SeparatesTheUrlFromTheCredential(t *testing.T) { + cases := []struct { + name string + err error + want string + }{ + { + name: "401 is the credential", + err: &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 401}, + want: "the provider rejected the credential", + }, + { + name: "403 is the credential", + err: &modeldiscovery.VendorStatusError{Provider: "Bedrock", Status: 403}, + want: "the provider rejected the credential", + }, + { + // The host authenticated us fine and then said it has no such + // endpoint, which is the URL being wrong rather than the key. + name: "404 is the url", + err: &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 404}, + want: "the upstream url did not answer a model listing", + }, + { + name: "405 is the url", + err: &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 405}, + want: "the upstream url did not answer a model listing", + }, + { + name: "500 is the vendor", + err: &modeldiscovery.VendorStatusError{Provider: "Anthropic", Status: 500}, + want: "the provider returned an error", + }, + { + name: "503 is the vendor", + err: &modeldiscovery.VendorStatusError{Provider: "Anthropic", Status: 503}, + want: "the provider returned an error", + }, + { + name: "429 is the vendor", + err: &modeldiscovery.VendorStatusError{Provider: "Anthropic", Status: 429}, + want: "the provider returned an error", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, blocking := credentialCheckFailure(tc.err) + require.True(t, blocking, "a vendor refusal must block the write") + require.Equal(t, tc.want, got) + }) + } +} + +// TestCredentialCheckFailure_NamesTheTransportFault covers the failures that +// never reached the vendor. The distinction inside them is worth keeping: a +// refused connection is a wrong port and an unknown host is a wrong hostname, +// and an operator staring at a URL they believe in needs to be told which. +func TestCredentialCheckFailure_NamesTheTransportFault(t *testing.T) { + cases := []struct { + name string + err error + want string + }{ + { + name: "unknown host", + err: &net.DNSError{Err: "no such host", Name: "api.example.com", IsNotFound: true}, + want: "the upstream url could not be reached: no such host", + }, + { + name: "dns failure that is not a missing name", + err: &net.DNSError{Err: "server misbehaving", Name: "api.example.com"}, + want: "the upstream url could not be reached: dns lookup failed", + }, + { + name: "connection refused", + err: &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, + want: "the upstream url could not be reached: connection refused", + }, + { + name: "host unreachable", + err: &net.OpError{Op: "dial", Net: "tcp", Err: syscall.EHOSTUNREACH}, + want: "the upstream url could not be reached: host unreachable", + }, + { + name: "timeout", + err: fmt.Errorf("dial: %w", os.ErrDeadlineExceeded), + want: "the upstream url could not be reached: connection timed out", + }, + { + name: "context deadline", + err: fmt.Errorf("dial: %w", context.DeadlineExceeded), + want: "the upstream url could not be reached: connection timed out", + }, + { + name: "untrusted certificate", + err: &tls.CertificateVerificationError{}, + want: "the upstream url could not be reached: tls certificate not trusted", + }, + { + name: "plaintext service on an https url", + err: tls.RecordHeaderError{Msg: "first record does not look like a TLS handshake"}, + want: "the upstream url could not be reached: not a tls endpoint", + }, + { + // Nothing we recognise. Better to say only that it could not be + // reached than to paste a Go error into the provider form. + name: "cause we do not recognise", + err: errors.New("something went sideways"), + want: "the upstream url could not be reached", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + wrapped := &modeldiscovery.UnreachableError{Provider: "OpenAI", Err: tc.err} + got, blocking := credentialCheckFailure(wrapped) + require.True(t, blocking, "an unreachable upstream must block the write") + require.Equal(t, tc.want, got) + }) + } +} + +// TestCredentialCheckFailure_AnAnsweringUrlThatIsNotTheApi covers the case a +// status probe would wave through: the host is up, the credential was accepted +// or not required, and the body is a login page. Reusing the discovery parser +// for the check is what catches it. +func TestCredentialCheckFailure_AnAnsweringUrlThatIsNotTheApi(t *testing.T) { + err := fmt.Errorf("%w: decode model listing: unexpected token", modeldiscovery.ErrUnparseableListing) + + got, blocking := credentialCheckFailure(err) + require.True(t, blocking) + require.Equal(t, "the upstream url answered, but not with a model listing", got) +} + +// TestCredentialCheckFailure_WhatCannotBeCheckedIsNotAFailure pins the +// difference between "this record is wrong" and "we have no way to ask". A +// gateway with no listing endpoint, a Bedrock record pointed at a proxy, and a +// self-hosted endpoint the proxy reaches through the tunnel are all legitimate +// providers. Blocking them would make the feature a lockout. +func TestCredentialCheckFailure_WhatCannotBeCheckedIsNotAFailure(t *testing.T) { + cases := map[string]error{ + "no listing endpoint": modeldiscovery.ErrNoDiscovery, + "no derivable host": fmt.Errorf("%w: %w: bedrock", modeldiscovery.ErrInvalidRequest, modeldiscovery.ErrNoDiscoveryHost), + "private upstream": fmt.Errorf("%w: %w: 10.0.0.5", modeldiscovery.ErrInvalidRequest, modeldiscovery.ErrPrivateHost), + } + + for name, err := range cases { + t.Run(name, func(t *testing.T) { + message, blocking := credentialCheckFailure(err) + require.False(t, blocking, "a provider we cannot check must still save") + require.Empty(t, message) + }) + } +} + +// TestCredentialCheckFailure_AnUnrecognisedFailureStillBlocks covers a fault of +// ours rather than the vendor's — a malformed request this code built, or a +// catalog entry whose parser does not match its endpoint. The record went +// unverified either way, and silently saving what we could not check is the +// thing this feature exists to prevent. +func TestCredentialCheckFailure_AnUnrecognisedFailureStillBlocks(t *testing.T) { + message, blocking := credentialCheckFailure(errors.New("no parser for listing shape \"\"")) + require.True(t, blocking) + require.Equal(t, "the provider could not be checked", message) +} + +// newCheckedProvider returns a record shaped the way the handler guarantees +// one: a known catalog id, a public upstream and a key. +func newCheckedProvider(accountID string) *types.Provider { + provider := types.NewProvider(accountID) + provider.ProviderID = "openai_api" + provider.Name = "openai" + provider.UpstreamURL = "https://api.openai.com" + provider.APIKey = "sk-good" + provider.Enabled = true + return provider +} + +// TestCreateProvider_RefusesARecordTheVendorRejects is the whole point of the +// feature: a key with a character missing used to save cleanly and surface +// minutes later as a failed request with nothing pointing back at the record. +func TestCreateProvider_RefusesARecordTheVendorRejects(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.vendor.err = &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 401} + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + _, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + + require.Error(t, err) + require.Contains(t, err.Error(), "the provider rejected the credential") + + var sErr *status.Error + require.ErrorAs(t, err, &sErr) + require.Equal(t, status.InvalidArgument, sErr.Type(), "the refusal must reach the caller as a 422") + + stored, err := f.store.GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, "account1") + require.NoError(t, err) + require.Empty(t, stored, "a record that failed its check must not be written") +} + +// TestCreateProvider_ChecksTheCredentialItWasGiven pins what the vendor is +// asked with, since a check run against the wrong upstream or a stale key +// would pass while proving nothing. +func TestCreateProvider_ChecksTheCredentialItWasGiven(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + _, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + + asked := f.vendor.only(t) + require.Equal(t, "openai_api", asked.CatalogID) + require.Equal(t, "https://api.openai.com", asked.UpstreamURL) + require.Equal(t, "sk-good", asked.APIKey) +} + +// TestUpdateProvider_AUrlOnlyChangeIsCheckedAgainstTheStoredKey covers the +// case that shaped where the check sits. The key never returns to the browser, +// so an operator editing only the URL has none to offer — the stored one is +// the only credential there is, and the new URL still has to be proven with +// it. +func TestUpdateProvider_AUrlOnlyChangeIsCheckedAgainstTheStoredKey(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + f.vendor.requests = nil + + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Update, true) + edit := newCheckedProvider("account1") + edit.ID = created.ID + edit.UpstreamURL = "https://gateway.example.com" + edit.APIKey = "" // the form sends no key when it was not retyped + + _, err = f.manager.UpdateProvider(ctx, "user1", edit) + require.NoError(t, err) + + asked := f.vendor.only(t) + require.Equal(t, "https://gateway.example.com", asked.UpstreamURL, "the new url must be what gets tested") + require.Equal(t, "sk-good", asked.APIKey, "and the stored key must be what tests it") +} + +// TestUpdateProvider_AFailedRotationLeavesTheWorkingKeyInPlace is the +// half-applied state the check must never produce: refusing the new key while +// having already replaced the old one would take the provider down. +func TestUpdateProvider_AFailedRotationLeavesTheWorkingKeyInPlace(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + + f.vendor.err = &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 403} + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Update, true) + rotation := newCheckedProvider("account1") + rotation.ID = created.ID + rotation.APIKey = "sk-typo" + + _, err = f.manager.UpdateProvider(ctx, "user1", rotation) + require.Error(t, err) + require.Contains(t, err.Error(), "the provider rejected the credential") + + stored, err := f.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, "account1", created.ID) + require.NoError(t, err) + require.Equal(t, "sk-good", stored.APIKey, "the rejected key must not have replaced the working one") +} + +// TestUpdateProvider_AnEditTouchingNeitherFieldAsksNoVendor keeps renames, +// model rows and price edits off the vendor's doorstep. They have nothing new +// to prove, and making them wait on a vendor — or fail because one is having a +// bad day — would be a tax on edits that carry no risk. +func TestUpdateProvider_AnEditTouchingNeitherFieldAsksNoVendor(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + f.vendor.requests = nil + // Any call at all now would fail the update, which is what makes the + // assertion below load-bearing rather than decorative. + f.vendor.err = &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 500} + + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Update, true) + rename := newCheckedProvider("account1") + rename.ID = created.ID + rename.Name = "openai-renamed" + rename.APIKey = "" + + _, err = f.manager.UpdateProvider(ctx, "user1", rename) + require.NoError(t, err, "an edit that changes neither url nor key must not be checked") + require.Zero(t, f.vendor.calls(), "and must not reach the vendor at all") +} + +// TestCreateProvider_AProviderWeCannotCheckStillSaves covers the eleven +// catalog entries with no listing endpoint, a Bedrock record behind a proxy, +// and a self-hosted endpoint on a private network. None of those are evidence +// the record is wrong, and refusing them would make this a lockout. +func TestCreateProvider_AProviderWeCannotCheckStillSaves(t *testing.T) { + cases := map[string]error{ + "gateway with no listing endpoint": modeldiscovery.ErrNoDiscovery, + "bedrock behind a proxy": fmt.Errorf("%w: %w", modeldiscovery.ErrInvalidRequest, modeldiscovery.ErrNoDiscoveryHost), + "self-hosted on a private network": fmt.Errorf("%w: %w", modeldiscovery.ErrInvalidRequest, modeldiscovery.ErrPrivateHost), + } + + for name, vendorErr := range cases { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.vendor.err = vendorErr + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + require.NotNil(t, created) + + stored, err := f.store.GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, "account1") + require.NoError(t, err) + require.Len(t, stored, 1, "a provider we cannot check must still be written") + }) + } +} + +// TestDiscoveryFailure_TellsTheOperatorWhatWentWrong covers the button, not the +// save. Pressing "Load models from provider" against a bad key used to answer +// "internal server error", which names neither the thing that failed nor +// anything the operator could act on — every outcome here is their key or their +// URL. +func TestDiscoveryFailure_TellsTheOperatorWhatWentWrong(t *testing.T) { + cases := map[string]struct { + err error + want string + }{ + "refused credential": { + err: &modeldiscovery.VendorStatusError{Provider: "Bedrock", Status: 403}, + want: "the provider rejected the credential", + }, + "upstream that is not the api": { + err: &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 404}, + want: "the upstream url did not answer a model listing", + }, + "upstream that does not resolve": { + err: &modeldiscovery.UnreachableError{ + Provider: "OpenAI", + Err: &net.DNSError{Err: "no such host", Name: "api.example.com", IsNotFound: true}, + }, + want: "the upstream url could not be reached: no such host", + }, + "vendor having a bad day": { + err: &modeldiscovery.VendorStatusError{Provider: "Anthropic", Status: 503}, + want: "the provider returned an error", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + err := discoveryFailure(context.Background(), "openai_api", tc.err) + require.EqualError(t, err, tc.want) + + var sErr *status.Error + require.ErrorAs(t, err, &sErr) + require.Equal(t, status.InvalidArgument, sErr.Type(), + "a failure the operator caused must not read as a server fault") + }) + } +} + +// TestDiscoveryFailure_LeavesTheCatalogFactsAlone keeps the two outcomes the +// handler already maps. A provider with no listing endpoint is a fact about the +// catalog entry, and the caller falls back to the catalog's own models rather +// than showing an error at all — rewriting it as a refusal would turn a normal +// path into one. +func TestDiscoveryFailure_LeavesTheCatalogFactsAlone(t *testing.T) { + for name, err := range map[string]error{ + "no listing endpoint": modeldiscovery.ErrNoDiscovery, + "bad request": fmt.Errorf("%w: unknown catalog provider", modeldiscovery.ErrInvalidRequest), + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, err, discoveryFailure(context.Background(), "openai_api", err), + "the handler's own mapping must still see the original error") + }) + } +} + +// TestDiscoverProviderModels_SurfacesTheVendorRefusal drives the manager rather +// than the classifier, so a future refactor that stops translating on this path +// fails here rather than silently going back to 500s. +func TestDiscoverProviderModels_SurfacesTheVendorRefusal(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.vendor.err = &modeldiscovery.VendorStatusError{Provider: "OpenAI", Status: 401} + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + _, err := f.manager.DiscoverProviderModels(ctx, "account1", "user1", modeldiscovery.Request{ + CatalogID: "openai_api", + UpstreamURL: "https://api.openai.com", + APIKey: "sk-wrong", + }, "") + + require.EqualError(t, err, "the provider rejected the credential") +} + +// TestDiscoverProviderModels_ListsAgainstTheUrlOnTheForm covers the edit the +// operator cannot otherwise make: the upstream has been retyped and the +// credential has not, because the API never returned it to be retyped. Naming +// the record supplies the key; the request supplies the URL under test. +func TestDiscoverProviderModels_ListsAgainstTheUrlOnTheForm(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + // Twice: the create, and the listing, which is gated on Create too. + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + f.vendor.requests = nil + + _, err = f.manager.DiscoverProviderModels(ctx, "account1", "user1", modeldiscovery.Request{ + CatalogID: "openai_api", + UpstreamURL: "https://gateway.example.com", + }, created.ID) + require.NoError(t, err) + + asked := f.vendor.only(t) + require.Equal(t, "https://gateway.example.com", asked.UpstreamURL, "the typed url must be the one listed against") + require.Equal(t, "sk-good", asked.APIKey, "and the stored key must be what lists it") +} + +// TestDiscoverProviderModels_FallsBackToTheStoredUrl keeps the plain refresh +// working: a request naming only the record still reaches the saved upstream. +func TestDiscoverProviderModels_FallsBackToTheStoredUrl(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + stored := f.vendor.only(t).UpstreamURL + f.vendor.requests = nil + + _, err = f.manager.DiscoverProviderModels(ctx, "account1", "user1", modeldiscovery.Request{ + CatalogID: "openai_api", + }, created.ID) + require.NoError(t, err) + + require.Equal(t, stored, f.vendor.only(t).UpstreamURL) +} + +// TestUpdateProvider_MovingARecordToAnotherVendorIsChecked covers the edit that +// changes neither field the vendor judges and still invalidates both. The +// catalog entry decides which vendor is asked and under which auth header, so +// the unchanged credential is now being offered somewhere it has never been +// accepted. +func TestUpdateProvider_MovingARecordToAnotherVendorIsChecked(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + created, err := f.manager.CreateProvider(ctx, "user1", newCheckedProvider("account1")) + require.NoError(t, err) + f.vendor.requests = nil + + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Update, true) + edit := newCheckedProvider("account1") + edit.ID = created.ID + edit.ProviderID = "anthropic_api" + edit.APIKey = "" + + _, err = f.manager.UpdateProvider(ctx, "user1", edit) + require.NoError(t, err) + + require.Equal(t, "anthropic_api", f.vendor.only(t).CatalogID, + "the new vendor is the one that has to accept the key") +} + +// TestCreateProvider_ASkipTlsRecordIsNotCheckedAgainstItsCertificate covers the +// lockout the check would otherwise be: the flag exists for a self-hosted +// endpoint behind a certificate nothing public can verify, and discovery +// verifies certificates. Refusing the save would reject the record for the one +// reason the operator already declared they accept. +func TestCreateProvider_ASkipTlsRecordIsNotCheckedAgainstItsCertificate(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.vendor.err = &modeldiscovery.UnreachableError{ + Provider: "OpenAI", + Err: &tls.CertificateVerificationError{}, + } + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + provider := newCheckedProvider("account1") + provider.SkipTLSVerification = true + + created, err := f.manager.CreateProvider(ctx, "user1", provider) + require.NoError(t, err, "a record we were told not to verify must still save") + require.NotEmpty(t, created.ID) + require.Zero(t, f.vendor.calls(), "and the vendor must not be asked at all") +} + +// TestCreateProvider_TheStoredKeyIsTheOneThatWasChecked pins the two halves to +// one value. The vendor call trims the credential before building its auth +// header; the synthesiser substitutes the stored one verbatim. A key pasted +// with surrounding whitespace would otherwise pass its check and then fail +// every request the provider serves. +func TestCreateProvider_TheStoredKeyIsTheOneThatWasChecked(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + provider := newCheckedProvider("account1") + provider.APIKey = " sk-good\n" + + created, err := f.manager.CreateProvider(ctx, "user1", provider) + require.NoError(t, err) + + require.Equal(t, "sk-good", f.vendor.only(t).APIKey, "the vendor is asked about the trimmed key") + + stored, err := f.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, "account1", created.ID) + require.NoError(t, err) + require.Equal(t, "sk-good", stored.APIKey, "and that is the one the proxy will send") +} + +// TestUpdateProvider_TurningTlsVerificationBackOnChecksTheRecord covers the +// hole the skip-TLS exemption opens on its own. Such a record is stored without +// ever being checked, so the moment verification is switched back on is the +// first moment it can be checked at all — and none of the three fields the +// re-check usually watches has to move for that to happen. +func TestUpdateProvider_TurningTlsVerificationBackOnChecksTheRecord(t *testing.T) { + ctx := context.Background() + f := newBootstrapFixture(t) + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) + + unchecked := newCheckedProvider("account1") + unchecked.SkipTLSVerification = true + created, err := f.manager.CreateProvider(ctx, "user1", unchecked) + require.NoError(t, err) + require.Zero(t, f.vendor.calls(), "the create was exempt") + + f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Update, true) + edit := newCheckedProvider("account1") + edit.ID = created.ID + edit.APIKey = "" + edit.SkipTLSVerification = false + + _, err = f.manager.UpdateProvider(ctx, "user1", edit) + require.NoError(t, err) + require.Equal(t, 1, f.vendor.calls(), "switching verification on must check what was never checked") +} diff --git a/management/internals/modules/agentnetwork/handlers/providers_handler.go b/management/internals/modules/agentnetwork/handlers/providers_handler.go index ef4b93dac..81a6d6eb0 100644 --- a/management/internals/modules/agentnetwork/handlers/providers_handler.go +++ b/management/internals/modules/agentnetwork/handlers/providers_handler.go @@ -340,6 +340,14 @@ func validate(req *api.AgentNetworkProviderRequest, requireAPIKey bool) error { if requireAPIKey && (req.ApiKey == nil || strings.TrimSpace(*req.ApiKey) == "") { return status.Errorf(status.InvalidArgument, "api_key is required") } + // An update omits api_key to keep the stored credential. A key that is + // present but blank is not that: Provider.FromAPIRequest drops it exactly + // as if it were absent, so a rotation the operator believes they performed + // would answer 200 having changed nothing. Refuse it here, where the + // request still carries the difference between absent and blank. + if req.ApiKey != nil && strings.TrimSpace(*req.ApiKey) == "" { + return status.Errorf(status.InvalidArgument, "api_key must be omitted to keep the stored credential rather than sent blank") + } if req.Models != nil { for i, m := range *req.Models { if err := validateModel(i, m); err != nil { diff --git a/management/internals/modules/agentnetwork/handlers/providers_handler_test.go b/management/internals/modules/agentnetwork/handlers/providers_handler_test.go index 05024cde9..033de3b8a 100644 --- a/management/internals/modules/agentnetwork/handlers/providers_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/providers_handler_test.go @@ -54,6 +54,39 @@ func TestValidate_ModelRates(t *testing.T) { } } +// TestValidate_ABlankApiKeyIsNotTheSameAsAnOmittedOne covers the one shape the +// manager's own guard cannot see. Provider.FromAPIRequest assigns the key only +// when it trims to something, so a request carrying " " arrives at +// UpdateProvider indistinguishable from one that omitted it — the stored +// credential is kept and the write answers 200, telling an operator who thinks +// they just rotated a key that it worked. +// +// The request still knows the difference, so the refusal belongs here. +func TestValidate_ABlankApiKeyIsNotTheSameAsAnOmittedOne(t *testing.T) { + req := func(key *string) *api.AgentNetworkProviderRequest { + return &api.AgentNetworkProviderRequest{ + ProviderId: "openai_api", + Name: "OpenAI", + UpstreamUrl: "https://api.openai.com", + ApiKey: key, + } + } + + blank := " " + err := validate(req(&blank), false) + require.Error(t, err, "a blank api_key on update must not be read as 'keep what is stored'") + assert.Contains(t, err.Error(), "api_key") + + require.NoError(t, validate(req(nil), false), "an omitted api_key is how an update keeps the stored credential") + + // Create already refuses this, and keeps its own message: a caller who sent + // no usable key is told the field is required rather than being told how to + // preserve a credential that does not exist yet. + err = validate(req(&blank), true) + require.Error(t, err) + assert.Contains(t, err.Error(), "api_key is required") +} + // TestProviderHandler_UpdateReplacesFullState pins the update contract shared // with the other PUT endpoints: the request replaces the provider's mutable // state, so optional fields absent from the JSON land as their zero values. @@ -64,10 +97,13 @@ func TestValidate_ModelRates(t *testing.T) { func TestProviderHandler_UpdateReplacesFullState(t *testing.T) { f := newAgentNetworkHandlerFixture(t) + // A private upstream: the save-time credential check leaves it unchecked + // rather than spending "sk-test" against the real api.openai.com, which + // the vendor refuses. create := `{ "provider_id": "openai_api", "name": "openai", - "upstream_url": "https://api.openai.com", + "upstream_url": "https://10.255.255.1", "api_key": "sk-test", "enabled": true, "metadata_disabled": true, @@ -84,7 +120,7 @@ func TestProviderHandler_UpdateReplacesFullState(t *testing.T) { // Minimal update: only the required fields, no api_key. Everything // optional must land as its zero value. - update := `{"provider_id": "openai_api", "name": "openai-renamed", "upstream_url": "https://api.openai.com", "enabled": true}` + update := `{"provider_id": "openai_api", "name": "openai-renamed", "upstream_url": "https://10.255.255.1", "enabled": true}` rec = f.do(t, nethttp.MethodPut, "/agent-network/providers/"+created.Id, update) require.Equal(t, nethttp.StatusOK, rec.Code, "update without api_key must succeed (key is preserved): %s", rec.Body.String()) diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen.go b/management/internals/modules/agentnetwork/labelgen/labelgen.go index 549767096..bd5b0129d 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen.go @@ -3,9 +3,10 @@ package labelgen import ( "fmt" - "math/rand" "sort" "sync" + + "github.com/netbirdio/netbird/management/server/util" ) // pickAttempts caps the random retries before falling back to the @@ -40,16 +41,15 @@ func uniqueWords() []string { // PickUnique selects a label not already in `taken`. It tries up to // pickAttempts random picks; on exhaustion it scans the deduplicated // wordlist for any remaining free entry, and if none is left appends -// `-` to a deterministic word and returns. The caller -// is responsible for seeding rng (math/rand). -func PickUnique(rng *rand.Rand, taken map[string]struct{}, fallbackSuffix string) string { +// `-` to a random word and returns. +func PickUnique(taken map[string]struct{}, fallbackSuffix string) string { pool := uniqueWords() if len(pool) == 0 { return fallbackSuffix } for i := 0; i < pickAttempts; i++ { - w := pool[rng.Intn(len(pool))] + w := pool[util.RandIntn(len(pool))] if _, ok := taken[w]; !ok { return w } @@ -61,7 +61,7 @@ func PickUnique(rng *rand.Rand, taken map[string]struct{}, fallbackSuffix string } } - w := pool[rng.Intn(len(pool))] + w := pool[util.RandIntn(len(pool))] return fmt.Sprintf("%s-%s", w, fallbackSuffix) } @@ -74,10 +74,10 @@ func PickUnique(rng *rand.Rand, taken map[string]struct{}, fallbackSuffix string // a noun spans len(adjectives) * 857 instead. Uniqueness is enforced by a // database constraint and retried by the caller, rather than guessed from a // pre-read set that a concurrent allocation can invalidate. -func PickTuple(rng *rand.Rand) string { +func PickTuple() string { nouns := uniqueWords() if len(nouns) == 0 || len(adjectives) == 0 { return "" } - return adjectives[rng.Intn(len(adjectives))] + "-" + nouns[rng.Intn(len(nouns))] + return adjectives[util.RandIntn(len(adjectives))] + "-" + nouns[util.RandIntn(len(nouns))] } diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go index 7e12fc133..dda0f09ba 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go @@ -1,7 +1,7 @@ package labelgen import ( - "math/rand" + "slices" "strings" "testing" @@ -9,19 +9,12 @@ import ( "github.com/stretchr/testify/require" ) -// TestPickUnique_DeterministicWithSeededRng locks the property the -// caller relies on: same seed + same taken set → same pick. Without -// that, the bootstrap flow can't reproduce a label across retries. -func TestPickUnique_DeterministicWithSeededRng(t *testing.T) { - taken := map[string]struct{}{} +// TestPickUnique_ReturnsWordFromPool confirms a pick against an empty +// taken set is always drawn verbatim from the wordlist. +func TestPickUnique_ReturnsWordFromPool(t *testing.T) { + got := PickUnique(map[string]struct{}{}, "abcd") - rngA := rand.New(rand.NewSource(42)) - rngB := rand.New(rand.NewSource(42)) - - a := PickUnique(rngA, taken, "abcd") - b := PickUnique(rngB, taken, "abcd") - - assert.Equal(t, a, b, "Same seed and taken set must produce identical pick") + assert.True(t, slices.Contains(uniqueWords(), got), "Pick %q must be drawn from the wordlist", got) } // TestPickUnique_AvoidsTakenWordsWhenMostAreReserved seeds taken with @@ -46,8 +39,7 @@ func TestPickUnique_AvoidsTakenWordsWhenMostAreReserved(t *testing.T) { taken[w] = struct{}{} } - rng := rand.New(rand.NewSource(7)) - got := PickUnique(rng, taken, "abcd") + got := PickUnique(taken, "abcd") _, isFree := free[got] assert.True(t, isFree, "PickUnique must return one of the free words; got %q", got) @@ -65,8 +57,7 @@ func TestPickUnique_FallsBackWhenAllReserved(t *testing.T) { taken[w] = struct{}{} } - rng := rand.New(rand.NewSource(99)) - got := PickUnique(rng, taken, "abcd") + got := PickUnique(taken, "abcd") assert.True(t, strings.HasSuffix(got, "-abcd"), "Exhausted pool must produce -; got %q", got) @@ -114,9 +105,8 @@ func TestPickTuple_ShapeAndPoolMembership(t *testing.T) { inAdjectives[a] = struct{}{} } - rng := rand.New(rand.NewSource(7)) for i := 0; i < 200; i++ { - got := PickTuple(rng) + got := PickTuple() parts := strings.Split(got, "-") require.Len(t, parts, 2, "PickTuple must produce exactly two hyphen-joined words; got %q", got) @@ -158,22 +148,13 @@ func TestAdjectives_AreDNSSafeAndDeduplicated(t *testing.T) { assert.Greater(t, len(adjectives), 150, "Adjective pool too small to give a useful namespace") } -// TestPickTuple_DeterministicWithSeededRng documents that generation is a pure -// function of the rng, which is what makes allocation retries reproducible in tests. -func TestPickTuple_DeterministicWithSeededRng(t *testing.T) { - a := PickTuple(rand.New(rand.NewSource(42))) - b := PickTuple(rand.New(rand.NewSource(42))) - assert.Equal(t, a, b, "Same seed must yield the same tuple") -} - // TestPickTuple_SpansALargeNamespace guards the reason we moved to tuples: a // single-word pool caps the GLOBAL namespace at 857. Drawing many tuples must // yield overwhelmingly distinct values. func TestPickTuple_SpansALargeNamespace(t *testing.T) { - rng := rand.New(rand.NewSource(11)) seen := make(map[string]struct{}, 2000) for i := 0; i < 2000; i++ { - seen[PickTuple(rng)] = struct{}{} + seen[PickTuple()] = struct{}{} } assert.Greater(t, len(seen), 1900, "2000 draws should be nearly all distinct across a ~200k namespace; got %d unique", len(seen)) diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index 98aca7f5d..efcc944be 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "slices" "strings" "sync" @@ -133,24 +132,34 @@ type managerImpl struct { proxyController proxy.Controller // modelDiscovery queries vendors for the models a credential can reach. - // A field rather than a package call so tests can drive it without - // reaching the network. + // An interface rather than the concrete client because it is now on a + // write path: the credential check runs inside CreateProvider and + // UpdateProvider, so every test that saves a provider would otherwise + // reach a vendor over the network to do it. // // One instance serves every request for the process's lifetime, so its // fields must stay read-only after construction: lazy initialisation // inside Fetch or httpClient would race across request goroutines. - modelDiscovery *modeldiscovery.Client + modelDiscovery ModelLister // reconcileCache holds the last set of synthesised proxy mappings // per account, each paired with the proxy that served it, so a change // of serving proxy can be diffed without re-deriving it. reconcileMu sync.Mutex reconcileCache map[string]map[string]syntheticMapping +} - // labelRngMu guards labelRng. PickUnique consumes math/rand.Source - // state; concurrent provider creates would otherwise race. - labelRngMu sync.Mutex - labelRng *rand.Rand +// ManagerOption replaces a manager dependency at construction. Production +// passes none; each option exists for something a test cannot let run for +// real. +type ManagerOption func(*managerImpl) + +// WithModelLister replaces the vendor call behind the provider credential +// check. A test that saves a provider needs this — the check runs inside +// CreateProvider and UpdateProvider, so the write path reaches a vendor +// without it. +func WithModelLister(lister ModelLister) ManagerOption { + return func(m *managerImpl) { m.modelDiscovery = lister } } // NewManager constructs the persistent Agent Network manager. The @@ -163,16 +172,20 @@ func NewManager( permissionsManager permissions.Manager, accountManager account.Manager, proxyController proxy.Controller, + opts ...ManagerOption, ) Manager { - return &managerImpl{ + m := &managerImpl{ store: store, accountManager: accountManager, permissionsManager: permissionsManager, proxyController: proxyController, modelDiscovery: &modeldiscovery.Client{}, reconcileCache: make(map[string]map[string]syntheticMapping), - labelRng: rand.New(rand.NewSource(time.Now().UnixNano())), } + for _, opt := range opts { + opt(m) + } + return m } // GetAllProviders returns the account's providers for callers holding the @@ -297,9 +310,11 @@ func (m *managerImpl) redactProvidersForViewer(ctx context.Context, accountID, u // DiscoverProviderModels asks the vendor which models a credential can reach. // -// recordID, when set, names an existing provider whose stored credential and -// upstream are used instead of the ones in req — so the dashboard can refresh -// the list without ever holding the key. +// recordID, when set, names an existing provider whose stored credential is +// used instead of the one in req — so the dashboard can refresh the list +// without ever holding the key. An upstream in req overrides the stored one, +// which is what lets a form list against a URL the operator has typed but not +// saved yet, using the credential they cannot retype. // // Gated on Create rather than Read: this spends the operator's credential // against a third party, which is not something a read-only role should be @@ -320,11 +335,29 @@ func (m *managerImpl) DiscoverProviderModels(ctx context.Context, accountID, use // name a different one would run a provider's credential against // whichever vendor endpoint they picked. req.CatalogID = record.ProviderID - req.UpstreamURL = record.UpstreamURL req.APIKey = record.APIKey + // The upstream is the one field the caller may override, so that a URL + // typed into the form can be listed against before it is saved. + // + // It sends the stored credential to a host the caller named, which is + // a capability they already have: the same permission set updates the + // record's upstream, and that write runs this same check against + // whatever it is pointed at. What it would not otherwise be is silent, + // since the write leaves an activity event behind — so the override is + // recorded here. + if strings.TrimSpace(req.UpstreamURL) == "" { + req.UpstreamURL = record.UpstreamURL + } else if req.UpstreamURL != record.UpstreamURL { + log.WithContext(ctx).Infof("agent network provider %s listed against caller-supplied upstream %s by user %s", + recordID, req.UpstreamURL, userID) + } } - return m.modelDiscovery.Fetch(ctx, req) + models, err := m.modelDiscovery.Fetch(ctx, req) + if err != nil { + return nil, discoveryFailure(ctx, req.CatalogID, err) + } + return models, nil } // CreateProvider persists a new provider for the account. Providers have no @@ -342,6 +375,18 @@ func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provide if strings.TrimSpace(provider.APIKey) == "" { return nil, status.Errorf(status.InvalidArgument, "api_key is required when creating an agent network provider") } + // Stored as it will be sent. The vendor call below trims the key before + // building the auth header while the synthesiser substitutes the stored + // value verbatim, so a key pasted with surrounding whitespace would pass + // its check and then fail every request the provider serves. + provider.APIKey = strings.TrimSpace(provider.APIKey) + + // Before anything is persisted: a record whose upstream or credential does + // not work is rejected here rather than discovered later as a failed + // request with nothing pointing back at it. + if err := m.checkProviderCredential(ctx, provider); err != nil { + return nil, err + } if provider.ID == "" { fresh := types.NewProvider(provider.AccountID) @@ -377,11 +422,47 @@ func (m *managerImpl) UpdateProvider(ctx context.Context, userID string, provide // Preserve the API key if the caller didn't rotate it. A // whitespace-only value is treated as "not rotated" rather than a // real key, but it must not silently overwrite a valid stored key. - if provider.APIKey == "" { - provider.APIKey = existing.APIKey - } else if strings.TrimSpace(provider.APIKey) == "" { + switch trimmed := strings.TrimSpace(provider.APIKey); { + case provider.APIKey == "": + // Trimmed on the way through: a record stored before keys were + // normalised carries whitespace the proxy still sends, and an edit + // that preserves the key is the occasion to repair it. Doing so makes + // the comparison below see a change, which is correct — that key has + // never been tested in the form it is about to be sent in. + provider.APIKey = strings.TrimSpace(existing.APIKey) + case trimmed == "": return nil, status.Errorf(status.InvalidArgument, "api_key must be non-blank when rotating an agent network provider") + default: + // See CreateProvider: the key is stored in the form the proxy will + // send, so the check below tests what the provider will actually use. + provider.APIKey = trimmed } + + // Only the fields the vendor would judge are worth a round-trip. This same + // call carries renames, model rows and price edits, and none of those + // should wait on a vendor — or be refused because one is having a bad day. + // + // The catalog entry counts as one of them: it decides which vendor is + // asked, under which auth header, so moving a record from one to another + // sends an unchanged credential somewhere it has never been accepted. + // + // The comparison runs after the merge above, so an update that changes only + // the URL reads as unchanged on the key and is checked against the stored + // one, which is the only credential the operator has to offer here. + // + // Turning TLS verification back on is the fourth: the record was stored + // unchecked precisely because that flag was set, so this is the first + // moment it can be checked at all, and nothing else about it need change + // for that to be true. + if provider.UpstreamURL != existing.UpstreamURL || + provider.APIKey != existing.APIKey || + provider.ProviderID != existing.ProviderID || + (existing.SkipTLSVerification && !provider.SkipTLSVerification) { + if err := m.checkProviderCredential(ctx, provider); err != nil { + return nil, err + } + } + // Always preserve the session keypair across updates so existing // session cookies stay valid. The keys are server-managed and // never surfaced through the API. @@ -986,9 +1067,7 @@ func (m *managerImpl) bootstrapLabeled(ctx context.Context, settings *types.Sett } for attempt := 1; attempt <= maxDomainAllocationAttempts; attempt++ { - m.labelRngMu.Lock() - label := labelgen.PickTuple(m.labelRng) - m.labelRngMu.Unlock() + label := labelgen.PickTuple() if label == "" { // Only reachable if either word pool were emptied. An empty label // would produce a broken endpoint like ".example.com", so fail diff --git a/management/internals/modules/agentnetwork/modeldiscovery/discovery.go b/management/internals/modules/agentnetwork/modeldiscovery/discovery.go index c9f2b09df..d9779a084 100644 --- a/management/internals/modules/agentnetwork/modeldiscovery/discovery.go +++ b/management/internals/modules/agentnetwork/modeldiscovery/discovery.go @@ -123,14 +123,28 @@ func (c *Client) Fetch(ctx context.Context, req Request) ([]Model, error) { return nil, ErrNoDiscovery } - endpoint, err := c.discoveryURL(entry, req) + // One deadline over the whole operation. Both host lookups and the request + // itself run under it, so a vendor cannot be slow twice, and a caller that + // gives up is not left waiting on a resolver. + ctx, cancel := context.WithTimeout(ctx, fetchTimeout) + defer cancel() + + // An entry with a listing host of its own answers from somewhere other + // than the upstream on the record — Bedrock lists from the control plane + // and infers on the runtime host. Reaching the listing therefore proves + // nothing about the host requests will actually go to, so that one is + // checked separately or not at all. + if entry.Discovery.Host != "" { + if err := c.checkUpstreamHost(ctx, entry, req.UpstreamURL); err != nil { + return nil, err + } + } + + endpoint, err := c.discoveryURL(ctx, entry, req) if err != nil { return nil, err } - ctx, cancel := context.WithTimeout(ctx, fetchTimeout) - defer cancel() - httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, fmt.Errorf("build discovery request: %w", err) @@ -145,7 +159,7 @@ func (c *Client) Fetch(ctx context.Context, req Request) ([]Model, error) { resp, err := c.httpClient().Do(httpReq) if err != nil { - return nil, fmt.Errorf("reach %s: %w", entry.Name, err) + return nil, &UnreachableError{Provider: entry.Name, Err: err} } defer func() { _ = resp.Body.Close() }() @@ -156,7 +170,7 @@ func (c *Client) Fetch(ctx context.Context, req Request) ([]Model, error) { if resp.StatusCode != http.StatusOK { // Surface the vendor's own status. An operator whose key lacks a scope // needs to see 403 rather than a generic failure. - return nil, fmt.Errorf("%s returned %d for its model listing", entry.Name, resp.StatusCode) + return nil, &VendorStatusError{Provider: entry.Name, Status: resp.StatusCode} } ids, err := parseListing(entry.Discovery.Shape, body) @@ -175,12 +189,16 @@ func (c *Client) Fetch(ctx context.Context, req Request) ([]Model, error) { // management holds credentials for every provider, and an upstream pointed at // an internal address would turn this endpoint into a probe of the management // server's own network. -func (c *Client) discoveryURL(entry catalog.Provider, req Request) (string, error) { +func (c *Client) discoveryURL(ctx context.Context, entry catalog.Provider, req Request) (string, error) { host := entry.Discovery.Host if host == "" { parsed, err := url.Parse(strings.TrimSpace(req.UpstreamURL)) if err != nil || parsed.Host == "" { - return "", fmt.Errorf("%w: provider upstream %q is not a usable URL", ErrInvalidRequest, req.UpstreamURL) + // The URL is left out of the message on purpose: it reaches the + // operator through an endpoint that does not lowercase it, but the + // rest of this feature's copy never echoes what they typed, and one + // path that does is the one that ends up quoted in a bug report. + return "", fmt.Errorf("%w: the provider upstream is not a usable URL", ErrInvalidRequest) } host = parsed.Host } @@ -193,19 +211,47 @@ func (c *Client) discoveryURL(entry catalog.Provider, req Request) (string, erro region = RegionFromUpstream(entry, req.UpstreamURL) } if region == "" { - return "", fmt.Errorf("%w: %s discovery needs a region, and none could be read from the provider upstream", - ErrInvalidRequest, entry.Name) + return "", fmt.Errorf("%w: %w: %s discovery needs a region, and none could be read from the provider upstream", + ErrInvalidRequest, ErrNoDiscoveryHost, entry.Name) } host = strings.ReplaceAll(host, catalog.RegionPlaceholder, region) } target := &url.URL{Scheme: "https", Host: host, Path: entry.Discovery.Path, RawQuery: entry.Discovery.Query} - if err := c.checkPublicHost(target.Hostname()); err != nil { + if err := c.classifyHost(ctx, entry, target.Hostname()); err != nil { return "", err } return target.String(), nil } +// checkUpstreamHost verifies the host the operator configured, for entries +// whose listing lives elsewhere and so cannot vouch for it. +// +// A name that does not resolve is the record being wrong. One that resolves +// privately is not: an upstream behind a proxy is a supported configuration, +// and ErrPrivateHost carries that difference on to the caller, which treats it +// as unverifiable rather than as a failure. +func (c *Client) checkUpstreamHost(ctx context.Context, entry catalog.Provider, upstreamURL string) error { + parsed, err := url.Parse(strings.TrimSpace(upstreamURL)) + if err != nil || parsed.Hostname() == "" { + return fmt.Errorf("%w: the provider upstream is not a usable URL", ErrInvalidRequest) + } + return c.classifyHost(ctx, entry, parsed.Hostname()) +} + +// classifyHost renders a failed host check as the two outcomes the caller +// distinguishes. A host that refuses to resolve is the commonest way for an +// upstream to be wrong and has to arrive as unreachable rather than as an +// unclassified fault. ErrPrivateHost means something else entirely — not a bad +// host, one we decline to dial. +func (c *Client) classifyHost(ctx context.Context, entry catalog.Provider, host string) error { + err := c.checkPublicHost(ctx, host) + if err == nil || errors.Is(err, ErrPrivateHost) { + return err + } + return &UnreachableError{Provider: entry.Name, Err: err} +} + // RegionFromUpstream recovers the region an operator embedded in the provider // upstream, by matching it against the catalog's own host template. Bedrock's // template is "bedrock-runtime..amazonaws.com" and Vertex's is @@ -243,7 +289,7 @@ func RegionFromUpstream(entry catalog.Provider, upstreamURL string) string { // checkPublicHost refuses hosts that resolve to an address the management // server should never be asked to reach on an operator's behalf. -func (c *Client) checkPublicHost(host string) error { +func (c *Client) checkPublicHost(ctx context.Context, host string) error { if c.AllowPrivateHosts { return nil } @@ -254,9 +300,6 @@ func (c *Client) checkPublicHost(host string) error { if resolver == nil { resolver = net.DefaultResolver } - ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) - defer cancel() - addrs, err := resolver.LookupNetIP(ctx, "ip", host) if err != nil { return fmt.Errorf("resolve discovery host %q: %w", host, err) @@ -265,7 +308,7 @@ func (c *Client) checkPublicHost(host string) error { // loopback address is still a way to reach loopback. for _, addr := range addrs { if !isPublic(addr) { - return fmt.Errorf("%w: discovery host %q resolves to a non-public address", ErrInvalidRequest, host) + return fmt.Errorf("%w: %w: discovery host %q resolves to a non-public address", ErrInvalidRequest, ErrPrivateHost, host) } } return nil @@ -465,6 +508,13 @@ func guardDialAddress(address string) error { return fmt.Errorf("discovery dial address %q is not an IP", host) } if !isPublic(addr) { + // Deliberately not ErrPrivateHost, which means "this upstream is on a + // private network, so we cannot check it" and lets a save through + // unchecked. checkPublicHost has already cleared the target by the + // time anything is dialled, so an address refused here is not the + // operator's upstream: it is a rebinding attempt, or an HTTP proxy in + // the path. Neither may quietly skip the check — one is hostile, and + // the other would silently disable this on every provider. return fmt.Errorf("discovery refused to dial non-public address %s", addr) } return nil diff --git a/management/internals/modules/agentnetwork/modeldiscovery/discovery_test.go b/management/internals/modules/agentnetwork/modeldiscovery/discovery_test.go index 59b21a2fe..62b34e754 100644 --- a/management/internals/modules/agentnetwork/modeldiscovery/discovery_test.go +++ b/management/internals/modules/agentnetwork/modeldiscovery/discovery_test.go @@ -2,7 +2,9 @@ package modeldiscovery import ( "context" + "errors" "io" + "net" "net/http" "net/http/httptest" "net/netip" @@ -315,7 +317,7 @@ func TestHostGuardRejectsNonPublicAddresses(t *testing.T) { func TestHostGuardResolvesAndRejectsLocalhost(t *testing.T) { cl := &Client{} - err := cl.checkPublicHost("localhost") + err := cl.checkPublicHost(context.Background(), "localhost") require.Error(t, err, "a name resolving to loopback must be refused, not just a literal address") assert.Contains(t, err.Error(), "non-public") } @@ -557,3 +559,120 @@ func TestBedrockProfilesFromAnyGeographyArrivePriced(t *testing.T) { // only form that works at invoke time. assert.Equal(t, "jp.anthropic.claude-sonnet-5-20260514-v1:0", models[0].ID) } + +// TestFetch_AHostThatWillNotResolveIsUnreachable closes a gap the live suite +// found. The SSRF guard resolves the host before any request is built, so a +// name that does not resolve fails there rather than at the transport — and +// that error used to reach the caller unclassified. A wrong hostname is the +// commonest way for an upstream to be wrong, so it has to arrive as +// "unreachable" and not as an unrecognised fault. +func TestFetch_AHostThatWillNotResolveIsUnreachable(t *testing.T) { + // A resolver whose dial always fails, so the lookup errors without the + // test depending on real DNS. + refusing := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + return nil, errors.New("resolver unavailable") + }, + } + client := &Client{Resolver: refusing} + + _, err := client.Fetch(context.Background(), Request{ + CatalogID: "openai_api", + UpstreamURL: "https://not-a-real-vendor-host.example.invalid", + APIKey: "sk-test", + }) + + require.Error(t, err) + var unreachable *UnreachableError + require.ErrorAs(t, err, &unreachable, "a host that will not resolve must classify as unreachable") + require.NotErrorIs(t, err, ErrPrivateHost, "it is not a host we declined to dial") +} + +// TestFetch_AProxyInThePathDoesNotSilentlyDisableTheCheck pins a fail-open the +// dial-time guard can produce. checkPublicHost clears the target before +// anything is dialled, so a private address refused at the socket is never the +// operator's upstream — it is a rebinding attempt, or an HTTP proxy the +// management server egresses through. Reporting either as ErrPrivateHost would +// read as "this provider cannot be checked" and let every save through +// unchecked, which is how a proxied deployment would install this feature and +// have it quietly do nothing. +func TestFetch_AProxyInThePathDoesNotSilentlyDisableTheCheck(t *testing.T) { + // A transport that refuses at the socket exactly as the guard does, with a + // loopback address standing in for the proxy the dial went to. + // AllowPrivateHosts short-circuits the resolve-stage check only; the + // injected transport below is still what the request goes through. Without + // it this test resolves api.openai.com for real, and on a runner with no + // egress that lookup fails as an UnreachableError too — so it would pass + // while never reaching the socket guard it is named for. + client := &Client{AllowPrivateHosts: true, HTTPClient: &http.Client{ + Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, guardDialAddress("127.0.0.1:38599") + }), + CheckRedirect: refuseRedirect, + }} + + _, err := client.Fetch(context.Background(), Request{ + CatalogID: "openai_api", + UpstreamURL: "https://api.openai.com", + APIKey: "sk-test", + }) + + require.Error(t, err) + require.NotErrorIs(t, err, ErrPrivateHost, + "a refusal at the socket must not read as an upstream we cannot check") + var unreachable *UnreachableError + require.ErrorAs(t, err, &unreachable, "it is the vendor we failed to reach") +} + +// roundTripFunc adapts a function to http.RoundTripper. +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// TestFetch_TheUpstreamIsCheckedWhenTheListingCannotVouchForIt covers the hole +// a separate listing host leaves. Bedrock lists from the control plane, so a +// record whose runtime upstream does not exist reaches a perfectly good +// listing and saves — the requests it then serves go nowhere. +// +// Both halves matter. A runtime host that cannot be resolved is the record +// being wrong, and blocks. A proxied one resolves and only leaves the region +// underivable, which stays the unverifiable outcome it already was. +func TestFetch_TheUpstreamIsCheckedWhenTheListingCannotVouchForIt(t *testing.T) { + refusing := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + return nil, errors.New("resolver unavailable") + }, + } + client := &Client{Resolver: refusing} + + _, err := client.Fetch(context.Background(), Request{ + CatalogID: "bedrock_api", + // Matches no catalog template, so nothing here reaches the control + // plane the listing comes from: without its own check this upstream + // was never contacted at all. + UpstreamURL: "https://bedrock.typo.example.invalid", + APIKey: "aws-bearer", + }) + + require.Error(t, err) + var unreachable *UnreachableError + require.ErrorAs(t, err, &unreachable, "a runtime host that will not resolve must block the save") +} + +// TestFetch_AListingHostOfItsOwnDoesNotReachThroughTheUpstream keeps the check +// above from reading the operator's upstream as the place to list from. +func TestFetch_AListingHostOfItsOwnDoesNotReachThroughTheUpstream(t *testing.T) { + cl, tr := newStubClient(http.StatusOK, bedrockListing) + + _, err := cl.Fetch(context.Background(), Request{ + CatalogID: "bedrock_api", + UpstreamURL: "https://bedrock-runtime.eu-central-1.amazonaws.com", + APIKey: "aws-bearer", + }) + require.NoError(t, err) + + assert.Equal(t, "bedrock.eu-central-1.amazonaws.com", tr.got.URL.Host, + "checking the runtime host must not turn it into the listing host") +} diff --git a/management/internals/modules/agentnetwork/modeldiscovery/failure.go b/management/internals/modules/agentnetwork/modeldiscovery/failure.go new file mode 100644 index 000000000..41e4bfb56 --- /dev/null +++ b/management/internals/modules/agentnetwork/modeldiscovery/failure.go @@ -0,0 +1,114 @@ +package modeldiscovery + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "os" + "syscall" +) + +// Fetch serves two callers with different needs: the model picker, which only +// needs to know it failed, and the provider credential check, which has to +// tell an operator whether the URL or the key is at fault. Each failure +// carries a type so the second does not have to branch on a message. + +// VendorStatusError reports a listing answered with something other than 200. +// Only the vendor's own code separates a refused credential (401, 403) from a +// URL that does not serve this API (404, 405) from an unwell vendor (5xx). +type VendorStatusError struct { + Provider string + Status int +} + +func (e *VendorStatusError) Error() string { + return fmt.Sprintf("%s returned %d for its model listing", e.Provider, e.Status) +} + +// UnreachableError reports that the request never reached the vendor: the +// name did not resolve, the connection was refused, TLS failed, or it timed +// out. Nothing was authenticated, so only the URL is implicated. +type UnreachableError struct { + Provider string + Err error +} + +func (e *UnreachableError) Error() string { + return fmt.Sprintf("reach %s: %v", e.Provider, e.Err) +} + +func (e *UnreachableError) Unwrap() error { return e.Err } + +// Reason names the transport failure in words an operator can act on: a wrong +// port and a wrong hostname fail differently and are worth telling apart. +// Empty means unrecognised, and the caller should say only that the host could +// not be reached rather than paste a Go error into the UI. +func (e *UnreachableError) Reason() string { + err := e.Err + + var dns *net.DNSError + if errors.As(err, &dns) { + if dns.IsNotFound { + return "no such host" + } + // Named apart from the dial timeout below. A resolver that never + // answered and an upstream that never answered send an operator to + // different places, and the generic "connection timed out" would + // describe a connection that was never attempted. + if dns.IsTimeout { + return "dns lookup timed out" + } + return "dns lookup failed" + } + + // Timeouts are checked before the syscall cases: a dial that times out is + // reported as a net.OpError wrapping a timeout, and the operator needs to + // hear "timed out" rather than the syscall underneath it. + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, os.ErrDeadlineExceeded) { + return "connection timed out" + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return "connection timed out" + } + + if errors.Is(err, syscall.ECONNREFUSED) { + return "connection refused" + } + if errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) { + return "host unreachable" + } + + var certErr *tls.CertificateVerificationError + if errors.As(err, &certErr) { + return "tls certificate not trusted" + } + var recordErr tls.RecordHeaderError + if errors.As(err, &recordErr) { + return "not a tls endpoint" + } + + return "" +} + +// ErrUnparseableListing marks a 200 whose body is not a listing in the shape +// the catalog declared. Distinct from a status refusal: the host answered and +// authenticated fine, it is just not the API — a login page, say. +var ErrUnparseableListing = errors.New("response is not a model listing") + +// ErrNoDiscoveryHost marks a provider whose listing host cannot be derived +// from the record: Bedrock's control-plane host comes from the region in the +// upstream, so a proxied endpoint leaves nowhere to send it, and inventing one +// would spend the credential somewhere never configured. +// +// Wraps ErrInvalidRequest so the discovery endpoint still answers 400, while a +// credential check can read it as "cannot be checked" rather than "broken". +var ErrNoDiscoveryHost = errors.New("provider has no derivable discovery host") + +// ErrPrivateHost marks an upstream resolving somewhere management will not +// dial. A self-hosted endpoint on a private network is a legitimate provider +// the proxy reaches through the tunnel, so this means the check cannot run, +// not that the record is wrong. +var ErrPrivateHost = errors.New("discovery host is not publicly routable") diff --git a/management/internals/modules/agentnetwork/modeldiscovery/parse.go b/management/internals/modules/agentnetwork/modeldiscovery/parse.go index 83048cb8a..67a10bf36 100644 --- a/management/internals/modules/agentnetwork/modeldiscovery/parse.go +++ b/management/internals/modules/agentnetwork/modeldiscovery/parse.go @@ -43,7 +43,7 @@ func parseOpenAIData(body []byte) ([]listedModel, error) { } `json:"data"` } if err := json.Unmarshal(body, &doc); err != nil { - return nil, fmt.Errorf("decode model listing: %w", err) + return nil, fmt.Errorf("%w: decode model listing: %w", ErrUnparseableListing, err) } out := make([]listedModel, 0, len(doc.Data)) for _, entry := range doc.Data { @@ -71,7 +71,7 @@ func parseBedrockInferenceProfiles(body []byte) ([]listedModel, error) { } `json:"inferenceProfileSummaries"` } if err := json.Unmarshal(body, &doc); err != nil { - return nil, fmt.Errorf("decode inference-profile listing: %w", err) + return nil, fmt.Errorf("%w: decode inference-profile listing: %w", ErrUnparseableListing, err) } out := make([]listedModel, 0, len(doc.Summaries)) for _, entry := range doc.Summaries { @@ -98,7 +98,7 @@ func parseVertexPublisherModels(body []byte) ([]listedModel, error) { } `json:"publisherModels"` } if err := json.Unmarshal(body, &doc); err != nil { - return nil, fmt.Errorf("decode publisher-model listing: %w", err) + return nil, fmt.Errorf("%w: decode publisher-model listing: %w", ErrUnparseableListing, err) } out := make([]listedModel, 0, len(doc.Models)) for _, entry := range doc.Models { diff --git a/management/internals/modules/agentnetwork/policyselect.go b/management/internals/modules/agentnetwork/policyselect.go index 9bb893b36..7d21d7bc9 100644 --- a/management/internals/modules/agentnetwork/policyselect.go +++ b/management/internals/modules/agentnetwork/policyselect.go @@ -164,23 +164,12 @@ func (m *managerImpl) SelectPolicyForRequest(ctx context.Context, in PolicySelec } candidates := filterApplicablePolicies(policies, in) - // Model-allowlist gate scoped to the matched policies: keep candidates whose - // guardrails permit the model (none enabled = unrestricted), deny when - // policies apply but none permits it. Skip the load when none has a guardrail. - if len(candidates) > 0 && anyPolicyHasGuardrails(candidates) { - guardrailsByID, gErr := m.loadGuardrailsByID(ctx, in.AccountID) - if gErr != nil { - return nil, gErr - } - permitted := filterModelPermittedPolicies(candidates, guardrailsByID, in.Model) - if len(permitted) == 0 { - return &PolicySelectionResult{ - Allow: false, - DenyCode: denyCodeModelBlocked, - DenyReason: modelBlockedReason(in.Model), - }, nil - } - candidates = permitted + candidates, denied, err := m.applyModelGate(ctx, in, candidates) + if err != nil { + return nil, err + } + if denied != nil { + return denied, nil } // Prefetch every consumption counter the ceiling + candidate policies will @@ -285,6 +274,59 @@ func anyPolicyHasGuardrails(policies []*types.Policy) bool { return false } +// applyModelGate is the model-allowlist gate scoped to the matched policies: +// it keeps the candidates whose guardrails permit the model (none enabled = +// unrestricted) and returns a deny result when policies apply but none +// permits it. The guardrail load is skipped when no candidate references a +// guardrail, and the provider's catalog id — which picks the model-id +// normalizer — is resolved only when a candidate actually restricts models: +// with no enabled allowlist every candidate is unrestricted, and a +// provider-store failure must not fail a request the gate would have waved +// through. +func (m *managerImpl) applyModelGate(ctx context.Context, in PolicySelectionInput, candidates []*types.Policy) ([]*types.Policy, *PolicySelectionResult, error) { + if len(candidates) == 0 || !anyPolicyHasGuardrails(candidates) { + return candidates, nil, nil + } + guardrailsByID, err := m.loadGuardrailsByID(ctx, in.AccountID) + if err != nil { + return nil, nil, err + } + if !anyEnabledModelAllowlist(candidates, guardrailsByID) { + return candidates, nil, nil + } + catalogID, err := m.providerCatalogID(ctx, in.AccountID, in.ProviderID) + if err != nil { + return nil, nil, err + } + permitted := filterModelPermittedPolicies(candidates, guardrailsByID, in.Model, catalogID) + if len(permitted) == 0 { + return nil, &PolicySelectionResult{ + Allow: false, + DenyCode: denyCodeModelBlocked, + DenyReason: modelBlockedReason(in.Model), + }, nil + } + return permitted, nil, nil +} + +// anyEnabledModelAllowlist reports whether any policy references a guardrail +// whose model allowlist is enabled — the only case the model gate restricts +// anything. Disabled allowlists, stale guardrail references, and guardrails +// carrying only other checks all leave every candidate unrestricted. +func anyEnabledModelAllowlist(policies []*types.Policy, byID map[string]*types.Guardrail) bool { + for _, p := range policies { + if p == nil { + continue + } + for _, gID := range p.GuardrailIDs { + if g, ok := byID[gID]; ok && g != nil && g.Checks.ModelAllowlist.Enabled { + return true + } + } + } + return false +} + // loadGuardrailsByID loads the account's guardrails indexed by ID. Used by the // model-allowlist gate to resolve each candidate policy's attached guardrails. func (m *managerImpl) loadGuardrailsByID(ctx context.Context, accountID string) (map[string]*types.Guardrail, error) { @@ -301,12 +343,33 @@ func (m *managerImpl) loadGuardrailsByID(ctx context.Context, accountID string) return byID, nil } +// providerCatalogID resolves a provider record id to its catalog provider +// id, the key the model-id normalizers are picked by. A missing provider +// resolves to the empty catalog id — the compare then runs verbatim-only, +// which can never widen an allowlist — while a store failure propagates +// rather than degrading a security decision. +func (m *managerImpl) providerCatalogID(ctx context.Context, accountID, providerID string) (string, error) { + if providerID == "" { + return "", nil + } + provider, err := m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID) + switch { + case err == nil: + return provider.ProviderID, nil + case isNotFound(err): + return "", nil + default: + return "", fmt.Errorf("get provider: %w", err) + } +} + // filterModelPermittedPolicies returns the subset of policies whose guardrails -// permit the model. Order is preserved so downstream scoring is unaffected. -func filterModelPermittedPolicies(policies []*types.Policy, byID map[string]*types.Guardrail, model string) []*types.Policy { +// permit the model on the provider with the given catalog id. Order is +// preserved so downstream scoring is unaffected. +func filterModelPermittedPolicies(policies []*types.Policy, byID map[string]*types.Guardrail, model, catalogProviderID string) []*types.Policy { out := make([]*types.Policy, 0, len(policies)) for _, p := range policies { - if policyPermitsModel(p, byID, model) { + if policyPermitsModel(p, byID, model, catalogProviderID) { out = append(out, p) } } @@ -316,8 +379,13 @@ func filterModelPermittedPolicies(policies []*types.Policy, byID map[string]*typ // policyPermitsModel reports whether a policy permits the model. No // allowlist-enabled guardrail = unrestricted (permits any, incl. empty); // otherwise the model must be in the union of its allowlists, so an -// empty/undetermined model fails closed. -func policyPermitsModel(p *types.Policy, byID map[string]*types.Guardrail, model string) bool { +// empty/undetermined model fails closed. An entry matches on its own +// normalised form or, for a path-style provider, its canonical form: the +// parser emits the canonical id for path-routed requests, while an +// allowlist may hold the raw declared id the dashboard's picker copies +// from the provider. The catalog id picks the normalizer, so a plain +// provider's entries always compare verbatim. +func policyPermitsModel(p *types.Policy, byID map[string]*types.Guardrail, model, catalogProviderID string) bool { if p == nil { return false } @@ -333,7 +401,7 @@ func policyPermitsModel(p *types.Policy, byID map[string]*types.Guardrail, model continue } for _, allowed := range g.Checks.ModelAllowlist.Models { - if normaliseModelID(allowed) == wanted { + if normaliseModelID(allowed) == wanted || canonicalModelKey(catalogProviderID, allowed) == wanted { return true } } diff --git a/management/internals/modules/agentnetwork/policyselect_model_test.go b/management/internals/modules/agentnetwork/policyselect_model_test.go index 7ae13e4ef..c70816b98 100644 --- a/management/internals/modules/agentnetwork/policyselect_model_test.go +++ b/management/internals/modules/agentnetwork/policyselect_model_test.go @@ -6,12 +6,13 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/shared/management/status" ) // guardedPolicy builds an enabled, uncapped policy that authorises sourceGroups @@ -53,6 +54,17 @@ func expectGuardrails(mockStore *store.MockStore, account string, guardrails ... Return(guardrails, nil) } +// expectProviderCatalog resolves the destination provider to the given +// catalog provider id, which picks the model-id normalizer the allowlist +// gate compares through. AnyTimes: the lookup runs only when the guardrail +// gate is reached. +func expectProviderCatalog(mockStore *store.MockStore, account, providerID, catalog string) { + mockStore.EXPECT(). + GetAgentNetworkProviderByID(gomock.Any(), gomock.Any(), account, providerID). + Return(&types.Provider{ID: providerID, AccountID: account, ProviderID: catalog}, nil). + AnyTimes() +} + // TestSelectPolicy_ModelBlockedByAllowlist proves the authoritative allowlist // decision: a policy authorises the (provider, group) but restricts the model, // and the requested model isn't on the list, so the request is denied. @@ -63,6 +75,7 @@ func TestSelectPolicy_ModelBlockedByAllowlist(t *testing.T) { policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") expectPolicies(mockStore, "acc-1", policy) expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ AccountID: "acc-1", @@ -86,6 +99,7 @@ func TestSelectPolicy_ModelAllowedByAllowlist(t *testing.T) { policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") expectPolicies(mockStore, "acc-1", policy) expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o", "claude-opus-4")) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -109,6 +123,7 @@ func TestSelectPolicy_CaseInsensitiveModelMatch(t *testing.T) { policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") expectPolicies(mockStore, "acc-1", policy) expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", " GPT-4o ")) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -132,6 +147,7 @@ func TestSelectPolicy_UnguardedPolicyIsUnrestricted(t *testing.T) { open := guardedPolicy("pol-open", "acc-1", []string{"grp-eng"}, "prov-1") // no guardrail expectPolicies(mockStore, "acc-1", restricted, open) expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -159,6 +175,7 @@ func TestSelectPolicy_AllowlistDoesNotLeakAcrossGroups(t *testing.T) { allowlistGuardrail("g-a", "acc-1", "gpt-4o"), allowlistGuardrail("g-b", "acc-1", "claude-opus-4"), ) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ AccountID: "acc-1", @@ -181,6 +198,7 @@ func TestSelectPolicy_UndeterminedModelFailsClosed(t *testing.T) { policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") expectPolicies(mockStore, "acc-1", policy) expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ AccountID: "acc-1", @@ -210,6 +228,8 @@ func TestSelectPolicy_DisabledAllowlistDoesNotRestrict(t *testing.T) { } expectPolicies(mockStore, "acc-1", policy) expectGuardrails(mockStore, "acc-1", disabled) + // Deliberately no provider expectation: with no enabled allowlist the + // gate must skip the catalog-id lookup entirely. expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -235,6 +255,7 @@ func TestSelectPolicy_UnionAcrossPolicyGuardrails(t *testing.T) { allowlistGuardrail("g-1", "acc-1", "gpt-4o"), allowlistGuardrail("g-2", "acc-1", "claude-opus-4"), ) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -281,6 +302,8 @@ func TestSelectPolicy_MissingGuardrailReferenceTreatedAsUnrestricted(t *testing. policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-missing") expectPolicies(mockStore, "acc-1", policy) expectGuardrails(mockStore, "acc-1") + // Deliberately no provider expectation: an orphaned guardrail reference + // restricts nothing, so the gate must skip the catalog-id lookup. expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -314,6 +337,7 @@ func TestSelectPolicy_PartialCandidatesPermittedAfterModelFilter(t *testing.T) { allowlistGuardrail("g-restrict", "acc-1", "gpt-4o"), allowlistGuardrail("g-permit", "acc-1", "claude-opus-4"), ) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") expectConsumptionBatch(mockStore, nil) res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ @@ -327,3 +351,159 @@ func TestSelectPolicy_PartialCandidatesPermittedAfterModelFilter(t *testing.T) { assert.Equal(t, "pol-small", res.SelectedPolicyID, "the model filter must exclude pol-big before cap scoring") } + +// TestSelectPolicy_RawDeclaredAllowlistPermitsCanonicalModel proves an +// allowlist holding the raw vendor-issued id — the form the dashboard's +// picker copies from a provider's declared models — permits the request: +// the parser emits the path-style canonical id, so the entry must match +// through the same canonicalization. +func TestSelectPolicy_RawDeclaredAllowlistPermitsCanonicalModel(t *testing.T) { + cases := []struct { + name string + catalog string + entry string + request string + }{ + {"bedrock raw region/version form", "bedrock_api", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "anthropic.claude-sonnet-4-5"}, + {"vertex raw @version form", "vertex_ai_api", "claude-sonnet-4-5@20250929", "claude-sonnet-4-5"}, + {"vertex raw dated @version form", "vertex_ai_api", "gpt-4o@2024-08-06", "gpt-4o"}, + {"bedrock raw form with case and whitespace", "bedrock_api", " EU.Anthropic.Claude-Sonnet-4-5-20250929-V1:0 ", "anthropic.claude-sonnet-4-5"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", tc.entry)) + expectProviderCatalog(mockStore, "acc-1", "prov-1", tc.catalog) + expectConsumptionBatch(mockStore, nil) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: tc.request, + }) + require.NoError(t, err) + assert.True(t, res.Allow, "the raw declared allowlist entry must permit its canonical model") + assert.Equal(t, "pol-A", res.SelectedPolicyID) + }) + } + + // A model outside the allowlist stays denied under the same entry shape. + t.Run("unrelated canonical model stays denied", func(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0")) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "bedrock_api") + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "anthropic.claude-opus-4-8", + }) + require.NoError(t, err) + assert.False(t, res.Allow, "a model the allowlist never names must stay denied") + assert.Equal(t, denyCodeModelBlocked, res.DenyCode) + }) +} + +// TestSelectPolicy_PlainProviderEntriesStayVerbatim proves the canonical-form +// compare never relaxes an allowlist on a body-routed provider: its catalog +// id selects no normalizer, so a suffix that would be stripped under Bedrock +// ("-v2") or Vertex ("@...") stays part of the entry and must NOT also admit +// the stripped id — on this provider that is a different model. +func TestSelectPolicy_PlainProviderEntriesStayVerbatim(t *testing.T) { + cases := []struct { + name string + entry string + request string + }{ + {"a -vN suffix is not a Bedrock version tag here", "claude-3-5-sonnet-v2", "claude-3-5-sonnet"}, + {"an @word suffix is not a Vertex version tag here", "custom-model@team", "custom-model"}, + {"an @digits suffix is not a Vertex version tag here", "custom-model@2024", "custom-model"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", tc.entry)) + expectProviderCatalog(mockStore, "acc-1", "prov-1", "openai_api") + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: tc.request, + }) + require.NoError(t, err) + assert.False(t, res.Allow, "a plain provider's allowlist entry must not widen to its stripped form") + assert.Equal(t, denyCodeModelBlocked, res.DenyCode) + }) + } +} + +// TestSelectPolicy_MissingProviderRecordComparesVerbatim proves a provider the +// store no longer holds degrades to the verbatim-only compare — the raw entry +// still matches itself, and nothing widens — rather than erroring or guessing +// a normalizer. +func TestSelectPolicy_MissingProviderRecordComparesVerbatim(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "eu.anthropic.claude-sonnet-4-5-20250929-v1:0")) + mockStore.EXPECT(). + GetAgentNetworkProviderByID(gomock.Any(), gomock.Any(), "acc-1", "prov-1"). + Return(nil, status.Errorf(status.NotFound, "provider not found")). + AnyTimes() + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "anthropic.claude-sonnet-4-5", + }) + require.NoError(t, err) + assert.False(t, res.Allow, "without the provider record the compare runs verbatim and must not widen") + assert.Equal(t, denyCodeModelBlocked, res.DenyCode) +} + +// TestSelectPolicy_ProviderLookupErrorPropagates proves a store failure while +// resolving the provider's catalog id surfaces as an error — the model gate is +// a security decision and must not silently degrade. +func TestSelectPolicy_ProviderLookupErrorPropagates(t *testing.T) { + ctrl := gomock.NewController(t) + mgr, mockStore := newSelectorMgr(t, ctrl) + + policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1") + expectPolicies(mockStore, "acc-1", policy) + expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o")) + mockStore.EXPECT(). + GetAgentNetworkProviderByID(gomock.Any(), gomock.Any(), "acc-1", "prov-1"). + Return(nil, errors.New("store unavailable")) + + res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{ + AccountID: "acc-1", + UserID: "user-1", + GroupIDs: []string{"grp-eng"}, + ProviderID: "prov-1", + Model: "gpt-4o", + }) + require.Error(t, err, "a provider-lookup failure must surface as an error") + assert.Nil(t, res) +} diff --git a/management/internals/modules/agentnetwork/settings_bootstrap_test.go b/management/internals/modules/agentnetwork/settings_bootstrap_test.go index fc6fd8b82..fea62353e 100644 --- a/management/internals/modules/agentnetwork/settings_bootstrap_test.go +++ b/management/internals/modules/agentnetwork/settings_bootstrap_test.go @@ -26,6 +26,10 @@ type bootstrapFixture struct { manager Manager store store.Store perms *permissions.MockManager + // vendor stands in for the provider credential check's vendor call, which + // runs on every provider write. Without it these tests would reach a real + // vendor to save a record. + vendor *stubLister } func newBootstrapFixture(t *testing.T) *bootstrapFixture { @@ -47,10 +51,12 @@ func newBootstrapFixture(t *testing.T) *bootstrapFixture { accounts.EXPECT().UpdateAccountPeers(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() accounts.EXPECT().BufferUpdateAccountPeers(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + vendor := &stubLister{} return &bootstrapFixture{ - manager: NewManager(st, perms, accounts, nil), + manager: NewManager(st, perms, accounts, nil, WithModelLister(vendor)), store: st, perms: perms, + vendor: vendor, } } @@ -211,6 +217,7 @@ func TestCreateProviderHasNoSettingsSideEffects(t *testing.T) { f.expectPermission("account1", "user1", modules.AgentNetworkProviders, operations.Create, true) provider := types.NewProvider("account1") + provider.ProviderID = "openai_api" provider.Name = "openai" provider.UpstreamURL = "https://api.openai.com" provider.APIKey = "sk-test" diff --git a/management/internals/modules/agentnetwork/synthesizer.go b/management/internals/modules/agentnetwork/synthesizer.go index b838ac547..212ec7f84 100644 --- a/management/internals/modules/agentnetwork/synthesizer.go +++ b/management/internals/modules/agentnetwork/synthesizer.go @@ -211,18 +211,19 @@ func SynthesizeServices(ctx context.Context, s store.Store, accountID string) ([ } groupIndex := indexProviderGroups(enabledPolicies) + catalogByProvider := catalogIDsByProvider(enabledProviders) // The proxy guardrail is a per-provider fail-closed backstop; the // authoritative per-policy/group decision is management's // SelectPolicyForRequest. A provider lands in that map only when every // authorising policy restricts models. - providerAllowlists := buildProviderAllowlists(enabledPolicies, guardrailsByID) + providerAllowlists := buildProviderAllowlists(enabledPolicies, guardrailsByID, catalogByProvider) // Discovery gets the finer view: per policy rather than flattened per // provider, so a listing can be bounded to what the calling groups may // actually use instead of the union across everyone who reaches the // provider. - modelPolicies := buildModelPolicies(enabledPolicies, guardrailsByID) + modelPolicies := buildModelPolicies(enabledPolicies, guardrailsByID, catalogByProvider) routerCfgJSON, err := buildRouterConfigJSON(enabledProviders, groupIndex, modelPolicies) if err != nil { @@ -907,7 +908,9 @@ func marshalGuardrailConfig(providerAllowlists map[string][]string, capture Merg // buildProviderAllowlists returns the proxy's per-provider backstop: a provider // is included only when every authorising policy restricts models (their union); // if any leaves it unrestricted it is omitted, so management decides per group. -func buildProviderAllowlists(policies []*types.Policy, byID map[string]*types.Guardrail) map[string][]string { +// Entries carry their provider-specific canonical form alongside the verbatim +// one, resolved through catalogByProvider. +func buildProviderAllowlists(policies []*types.Policy, byID map[string]*types.Guardrail, catalogByProvider map[string]string) map[string][]string { type providerAcc struct { models map[string]struct{} anyUnrestricted bool @@ -931,7 +934,7 @@ func buildProviderAllowlists(policies []*types.Policy, byID map[string]*types.Gu acc.anyUnrestricted = true continue } - for _, m := range models { + for _, m := range expandModelsForProvider(models, catalogByProvider[providerID]) { acc.models[m] = struct{}{} } } @@ -952,8 +955,10 @@ func buildProviderAllowlists(policies []*types.Policy, byID map[string]*types.Gu } // policyModelAllowlist reports whether a policy restricts models (has an -// allowlist-enabled guardrail) and the union of allowed models. Models are -// verbatim; the proxy factory lowercases/trims them at decode time. +// allowlist-enabled guardrail) and the union of allowed models, verbatim. +// Consumers expand the entries per destination provider with +// expandModelsForProvider — the canonical form is provider-specific — and +// the proxy factory lowercases/trims them at decode time. func policyModelAllowlist(p *types.Policy, byID map[string]*types.Guardrail) (bool, []string) { restricted := false var models []string @@ -972,6 +977,45 @@ func policyModelAllowlist(p *types.Policy, byID map[string]*types.Guardrail) (bo return restricted, models } +// expandModelsForProvider returns the allowlist entries for one destination +// provider: each entry verbatim plus, when it differs, its canonical form +// under that provider's catalog id — the id the proxy's parser emits at +// request time — deduplicated. The proxy-side compares (guardrail backstop, +// per-group router rules) then admit an allowlist however the operator +// wrote it, raw declared id or canonical, while a plain provider's entries +// stay verbatim and can never widen. +func expandModelsForProvider(models []string, catalogProviderID string) []string { + out := make([]string, 0, len(models)) + seen := make(map[string]struct{}, len(models)) + add := func(m string) { + if m == "" { + return + } + if _, dup := seen[m]; dup { + return + } + seen[m] = struct{}{} + out = append(out, m) + } + for _, m := range models { + add(m) + add(canonicalModelKey(catalogProviderID, m)) + } + return out +} + +// catalogIDsByProvider indexes providers' catalog ids by provider record id, +// the lookup the per-provider allowlist expansion keys the normalizer on. +func catalogIDsByProvider(providers []*types.Provider) map[string]string { + out := make(map[string]string, len(providers)) + for _, p := range providers { + if p != nil { + out[p.ID] = p.ProviderID + } + } + return out +} + // buildAccountService composes the per-account gateway Service. The // target carries the noop placeholder URL — the router middleware // rewrites every request to the matched provider's upstream before the @@ -1180,23 +1224,25 @@ type routerModelPolicy struct { // models — a picker full of entries the next request refuses. Keeping the // source groups alongside the models lets the router answer it at request time, // where it knows the caller's groups. -func buildModelPolicies(policies []*types.Policy, byID map[string]*types.Guardrail) map[string][]routerModelPolicy { +func buildModelPolicies(policies []*types.Policy, byID map[string]*types.Guardrail, catalogByProvider map[string]string) map[string][]routerModelPolicy { out := make(map[string][]routerModelPolicy) for _, p := range policies { if p == nil || len(p.SourceGroups) == 0 { continue } restricted, models := policyModelAllowlist(p, byID) - rule := routerModelPolicy{GroupIDs: append([]string(nil), p.SourceGroups...)} - if restricted { - // Never nil when restricted: an allowlist permitting nothing must - // stay distinguishable from no allowlist at all. - rule.Models = append([]string{}, models...) - } for _, providerID := range p.DestinationProviderIDs { if providerID == "" { continue } + rule := routerModelPolicy{GroupIDs: append([]string(nil), p.SourceGroups...)} + if restricted { + // Never nil when restricted: an allowlist permitting nothing + // must stay distinguishable from no allowlist at all. The + // expansion is per provider — the canonical form of an entry + // depends on the destination's catalog id. + rule.Models = append([]string{}, expandModelsForProvider(models, catalogByProvider[providerID])...) + } out[providerID] = append(out[providerID], rule) } } diff --git a/management/internals/modules/agentnetwork/synthesizer_provider_allowlist_test.go b/management/internals/modules/agentnetwork/synthesizer_provider_allowlist_test.go index a27cd2ae4..22833ec10 100644 --- a/management/internals/modules/agentnetwork/synthesizer_provider_allowlist_test.go +++ b/management/internals/modules/agentnetwork/synthesizer_provider_allowlist_test.go @@ -33,7 +33,7 @@ func TestBuildProviderAllowlists(t *testing.T) { policyForProviders("p1", []string{"g-4o"}, "prov-x"), policyForProviders("p2", []string{"g-opus"}, "prov-x"), } - got := buildProviderAllowlists(policies, byID) + got := buildProviderAllowlists(policies, byID, nil) assert.Equal(t, map[string][]string{"prov-x": {"claude-opus-4", "gpt-4o"}}, got, "a provider every policy restricts carries the sorted union of their models") }) @@ -43,7 +43,7 @@ func TestBuildProviderAllowlists(t *testing.T) { policyForProviders("p1", []string{"g-4o"}, "prov-x"), policyForProviders("p2", nil, "prov-x"), // no guardrail } - got := buildProviderAllowlists(policies, byID) + got := buildProviderAllowlists(policies, byID, nil) assert.NotContains(t, got, "prov-x", "a provider reachable by an un-guardrailed policy must be omitted so the proxy treats it as unrestricted") }) @@ -52,7 +52,7 @@ func TestBuildProviderAllowlists(t *testing.T) { policies := []*types.Policy{ policyForProviders("p1", []string{"g-disabled"}, "prov-x"), } - got := buildProviderAllowlists(policies, byID) + got := buildProviderAllowlists(policies, byID, nil) assert.NotContains(t, got, "prov-x", "a policy whose only guardrail has a disabled allowlist is unrestricted") }) @@ -62,7 +62,7 @@ func TestBuildProviderAllowlists(t *testing.T) { policyForProviders("p1", []string{"g-4o"}, "prov-x"), policyForProviders("p2", []string{"g-opus"}, "prov-y"), } - got := buildProviderAllowlists(policies, byID) + got := buildProviderAllowlists(policies, byID, nil) assert.Equal(t, []string{"gpt-4o"}, got["prov-x"], "prov-x keeps only its own model") assert.Equal(t, []string{"claude-opus-4"}, got["prov-y"], "prov-y keeps only its own model") }) @@ -71,7 +71,7 @@ func TestBuildProviderAllowlists(t *testing.T) { policies := []*types.Policy{ policyForProviders("p1", []string{"g-4o"}, "prov-x", "prov-y"), } - got := buildProviderAllowlists(policies, byID) + got := buildProviderAllowlists(policies, byID, nil) assert.Equal(t, []string{"gpt-4o"}, got["prov-x"]) assert.Equal(t, []string{"gpt-4o"}, got["prov-y"]) }) @@ -80,7 +80,7 @@ func TestBuildProviderAllowlists(t *testing.T) { policies := []*types.Policy{ policyForProviders("p1", []string{"g-4o", "g-opus"}, "prov-x"), } - got := buildProviderAllowlists(policies, byID) + got := buildProviderAllowlists(policies, byID, nil) assert.ElementsMatch(t, []string{"claude-opus-4", "gpt-4o"}, got["prov-x"], "a policy's own multiple allowlist guardrails union together") }) @@ -89,7 +89,7 @@ func TestBuildProviderAllowlists(t *testing.T) { empty := map[string]*types.Guardrail{"g-empty": allowlistGuardrail("g-empty", "acc-1")} got := buildProviderAllowlists([]*types.Policy{ policyForProviders("p1", []string{"g-empty"}, "prov-x"), - }, empty) + }, empty, nil) assert.Equal(t, map[string][]string{"prov-x": {}}, got, "an enabled-but-empty allowlist is restricted with an empty set, not unrestricted") }) @@ -124,7 +124,7 @@ func TestBuildModelPolicies(t *testing.T) { policyForGroups("p1", []string{"grp-eng"}, []string{"g-4o"}, "prov-x"), policyForGroups("p2", []string{"grp-sales"}, []string{"g-opus"}, "prov-x"), } - got := buildModelPolicies(policies, byID) + got := buildModelPolicies(policies, byID, nil) assert.Equal(t, []routerModelPolicy{ {GroupIDs: []string{"grp-eng"}, Models: []string{"gpt-4o"}}, {GroupIDs: []string{"grp-sales"}, Models: []string{"claude-opus-4"}}, @@ -137,14 +137,14 @@ func TestBuildModelPolicies(t *testing.T) { policyForGroups("p1", []string{"grp-eng"}, []string{"g-4o"}, "prov-x"), policyForGroups("p2", []string{"grp-admin"}, nil, "prov-x"), } - got := buildModelPolicies(policies, byID) + got := buildModelPolicies(policies, byID, nil) assert.Nil(t, got["prov-x"][1].Models, "no allowlist must reach the router as nil, which lifts the restriction for its groups") }) t.Run("a disabled allowlist is not a restriction", func(t *testing.T) { policies := []*types.Policy{policyForGroups("p1", []string{"grp-eng"}, []string{"g-disabled"}, "prov-x")} - got := buildModelPolicies(policies, byID) + got := buildModelPolicies(policies, byID, nil) assert.Nil(t, got["prov-x"][0].Models, "a guardrail with the allowlist check off restricts nothing") }) @@ -154,7 +154,7 @@ func TestBuildModelPolicies(t *testing.T) { "g-empty": {ID: "g-empty", Checks: types.GuardrailChecks{ModelAllowlist: types.GuardrailModelAllowlist{Enabled: true}}}, } policies := []*types.Policy{policyForGroups("p1", []string{"grp-eng"}, []string{"g-empty"}, "prov-x")} - got := buildModelPolicies(policies, byIDEmpty) + got := buildModelPolicies(policies, byIDEmpty, nil) require.NotNil(t, got["prov-x"][0].Models, "an empty allowlist must not arrive as nil — that would read as unrestricted") assert.Empty(t, got["prov-x"][0].Models) @@ -162,7 +162,71 @@ func TestBuildModelPolicies(t *testing.T) { t.Run("a policy binding no groups is skipped", func(t *testing.T) { policies := []*types.Policy{policyForGroups("p1", nil, []string{"g-4o"}, "prov-x")} - assert.Empty(t, buildModelPolicies(policies, byID), + assert.Empty(t, buildModelPolicies(policies, byID, nil), "a policy with no source groups authorises nobody, so it bounds nobody's listing") }) } + +// TestSynthesizedAllowlists_ExpandDeclaredIDsPerProvider proves the +// synthesized allowlists carry the canonical form alongside a raw declared +// entry — under the destination provider's own catalog id, never another's — +// so the proxy-side compares (guardrail backstop, per-group router rules) +// admit the allowlist however the operator wrote it, while a plain provider's +// "-vN"- or "@"-suffixed entries stay verbatim and cannot widen. +func TestSynthesizedAllowlists_ExpandDeclaredIDsPerProvider(t *testing.T) { + byID := map[string]*types.Guardrail{ + "g-raw": allowlistGuardrail("g-raw", "acc-1", + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5@20250929", + "gpt-4o"), + } + catalogByProvider := map[string]string{ + "prov-bedrock": "bedrock_api", + "prov-vertex": "vertex_ai_api", + "prov-plain": "openai_api", + } + policies := []*types.Policy{ + policyForGroups("p1", []string{"grp-eng"}, []string{"g-raw"}, + "prov-bedrock", "prov-vertex", "prov-plain"), + } + + t.Run("guardrail backstop expands under each provider's own normalizer", func(t *testing.T) { + got := buildProviderAllowlists(policies, byID, catalogByProvider) + assert.ElementsMatch(t, []string{ + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-sonnet-4-5", + "claude-sonnet-4-5@20250929", + "gpt-4o", + }, got["prov-bedrock"], + "the Bedrock destination strips geography/version, but must not apply Vertex's @-strip") + assert.ElementsMatch(t, []string{ + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5@20250929", + "claude-sonnet-4-5", + "gpt-4o", + }, got["prov-vertex"], + "the Vertex destination strips @version, but must not apply Bedrock's suffix strip") + assert.ElementsMatch(t, []string{ + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5@20250929", + "gpt-4o", + }, got["prov-plain"], + "a body-routed provider keeps every entry verbatim — no alternate can widen it") + }) + + t.Run("router model rules expand the same way", func(t *testing.T) { + got := buildModelPolicies(policies, byID, catalogByProvider) + require.Len(t, got["prov-bedrock"], 1) + assert.Contains(t, got["prov-bedrock"][0].Models, "anthropic.claude-sonnet-4-5") + assert.NotContains(t, got["prov-bedrock"][0].Models, "claude-sonnet-4-5") + require.Len(t, got["prov-vertex"], 1) + assert.Contains(t, got["prov-vertex"][0].Models, "claude-sonnet-4-5") + assert.NotContains(t, got["prov-vertex"][0].Models, "anthropic.claude-sonnet-4-5") + require.Len(t, got["prov-plain"], 1) + assert.ElementsMatch(t, []string{ + "eu.anthropic.claude-sonnet-4-5-20250929-v1:0", + "claude-sonnet-4-5@20250929", + "gpt-4o", + }, got["prov-plain"][0].Models) + }) +} diff --git a/management/internals/modules/reverseproxy/service/manager/manager_test.go b/management/internals/modules/reverseproxy/service/manager/manager_test.go index 10893673e..dd0edec60 100644 --- a/management/internals/modules/reverseproxy/service/manager/manager_test.go +++ b/management/internals/modules/reverseproxy/service/manager/manager_test.go @@ -7,11 +7,10 @@ import ( "testing" "time" - cachestore "github.com/eko/gocache/lib/v4/store" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/metric/noop" + "go.uber.org/mock/gomock" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" proxymanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy/manager" @@ -31,7 +30,7 @@ import ( "github.com/netbirdio/netbird/shared/management/status" ) -func testCacheStore(t *testing.T) cachestore.StoreInterface { +func testCacheStore(t *testing.T) nbcache.Store { t.Helper() s, err := nbcache.NewStore(context.Background(), 30*time.Minute, 10*time.Minute, 100) require.NoError(t, err) @@ -295,6 +294,7 @@ func TestPersistNewService(t *testing.T) { assert.Equal(t, status.AlreadyExists, sErr.Type()) }) } + func TestPreserveExistingAuthSecrets(t *testing.T) { mgr := &Manager{} diff --git a/management/internals/modules/reverseproxy/service/service.go b/management/internals/modules/reverseproxy/service/service.go index b6438abde..ef92fbbf4 100644 --- a/management/internals/modules/reverseproxy/service/service.go +++ b/management/internals/modules/reverseproxy/service/service.go @@ -55,6 +55,8 @@ const ( SourceEphemeral = "ephemeral" ) +var ErrUnsupportedIPAddressUpstreamHost = errors.New("unsupported ip address for a direct upstream host") + type TargetOptions struct { SkipTLSVerify bool `json:"skip_tls_verify"` RequestTimeout time.Duration `json:"request_timeout,omitempty"` @@ -388,6 +390,7 @@ func (s *Service) ToProtoMapping(operation Operation, authToken string, oidcConf if s.Auth.BearerAuth != nil && s.Auth.BearerAuth.Enabled { auth.Oidc = true + auth.AllowedGroupIds = append([]string(nil), s.Auth.BearerAuth.DistributionGroups...) } for _, h := range s.Auth.HeaderAuths { @@ -961,8 +964,8 @@ func (s *Service) validateHTTPTargets() error { return err } case TargetTypeSubnet: - if target.Host == "" { - return fmt.Errorf("target %d has empty host but target_type is %q", i, target.TargetType) + if err := validateSubnetTarget(i, target); err != nil { + return err } case TargetTypeCluster: if err := validateClusterTarget(i, target); err != nil { @@ -985,6 +988,34 @@ func (s *Service) validateHTTPTargets() error { return nil } +func validateSubnetTarget(idx int, target *Target) error { + host := strings.TrimSpace(target.Host) + if host == "" { + return fmt.Errorf("target %d has empty host but target_type is %q", idx, target.TargetType) + } + if strings.ContainsAny(host, " \t/") { + return fmt.Errorf("target %d: host %q contains invalid characters", idx, host) + } + if _, _, err := net.SplitHostPort(host); err == nil { + return fmt.Errorf("target %d: host %q must not include a port (set target.port instead)", idx, host) + } + noBrackets := strings.TrimSuffix(strings.TrimPrefix(host, "["), "]") + maybeip, err := netip.ParseAddr(noBrackets) + if err != nil { // not an ip + return nil //nolint:nilerr + } + if maybeip.Zone() != "" { + return fmt.Errorf("invalid direct upstream host ip %s %w", maybeip.String(), ErrUnsupportedIPAddressUpstreamHost) + } + if !target.Options.DirectUpstream { + return nil + } + if maybeip.IsLoopback() || maybeip.IsMulticast() || maybeip.IsLinkLocalUnicast() { + return fmt.Errorf("invalid direct upstream host ip %s %w", maybeip.String(), ErrUnsupportedIPAddressUpstreamHost) + } + return nil +} + // validateClusterTarget cluster targets should not have empty hosts and should have direct upstream enabled. func validateClusterTarget(idx int, target *Target) error { host := strings.TrimSpace(target.Host) @@ -1019,6 +1050,15 @@ func validateDirectUpstreamHost(idx int, target *Target) error { if _, _, err := net.SplitHostPort(host); err == nil { return fmt.Errorf("target %d: host %q must not include a port (set target.port instead)", idx, host) } + noBrackets := strings.TrimSuffix(strings.TrimPrefix(host, "["), "]") + maybeip, err := netip.ParseAddr(noBrackets) + if err != nil { // not an ip + return nil //nolint:nilerr + } + if maybeip.Zone() != "" || maybeip.IsLoopback() || maybeip.IsMulticast() || maybeip.IsLinkLocalUnicast() { + return fmt.Errorf("invalid direct upstream host ip %s %w", maybeip.String(), ErrUnsupportedIPAddressUpstreamHost) + } + return nil } diff --git a/management/internals/modules/reverseproxy/service/service_test.go b/management/internals/modules/reverseproxy/service/service_test.go index a149ac609..84343bcdd 100644 --- a/management/internals/modules/reverseproxy/service/service_test.go +++ b/management/internals/modules/reverseproxy/service/service_test.go @@ -216,6 +216,64 @@ func TestValidateTargetOptions_CustomHeaders(t *testing.T) { }) } +func TestValidate_DirectUpstreamHost(t *testing.T) { + target := Target{TargetId: "id-1", TargetType: TargetTypePeer, Host: "10.0.0.1", Port: 80, Protocol: "http", Enabled: true, Options: TargetOptions{DirectUpstream: true}} + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "127.0.0.2")), ErrUnsupportedIPAddressUpstreamHost) + assert.NotNil(t, validateDirectUpstreamHost(0, targetWithHost(&target, "127.0.0.2:80")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "::1")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "::1%lo0")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "[::1]")), ErrUnsupportedIPAddressUpstreamHost) + assert.NotNil(t, validateDirectUpstreamHost(0, targetWithHost(&target, "[::1]:80")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "[::1%lo0]")), ErrUnsupportedIPAddressUpstreamHost) + assert.NotNil(t, validateDirectUpstreamHost(0, targetWithHost(&target, "[::1%lo0]:80")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "169.254.100.100")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "fe80::1")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "[fe80::1]")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "224.100.100.100")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "ff00::ffff")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateDirectUpstreamHost(0, targetWithHost(&target, "[ff00::ffff]")), ErrUnsupportedIPAddressUpstreamHost) + + // empty host + assert.Nil(t, validateDirectUpstreamHost(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: " "})) + // host with a space + assert.NotNil(t, validateDirectUpstreamHost(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: "with space"})) + // host with a tab + assert.NotNil(t, validateDirectUpstreamHost(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: "with\ttab"})) + // host with a slash + assert.NotNil(t, validateDirectUpstreamHost(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: "with/slash"})) +} + +func TestValidate_ValidateSubnetTarget(t *testing.T) { + target := Target{TargetId: "id-1", TargetType: TargetTypeSubnet, Host: "10.0.0.1", Port: 80, Protocol: "http", Enabled: true, Options: TargetOptions{DirectUpstream: true}} + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "127.0.0.2")), ErrUnsupportedIPAddressUpstreamHost) + assert.NotNil(t, validateSubnetTarget(0, targetWithHost(&target, "127.0.0.2:80")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "::1")), ErrUnsupportedIPAddressUpstreamHost) + assert.NotNil(t, validateSubnetTarget(0, targetWithHost(&target, "[::1]:80")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "::1%lo0")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "[::1%lo0]")), ErrUnsupportedIPAddressUpstreamHost) + assert.NotNil(t, validateSubnetTarget(0, targetWithHost(&target, "[::1%lo0]:80")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "169.254.100.100")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "fe80::1")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "[fe80::1]")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "224.100.100.100")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "ff00::ffff")), ErrUnsupportedIPAddressUpstreamHost) + assert.ErrorIs(t, validateSubnetTarget(0, targetWithHost(&target, "[ff00::ffff]")), ErrUnsupportedIPAddressUpstreamHost) + + // empty host + assert.NotNil(t, validateSubnetTarget(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: " "})) + // host with a space + assert.NotNil(t, validateSubnetTarget(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: "with space"})) + // host with a tab + assert.NotNil(t, validateSubnetTarget(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: "with\ttab"})) + // host with a slash + assert.NotNil(t, validateSubnetTarget(0, &Target{Options: TargetOptions{DirectUpstream: true}, Host: "with/slash"})) +} + +func targetWithHost(t *Target, host string) *Target { + t.Host = host + return t +} + func TestToProtoMapping_TargetOptions(t *testing.T) { rp := &Service{ ID: "svc-1", @@ -250,6 +308,44 @@ func TestToProtoMapping_TargetOptions(t *testing.T) { assert.Equal(t, int64(30), opts.RequestTimeout.Seconds) } +// TestToProtoMapping_AllowedGroupIds covers the list the proxy gates session +// cookies on: without it the proxy can only check a cookie's signature, which +// makes a token minted for a user outside the groups a bearer credential. +func TestToProtoMapping_AllowedGroupIds(t *testing.T) { + t.Run("distribution groups reach the proxy", func(t *testing.T) { + rp := &Service{ + ID: "svc-1", + AccountID: "acc-1", + Domain: "example.com", + Auth: AuthConfig{ + BearerAuth: &BearerAuthConfig{ + Enabled: true, + DistributionGroups: []string{"grp-1", "grp-2"}, + }, + }, + } + pm := rp.ToProtoMapping(Create, "token", proxy.OIDCValidationConfig{}) + + assert.True(t, pm.GetAuth().GetOidc()) + assert.Equal(t, []string{"grp-1", "grp-2"}, pm.GetAuth().GetAllowedGroupIds()) + }) + + t.Run("a service open to the account carries no groups", func(t *testing.T) { + rp := &Service{ + ID: "svc-1", + AccountID: "acc-1", + Domain: "example.com", + Auth: AuthConfig{ + BearerAuth: &BearerAuthConfig{Enabled: true}, + }, + } + pm := rp.ToProtoMapping(Create, "token", proxy.OIDCValidationConfig{}) + + assert.True(t, pm.GetAuth().GetOidc()) + assert.Empty(t, pm.GetAuth().GetAllowedGroupIds(), "an empty list must not restrict access") + }) +} + func TestToProtoMapping_NoOptionsWhenDefault(t *testing.T) { rp := &Service{ ID: "svc-1", diff --git a/management/internals/network_map_db/pgsql/dns.go b/management/internals/network_map_db/pgsql/dns.go index b22b43903..46ef0ddfa 100644 --- a/management/internals/network_map_db/pgsql/dns.go +++ b/management/internals/network_map_db/pgsql/dns.go @@ -15,6 +15,7 @@ const ( from zones left join records as r on r.zone_id = zones.id where zones.account_id=$1 and zones.enabled + order by zones.id ` ) diff --git a/management/internals/network_map_db/sqlite/network_router.go b/management/internals/network_map_db/sqlite/network_router.go index 8c4c31cd6..9fb0fde7e 100644 --- a/management/internals/network_map_db/sqlite/network_router.go +++ b/management/internals/network_map_db/sqlite/network_router.go @@ -11,9 +11,11 @@ import ( ) const ( + // Outer join: a groupless router must survive. GetNetworkRouterQuery = ` select public_id, peer, network_id, masquerade, metric, enabled, peer_groups, group_peers.peer_id - from network_routers, json_each(peer_groups) + from network_routers + left join json_each(network_routers.peer_groups) on true left join group_peers on group_peers.account_id=? and group_peers.group_id=json_each.value where network_routers.account_id=? ` diff --git a/management/internals/network_map_db/sqlite/user.go b/management/internals/network_map_db/sqlite/user.go index 0bdda372e..23c46bf76 100644 --- a/management/internals/network_map_db/sqlite/user.go +++ b/management/internals/network_map_db/sqlite/user.go @@ -42,17 +42,20 @@ func (sc *SqliteStoreConn) GetAllowedUsers(ctx context.Context, accountId string userIdIdx := make(map[string]struct{}) groupIdToUserIds := make(map[string][]string) for _, user := range users { + for _, allgid := range allGroupIds { + groupIdToUserIds[allgid] = append(groupIdToUserIds[allgid], user.ID) + } + userIdIdx[user.ID] = struct{}{} autogroups := make([]string, 0) + if user.AutoGroups == nil { + continue + } if err := json.Unmarshal(user.AutoGroups, &autogroups); err != nil { return nil, nil, err } - userIdIdx[user.ID] = struct{}{} for _, groupId := range autogroups { groupIdToUserIds[groupId] = append(groupIdToUserIds[groupId], user.ID) } - for _, allgid := range allGroupIds { - groupIdToUserIds[allgid] = append(groupIdToUserIds[allgid], user.ID) - } } return userIdIdx, groupIdToUserIds, nil diff --git a/management/internals/server/boot.go b/management/internals/server/boot.go index 0a4df3924..87a36a93b 100644 --- a/management/internals/server/boot.go +++ b/management/internals/server/boot.go @@ -21,8 +21,6 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" - cachestore "github.com/eko/gocache/lib/v4/store" - "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/formatter/hook" "github.com/netbirdio/netbird/management/internals/modules/agentnetwork" @@ -75,8 +73,8 @@ func (s *BaseServer) Metrics() telemetry.AppMetrics { // CacheStore returns a shared cache store backed by Redis or in-memory depending on the environment. // All consumers should reuse this store to avoid creating multiple Redis connections. -func (s *BaseServer) CacheStore() cachestore.StoreInterface { - return Create(s, func() cachestore.StoreInterface { +func (s *BaseServer) CacheStore() nbcache.Store { + return Create(s, func() nbcache.Store { cs, err := nbcache.NewStore(context.Background(), nbcache.DefaultStoreMaxTimeout, nbcache.DefaultStoreCleanupInterval, nbcache.DefaultStoreMaxConn) if err != nil { log.Fatalf("failed to create shared cache store: %v", err) diff --git a/management/internals/shared/grpc/pkce_verifier.go b/management/internals/shared/grpc/pkce_verifier.go index a1325256c..18155dc1d 100644 --- a/management/internals/shared/grpc/pkce_verifier.go +++ b/management/internals/shared/grpc/pkce_verifier.go @@ -5,22 +5,23 @@ import ( "fmt" "time" - "github.com/eko/gocache/lib/v4/cache" "github.com/eko/gocache/lib/v4/store" log "github.com/sirupsen/logrus" + + nbcache "github.com/netbirdio/netbird/management/server/cache" ) // PKCEVerifierStore manages PKCE verifiers for OAuth flows. // Supports both in-memory and Redis storage via NB_IDP_CACHE_REDIS_ADDRESS env var. type PKCEVerifierStore struct { - cache *cache.Cache[string] + cache nbcache.Store ctx context.Context } // NewPKCEVerifierStore creates a PKCE verifier store using the provided shared cache store. -func NewPKCEVerifierStore(ctx context.Context, cacheStore store.StoreInterface) *PKCEVerifierStore { +func NewPKCEVerifierStore(ctx context.Context, cacheStore nbcache.Store) *PKCEVerifierStore { return &PKCEVerifierStore{ - cache: cache.New[string](cacheStore), + cache: cacheStore, ctx: ctx, } } @@ -40,14 +41,14 @@ func (s *PKCEVerifierStore) Store(state, verifier string, ttl time.Duration) err // Returns the verifier and true if found, or empty string and false if not found. // This enforces single-use semantics for PKCE verifiers. func (s *PKCEVerifierStore) LoadAndDelete(state string) (string, bool) { - verifier, err := s.cache.Get(s.ctx, state) + verifier, found, err := s.cache.GetDel(s.ctx, state) if err != nil { - log.Debugf("PKCE verifier not found for state") + log.Warnf("Failed to consume PKCE verifier: %v", err) return "", false } - - if err := s.cache.Delete(s.ctx, state); err != nil { - log.Warnf("Failed to delete PKCE verifier for state: %v", err) + if !found { + log.Debug("PKCE verifier not found for state") + return "", false } return verifier, true diff --git a/management/internals/shared/grpc/pkce_verifier_test.go b/management/internals/shared/grpc/pkce_verifier_test.go new file mode 100644 index 000000000..e7175b6c5 --- /dev/null +++ b/management/internals/shared/grpc/pkce_verifier_test.go @@ -0,0 +1,85 @@ +package grpc + +import ( + "context" + "testing" + "time" +) + +func TestPKCEVerifierStoreLoadAndDelete(t *testing.T) { + const ( + state = "state" + verifier = "verifier" + attempts = 64 + ) + + t.Run("exactly one concurrent caller consumes the verifier", func(t *testing.T) { + store := NewPKCEVerifierStore(context.Background(), testCacheStore(t)) + if err := store.Store(state, verifier, time.Minute); err != nil { + t.Fatalf("couldn't store PKCE verifier: %s", err) + } + + start := make(chan struct{}) + type result struct { + verifier string + found bool + } + results := make(chan result, attempts) + for range attempts { + go func() { + <-start + verifier, found := store.LoadAndDelete(state) + results <- result{verifier: verifier, found: found} + }() + } + close(start) + + winners := 0 + for range attempts { + result := <-results + if result.found { + winners++ + if result.verifier != verifier { + t.Fatalf("unexpected verifier: got %q, expected %q", result.verifier, verifier) + } + } + } + if winners != 1 { + t.Fatalf("expected exactly one PKCE verifier consumer, got %d", winners) + } + }) + + t.Run("replayed state is rejected", func(t *testing.T) { + store := NewPKCEVerifierStore(context.Background(), testCacheStore(t)) + if err := store.Store(state, verifier, time.Minute); err != nil { + t.Fatalf("couldn't store PKCE verifier: %s", err) + } + + if got, found := store.LoadAndDelete(state); !found || got != verifier { + t.Fatalf("first load should return the verifier, got %q, found %t", got, found) + } + if got, found := store.LoadAndDelete(state); found { + t.Fatalf("replayed state should not resolve, got %q", got) + } + }) + + t.Run("unknown state is rejected", func(t *testing.T) { + store := NewPKCEVerifierStore(context.Background(), testCacheStore(t)) + + if got, found := store.LoadAndDelete("never-stored"); found { + t.Fatalf("unknown state should not resolve, got %q", got) + } + }) + + t.Run("expired verifier is rejected", func(t *testing.T) { + store := NewPKCEVerifierStore(context.Background(), testCacheStore(t)) + if err := store.Store(state, verifier, 50*time.Millisecond); err != nil { + t.Fatalf("couldn't store PKCE verifier: %s", err) + } + + time.Sleep(100 * time.Millisecond) + if got, found := store.LoadAndDelete(state); found { + t.Fatalf("expired verifier should not resolve, got %q", got) + } + }) +} diff --git a/management/internals/shared/grpc/proxy.go b/management/internals/shared/grpc/proxy.go index cee50b270..2fc969ad0 100644 --- a/management/internals/shared/grpc/proxy.go +++ b/management/internals/shared/grpc/proxy.go @@ -1651,6 +1651,10 @@ var ( // ErrUserBlocked reports a blocked user, who may not hold a proxy session. ErrUserBlocked = errors.New("user blocked") + // ErrUserNotInGroup reports a user outside the service's distribution + // groups, who may not hold a proxy session for it. + ErrUserNotInGroup = errors.New("user not in allowed groups") + errUserUnresolved = errors.New("user could not be resolved") ) @@ -1689,8 +1693,10 @@ func sameAccount(userAccountID, serviceAccountID string) bool { // GenerateSessionToken creates a signed session JWT for the given domain and // user. The user's group memberships are embedded in the token so policy-aware // middlewares on the proxy can authorise without an extra management round-trip. -// A user the store cannot resolve, or whose account is pending approval or -// blocked, gets no token at all, so the browser never receives a session cookie. +// A user the store cannot resolve, whose account is pending approval or blocked, +// or who is outside the service's distribution groups, gets no token at all: the +// token is a bearer credential for the service, so authorisation has to run +// before it is signed rather than only when the proxy presents it back. func (s *ProxyServiceServer) GenerateSessionToken(ctx context.Context, domain, userID string, method proxyauth.Method) (string, error) { service, err := s.getServiceByDomain(ctx, domain) if err != nil { @@ -1726,6 +1732,14 @@ func (s *ProxyServiceServer) GenerateSessionToken(ctx context.Context, domain, u return "", fmt.Errorf("session token for user %s: %w", userID, err) } + if err := s.checkGroupAccess(service, user); err != nil { + log.WithContext(ctx).WithFields(log.Fields{ + "domain": domain, + "user_id": userID, + }).Debug("GenerateSessionToken: user not in the service's distribution groups") + return "", fmt.Errorf("session token for user %s: %w", userID, ErrUserNotInGroup) + } + groupIDs, groupNames := pairGroupIDsAndNames(userGroups) token, err := sessionkey.SignToken( diff --git a/management/internals/shared/grpc/proxy_test.go b/management/internals/shared/grpc/proxy_test.go index 0379edc6d..29b7c9523 100644 --- a/management/internals/shared/grpc/proxy_test.go +++ b/management/internals/shared/grpc/proxy_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - cachestore "github.com/eko/gocache/lib/v4/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -21,7 +20,7 @@ import ( "github.com/netbirdio/netbird/shared/management/proto" ) -func testCacheStore(t *testing.T) cachestore.StoreInterface { +func testCacheStore(t *testing.T) nbcache.Store { t.Helper() s, err := nbcache.NewStore(context.Background(), 30*time.Minute, 10*time.Minute, 100) require.NoError(t, err) diff --git a/management/internals/shared/grpc/server.go b/management/internals/shared/grpc/server.go index 844ae42db..a9cc0ad36 100644 --- a/management/internals/shared/grpc/server.go +++ b/management/internals/shared/grpc/server.go @@ -247,17 +247,8 @@ func (s *Server) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_S sRealIP := realIP.String() peerMeta := extractPeerMeta(ctx, syncReq.GetMeta()) - userID, err := s.accountManager.GetUserIDByPeerKey(ctx, peerKey.String()) - if err != nil { - s.syncSem.Add(-1) - if errStatus, ok := internalStatus.FromError(err); ok && errStatus.Type() == internalStatus.NotFound { - return status.Errorf(codes.PermissionDenied, "peer is not registered") - } - return mapError(ctx, err) - } - metahashed := metaHash(peerMeta) - if userID == "" && !s.loginFilter.allowLogin(peerKey.String(), metahashed) { + if !s.loginFilter.allowLogin(peerKey.String(), metahashed) { if s.appMetrics != nil { s.appMetrics.GRPCMetrics().CountSyncRequestBlocked() } diff --git a/management/internals/shared/grpc/validate_session_test.go b/management/internals/shared/grpc/validate_session_test.go index 03f200414..4e70e61e4 100644 --- a/management/internals/shared/grpc/validate_session_test.go +++ b/management/internals/shared/grpc/validate_session_test.go @@ -431,6 +431,57 @@ func TestValidateSession_MissingToken(t *testing.T) { assert.Contains(t, resp.DeniedReason, "missing") } +// TestGenerateSessionToken_UserNotInAllowedGroupGetsNoToken is the regression +// guard for the group-authorisation bypass: the callback used to hand a signed +// token to a user the service denies, and the proxy honoured that token as soon +// as the user moved it into the nb_session cookie themselves. Authorisation has +// to run before the token is signed. +func TestGenerateSessionToken_UserNotInAllowedGroupGetsNoToken(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + token, err := setup.proxyService.GenerateSessionToken(context.Background(), "restricted-proxy.example.com", "nonGroupUserId", auth.MethodOIDC) + + require.Error(t, err, "a user outside the distribution groups must not receive a token") + assert.ErrorIs(t, err, ErrUserNotInGroup, "the callback maps this sentinel onto the access denied page") + assert.Empty(t, token, "no token may reach the browser") +} + +func TestGenerateSessionToken_UserInAllowedGroupGetsTokenWithGroups(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + ctx := context.Background() + svc, err := setup.store.GetServiceByID(ctx, store.LockingStrengthNone, "testAccountId", "restrictedProxyId") + require.NoError(t, err) + + token, err := setup.proxyService.GenerateSessionToken(ctx, "restricted-proxy.example.com", "allowedUserId", auth.MethodOIDC) + require.NoError(t, err) + require.NotEmpty(t, token) + + pubKey, err := base64.StdEncoding.DecodeString(svc.SessionPublicKey) + require.NoError(t, err) + + userID, _, method, groups, _, err := auth.ValidateSessionJWT(token, "restricted-proxy.example.com", pubKey) + require.NoError(t, err) + assert.Equal(t, "allowedUserId", userID) + assert.Equal(t, auth.MethodOIDC.String(), method) + assert.Equal(t, []string{"allowedGroupId"}, groups, "the proxy gates the cookie on this claim, so it must carry the matched group") +} + +// TestGenerateSessionToken_UnrestrictedServiceAllowsAnyAccountUser keeps the new +// gate scoped: a service without distribution groups is open to every user of +// its account, as before. +func TestGenerateSessionToken_UnrestrictedServiceAllowsAnyAccountUser(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + token, err := setup.proxyService.GenerateSessionToken(context.Background(), "test-proxy.example.com", "nonGroupUserId", auth.MethodOIDC) + + require.NoError(t, err, "an unrestricted service must keep working for any user of the account") + assert.NotEmpty(t, token) +} + type testValidateSessionServiceManager struct { store store.Store } diff --git a/management/server/account.go b/management/server/account.go index 4fe0e5338..3ceef79db 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "net" "net/netip" "os" @@ -15,10 +14,6 @@ import ( "sync" "time" - "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" - "github.com/netbirdio/netbird/management/server/job" - "github.com/netbirdio/netbird/shared/auth" - cacheStore "github.com/eko/gocache/lib/v4/store" "github.com/eko/gocache/store/redis/v4" "github.com/rs/xid" @@ -30,6 +25,7 @@ import ( "github.com/netbirdio/netbird/formatter/hook" "github.com/netbirdio/netbird/idp/dex" "github.com/netbirdio/netbird/management/internals/controllers/network_map" + "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" nbconfig "github.com/netbirdio/netbird/management/internals/server/config" "github.com/netbirdio/netbird/management/server/account" "github.com/netbirdio/netbird/management/server/activity" @@ -40,6 +36,7 @@ import ( "github.com/netbirdio/netbird/management/server/idp" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator" "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" + "github.com/netbirdio/netbird/management/server/job" nbpeer "github.com/netbirdio/netbird/management/server/peer" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/permissions/modules" @@ -51,6 +48,7 @@ import ( "github.com/netbirdio/netbird/management/server/types" "github.com/netbirdio/netbird/management/server/util" "github.com/netbirdio/netbird/route" + "github.com/netbirdio/netbird/shared/auth" nbdomain "github.com/netbirdio/netbird/shared/management/domain" "github.com/netbirdio/netbird/shared/management/networkmap/nmdata" "github.com/netbirdio/netbird/shared/management/status" @@ -65,7 +63,7 @@ const ( type userLoggedInOnce bool func cacheEntryExpiration() time.Duration { - r := rand.Intn(int(nbcache.DefaultIDPCacheExpirationMax.Milliseconds()-nbcache.DefaultIDPCacheExpirationMin.Milliseconds())) + int(nbcache.DefaultIDPCacheExpirationMin.Milliseconds()) + r := util.RandIntn(int(nbcache.DefaultIDPCacheExpirationMax.Milliseconds()-nbcache.DefaultIDPCacheExpirationMin.Milliseconds())) + int(nbcache.DefaultIDPCacheExpirationMin.Milliseconds()) return time.Duration(r) * time.Millisecond } @@ -239,6 +237,10 @@ func BuildManager( log.WithContext(ctx).Error(err) } + if IsEmbeddedIdp(idpManager) && accountsCounter > 1 { + log.WithContext(ctx).Warnf("embedded IdP requires a single account, found %d", accountsCounter) + } + // enable single account mode only if configured by user and number of existing accounts is not grater than 1 am.singleAccountMode = singleAccountModeDomain != "" && accountsCounter <= 1 if am.singleAccountMode { @@ -1593,7 +1595,10 @@ func (am *DefaultAccountManager) updateUserAuthWithSingleMode(ctx context.Contex if err != nil { return err } - userAuth.Domain = domain + // Keep the configured single account domain when the existing account has none + if domain != "" { + userAuth.Domain = domain + } log.WithContext(ctx).Debugf("overriding JWT Domain and DomainCategory claims since single account mode is enabled") return nil @@ -1838,6 +1843,7 @@ func (am *DefaultAccountManager) getAccountIDWithAuthorizationClaims(ctx context return am.addNewPrivateAccount(ctx, domainAccountID, userAuth) } + func (am *DefaultAccountManager) getPrivateDomainWithGlobalLock(ctx context.Context, domain string) (string, context.CancelFunc, error) { domainAccountID, err := am.Store.GetAccountIDByPrivateDomain(ctx, store.LockingStrengthNone, domain) if handleNotFound(err) != nil { @@ -2470,8 +2476,7 @@ func (am *DefaultAccountManager) ensureIPv6Subnet(ctx context.Context, transacti return transaction.UpdateAccountNetworkV6(ctx, accountID, network.NetV6) } if network.NetV6.IP == nil { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - network.NetV6 = types.AllocateIPv6Subnet(r) + network.NetV6 = types.AllocateIPv6Subnet() // Sync settings to match the allocated subnet so SaveAccountSettings persists it. ones, _ := network.NetV6.Mask.Size() diff --git a/management/server/agentnetwork_budgetrule_realstack_test.go b/management/server/agentnetwork_budgetrule_realstack_test.go index 95f9c35dc..b046581e8 100644 --- a/management/server/agentnetwork_budgetrule_realstack_test.go +++ b/management/server/agentnetwork_budgetrule_realstack_test.go @@ -96,10 +96,14 @@ func TestAgentNetwork_UpdateSettings_PreservesImmutableAndTogglesCollection(t *t assert.False(t, before.EnablePromptCollection, "prompt collection defaults off") _, err = mgr.CreateProvider(ctx, adminUserID, &agenttypes.Provider{ - AccountID: accountID, - ProviderID: "openai_api", - Name: "openai", - UpstreamURL: "https://api.openai.com", + AccountID: accountID, + ProviderID: "openai_api", + Name: "openai", + // A private address: the save-time credential check leaves it + // unchecked rather than spending a dummy key against the real + // api.openai.com, which the vendor refuses and which would make + // this test depend on the runner having egress. + UpstreamURL: "https://10.255.255.1", APIKey: "sk-test", Enabled: true, Models: []agenttypes.ProviderModel{{ID: "gpt-5.4"}}, diff --git a/management/server/agentnetwork_realstack_test.go b/management/server/agentnetwork_realstack_test.go index d4efb1607..d438ffbdd 100644 --- a/management/server/agentnetwork_realstack_test.go +++ b/management/server/agentnetwork_realstack_test.go @@ -101,10 +101,14 @@ func TestAgentNetwork_ProviderCRUD_FansOutToProxyAndClientPeers(t *testing.T) { drain(proxyCh) provider, err := agentMgr.CreateProvider(ctx, adminUserID, &agenttypes.Provider{ - AccountID: accountID, - ProviderID: "openai_api", - Name: "openai-test", - UpstreamURL: "https://api.openai.com", + AccountID: accountID, + ProviderID: "openai_api", + Name: "openai-test", + // A private address: the save-time credential check leaves it + // unchecked rather than spending a dummy key against the real + // api.openai.com, which the vendor refuses and which would make + // this test depend on the runner having egress. + UpstreamURL: "https://10.255.255.1", APIKey: "sk-test-key", Enabled: true, Models: []agenttypes.ProviderModel{{ID: "gpt-5.4"}}, diff --git a/management/server/auth/session.go b/management/server/auth/session.go index 7621a1c10..778146589 100644 --- a/management/server/auth/session.go +++ b/management/server/auth/session.go @@ -7,9 +7,6 @@ import ( "errors" "fmt" "time" - - "github.com/eko/gocache/lib/v4/cache" - "github.com/eko/gocache/lib/v4/store" ) const ( @@ -22,12 +19,17 @@ var ( ErrTokenExpired = errors.New("JWT expired") ) -type SessionStore struct { - cache *cache.Cache[string] +// TokenCache atomically records used JWTs until their expiration. +type TokenCache interface { + SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) } -func NewSessionStore(cacheStore store.StoreInterface) *SessionStore { - return &SessionStore{cache: cache.New[string](cacheStore)} +type SessionStore struct { + cache TokenCache +} + +func NewSessionStore(cacheStore TokenCache) *SessionStore { + return &SessionStore{cache: cacheStore} } // RegisterToken records a JWT until its exp time and rejects reuse. @@ -38,20 +40,14 @@ func (s *SessionStore) RegisterToken(ctx context.Context, token string, expiresA } key := usedTokenKeyPrefix + hashToken(token) - _, err := s.cache.Get(ctx, key) - if err == nil { + created, err := s.cache.SetNX(ctx, key, usedTokenMarker, ttl) + if err != nil { + return fmt.Errorf("store used token entry: %w", err) + } + if !created { return ErrTokenAlreadyUsed } - var notFound *store.NotFound - if !errors.As(err, ¬Found) { - return fmt.Errorf("failed to lookup used token entry: %w", err) - } - - if err := s.cache.Set(ctx, key, usedTokenMarker, store.WithExpiration(ttl)); err != nil { - return fmt.Errorf("failed to store used token entry: %w", err) - } - return nil } diff --git a/management/server/auth/session_test.go b/management/server/auth/session_test.go index 3a7d85f4c..7c82dfc43 100644 --- a/management/server/auth/session_test.go +++ b/management/server/auth/session_test.go @@ -2,6 +2,7 @@ package auth import ( "context" + "errors" "testing" "time" @@ -38,6 +39,39 @@ func TestSessionStore_RegisterSameTokenTwiceIsRejected(t *testing.T) { assert.ErrorIs(t, err, ErrTokenAlreadyUsed) } +func TestSessionStore_ConcurrentRegistrationAllowsOneCaller(t *testing.T) { + s := newTestSessionStore(t) + ctx := context.Background() + const attempts = 100 + + start := make(chan struct{}) + results := make(chan error, attempts) + for range attempts { + go func() { + <-start + results <- s.RegisterToken(ctx, "token", time.Now().Add(time.Hour)) + }() + } + close(start) + + succeeded := 0 + alreadyUsed := 0 + for range attempts { + err := <-results + switch { + case err == nil: + succeeded++ + case errors.Is(err, ErrTokenAlreadyUsed): + alreadyUsed++ + default: + require.NoError(t, err, "concurrent registration returned an unexpected error") + } + } + + assert.Equal(t, 1, succeeded, "exactly one concurrent caller should register the token") + assert.Equal(t, attempts-1, alreadyUsed, "every other caller should be rejected as already used") +} + func TestSessionStore_RegisterDifferentTokensAreIndependent(t *testing.T) { s := newTestSessionStore(t) ctx := context.Background() @@ -72,6 +106,23 @@ func TestSessionStore_EntryEvictsAtTTLAndAllowsReRegistration(t *testing.T) { require.NoError(t, s.RegisterToken(ctx, token, time.Now().Add(time.Hour))) } +type failingTokenCache struct { + err error +} + +func (f failingTokenCache) SetNX(context.Context, string, string, time.Duration) (bool, error) { + return false, f.err +} + +func TestSessionStore_CacheErrorIsReturned(t *testing.T) { + cacheErr := errors.New("cache unavailable") + s := NewSessionStore(failingTokenCache{err: cacheErr}) + + err := s.RegisterToken(context.Background(), "token", time.Now().Add(time.Hour)) + require.Error(t, err, "cache failure should be surfaced to the caller") + assert.ErrorIs(t, err, cacheErr, "cache error should be wrapped, not replaced") +} + func TestHashToken_StableAndDoesNotLeak(t *testing.T) { a := hashToken("tokenA") b := hashToken("tokenB") diff --git a/management/server/cache/memory.go b/management/server/cache/memory.go new file mode 100644 index 000000000..f140f3ec8 --- /dev/null +++ b/management/server/cache/memory.go @@ -0,0 +1,57 @@ +package cache + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/eko/gocache/lib/v4/store" + gocachestore "github.com/eko/gocache/store/go_cache/v4" + gocache "github.com/patrickmn/go-cache" +) + +type goCacheStore struct { + store.StoreInterface + client *gocache.Cache + mu sync.Mutex +} + +func newMemoryStore(maxTimeout, cleanupInterval time.Duration) Store { + client := gocache.New(maxTimeout, cleanupInterval) + return &goCacheStore{ + StoreInterface: gocachestore.NewGoCache(client), + client: client, + } +} + +func (s *goCacheStore) SetNX(_ context.Context, key, value string, ttl time.Duration) (bool, error) { + // Add only returns an error when a non-expired entry already exists. + if err := s.client.Add(key, value, ttl); err != nil { + return false, nil //nolint:nilerr + } + return true, nil +} + +// GetDel reads the value under key and removes it. go-cache has no native read-and-delete +// and releases its own lock between the two calls, so mu holds the pair together and no +// value is consumed twice. +// +// Writes do not take mu: a Set landing mid-pair is lost, since GetDel returns the prior +// value and deletes the new one. Callers must write a consumed key only once. +func (s *goCacheStore) GetDel(_ context.Context, key string) (string, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + + value, found := s.client.Get(key) + if !found { + return "", false, nil + } + s.client.Delete(key) + + str, ok := value.(string) + if !ok { + return "", false, fmt.Errorf("cached value is %T, not a string", value) + } + return str, true, nil +} diff --git a/management/server/cache/memory_test.go b/management/server/cache/memory_test.go new file mode 100644 index 000000000..363504921 --- /dev/null +++ b/management/server/cache/memory_test.go @@ -0,0 +1,76 @@ +package cache_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/management/server/cache" +) + +func TestMemoryStore(t *testing.T) { + memStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100) + require.NoError(t, err, "couldn't create memory store") + + ctx := context.Background() + key, value := "testing", "tested" + err = memStore.Set(ctx, key, value) + assert.NoError(t, err, "couldn't set testing data") + + result, err := memStore.Get(ctx, key) + assert.NoError(t, err, "couldn't get testing data") + assert.Equal(t, value, result, "value returned doesn't match testing data") + + created, err := memStore.SetNX(ctx, "conditional", value, 100*time.Millisecond) + require.NoError(t, err, "couldn't conditionally set testing data") + require.True(t, created, "first conditional set should create the entry") + + created, err = memStore.SetNX(ctx, "conditional", value, 100*time.Millisecond) + require.NoError(t, err, "couldn't conditionally check testing data") + require.False(t, created, "second conditional set should not replace the entry") + + // test expiration + time.Sleep(300 * time.Millisecond) + _, err = memStore.Get(ctx, key) + assert.Error(t, err, "value should not be found") +} + +func TestMemoryStoreGetDel(t *testing.T) { + ctx := context.Background() + newStore := func(t *testing.T) cache.Store { + t.Helper() + memStore, err := cache.NewStore(ctx, time.Minute, time.Minute, 100) + require.NoError(t, err, "couldn't create memory store") + + return memStore + } + + const ( + key = "consume" + value = "verifier" + ) + + t.Run("exactly one concurrent caller consumes the key", func(t *testing.T) { + memStore := newStore(t) + require.NoError(t, memStore.Set(ctx, key, value), "couldn't set testing data") + + assertGetDelConsumedOnce(ctx, t, []cache.Store{memStore}, key, value) + assertGetDelMisses(ctx, t, memStore, key) + }) + + t.Run("missing key is not an error", func(t *testing.T) { + assertGetDelMisses(ctx, t, newStore(t), "never-set") + }) + + t.Run("expired key is not found", func(t *testing.T) { + memStore := newStore(t) + _, err := memStore.SetNX(ctx, key, value, 50*time.Millisecond) + require.NoError(t, err, "couldn't set testing data") + + time.Sleep(100 * time.Millisecond) + assertGetDelMisses(ctx, t, memStore, key) + }) +} diff --git a/management/server/cache/redis.go b/management/server/cache/redis.go new file mode 100644 index 000000000..0cd921c92 --- /dev/null +++ b/management/server/cache/redis.go @@ -0,0 +1,63 @@ +package cache + +import ( + "context" + "errors" + "fmt" + "math" + "time" + + "github.com/eko/gocache/lib/v4/store" + redisstore "github.com/eko/gocache/store/redis/v4" + "github.com/redis/go-redis/v9" + log "github.com/sirupsen/logrus" +) + +type redisStore struct { + store.StoreInterface + client *redis.Client +} + +func getRedisStore(ctx context.Context, redisEnvAddr string, maxConn int) (Store, error) { + options, err := redis.ParseURL(redisEnvAddr) + if err != nil { + return nil, fmt.Errorf("parsing redis cache url: %s", err) + } + + options.MaxIdleConns = int(math.Ceil(float64(maxConn) * 0.5)) // 50% of max conns + options.MinIdleConns = int(math.Ceil(float64(maxConn) * 0.1)) // 10% of max conns + options.MaxActiveConns = maxConn + options.ConnMaxIdleTime = 30 * time.Minute + options.ConnMaxLifetime = 0 + options.PoolTimeout = 10 * time.Second + redisClient := redis.NewClient(options) + subCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + _, err = redisClient.Ping(subCtx).Result() + if err != nil { + return nil, err + } + + log.WithContext(subCtx).Infof("using redis cache at %s", redisEnvAddr) + + return &redisStore{ + StoreInterface: redisstore.NewRedis(redisClient), + client: redisClient, + }, nil +} + +func (s *redisStore) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + return s.client.SetNX(ctx, key, value, ttl).Result() +} + +func (s *redisStore) GetDel(ctx context.Context, key string) (string, bool, error) { + value, err := s.client.GetDel(ctx, key).Result() + if errors.Is(err, redis.Nil) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return value, true, nil +} diff --git a/management/server/cache/redis_test.go b/management/server/cache/redis_test.go new file mode 100644 index 000000000..994ec7490 --- /dev/null +++ b/management/server/cache/redis_test.go @@ -0,0 +1,153 @@ +package cache_test + +import ( + "context" + "testing" + "time" + + "github.com/eko/gocache/lib/v4/store" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + testcontainersredis "github.com/testcontainers/testcontainers-go/modules/redis" + + "github.com/netbirdio/netbird/management/server/cache" +) + +func startRedis(t *testing.T) string { + t.Helper() + + ctx := context.Background() + redisContainer, err := testcontainersredis.Run(ctx, "redis:7") + require.NoError(t, err, "couldn't start redis container") + + t.Cleanup(func() { + if err := redisContainer.Terminate(ctx); err != nil { + t.Logf("failed to terminate container: %s", err) + } + }) + + redisURL, err := redisContainer.ConnectionString(ctx) + require.NoError(t, err, "couldn't get connection string") + + t.Setenv(cache.RedisStoreEnvVar, redisURL) + return redisURL +} + +func newRedisStore(t *testing.T) cache.Store { + t.Helper() + + redisStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100) + require.NoError(t, err) + + return redisStore +} + +func TestRedisStoreConnectionFailure(t *testing.T) { + t.Setenv(cache.RedisStoreEnvVar, "redis://127.0.0.1:6379") + _, err := cache.NewStore(context.Background(), 10*time.Millisecond, 30*time.Millisecond, 100) + require.Error(t, err, "getting redis cache store should return error") +} + +func TestRedisStoreConnectionSuccess(t *testing.T) { + ctx := context.Background() + redisURL := startRedis(t) + redisStore := newRedisStore(t) + + key, value := "testing", "tested" + err := redisStore.Set(ctx, key, value, store.WithExpiration(100*time.Millisecond)) + assert.NoError(t, err, "couldn't set testing data") + + result, err := redisStore.Get(ctx, key) + assert.NoError(t, err, "couldn't get testing data") + assert.Equal(t, value, result, "value returned doesn't match testing data") + + options, err := redis.ParseURL(redisURL) + require.NoError(t, err, "parsing redis cache url") + + redisClient := redis.NewClient(options) + r, err := redisClient.Get(ctx, key).Result() + assert.NoError(t, err, "couldn't get testing data from redis") + assert.Equal(t, value, r, "value returned from redis doesn't match testing data") + + // test expiration + time.Sleep(300 * time.Millisecond) + _, err = redisStore.Get(ctx, key) + assert.Error(t, err, "value should not be found") +} + +func TestRedisStoreSetNX(t *testing.T) { + ctx := context.Background() + redisURL := startRedis(t) + redisStore, secondRedisStore := newRedisStore(t), newRedisStore(t) + + const ( + key = "conditional" + value = "tested" + ) + + start := make(chan struct{}) + type setResult struct { + created bool + err error + } + results := make(chan setResult, 2) + for _, cacheStore := range []cache.Store{redisStore, secondRedisStore} { + go func() { + <-start + created, err := cacheStore.SetNX(ctx, key, value, time.Minute) + results <- setResult{created: created, err: err} + }() + } + close(start) + + created := 0 + for range 2 { + result := <-results + require.NoError(t, result.err, "conditional redis set failed") + if result.created { + created++ + } + } + require.Equal(t, 1, created, "expected exactly one redis client to create the entry") + + options, err := redis.ParseURL(redisURL) + require.NoError(t, err, "parsing redis cache url") + + ttl, err := redis.NewClient(options).PTTL(ctx, key).Result() + require.NoError(t, err, "couldn't read entry TTL") + require.Positive(t, ttl, "created entry should have a positive TTL") +} + +func TestRedisStoreGetDel(t *testing.T) { + ctx := context.Background() + startRedis(t) + redisStore, secondRedisStore := newRedisStore(t), newRedisStore(t) + + const ( + key = "consume" + value = "verifier" + ) + + t.Run("exactly one caller across independent clients consumes the key", func(t *testing.T) { + // A generous TTL: the key is consumed explicitly, so expiry racing the + // concurrent callers would only make the test flaky on a loaded runner. + err := redisStore.Set(ctx, key, value, store.WithExpiration(time.Minute)) + require.NoError(t, err, "couldn't set value to consume") + + assertGetDelConsumedOnce(ctx, t, []cache.Store{redisStore, secondRedisStore}, key, value) + assertGetDelMisses(ctx, t, secondRedisStore, key) + }) + + t.Run("missing key is not an error", func(t *testing.T) { + assertGetDelMisses(ctx, t, redisStore, "never-set") + }) + + t.Run("expired key is not found", func(t *testing.T) { + err := redisStore.Set(ctx, key, value, store.WithExpiration(50*time.Millisecond)) + require.NoError(t, err, "couldn't set value to consume") + + time.Sleep(100 * time.Millisecond) + assertGetDelMisses(ctx, t, redisStore, key) + }) +} diff --git a/management/server/cache/store.go b/management/server/cache/store.go index 2ca8e8603..a0c093e5d 100644 --- a/management/server/cache/store.go +++ b/management/server/cache/store.go @@ -2,17 +2,10 @@ package cache import ( "context" - "fmt" - "math" "os" "time" "github.com/eko/gocache/lib/v4/store" - gocache_store "github.com/eko/gocache/store/go_cache/v4" - redis_store "github.com/eko/gocache/store/redis/v4" - gocache "github.com/patrickmn/go-cache" - "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" ) // RedisStoreEnvVar is the environment variable that determines if a redis store should be used. @@ -31,15 +24,23 @@ const ( DefaultStoreMaxConn = 1000 ) +// Store extends the shared cache interface with conditional and consuming operations. +type Store interface { + store.StoreInterface + // SetNX stores a value with a TTL only when the key does not exist. + SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) + // GetDel reads a value and removes it, so only one caller can consume a key. + GetDel(ctx context.Context, key string) (value string, found bool, err error) +} + // NewStore creates a new cache store with the given max timeout and cleanup interval. It checks for the environment Variable RedisStoreEnvVar // to determine if a redis store should be used. If the environment variable is set, it will attempt to connect to the redis store. -func NewStore(ctx context.Context, maxTimeout, cleanupInterval time.Duration, maxConn int) (store.StoreInterface, error) { +func NewStore(ctx context.Context, maxTimeout, cleanupInterval time.Duration, maxConn int) (Store, error) { redisAddr := GetAddrFromEnv() if redisAddr != "" { return getRedisStore(ctx, redisAddr, maxConn) } - goc := gocache.New(maxTimeout, cleanupInterval) - return gocache_store.NewGoCache(goc), nil + return newMemoryStore(maxTimeout, cleanupInterval), nil } // GetAddrFromEnv returns the redis address from the environment variable RedisStoreEnvVar or its legacy counterpart. @@ -50,29 +51,3 @@ func GetAddrFromEnv() string { } return addr } - -func getRedisStore(ctx context.Context, redisEnvAddr string, maxConn int) (store.StoreInterface, error) { - options, err := redis.ParseURL(redisEnvAddr) - if err != nil { - return nil, fmt.Errorf("parsing redis cache url: %s", err) - } - - options.MaxIdleConns = int(math.Ceil(float64(maxConn) * 0.5)) // 50% of max conns - options.MinIdleConns = int(math.Ceil(float64(maxConn) * 0.1)) // 10% of max conns - options.MaxActiveConns = maxConn - options.ConnMaxIdleTime = 30 * time.Minute - options.ConnMaxLifetime = 0 - options.PoolTimeout = 10 * time.Second - redisClient := redis.NewClient(options) - subCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - - _, err = redisClient.Ping(subCtx).Result() - if err != nil { - return nil, err - } - - log.WithContext(subCtx).Infof("using redis cache at %s", redisEnvAddr) - - return redis_store.NewRedis(redisClient), nil -} diff --git a/management/server/cache/store_test.go b/management/server/cache/store_test.go index b869170f0..a59be8393 100644 --- a/management/server/cache/store_test.go +++ b/management/server/cache/store_test.go @@ -3,101 +3,53 @@ package cache_test import ( "context" "testing" - "time" - "github.com/eko/gocache/lib/v4/store" - "github.com/redis/go-redis/v9" - testcontainersredis "github.com/testcontainers/testcontainers-go/modules/redis" + "github.com/stretchr/testify/require" "github.com/netbirdio/netbird/management/server/cache" ) -func TestMemoryStore(t *testing.T) { - memStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100) - if err != nil { - t.Fatalf("couldn't create memory store: %s", err) - } - ctx := context.Background() - key, value := "testing", "tested" - err = memStore.Set(ctx, key, value) - if err != nil { - t.Errorf("couldn't set testing data: %s", err) - } - result, err := memStore.Get(ctx, key) - if err != nil { - t.Errorf("couldn't get testing data: %s", err) - } - if value != result.(string) { - t.Errorf("value returned doesn't match testing data, got %s, expected %s", result, value) - } - // test expiration - time.Sleep(300 * time.Millisecond) - _, err = memStore.Get(ctx, key) - if err == nil { - t.Error("value should not be found") - } -} +func assertGetDelConsumedOnce(ctx context.Context, t *testing.T, stores []cache.Store, key, value string) { + t.Helper() -func TestRedisStoreConnectionFailure(t *testing.T) { - t.Setenv(cache.RedisStoreEnvVar, "redis://127.0.0.1:6379") - _, err := cache.NewStore(context.Background(), 10*time.Millisecond, 30*time.Millisecond, 100) - if err == nil { - t.Fatal("getting redis cache store should return error") - } -} + const getDelAttempts = 64 -func TestRedisStoreConnectionSuccess(t *testing.T) { - ctx := context.Background() - redisContainer, err := testcontainersredis.Run(ctx, "redis:7") - if err != nil { - t.Fatalf("couldn't start redis container: %s", err) + type getDelResult struct { + value string + found bool + err error } - defer func() { - if err := redisContainer.Terminate(ctx); err != nil { - t.Logf("failed to terminate container: %s", err) + + start := make(chan struct{}) + results := make(chan getDelResult, getDelAttempts) + for i := range getDelAttempts { + cacheStore := stores[i%len(stores)] + go func() { + <-start + value, found, err := cacheStore.GetDel(ctx, key) + results <- getDelResult{value: value, found: found, err: err} + }() + } + close(start) + + consumers := 0 + for range getDelAttempts { + result := <-results + require.NoError(t, result.err, "concurrent GetDel failed") + if !result.found { + continue } - }() - redisURL, err := redisContainer.ConnectionString(ctx) - if err != nil { - t.Fatalf("couldn't get connection string: %s", err) - } - - t.Setenv(cache.RedisStoreEnvVar, redisURL) - redisStore, err := cache.NewStore(context.Background(), 100*time.Millisecond, 300*time.Millisecond, 100) - if err != nil { - t.Fatalf("couldn't create redis store: %s", err) - } - - key, value := "testing", "tested" - err = redisStore.Set(ctx, key, value, store.WithExpiration(100*time.Millisecond)) - if err != nil { - t.Errorf("couldn't set testing data: %s", err) - } - result, err := redisStore.Get(ctx, key) - if err != nil { - t.Errorf("couldn't get testing data: %s", err) - } - if value != result.(string) { - t.Errorf("value returned doesn't match testing data, got %s, expected %s", result, value) - } - - options, err := redis.ParseURL(redisURL) - if err != nil { - t.Errorf("parsing redis cache url: %s", err) - } - - redisClient := redis.NewClient(options) - r, e := redisClient.Get(ctx, key).Result() - if e != nil { - t.Errorf("couldn't get testing data from redis: %s", e) - } - if value != r { - t.Errorf("value returned from redis doesn't match testing data, got %s, expected %s", r, value) - } - // test expiration - time.Sleep(300 * time.Millisecond) - _, err = redisStore.Get(ctx, key) - if err == nil { - t.Error("value should not be found") + consumers++ + require.Equal(t, value, result.value, "consumed value doesn't match testing data") } + require.Equal(t, 1, consumers, "expected exactly one consumer") +} + +func assertGetDelMisses(ctx context.Context, t *testing.T, cacheStore cache.Store, key string) { + t.Helper() + + value, found, err := cacheStore.GetDel(ctx, key) + require.NoError(t, err, "GetDel on a missing key should not error") + require.False(t, found, "GetDel should not find key %q, got value %q", key, value) + require.Empty(t, value, "GetDel should return an empty value when not found") } diff --git a/management/server/group_ipv6_test.go b/management/server/group_ipv6_test.go index dfb436060..2679aa7c2 100644 --- a/management/server/group_ipv6_test.go +++ b/management/server/group_ipv6_test.go @@ -2,7 +2,6 @@ package server import ( "context" - "math/rand" "testing" "time" @@ -28,7 +27,7 @@ func TestGroupIPv6Assignment(t *testing.T) { require.NoError(t, err) // Allocate IPv6 subnet for the account - account.Network.NetV6 = types.AllocateIPv6Subnet(rand.New(rand.NewSource(time.Now().UnixNano()))) + account.Network.NetV6 = types.AllocateIPv6Subnet() require.NoError(t, am.Store.SaveAccount(ctx, account)) // Create setup key diff --git a/management/server/http/handlers/proxy/auth.go b/management/server/http/handlers/proxy/auth.go index 62725e8d4..0f4b72e14 100644 --- a/management/server/http/handlers/proxy/auth.go +++ b/management/server/http/handlers/proxy/auth.go @@ -100,9 +100,10 @@ func (h *AuthCallbackHandler) handleCallback(w http.ResponseWriter, r *http.Requ return } - // Group validation is performed by the proxy via ValidateSession gRPC call. - // This allows the proxy to show 403 pages directly without redirect dance. - + // GenerateSessionToken applies the service's group and account-status gates, + // so a user without access never receives a token. The proxy re-checks the + // installed cookie against the service's allowed groups, and renders the + // denial page from the error carried back in the redirect. sessionToken, err := h.proxyService.GenerateSessionToken(r.Context(), redirectURL.Hostname(), userID, auth.MethodOIDC) if err != nil { log.WithError(err).Error("Failed to create session token") @@ -136,6 +137,9 @@ func sessionTokenErrorDescription(err error) string { if errors.Is(err, nbgrpc.ErrUserBlocked) { return "Your account is blocked" } + if errors.Is(err, nbgrpc.ErrUserNotInGroup) { + return "You are not authorized to access this service" + } return "Service configuration error" } diff --git a/management/server/identity_provider_test.go b/management/server/identity_provider_test.go index eef69dc14..ecc47337c 100644 --- a/management/server/identity_provider_test.go +++ b/management/server/identity_provider_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/netbirdio/netbird/management/internals/controllers/network_map/controller" "github.com/netbirdio/netbird/management/internals/controllers/network_map/update_channel" @@ -34,6 +34,20 @@ import ( func createManagerWithEmbeddedIdP(t testing.TB) (*DefaultAccountManager, *update_channel.PeersUpdateManager, error) { t.Helper() + return createManagerWithEmbeddedIdPMode(t, "netbird.selfhosted") +} + +func createManagerWithEmbeddedIdPMode(t testing.TB, singleAccountModeDomain string) (*DefaultAccountManager, *update_channel.PeersUpdateManager, error) { + t.Helper() + return createManagerWithEmbeddedIdPModeAndSetup(t, singleAccountModeDomain, nil) +} + +func createManagerWithEmbeddedIdPModeAndSetup( + t testing.TB, + singleAccountModeDomain string, + setupStore func(context.Context, store.Store) error, +) (*DefaultAccountManager, *update_channel.PeersUpdateManager, error) { + t.Helper() ctx := context.Background() @@ -43,6 +57,11 @@ func createManagerWithEmbeddedIdP(t testing.TB) (*DefaultAccountManager, *update return nil, nil, err } t.Cleanup(cleanUp) + if setupStore != nil { + if err := setupStore(ctx, testStore); err != nil { + return nil, nil, err + } + } // Create embedded IdP manager embeddedConfig := &idp.EmbeddedIdPConfig{ @@ -93,7 +112,7 @@ func createManagerWithEmbeddedIdP(t testing.TB) (*DefaultAccountManager, *update updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := NewAccountRequestBuffer(ctx, testStore) networkMapController := controller.NewController(ctx, testStore, metrics, updateManager, requestBuffer, MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", port_forwarding.NewControllerMock(), ephemeral_manager.NewEphemeralManager(testStore, peersManager), &config.Config{}, nil) - manager, err := BuildManager(ctx, &config.Config{}, testStore, networkMapController, job.NewJobManager(nil, testStore, peersManager), idpManager, "", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false, cacheStore) + manager, err := BuildManager(ctx, &config.Config{}, testStore, networkMapController, job.NewJobManager(nil, testStore, peersManager), idpManager, singleAccountModeDomain, eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false, cacheStore) if err != nil { return nil, nil, err } @@ -196,6 +215,23 @@ func TestDefaultAccountManager_GetIdentityProvider_NotFound(t *testing.T) { assert.Contains(t, err.Error(), "not found") } +func TestUpdateUserAuthWithSingleModeKeepsConfiguredDomain(t *testing.T) { + ctx := context.Background() + manager, _, err := createManagerWithEmbeddedIdPModeAndSetup(t, "netbird.selfhosted", func(ctx context.Context, testStore store.Store) error { + // An account with no domain, as left behind by an IdP that emitted no domain claims. + return testStore.SaveAccount(ctx, newAccountWithId(ctx, "account-1", "user-1", "", "", "", false)) + }) + require.NoError(t, err) + require.True(t, manager.singleAccountMode) + + userAuth := auth.UserAuth{UserId: "user-2"} + require.NoError(t, manager.updateUserAuthWithSingleMode(ctx, &userAuth)) + + assert.Equal(t, "netbird.selfhosted", userAuth.Domain, + "An empty account domain must not clear the configured single account domain") + assert.Equal(t, types.PrivateCategory, userAuth.DomainCategory) +} + func TestDefaultAccountManager_UpdateIdentityProvider_Validation(t *testing.T) { manager, _, err := createManager(t) require.NoError(t, err) diff --git a/management/server/idp/migration/migration.go b/management/server/idp/migration/migration.go index 01cadb86d..bec0de84c 100644 --- a/management/server/idp/migration/migration.go +++ b/management/server/idp/migration/migration.go @@ -10,6 +10,8 @@ import ( "errors" "fmt" "os" + "regexp" + "strings" log "github.com/sirupsen/logrus" @@ -25,8 +27,10 @@ type Server interface { EventStore() EventStore // may return nil } -const idpSeedInfoKey = "IDP_SEED_INFO" -const dryRunEnvKey = "NB_IDP_MIGRATION_DRY_RUN" +const ( + idpSeedInfoKey = "IDP_SEED_INFO" + dryRunEnvKey = "NB_IDP_MIGRATION_DRY_RUN" +) func isDryRun() bool { return os.Getenv(dryRunEnvKey) == "true" @@ -233,3 +237,163 @@ func PopulateUserInfo(s Server, idpManager idp.Manager, dryRun bool) error { return nil } + +const DefaultSingleAccountDomain = "netbird.selfhosted" + +var ( + ErrMultipleAccounts = errors.New("the embedded IdP supports a single account only") + ErrUnusableDomain = errors.New("domain cannot be resolved in single account mode") + ErrDomainConflict = errors.New("requested domain conflicts with the account domain") +) + +var resolvableDomainRegexp = regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`) + +// RequireSingleAccount refuses to migrate an instance that holds more than one account. +func RequireSingleAccount(s Server) error { + accountsCounter, err := s.Store().GetAccountsCounter(context.Background()) + if err != nil { + return fmt.Errorf("failed to count accounts: %w", err) + } + + if accountsCounter > 1 { + return errMultipleAccounts(accountsCounter) + } + + return nil +} + +func errMultipleAccounts(accountsCounter int64) error { + return fmt.Errorf("%w: this instance has %d accounts. Identity provider connectors are stored without "+ + "an account scope, so every account would share and be able to manage the same connectors. "+ + "Consolidate this instance to a single account, or keep using an external IdP, before migrating", + ErrMultipleAccounts, accountsCounter) +} + +func NormalizeSingleAccountDomain(singleAccountDomain string) (string, error) { + if singleAccountDomain == "" { + singleAccountDomain = DefaultSingleAccountDomain + } + + singleAccountDomain = strings.ToLower(singleAccountDomain) + if !resolvableDomainRegexp.MatchString(singleAccountDomain) { + return "", fmt.Errorf("%w: %q must contain at least one dot and only lowercase letters, digits and "+ + "hyphens, otherwise users cannot join the existing account", ErrUnusableDomain, singleAccountDomain) + } + + return singleAccountDomain, nil +} + +// resolveAccountDomain picks the domain the account should end up with. The account keeps a usable +// domain of its own, the configured one only fills a blank. Anything else is a conflict to report. +func resolveAccountDomain(accountID, accountDomain, singleAccountDomain string, requested bool) (string, error) { + accountDomain = strings.ToLower(accountDomain) + + if accountDomain == "" { + return singleAccountDomain, nil + } + + if !resolvableDomainRegexp.MatchString(accountDomain) { + return "", fmt.Errorf("%w: account %s has domain %q, which must contain at least one dot and only "+ + "lowercase letters, digits and hyphens. Correct the account domain before migrating", + ErrUnusableDomain, accountID, accountDomain) + } + + if requested && accountDomain != singleAccountDomain { + return "", fmt.Errorf("%w: account %s already uses domain %q but %q was requested. Re-run without "+ + "--single-account-mode-domain to keep %q, or correct the account domain first", + ErrDomainConflict, accountID, accountDomain, singleAccountDomain, accountDomain) + } + + return accountDomain, nil +} + +// EnsureSingleAccountDomain gives the remaining account the domain attributes single account mode +// resolves against, so users can still join it after the migration. +func EnsureSingleAccountDomain(s Server, singleAccountDomain string) error { + plan, err := planSingleAccountDomain(s, singleAccountDomain) + if err != nil { + return err + } + if plan.skip { + return nil + } + + if isDryRun() { + log.Infof("[DRY RUN] would set account %s domain to %q, category to %q and mark it as the primary domain account "+ + "(currently domain=%q primary=%v)", plan.accountID, plan.domain, types.PrivateCategory, + plan.currentDomain, plan.isPrimary) + return nil + } + + if err := s.Store().UpdateAccountDomainAttributes(context.Background(), plan.accountID, plan.domain, + types.PrivateCategory, true); err != nil { + return fmt.Errorf("failed to update domain attributes of account %s: %w", plan.accountID, err) + } + + log.Infof("account %s now resolves in single account mode with domain %q", plan.accountID, plan.domain) + return nil +} + +// CheckSingleAccountDomain reports whether EnsureSingleAccountDomain would succeed, without writing. +func CheckSingleAccountDomain(s Server, singleAccountDomain string) error { + _, err := planSingleAccountDomain(s, singleAccountDomain) + return err +} + +type singleAccountDomainPlan struct { + accountID string + domain string + currentDomain string + isPrimary bool + skip bool +} + +// planSingleAccountDomain decides what the account's domain attributes should become. It reads +// only, so it can run both as a preflight and as the first half of the update. +func planSingleAccountDomain(s Server, singleAccountDomain string) (singleAccountDomainPlan, error) { + ctx := context.Background() + + // An empty value means the operator did not pick a domain, so the default is only a fallback. + requested := singleAccountDomain != "" + + singleAccountDomain, err := NormalizeSingleAccountDomain(singleAccountDomain) + if err != nil { + return singleAccountDomainPlan{}, err + } + + accountsCounter, err := s.Store().GetAccountsCounter(ctx) + if err != nil { + return singleAccountDomainPlan{}, fmt.Errorf("failed to count accounts: %w", err) + } + // The count is checked again here: it is read long after RequireSingleAccount, and marking an + // arbitrary account as the primary one for the domain would be wrong. + switch { + case accountsCounter == 0: + log.Info("no accounts yet, nothing to prepare for single account mode") + return singleAccountDomainPlan{skip: true}, nil + case accountsCounter > 1: + return singleAccountDomainPlan{}, errMultipleAccounts(accountsCounter) + } + + accountID, err := s.Store().GetAnyAccountID(ctx) + if err != nil { + return singleAccountDomainPlan{}, fmt.Errorf("failed to get the existing account: %w", err) + } + + isPrimary, accountDomain, err := s.Store().IsPrimaryAccount(ctx, accountID) + if err != nil { + return singleAccountDomainPlan{}, fmt.Errorf("failed to read domain attributes of account %s: %w", accountID, err) + } + + domain, err := resolveAccountDomain(accountID, accountDomain, singleAccountDomain, requested) + if err != nil { + return singleAccountDomainPlan{}, err + } + + return singleAccountDomainPlan{ + accountID: accountID, + domain: domain, + currentDomain: accountDomain, + isPrimary: isPrimary, + }, nil +} diff --git a/management/server/idp/migration/migration_test.go b/management/server/idp/migration/migration_test.go index 2ff71347e..f6a436015 100644 --- a/management/server/idp/migration/migration_test.go +++ b/management/server/idp/migration/migration_test.go @@ -24,6 +24,17 @@ type testStore struct { checkSchemaFunc func(checks []SchemaCheck) []SchemaError updateCalls []updateUserIDCall updateInfoCalls []updateUserInfoCall + + accountsCounter int64 + accounts map[string]*types.Account + domainAttrCalls []domainAttrCall +} + +type domainAttrCall struct { + AccountID string + Domain string + Category string + IsPrimary bool } type updateUserIDCall struct { @@ -38,6 +49,35 @@ type updateUserInfoCall struct { Name string } +func (s *testStore) GetAccountsCounter(context.Context) (int64, error) { + return s.accountsCounter, nil +} + +func (s *testStore) GetAnyAccountID(context.Context) (string, error) { + for id := range s.accounts { + return id, nil + } + return "", fmt.Errorf("no accounts") +} + +func (s *testStore) IsPrimaryAccount(_ context.Context, accountID string) (bool, string, error) { + account, ok := s.accounts[accountID] + if !ok { + return false, "", fmt.Errorf("account %s not found", accountID) + } + return account.IsDomainPrimaryAccount, account.Domain, nil +} + +func (s *testStore) UpdateAccountDomainAttributes(_ context.Context, accountID, domain, category string, isPrimaryDomain bool) error { + s.domainAttrCalls = append(s.domainAttrCalls, domainAttrCall{accountID, domain, category, isPrimaryDomain}) + if account, ok := s.accounts[accountID]; ok { + account.Domain = domain + account.DomainCategory = category + account.IsDomainPrimaryAccount = isPrimaryDomain + } + return nil +} + func (s *testStore) ListUsers(ctx context.Context) ([]*types.User, error) { return s.listUsersFunc(ctx) } @@ -826,3 +866,212 @@ func TestCheckSchema_MockStore(t *testing.T) { assert.Equal(t, "email", errs[0].Column) }) } + +func TestRequireSingleAccount(t *testing.T) { + tests := []struct { + name string + accounts int64 + expectErr bool + }{ + {name: "fresh install", accounts: 0}, + {name: "single account", accounts: 1}, + {name: "multiple accounts", accounts: 3, expectErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := &testServer{store: &testStore{accountsCounter: tt.accounts}} + + err := RequireSingleAccount(srv) + if !tt.expectErr { + require.NoError(t, err) + return + } + + require.Error(t, err) + assert.ErrorIs(t, err, ErrMultipleAccounts) + }) + } +} + +func TestEnsureSingleAccountDomain(t *testing.T) { + tests := []struct { + name string + account *types.Account + requestedDomain string + expectedDomain string + }{ + { + name: "account migrated from an IdP without domain claims", + account: &types.Account{Id: "account-1"}, + expectedDomain: DefaultSingleAccountDomain, + }, + { + name: "requested domain is applied to an account without one", + account: &types.Account{Id: "account-1"}, + requestedDomain: "corp.example.com", + expectedDomain: "corp.example.com", + }, + { + name: "account keeps its own domain", + account: &types.Account{Id: "account-1", Domain: "acme.com"}, + expectedDomain: "acme.com", + }, + { + name: "requesting the domain the account already has is not a conflict", + account: &types.Account{Id: "account-1", Domain: "acme.com"}, + requestedDomain: "acme.com", + expectedDomain: "acme.com", + }, + { + name: "already resolvable account is rewritten with the same values", + account: &types.Account{ + Id: "account-1", + Domain: "acme.com", + DomainCategory: types.PrivateCategory, + IsDomainPrimaryAccount: true, + }, + expectedDomain: "acme.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := &testStore{ + accountsCounter: 1, + accounts: map[string]*types.Account{tt.account.Id: tt.account}, + } + + require.NoError(t, EnsureSingleAccountDomain(&testServer{store: store}, tt.requestedDomain)) + + require.Len(t, store.domainAttrCalls, 1) + assert.Equal(t, domainAttrCall{ + AccountID: tt.account.Id, + Domain: tt.expectedDomain, + Category: types.PrivateCategory, + IsPrimary: true, + }, store.domainAttrCalls[0]) + }) + } +} + +func TestEnsureSingleAccountDomainDryRun(t *testing.T) { + t.Setenv(dryRunEnvKey, "true") + + store := &testStore{ + accountsCounter: 1, + accounts: map[string]*types.Account{"account-1": {Id: "account-1"}}, + } + + require.NoError(t, EnsureSingleAccountDomain(&testServer{store: store}, "")) + assert.Empty(t, store.domainAttrCalls, "Dry run must not write anything") +} + +func TestEnsureSingleAccountDomainRejectsUnresolvableDomains(t *testing.T) { + t.Run("account domain that cannot resolve is reported", func(t *testing.T) { + account := &types.Account{Id: "account-1", Domain: "corp"} + store := &testStore{ + accountsCounter: 1, + accounts: map[string]*types.Account{account.Id: account}, + } + + err := EnsureSingleAccountDomain(&testServer{store: store}, "") + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnusableDomain) + assert.Empty(t, store.domainAttrCalls, "A broken account domain must not be replaced silently") + }) + + t.Run("requested domain conflicting with the account domain is reported", func(t *testing.T) { + account := &types.Account{Id: "account-1", Domain: "acme.com"} + store := &testStore{ + accountsCounter: 1, + accounts: map[string]*types.Account{account.Id: account}, + } + + err := EnsureSingleAccountDomain(&testServer{store: store}, "corp.example.com") + require.Error(t, err) + assert.ErrorIs(t, err, ErrDomainConflict) + assert.Empty(t, store.domainAttrCalls, "A conflict must not overwrite the account domain") + }) + + t.Run("configured domain that cannot resolve is rejected", func(t *testing.T) { + store := &testStore{ + accountsCounter: 1, + accounts: map[string]*types.Account{"account-1": {Id: "account-1"}}, + } + + err := EnsureSingleAccountDomain(&testServer{store: store}, "corp") + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnusableDomain) + assert.Empty(t, store.domainAttrCalls) + }) + + t.Run("account appearing after the preflight is rejected", func(t *testing.T) { + store := &testStore{ + accountsCounter: 2, + accounts: map[string]*types.Account{ + "account-1": {Id: "account-1"}, + "account-2": {Id: "account-2"}, + }, + } + + err := EnsureSingleAccountDomain(&testServer{store: store}, "") + require.Error(t, err) + assert.ErrorIs(t, err, ErrMultipleAccounts) + assert.Empty(t, store.domainAttrCalls, "No account may be marked primary when several exist") + }) + + t.Run("fresh install with no accounts is a no-op", func(t *testing.T) { + store := &testStore{accountsCounter: 0, accounts: map[string]*types.Account{}} + + require.NoError(t, EnsureSingleAccountDomain(&testServer{store: store}, "")) + assert.Empty(t, store.domainAttrCalls) + }) +} + +func TestCheckSingleAccountDomain(t *testing.T) { + tests := []struct { + name string + account *types.Account + requested string + expectErr error + }{ + { + name: "usable account domain passes", + account: &types.Account{Id: "account-1", Domain: "acme.com"}, + }, + { + name: "empty account domain passes", + account: &types.Account{Id: "account-1"}, + }, + { + name: "unresolvable account domain fails", + account: &types.Account{Id: "account-1", Domain: "corp"}, + expectErr: ErrUnusableDomain, + }, + { + name: "conflicting request fails", + account: &types.Account{Id: "account-1", Domain: "acme.com"}, + requested: "corp.example.com", + expectErr: ErrDomainConflict, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := &testStore{ + accountsCounter: 1, + accounts: map[string]*types.Account{tt.account.Id: tt.account}, + } + + err := CheckSingleAccountDomain(&testServer{store: store}, tt.requested) + if tt.expectErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tt.expectErr) + } + + assert.Empty(t, store.domainAttrCalls, "The preflight must not write anything") + }) + } +} diff --git a/management/server/idp/migration/store.go b/management/server/idp/migration/store.go index e7cc54a41..868597a1d 100644 --- a/management/server/idp/migration/store.go +++ b/management/server/idp/migration/store.go @@ -60,6 +60,20 @@ type Store interface { // CheckSchema verifies that all tables and columns required by the migration // exist in the database. Returns a list of problems; an empty slice means OK. CheckSchema(checks []SchemaCheck) []SchemaError + + // GetAccountsCounter returns the total number of accounts in the store. + GetAccountsCounter(ctx context.Context) (int64, error) + + // GetAnyAccountID returns the ID of one of the existing accounts. + GetAnyAccountID(ctx context.Context) (string, error) + + // IsPrimaryAccount returns whether the account is the primary account for its domain, + // along with that domain. + IsPrimaryAccount(ctx context.Context, accountID string) (bool, string, error) + + // UpdateAccountDomainAttributes sets the domain, domain category and primary + // domain flag of an account. + UpdateAccountDomainAttributes(ctx context.Context, accountID string, domain string, category string, isPrimaryDomain bool) error } // RequiredEventSchema lists all tables and columns that the migration tool needs diff --git a/management/server/idp/util.go b/management/server/idp/util.go index ed82fb9e3..6545c2a69 100644 --- a/management/server/idp/util.go +++ b/management/server/idp/util.go @@ -2,11 +2,12 @@ package idp import ( "encoding/json" - "math/rand" "net/url" "os" "strings" "time" + + "github.com/netbirdio/netbird/management/server/util" ) var ( @@ -33,31 +34,32 @@ func GeneratePassword(passwordLength, minSpecialChar, minNum, minUpperCase int) //Set special character for i := 0; i < minSpecialChar; i++ { - random := rand.Intn(len(specialCharSet)) + random := util.RandIntn(len(specialCharSet)) password.WriteString(string(specialCharSet[random])) } //Set numeric for i := 0; i < minNum; i++ { - random := rand.Intn(len(numberSet)) + random := util.RandIntn(len(numberSet)) password.WriteString(string(numberSet[random])) } //Set uppercase for i := 0; i < minUpperCase; i++ { - random := rand.Intn(len(upperCharSet)) + random := util.RandIntn(len(upperCharSet)) password.WriteString(string(upperCharSet[random])) } remainingLength := passwordLength - minSpecialChar - minNum - minUpperCase for i := 0; i < remainingLength; i++ { - random := rand.Intn(len(allCharSet)) + random := util.RandIntn(len(allCharSet)) password.WriteString(string(allCharSet[random])) } inRune := []rune(password.String()) - rand.Shuffle(len(inRune), func(i, j int) { + for i := len(inRune) - 1; i > 0; i-- { + j := util.RandIntn(i + 1) inRune[i], inRune[j] = inRune[j], inRune[i] - }) + } return string(inRune) } diff --git a/management/server/mock_server/management_server_mock.go b/management/server/mock_server/management_server_mock.go index 45049f1fe..cd219b8a9 100644 --- a/management/server/mock_server/management_server_mock.go +++ b/management/server/mock_server/management_server_mock.go @@ -13,7 +13,7 @@ type ManagementServiceServerMock struct { proto.UnimplementedManagementServiceServer LoginFunc func(context.Context, *proto.EncryptedMessage) (*proto.EncryptedMessage, error) - SyncFunc func(*proto.EncryptedMessage, proto.ManagementService_SyncServer) + SyncFunc func(*proto.EncryptedMessage, proto.ManagementService_SyncServer) error GetServerKeyFunc func(context.Context, *proto.Empty) (*proto.ServerKeyResponse, error) IsHealthyFunc func(context.Context, *proto.Empty) (*proto.Empty, error) GetDeviceAuthorizationFlowFunc func(ctx context.Context, req *proto.EncryptedMessage) (*proto.EncryptedMessage, error) @@ -30,7 +30,7 @@ func (m ManagementServiceServerMock) Login(ctx context.Context, req *proto.Encry func (m ManagementServiceServerMock) Sync(msg *proto.EncryptedMessage, sync proto.ManagementService_SyncServer) error { if m.SyncFunc != nil { - return m.Sync(msg, sync) + return m.SyncFunc(msg, sync) } return status.Errorf(codes.Unimplemented, "method Sync not implemented") } diff --git a/management/server/types/network.go b/management/server/types/network.go index 72ca1af85..1ce6465b5 100644 --- a/management/server/types/network.go +++ b/management/server/types/network.go @@ -1,18 +1,18 @@ package types import ( + "crypto/rand" "encoding/binary" "fmt" - "math/rand" "net" "net/netip" "slices" "sync" - "time" "github.com/c-robinson/iplib" "github.com/rs/xid" + "github.com/netbirdio/netbird/management/server/util" "github.com/netbirdio/netbird/shared/management/status" ) @@ -47,14 +47,12 @@ func NewNetwork() *Network { n := iplib.NewNet4(net.ParseIP("100.64.0.0"), NetSize) sub, _ := n.Subnet(SubnetSize) - s := rand.NewSource(time.Now().UnixNano()) - r := rand.New(s) - intn := r.Intn(len(sub)) + intn := util.RandIntn(len(sub)) return &Network{ Identifier: xid.New().String(), Net: sub[intn].IPNet, - NetV6: AllocateIPv6Subnet(r), + NetV6: AllocateIPv6Subnet(), Dns: "", Serial: 0, } @@ -64,18 +62,13 @@ func NewNetwork() *Network { // The format follows RFC 4193 section 3.1: fd + 40-bit Global ID + 16-bit Subnet ID. // The Global ID and Subnet ID are randomized (simplified from the SHA-1 algorithm // in section 3.2.2), giving 2^56 possible /64 subnets across all accounts. -func AllocateIPv6Subnet(r *rand.Rand) net.IPNet { +func AllocateIPv6Subnet() net.IPNet { ip := make(net.IP, 16) ip[0] = 0xfd - // Bytes 1-5: 40-bit random Global ID - ip[1] = byte(r.Intn(256)) - ip[2] = byte(r.Intn(256)) - ip[3] = byte(r.Intn(256)) - ip[4] = byte(r.Intn(256)) - ip[5] = byte(r.Intn(256)) - // Bytes 6-7: 16-bit random Subnet ID - ip[6] = byte(r.Intn(256)) - ip[7] = byte(r.Intn(256)) + // Bytes 1-5: 40-bit random Global ID, bytes 6-7: 16-bit random Subnet ID + if _, err := rand.Read(ip[1:8]); err != nil { + panic(err) + } return net.IPNet{ IP: ip, @@ -109,10 +102,22 @@ func (n *Network) Copy() *Network { } } +// validateIPv4Prefix ensures the prefix is an IPv4 network with assignable host addresses. +func validateIPv4Prefix(prefix netip.Prefix) error { + if !prefix.IsValid() || !prefix.Addr().Is4() || prefix.Bits() < 1 || prefix.Bits() >= 31 { + return fmt.Errorf("invalid IPv4 subnet: %s", prefix.String()) + } + return nil +} + // AllocatePeerIP picks an available IP from a netip.Prefix. // This method considers already taken IPs and reuses IPs if there are gaps in takenIps. // E.g. if prefix=100.30.0.0/16 and takenIps=[100.30.0.1, 100.30.0.4] then the result would be 100.30.0.2 or 100.30.0.3. func AllocatePeerIP(prefix netip.Prefix, takenIps []netip.Addr) (netip.Addr, error) { + if err := validateIPv4Prefix(prefix); err != nil { + return netip.Addr{}, err + } + b := prefix.Masked().Addr().As4() baseIP := binary.BigEndian.Uint32(b[:]) hostBits := 32 - prefix.Bits() @@ -123,15 +128,17 @@ func AllocatePeerIP(prefix netip.Prefix, takenIps []netip.Addr) (netip.Addr, err taken[baseIP+totalIPs-1] = struct{}{} // reserve broadcast IP for _, ip := range takenIps { + if !ip.Is4() { + continue + } ab := ip.As4() taken[binary.BigEndian.Uint32(ab[:])] = struct{}{} } - rng := rand.New(rand.NewSource(time.Now().UnixNano())) maxAttempts := (int(totalIPs) - len(taken)) / 100 for i := 0; i < maxAttempts; i++ { - offset := uint32(rng.Intn(int(totalIPs-2))) + 1 + offset := uint32(util.RandIntn(int(totalIPs-2))) + 1 candidate := baseIP + offset if _, exists := taken[candidate]; !exists { return uint32ToIP(candidate), nil @@ -150,13 +157,16 @@ func AllocatePeerIP(prefix netip.Prefix, takenIps []netip.Addr) (netip.Addr, err // AllocateRandomPeerIP picks a random available IP from a netip.Prefix. func AllocateRandomPeerIP(prefix netip.Prefix) (netip.Addr, error) { + if err := validateIPv4Prefix(prefix); err != nil { + return netip.Addr{}, err + } + b := prefix.Masked().Addr().As4() baseIP := binary.BigEndian.Uint32(b[:]) hostBits := 32 - prefix.Bits() totalIPs := uint32(1 << hostBits) - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - offset := uint32(rng.Intn(int(totalIPs-2))) + 1 + offset := uint32(util.RandIntn(int(totalIPs-2))) + 1 candidate := baseIP + offset return uint32ToIP(candidate), nil @@ -172,23 +182,26 @@ func AllocateRandomPeerIPv6(prefix netip.Prefix) (netip.Addr, error) { ip := prefix.Addr().As16() - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - // Determine which byte the host bits start in firstHostByte := ones / 8 // If the prefix doesn't end on a byte boundary, handle the partial byte partialBits := ones % 8 + var rnd [16]byte + if _, err := rand.Read(rnd[firstHostByte:]); err != nil { + return netip.Addr{}, err + } + if partialBits > 0 { // Keep the network bits in the partial byte, randomize the rest hostMask := byte(0xff >> partialBits) - ip[firstHostByte] = (ip[firstHostByte] & ^hostMask) | (byte(rng.Intn(256)) & hostMask) + ip[firstHostByte] = (ip[firstHostByte] & ^hostMask) | (rnd[firstHostByte] & hostMask) firstHostByte++ } // Randomize remaining full host bytes for i := firstHostByte; i < 16; i++ { - ip[i] = byte(rng.Intn(256)) + ip[i] = rnd[i] } // Avoid all-zeros and all-ones host parts by checking only host bits. diff --git a/management/server/types/network_test.go b/management/server/types/network_test.go index d8a06dbbc..239f72426 100644 --- a/management/server/types/network_test.go +++ b/management/server/types/network_test.go @@ -143,6 +143,34 @@ func TestAllocatePeerIPVariousCIDRs(t *testing.T) { } } +func TestAllocateIPv4InvalidPrefixes(t *testing.T) { + prefixes := []netip.Prefix{ + {}, + netip.MustParsePrefix("0.0.0.0/0"), + netip.MustParsePrefix("192.168.1.0/31"), + netip.MustParsePrefix("192.168.1.1/32"), + netip.MustParsePrefix("fd12:3456:7890:abcd::/64"), + } + + for _, prefix := range prefixes { + t.Run(prefix.String(), func(t *testing.T) { + _, err := AllocatePeerIP(prefix, nil) + assert.Error(t, err) + + _, err = AllocateRandomPeerIP(prefix) + assert.Error(t, err) + }) + } +} + +func TestAllocatePeerIPIgnoresNonIPv4TakenIPs(t *testing.T) { + prefix := netip.MustParsePrefix("192.168.1.0/29") + + ip, err := AllocatePeerIP(prefix, []netip.Addr{netip.MustParseAddr("fd12:3456:7890:abcd::1")}) + require.NoError(t, err) + assert.True(t, prefix.Contains(ip)) +} + func TestGenerateIPs(t *testing.T) { ipNet := net.IPNet{IP: net.ParseIP("100.64.0.0"), Mask: net.IPMask{255, 255, 255, 0}} ips, ipsLen := generateIPs(&ipNet, map[string]struct{}{"100.64.0.0": {}}) diff --git a/management/server/util/util.go b/management/server/util/util.go index d85b55f02..38cd3de58 100644 --- a/management/server/util/util.go +++ b/management/server/util/util.go @@ -1,5 +1,20 @@ package util +import ( + "crypto/rand" + "math/big" +) + +// RandIntn returns a uniformly distributed int in [0, n) sourced from +// crypto/rand. It panics if n <= 0 or the platform randomness source fails. +func RandIntn(n int) int { + v, err := rand.Int(rand.Reader, big.NewInt(int64(n))) + if err != nil { + panic(err) + } + return int(v.Int64()) +} + // Difference returns the elements in `a` that aren't in `b`. func Difference(a, b []string) []string { mb := make(map[string]struct{}, len(b)) diff --git a/proxy/internal/auth/middleware.go b/proxy/internal/auth/middleware.go index 8abdf2923..1d46fd824 100644 --- a/proxy/internal/auth/middleware.go +++ b/proxy/internal/auth/middleware.go @@ -59,6 +59,11 @@ type DomainConfig struct { IPRestrictions *restrict.Filter // Private routes the domain through ValidateTunnelPeer; failure → 403. Private bool + // AllowedGroups holds the group ids that may reach the service through an + // OIDC identity. When non-empty, a session cookie is honoured only if its + // groups claim intersects this set. Empty means group membership does not + // restrict access. + AllowedGroups map[string]struct{} } type validationResult struct { @@ -316,6 +321,9 @@ func (mw *Middleware) handleOAuthCallbackError(w http.ResponseWriter, r *http.Re // forwardWithSessionCookie checks for a valid session cookie and, if found, // sets the user identity on the request context and forwards to the next handler. +// A signature-valid cookie is not on its own a grant: an OIDC session must also +// carry a group the service allows, so a token cannot be replayed past the +// group check that gated the login it came from. func (mw *Middleware) forwardWithSessionCookie(w http.ResponseWriter, r *http.Request, host string, config DomainConfig, next http.Handler) bool { cookie, err := r.Cookie(auth.SessionCookieName) if err != nil { @@ -335,6 +343,14 @@ func (mw *Middleware) forwardWithSessionCookie(w http.ResponseWriter, r *http.Re return false } + if !sessionGroupsAllowed(config.AllowedGroups, auth.Method(method), groups) { + mw.logger.WithFields(log.Fields{ + "host": host, + "user_id": userID, + }).Debug("session cookie rejected: groups claim does not intersect the service's allowed groups") + return false + } + if cd := proxy.CapturedDataFromContext(r.Context()); cd != nil { cd.SetUserID(userID) cd.SetUserEmail(email) @@ -625,7 +641,8 @@ func wasCredentialSubmitted(r *http.Request, method auth.Method) bool { // AddDomain registers authentication schemes for the given domain. With schemes a valid session public key is required. // private=true forces ValidateTunnelPeer enforcement (403 on failure) regardless of the schemes list. -func (mw *Middleware) AddDomain(domain string, schemes []Scheme, publicKeyB64 string, expiration time.Duration, accountID types.AccountID, serviceID types.ServiceID, ipRestrictions *restrict.Filter, private bool) error { +// allowedGroups restricts OIDC sessions to the given group ids; empty means unrestricted. +func (mw *Middleware) AddDomain(domain string, schemes []Scheme, publicKeyB64 string, expiration time.Duration, accountID types.AccountID, serviceID types.ServiceID, ipRestrictions *restrict.Filter, private bool, allowedGroups []string) error { if len(schemes) == 0 { mw.domainsMux.Lock() defer mw.domainsMux.Unlock() @@ -634,6 +651,7 @@ func (mw *Middleware) AddDomain(domain string, schemes []Scheme, publicKeyB64 st ServiceID: serviceID, IPRestrictions: ipRestrictions, Private: private, + AllowedGroups: groupSet(allowedGroups), } return nil } @@ -656,6 +674,7 @@ func (mw *Middleware) AddDomain(domain string, schemes []Scheme, publicKeyB64 st ServiceID: serviceID, IPRestrictions: ipRestrictions, Private: private, + AllowedGroups: groupSet(allowedGroups), } return nil } @@ -707,6 +726,50 @@ func (mw *Middleware) validateSessionToken(ctx context.Context, host, token stri return &validationResult{UserID: userID, UserEmail: email, Valid: true, Groups: groups, GroupNames: groupNames}, nil } +// groupSet builds the lookup set the cookie path consults, returning nil for an +// empty list so callers can test membership restriction with len(). +func groupSet(groups []string) map[string]struct{} { + if len(groups) == 0 { + return nil + } + set := make(map[string]struct{}, len(groups)) + for _, g := range groups { + if g != "" { + set[g] = struct{}{} + } + } + if len(set) == 0 { + return nil + } + return set +} + +// sessionGroupsAllowed reports whether a session token's groups claim satisfies +// the service's allowed groups. Only OIDC sessions are gated: password, PIN and +// header credentials carry no group identity and are authorised by the secret +// itself, which mirrors how management validates them. A token minted before the +// groups claim existed carries none and is therefore denied on a group-restricted +// service, which sends the user back through login for a fresh decision. A method +// this build doesn't know carries no such argument, so it is denied. +func sessionGroupsAllowed(allowed map[string]struct{}, method auth.Method, groups []string) bool { + if len(allowed) == 0 { + return true + } + switch method { + case auth.MethodPassword, auth.MethodPIN, auth.MethodHeader: + return true + case auth.MethodOIDC: + for _, g := range groups { + if _, ok := allowed[g]; ok { + return true + } + } + return false + default: + return false + } +} + // stripSessionTokenParam returns the request URI with the session_token query // parameter removed so it doesn't linger in the browser's address bar or history. func stripSessionTokenParam(u *url.URL) string { diff --git a/proxy/internal/auth/middleware_test.go b/proxy/internal/auth/middleware_test.go index 9220ce790..f1242f95e 100644 --- a/proxy/internal/auth/middleware_test.go +++ b/proxy/internal/auth/middleware_test.go @@ -66,7 +66,7 @@ func TestAddDomain_ValidKey(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - err := mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false) + err := mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil) require.NoError(t, err) mw.domainsMux.RLock() @@ -83,7 +83,7 @@ func TestAddDomain_EmptyKey(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - err := mw.AddDomain("example.com", []Scheme{scheme}, "", time.Hour, "", "", nil, false) + err := mw.AddDomain("example.com", []Scheme{scheme}, "", time.Hour, "", "", nil, false, nil) require.Error(t, err) assert.Contains(t, err.Error(), "invalid session public key size") @@ -97,7 +97,7 @@ func TestAddDomain_InvalidBase64(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - err := mw.AddDomain("example.com", []Scheme{scheme}, "not-valid-base64!!!", time.Hour, "", "", nil, false) + err := mw.AddDomain("example.com", []Scheme{scheme}, "not-valid-base64!!!", time.Hour, "", "", nil, false, nil) require.Error(t, err) assert.Contains(t, err.Error(), "decode session public key") @@ -112,7 +112,7 @@ func TestAddDomain_WrongKeySize(t *testing.T) { shortKey := base64.StdEncoding.EncodeToString([]byte("tooshort")) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - err := mw.AddDomain("example.com", []Scheme{scheme}, shortKey, time.Hour, "", "", nil, false) + err := mw.AddDomain("example.com", []Scheme{scheme}, shortKey, time.Hour, "", "", nil, false, nil) require.Error(t, err) assert.Contains(t, err.Error(), "invalid session public key size") @@ -125,7 +125,7 @@ func TestAddDomain_WrongKeySize(t *testing.T) { func TestAddDomain_NoSchemes_NoKeyRequired(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) - err := mw.AddDomain("example.com", nil, "", time.Hour, "", "", nil, false) + err := mw.AddDomain("example.com", nil, "", time.Hour, "", "", nil, false, nil) require.NoError(t, err, "domains with no auth schemes should not require a key") mw.domainsMux.RLock() @@ -141,8 +141,8 @@ func TestAddDomain_OverwritesPreviousConfig(t *testing.T) { scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp1.PublicKey, time.Hour, "", "", nil, false)) - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp2.PublicKey, 2*time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp1.PublicKey, time.Hour, "", "", nil, false, nil)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp2.PublicKey, 2*time.Hour, "", "", nil, false, nil)) mw.domainsMux.RLock() config := mw.domains["example.com"] @@ -158,7 +158,7 @@ func TestRemoveDomain(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) mw.RemoveDomain("example.com") @@ -182,7 +182,7 @@ func TestProtect_UnknownDomainPassesThrough(t *testing.T) { func TestProtect_DomainWithNoSchemesPassesThrough(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) - require.NoError(t, mw.AddDomain("example.com", nil, "", time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", nil, "", time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -199,7 +199,7 @@ func TestProtect_UnauthenticatedRequestIsBlocked(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) var backendCalled bool backend := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -220,7 +220,7 @@ func TestProtect_HostWithPortIsMatched(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) var backendCalled bool backend := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -241,7 +241,7 @@ func TestProtect_ValidSessionCookiePassesThrough(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) token, err := sessionkey.SignToken(kp.PrivateKey, "test-user", "", "example.com", auth.MethodPIN, nil, nil, time.Hour) require.NoError(t, err) @@ -274,7 +274,7 @@ func TestProtect_SessionCookieGroupsPropagate(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) groups := []string{"engineering", "sre"} token, err := sessionkey.SignToken(kp.PrivateKey, "test-user", "", "example.com", auth.MethodPIN, groups, nil, time.Hour) @@ -339,7 +339,7 @@ func TestProtect_PrivateService_TunnelPeerGroupsPropagate(t *testing.T) { kp := generateTestKeyPair(t) // Private service: no operator schemes — auth gates solely on the tunnel peer. - require.NoError(t, mw.AddDomain("agent.example.com", nil, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, true)) + require.NoError(t, mw.AddDomain("agent.example.com", nil, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, true, nil)) cd := proxy.NewCapturedData("") cd.SetClientIP(netip.MustParseAddr("100.90.1.14")) // CGNAT tunnel source @@ -379,7 +379,7 @@ func TestProtect_PrivateService_TunnelPeerDenied(t *testing.T) { }} mw := NewMiddleware(log.StandardLogger(), validator, nil) kp := generateTestKeyPair(t) - require.NoError(t, mw.AddDomain("agent.example.com", nil, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, true)) + require.NoError(t, mw.AddDomain("agent.example.com", nil, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, true, nil)) cd := proxy.NewCapturedData("") cd.SetClientIP(netip.MustParseAddr("100.90.1.14")) @@ -407,7 +407,7 @@ func TestProtect_ExpiredSessionCookieIsRejected(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) // Sign a token that expired 1 second ago. token, err := sessionkey.SignToken(kp.PrivateKey, "test-user", "", "example.com", auth.MethodPIN, nil, nil, -time.Second) @@ -433,7 +433,7 @@ func TestProtect_WrongDomainCookieIsRejected(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) // Token signed for a different domain audience. token, err := sessionkey.SignToken(kp.PrivateKey, "test-user", "", "other.com", auth.MethodPIN, nil, nil, time.Hour) @@ -460,7 +460,7 @@ func TestProtect_WrongKeyCookieIsRejected(t *testing.T) { kp2 := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp1.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp1.PublicKey, time.Hour, "", "", nil, false, nil)) // Token signed with a different private key. token, err := sessionkey.SignToken(kp2.PrivateKey, "test-user", "", "example.com", auth.MethodPIN, nil, nil, time.Hour) @@ -497,7 +497,7 @@ func TestProtect_SchemeAuthRedirectsWithCookie(t *testing.T) { return "", "pin", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) var backendCalled bool backend := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -550,7 +550,7 @@ func TestProtect_FailedAuthDoesNotSetCookie(t *testing.T) { return "", "pin", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -586,7 +586,7 @@ func TestProtect_MultipleSchemes(t *testing.T) { return "", "password", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{pinScheme, passwordScheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{pinScheme, passwordScheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) var backendCalled bool backend := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -616,7 +616,7 @@ func TestProtect_InvalidTokenFromSchemeReturns400(t *testing.T) { return "invalid-jwt-token", "", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -640,7 +640,7 @@ func TestAddDomain_RandomBytes32NotEd25519(t *testing.T) { key := base64.StdEncoding.EncodeToString(randomBytes) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - err = mw.AddDomain("example.com", []Scheme{scheme}, key, time.Hour, "", "", nil, false) + err = mw.AddDomain("example.com", []Scheme{scheme}, key, time.Hour, "", "", nil, false, nil) require.NoError(t, err, "any 32-byte key should be accepted at registration time") } @@ -649,10 +649,10 @@ func TestAddDomain_InvalidKeyDoesNotCorruptExistingConfig(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) // Attempt to overwrite with an invalid key. - err := mw.AddDomain("example.com", []Scheme{scheme}, "bad", time.Hour, "", "", nil, false) + err := mw.AddDomain("example.com", []Scheme{scheme}, "bad", time.Hour, "", "", nil, false, nil) require.Error(t, err) // The original valid config should still be intact. @@ -676,7 +676,7 @@ func TestProtect_FailedPinAuthCapturesAuthMethod(t *testing.T) { return "", "pin", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) capturedData := proxy.NewCapturedData("") handler := mw.Protect(newPassthroughHandler()) @@ -703,7 +703,7 @@ func TestProtect_FailedPasswordAuthCapturesAuthMethod(t *testing.T) { return "", "password", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) capturedData := proxy.NewCapturedData("") handler := mw.Protect(newPassthroughHandler()) @@ -730,7 +730,7 @@ func TestProtect_NoCredentialsDoesNotCaptureAuthMethod(t *testing.T) { return "", "pin", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) capturedData := proxy.NewCapturedData("") handler := mw.Protect(newPassthroughHandler()) @@ -818,7 +818,7 @@ func TestCheckIPRestrictions_UnparseableAddress(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) err := mw.AddDomain("example.com", nil, "", 0, "acc1", "svc1", - restrict.ParseFilter(restrict.FilterConfig{AllowedCIDRs: []string{"10.0.0.0/8"}}), false) + restrict.ParseFilter(restrict.FilterConfig{AllowedCIDRs: []string{"10.0.0.0/8"}}), false, nil) require.NoError(t, err) handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -854,7 +854,7 @@ func TestCheckIPRestrictions_UsesCapturedDataClientIP(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) err := mw.AddDomain("example.com", nil, "", 0, "acc1", "svc1", - restrict.ParseFilter(restrict.FilterConfig{AllowedCIDRs: []string{"203.0.113.0/24"}}), false) + restrict.ParseFilter(restrict.FilterConfig{AllowedCIDRs: []string{"203.0.113.0/24"}}), false, nil) require.NoError(t, err) handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -895,7 +895,7 @@ func TestCheckIPRestrictions_NilGeoWithCountryRules(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) err := mw.AddDomain("example.com", nil, "", 0, "acc1", "svc1", - restrict.ParseFilter(restrict.FilterConfig{AllowedCountries: []string{"US"}}), false) + restrict.ParseFilter(restrict.FilterConfig{AllowedCountries: []string{"US"}}), false, nil) require.NoError(t, err) handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -922,7 +922,7 @@ func TestCheckIPRestrictions_OverlayOriginSkipsCountryRules(t *testing.T) { restrict.ParseFilter(restrict.FilterConfig{ AllowedCIDRs: []string{"100.64.0.0/10"}, AllowedCountries: []string{"US"}, - }), false) + }), false, nil) require.NoError(t, err) handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -956,7 +956,7 @@ func TestCheckIPRestrictions_OverlayOriginRespectsCIDR(t *testing.T) { mw := NewMiddleware(log.StandardLogger(), nil, nil) err := mw.AddDomain("example.com", nil, "", 0, "acc1", "svc1", - restrict.ParseFilter(restrict.FilterConfig{AllowedCIDRs: []string{"100.64.0.0/16"}}), false) + restrict.ParseFilter(restrict.FilterConfig{AllowedCIDRs: []string{"100.64.0.0/16"}}), false, nil) require.NoError(t, err) handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -984,7 +984,7 @@ func TestProtect_OIDCOnlyRedirectsDirectly(t *testing.T) { return "", oidcURL, nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1013,7 +1013,7 @@ func TestProtect_OIDCWithOtherMethodShowsLoginPage(t *testing.T) { return "", "pin", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{oidcScheme, pinScheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{oidcScheme, pinScheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1043,7 +1043,7 @@ func TestProtect_HeaderAuth_ForwardsOnSuccess(t *testing.T) { kp := generateTestKeyPair(t) hdr := newHeaderScheme(t, "X-API-Key", "secret-key") - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) var backendCalled bool capturedData := proxy.NewCapturedData("") @@ -1079,7 +1079,7 @@ func TestProtect_HeaderAuth_MissingHeaderFallsThrough(t *testing.T) { hdr := newHeaderScheme(t, "X-API-Key", "secret-key") // Also add a PIN scheme so we can verify fallthrough behavior. pinScheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr, pinScheme}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr, pinScheme}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1096,7 +1096,7 @@ func TestProtect_HeaderAuth_WrongValueReturns401(t *testing.T) { kp := generateTestKeyPair(t) hdr := newHeaderScheme(t, "X-API-Key", "secret-key") - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) capturedData := proxy.NewCapturedData("") handler := mw.Protect(newPassthroughHandler()) @@ -1137,7 +1137,7 @@ func TestProtect_HeaderAuth_MatchesAnyConfiguredHeader(t *testing.T) { if tt.matchedLast { schemes = []Scheme{authz, apiKey} } - require.NoError(t, mw.AddDomain("example.com", schemes, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", schemes, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) var backendCalled bool handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -1166,7 +1166,7 @@ func TestProtect_HeaderAuth_RejectsWhenEveryPresentedHeaderFails(t *testing.T) { authz := newHeaderScheme(t, "Authorization", "Bearer proxy-secret") apiKey := newHeaderScheme(t, "X-Api-Key", "secret-key") - require.NoError(t, mw.AddDomain("example.com", []Scheme{authz, apiKey}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{authz, apiKey}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) var backendCalled bool handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -1209,7 +1209,7 @@ func TestProtect_HeaderAuth_ReportsUndecodableHash(t *testing.T) { kp := generateTestKeyPair(t) require.NoError(t, mw.AddDomain("example.com", []Scheme{NewHeader("X-Api-Key", tt.hashes)}, - kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1245,7 +1245,7 @@ func TestProtect_HeaderAuth_NoHashesFailsClosed(t *testing.T) { kp := generateTestKeyPair(t) hdr := NewHeader("X-API-Key", nil) - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) var backendCalled bool handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -1270,7 +1270,7 @@ func TestProtect_HeaderAuth_SubsequentRequestRequiresHeader(t *testing.T) { kp := generateTestKeyPair(t) hdr := newHeaderScheme(t, "X-API-Key", "secret-key") - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) var backendCalls int handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -1309,7 +1309,7 @@ func TestProtect_HeaderAuth_LegacySessionCookieIsIgnored(t *testing.T) { kp := generateTestKeyPair(t) hdr := newHeaderScheme(t, "X-API-Key", "secret-key") - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) // A token management would have minted for header auth before the upgrade. legacyToken, err := sessionkey.SignToken(kp.PrivateKey, auth.HeaderUserID, "", "example.com", auth.MethodHeader, nil, nil, time.Hour) @@ -1351,7 +1351,7 @@ func TestProtect_HeaderAuth_RepeatedValueIsMemoized(t *testing.T) { kp := generateTestKeyPair(t) hdr := newHeaderScheme(t, "X-API-Key", "key-a", "key-b") - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) @@ -1385,7 +1385,7 @@ func TestProtect_HeaderAuth_MultipleValuesSameHeader(t *testing.T) { kp := generateTestKeyPair(t) hdr := newHeaderScheme(t, "Authorization", "Bearer token-a", "Bearer token-b") - require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{hdr}, kp.PublicKey, time.Hour, "acc1", "svc1", nil, false, nil)) var backendCalled bool handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -1443,7 +1443,7 @@ func TestProtect_OIDCOnPlainHTTP_BlockedWith400(t *testing.T) { return "", "https://idp.example.com/authorize", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1467,7 +1467,7 @@ func TestProtect_OIDCOverTLS_NotBlocked(t *testing.T) { return "", "https://idp.example.com/authorize", nil }, } - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1487,7 +1487,7 @@ func TestProtect_NonOIDCSchemes_PlainHTTP_NotBlocked(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1517,7 +1517,7 @@ func TestProtect_TunnelPeerFastPath_RequiresInboundMarker(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) @@ -1552,7 +1552,7 @@ func TestProtect_TunnelPeerFastPath_TakesPathWithInboundMarker(t *testing.T) { kp := generateTestKeyPair(t) scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} - require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false)) + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "", "", nil, false, nil)) handler := mw.Protect(newPassthroughHandler()) diff --git a/proxy/internal/auth/session_groups_test.go b/proxy/internal/auth/session_groups_test.go new file mode 100644 index 000000000..6635b812b --- /dev/null +++ b/proxy/internal/auth/session_groups_test.go @@ -0,0 +1,167 @@ +package auth + +import ( + "context" + "crypto/tls" + "net/http" + "net/http/httptest" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/sessionkey" + "github.com/netbirdio/netbird/proxy/auth" + "github.com/netbirdio/netbird/proxy/internal/proxy" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// denyingSessionValidator mimics management for a user who completed OIDC login +// but is outside the service's distribution groups: ValidateSession denies. +type denyingSessionValidator struct { + calls int +} + +func (d *denyingSessionValidator) ValidateSession(context.Context, *proto.ValidateSessionRequest, ...grpc.CallOption) (*proto.ValidateSessionResponse, error) { + d.calls++ + return &proto.ValidateSessionResponse{Valid: false, UserId: "user-1", DeniedReason: "not_in_group"}, nil +} + +func (d *denyingSessionValidator) ValidateTunnelPeer(context.Context, *proto.ValidateTunnelPeerRequest, ...grpc.CallOption) (*proto.ValidateTunnelPeerResponse, error) { + return &proto.ValidateTunnelPeerResponse{Valid: false}, nil +} + +// TestProtect_SelfInstalledCookieCannotBypassGroupCheck is the regression guard +// for the group-authorisation bypass: a user denied at login still holds the raw +// session token from the ?session_token= redirect, so pasting it into the +// nb_session cookie must not buy access. The cookie path validated only the JWT +// signature, which turned the token management had already refused into a bearer +// credential for the service. +func TestProtect_SelfInstalledCookieCannotBypassGroupCheck(t *testing.T) { + validator := &denyingSessionValidator{} + mw := NewMiddleware(log.StandardLogger(), validator, nil) + kp := generateTestKeyPair(t) + + oidc := &stubScheme{method: auth.MethodOIDC, authFn: func(r *http.Request) (string, string, error) { + return r.URL.Query().Get("session_token"), "https://idp.example/authorize", nil + }} + require.NoError(t, mw.AddDomain("example.com", []Scheme{oidc}, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, false, []string{"grp-allowed"})) + + // The token a denied user gets to see: validly signed for this service and + // domain, but carrying no group the service allows. + token, err := sessionkey.SignToken(kp.PrivateKey, "user-1", "john.doe@example.com", "example.com", auth.MethodOIDC, nil, nil, time.Hour) + require.NoError(t, err) + + backendHits := 0 + handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + backendHits++ + w.WriteHeader(http.StatusOK) + })) + + t.Run("token in the callback URL is denied", func(t *testing.T) { + rec := serveWithCookie(t, handler, "https://example.com/?session_token="+token, nil) + + assert.Equal(t, http.StatusForbidden, rec.Code, "group check must deny the login") + assert.Empty(t, rec.Result().Cookies(), "a denied login must not install a session cookie") + }) + + t.Run("same token pasted into the session cookie is denied", func(t *testing.T) { + rec := serveWithCookie(t, handler, "https://example.com/", &http.Cookie{Name: auth.SessionCookieName, Value: token}) + + assert.NotEqual(t, http.StatusOK, rec.Code, "a self-installed cookie must not reach the backend") + assert.Equal(t, 0, backendHits, "backend must never be reached without an allowed group") + }) +} + +// TestProtect_SessionCookieWithAllowedGroupPassesThrough is the positive half of +// the group gate: a member of an allowed group keeps the cookie fast-path, with +// no management round-trip. +func TestProtect_SessionCookieWithAllowedGroupPassesThrough(t *testing.T) { + validator := &denyingSessionValidator{} + mw := NewMiddleware(log.StandardLogger(), validator, nil) + kp := generateTestKeyPair(t) + + oidc := &stubScheme{method: auth.MethodOIDC} + require.NoError(t, mw.AddDomain("example.com", []Scheme{oidc}, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, false, []string{"grp-other", "grp-allowed"})) + + token, err := sessionkey.SignToken(kp.PrivateKey, "user-2", "jane@example.com", "example.com", auth.MethodOIDC, + []string{"grp-unrelated", "grp-allowed"}, []string{"Unrelated", "Allowed"}, time.Hour) + require.NoError(t, err) + + handler := mw.Protect(newPassthroughHandler()) + rec := serveWithCookie(t, handler, "https://example.com/", &http.Cookie{Name: auth.SessionCookieName, Value: token}) + + assert.Equal(t, http.StatusOK, rec.Code, "a cookie carrying an allowed group must pass through") + assert.Equal(t, 0, validator.calls, "the cookie fast-path must not call management") +} + +// TestProtect_NonOIDCSessionCookieIgnoresGroupRestriction locks the scope of the +// gate: PIN, password and header credentials carry no group identity and are +// authorised by the secret itself, exactly as management validates them. +func TestProtect_NonOIDCSessionCookieIgnoresGroupRestriction(t *testing.T) { + mw := NewMiddleware(log.StandardLogger(), nil, nil) + kp := generateTestKeyPair(t) + + scheme := &stubScheme{method: auth.MethodPIN, promptID: "pin"} + require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, kp.PublicKey, time.Hour, "acct-1", "svc-1", nil, false, []string{"grp-allowed"})) + + token, err := sessionkey.SignToken(kp.PrivateKey, "pin-user", "", "example.com", auth.MethodPIN, nil, nil, time.Hour) + require.NoError(t, err) + + handler := mw.Protect(newPassthroughHandler()) + rec := serveWithCookie(t, handler, "https://example.com/", &http.Cookie{Name: auth.SessionCookieName, Value: token}) + + assert.Equal(t, http.StatusOK, rec.Code, "a PIN session must not be gated on OIDC group membership") +} + +func TestSessionGroupsAllowed(t *testing.T) { + allowed := groupSet([]string{"a", "b"}) + + tests := []struct { + name string + allowed map[string]struct{} + method auth.Method + groups []string + want bool + }{ + {"unrestricted service allows a groupless token", nil, auth.MethodOIDC, nil, true}, + {"restricted service allows an intersecting token", allowed, auth.MethodOIDC, []string{"c", "b"}, true}, + {"restricted service denies a disjoint token", allowed, auth.MethodOIDC, []string{"c"}, false}, + {"restricted service denies a groupless token", allowed, auth.MethodOIDC, nil, false}, + {"restricted service ignores a pin token", allowed, auth.MethodPIN, nil, true}, + {"restricted service ignores a password token", allowed, auth.MethodPassword, nil, true}, + {"restricted service ignores a header token", allowed, auth.MethodHeader, nil, true}, + {"restricted service denies an unknown method", allowed, auth.Method("totp"), []string{"a"}, false}, + {"restricted service denies a token with no method", allowed, auth.Method(""), []string{"a"}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, sessionGroupsAllowed(tc.allowed, tc.method, tc.groups)) + }) + } +} + +func TestGroupSetDropsEmptyEntries(t *testing.T) { + assert.Nil(t, groupSet(nil), "no groups means unrestricted") + assert.Nil(t, groupSet([]string{"", ""}), "blank ids must not restrict access to nothing reachable") + assert.Equal(t, map[string]struct{}{"a": {}}, groupSet([]string{"a", ""})) +} + +// serveWithCookie drives the middleware over TLS with captured data attached, +// optionally carrying a session cookie. +func serveWithCookie(t *testing.T, handler http.Handler, url string, cookie *http.Cookie) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, url, nil) + req.TLS = &tls.ConnectionState{} + if cookie != nil { + req.AddCookie(cookie) + } + req = req.WithContext(proxy.WithCapturedData(req.Context(), proxy.NewCapturedData(""))) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + return rec +} diff --git a/proxy/internal/auth/tunnel_lookup_test.go b/proxy/internal/auth/tunnel_lookup_test.go index 808aa8b41..066f5dc1a 100644 --- a/proxy/internal/auth/tunnel_lookup_test.go +++ b/proxy/internal/auth/tunnel_lookup_test.go @@ -44,7 +44,7 @@ func (s *stubSessionValidator) ValidateTunnelPeer(_ context.Context, in *proto.V func newTunnelMiddleware(t *testing.T, validator SessionValidator) *Middleware { t.Helper() mw := NewMiddleware(log.New(), validator, nil) - require.NoError(t, mw.AddDomain("svc.example", nil, "", 0, "acct-1", "svc-1", nil, false)) + require.NoError(t, mw.AddDomain("svc.example", nil, "", 0, "acct-1", "svc-1", nil, false, nil)) return mw } @@ -235,8 +235,8 @@ func TestForwardWithTunnelPeer_RoutesAccountIDIntoCacheKey(t *testing.T) { } mw := NewMiddleware(log.New(), validator, nil) - require.NoError(t, mw.AddDomain("svc-a.example", nil, "", 0, "acct-a", "svc-a", nil, false)) - require.NoError(t, mw.AddDomain("svc-b.example", nil, "", 0, "acct-b", "svc-b", nil, false)) + require.NoError(t, mw.AddDomain("svc-a.example", nil, "", 0, "acct-a", "svc-a", nil, false, nil)) + require.NoError(t, mw.AddDomain("svc-b.example", nil, "", 0, "acct-b", "svc-b", nil, false, nil)) // The fast-path requires the inbound-listener marker on the context. // The peerstore lookup itself is account-agnostic at this level @@ -299,7 +299,7 @@ func TestForwardWithTunnelPeer_LocalLookupShortCircuitDoesNotPopulateCache(t *te func TestPrivateService_FailsClosedOnTunnelPeerFailure(t *testing.T) { mw := NewMiddleware(log.New(), nil, nil) - require.NoError(t, mw.AddDomain("private.svc", nil, "", 0, "acct-1", "svc-1", nil, true)) + require.NoError(t, mw.AddDomain("private.svc", nil, "", 0, "acct-1", "svc-1", nil, true, nil)) called := false handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -328,7 +328,7 @@ func TestPrivateService_ForwardsOnTunnelPeerSuccess(t *testing.T) { }, } mw := NewMiddleware(log.New(), validator, nil) - require.NoError(t, mw.AddDomain("private.svc", nil, "", 0, "acct-1", "svc-1", nil, true)) + require.NoError(t, mw.AddDomain("private.svc", nil, "", 0, "acct-1", "svc-1", nil, true, nil)) called := false handler := mw.Protect(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go b/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go index badd358c5..03a17aa52 100644 --- a/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go +++ b/proxy/internal/middleware/builtin/llm_router/bedrock_route_test.go @@ -115,3 +115,20 @@ func TestRouter_BedrockNamespacedInferenceProfilesStripsPrefix(t *testing.T) { assert.Equal(t, "/bedrock", out.Mutations.RewriteUpstream.StripPathPrefix, "the namespace prefix must not reach the real Bedrock endpoint") } + +// TestRouteClaimsModel_VertexNormalizesCandidate is the Vertex counterpart of +// the Bedrock case above: the parser strips the "@version" suffix from the +// path model, so a provider registered with the versioned form must still +// match the normalized request model. +func TestRouteClaimsModel_VertexNormalizesCandidate(t *testing.T) { + route := ProviderRoute{Vertex: true, Models: []string{"claude-sonnet-4-5@20250929"}} + assert.True(t, routeClaimsModel(route, "claude-sonnet-4-5"), + "raw @version Vertex model must match the normalized request model") + assert.False(t, routeClaimsModel(route, "claude-opus-4-8"), + "a model outside the provider's list must not match") + + // Non-Vertex routes keep exact matching (no @version stripping). + openai := ProviderRoute{Models: []string{"gpt-4o@2024"}} + assert.False(t, routeClaimsModel(openai, "gpt-4o"), + "non-Vertex routes must not strip an @version suffix") +} diff --git a/proxy/internal/middleware/builtin/llm_router/middleware.go b/proxy/internal/middleware/builtin/llm_router/middleware.go index 6381f01c7..5c0a45119 100644 --- a/proxy/internal/middleware/builtin/llm_router/middleware.go +++ b/proxy/internal/middleware/builtin/llm_router/middleware.go @@ -331,6 +331,11 @@ func discoverableModels(route ProviderRoute, userGroups []string) ([]string, boo intersection[m] = struct{}{} } } + if route.Vertex { + if _, ok := permitted[llm.NormalizeVertexModel(m)]; ok { + intersection[m] = struct{}{} + } + } } return sortedModels(intersection), true } @@ -869,6 +874,11 @@ func routeClaimsModel(route ProviderRoute, model string) bool { if route.Bedrock && llm.NormalizeBedrockModel(candidate) == model { return true } + // Vertex likewise: the parser strips the "@version" suffix from the + // path model, while the operator may register the versioned form. + if route.Vertex && llm.NormalizeVertexModel(candidate) == model { + return true + } // A client may pin a dated Anthropic id ("claude-sonnet-4-5-20250929") // where the operator registered the undated one. Only an undated // registration absorbs a dated request: normalising both sides would diff --git a/proxy/management_integration_test.go b/proxy/management_integration_test.go index cb82813b0..df016e790 100644 --- a/proxy/management_integration_test.go +++ b/proxy/management_integration_test.go @@ -571,6 +571,7 @@ func TestIntegration_ProxyConnection_ReconnectDoesNotDuplicateState(t *testing.T proxytypes.ServiceID(mapping.GetId()), nil, mapping.GetPrivate(), + mapping.GetAuth().GetAllowedGroupIds(), ) require.NoError(t, err) diff --git a/proxy/server.go b/proxy/server.go index 38477fb87..5b652e61c 100644 --- a/proxy/server.go +++ b/proxy/server.go @@ -2069,7 +2069,7 @@ func (s *Server) updateMapping(ctx context.Context, mapping *proto.ProxyMapping) s.warnIfGeoUnavailable(mapping.GetDomain(), mapping.GetAccessRestrictions()) maxSessionAge := time.Duration(mapping.GetAuth().GetMaxSessionAgeSeconds()) * time.Second - if err := s.auth.AddDomain(mapping.GetDomain(), schemes, mapping.GetAuth().GetSessionKey(), maxSessionAge, accountID, svcID, ipRestrictions, mapping.GetPrivate()); err != nil { + if err := s.auth.AddDomain(mapping.GetDomain(), schemes, mapping.GetAuth().GetSessionKey(), maxSessionAge, accountID, svcID, ipRestrictions, mapping.GetPrivate(), mapping.GetAuth().GetAllowedGroupIds()); err != nil { return fmt.Errorf("auth setup for domain %s: %w", mapping.GetDomain(), err) } m := s.protoToMapping(ctx, mapping) diff --git a/shared/management/client/client.go b/shared/management/client/client.go index c48e1ed3e..13beabee6 100644 --- a/shared/management/client/client.go +++ b/shared/management/client/client.go @@ -12,7 +12,7 @@ import ( // Client is the interface for the management service client. type Client interface { io.Closer - Sync(ctx context.Context, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error + Sync(ctx context.Context, getInfo func(ctx context.Context) *system.Info, msgHandler func(msg *proto.SyncResponse) error) error Job(ctx context.Context, msgHandler func(msg *proto.JobRequest) *proto.JobResponse) error Register(setupKey string, jwtToken string, sysInfo *system.Info, sshKey []byte, dnsLabels domain.List) (*proto.LoginResponse, error) Login(sysInfo *system.Info, sshKey []byte, dnsLabels domain.List) (*proto.LoginResponse, error) diff --git a/shared/management/client/client_test.go b/shared/management/client/client_test.go index d91dab221..e6335dccb 100644 --- a/shared/management/client/client_test.go +++ b/shared/management/client/client_test.go @@ -2,9 +2,11 @@ package client import ( "context" + "fmt" "net" "os" "sync" + "sync/atomic" "testing" "time" @@ -305,7 +307,7 @@ func TestClient_Sync(t *testing.T) { defer cancel() go func() { - err = client.Sync(ctx, info, func(msg *mgmtProto.SyncResponse) error { + err = client.Sync(ctx, func(context.Context) *system.Info { return info }, func(msg *mgmtProto.SyncResponse) error { ch <- msg return nil }) @@ -397,6 +399,75 @@ func wgKeyFromBytes(raw []byte) string { return k.String() } +func TestClient_SyncGathersInfoOnEveryConnect(t *testing.T) { + s, lis, mgmtMockServer, serverKey := startMockManagement(t) + defer s.GracefulStop() + + testKey, err := wgtypes.GenerateKey() + require.NoError(t, err) + + hostnames := make(chan string, 2) + mgmtMockServer.SyncFunc = func(msg *mgmtProto.EncryptedMessage, _ mgmtProto.ManagementService_SyncServer) error { + peerKey, err := wgtypes.ParseKey(msg.GetWgPubKey()) + if err != nil { + t.Errorf("invalid peer key: %v", err) + return status.Error(codes.InvalidArgument, err.Error()) + } + syncReq := &mgmtProto.SyncRequest{} + if err := encryption.DecryptMessage(peerKey, serverKey, msg.Body, syncReq); err != nil { + t.Errorf("decrypt sync request: %v", err) + return status.Error(codes.InvalidArgument, err.Error()) + } + select { + case hostnames <- syncReq.GetMeta().GetHostname(): + default: + } + // Returning closes the stream, so the client reconnects and gathers again. + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client, err := NewClient(ctx, lis.Addr().String(), testKey, false) + require.NoError(t, err) + + var gathers atomic.Int32 + done := make(chan struct{}) + go func() { + defer close(done) + _ = client.Sync(ctx, func(ctx context.Context) *system.Info { + info := system.GetInfo(ctx) + info.Hostname = fmt.Sprintf("host-%d", gathers.Add(1)) + return info + }, func(*mgmtProto.SyncResponse) error { return nil }) + }() + + // A connect attempt can fail before it reaches the server, so the sequence + // numbers seen here may skip. What matters is that the reconnect carries a + // newly gathered info instead of the one sent on the previous stream. + var seen []int + for len(seen) < 2 { + select { + case got := <-hostnames: + var n int + _, err := fmt.Sscanf(got, "host-%d", &n) + require.NoError(t, err, "hostname should carry the gather sequence number") + seen = append(seen, n) + case <-time.After(10 * time.Second): + t.Fatalf("timeout waiting for the second sync request, got %v", seen) + } + } + assert.Greater(t, seen[1], seen[0], "the reconnect should carry a newly gathered info") + + cancel() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for Sync to return after cancel") + } +} + func Test_SystemMetaDataFromClient(t *testing.T) { s, lis, mgmtMockServer, serverKey := startMockManagement(t) defer s.GracefulStop() diff --git a/shared/management/client/grpc.go b/shared/management/client/grpc.go index efea47df0..ce2d07429 100644 --- a/shared/management/client/grpc.go +++ b/shared/management/client/grpc.go @@ -205,9 +205,9 @@ func (c *GrpcClient) ready() bool { // Sync wraps the real client's Sync endpoint call and takes care of retries and encryption/decryption of messages // Blocking request. The result will be sent via msgHandler callback function -func (c *GrpcClient) Sync(ctx context.Context, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error { +func (c *GrpcClient) Sync(ctx context.Context, getInfo func(ctx context.Context) *system.Info, msgHandler func(msg *proto.SyncResponse) error) error { return c.withMgmtStream(ctx, func(ctx context.Context, serverPubKey wgtypes.Key, backOff backoff.BackOff) error { - return c.handleSyncStream(ctx, serverPubKey, sysInfo, msgHandler, backOff) + return c.handleSyncStream(ctx, serverPubKey, getInfo, msgHandler, backOff) }) } @@ -424,11 +424,11 @@ func (c *GrpcClient) sendJobResponse( return nil } -func (c *GrpcClient) handleSyncStream(ctx context.Context, serverPubKey wgtypes.Key, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error, backOff backoff.BackOff) error { +func (c *GrpcClient) handleSyncStream(ctx context.Context, serverPubKey wgtypes.Key, getInfo func(ctx context.Context) *system.Info, msgHandler func(msg *proto.SyncResponse) error, backOff backoff.BackOff) error { ctx, cancelStream := context.WithCancel(ctx) defer cancelStream() - stream, err := c.connectToSyncStream(ctx, serverPubKey, sysInfo) + stream, err := c.connectToSyncStream(ctx, serverPubKey, getInfo(ctx)) if err != nil { log.Debugf("failed to open Management Service stream: %s", err) c.notifyDisconnected(err) diff --git a/shared/management/client/mock.go b/shared/management/client/mock.go index e57e314da..28278dcab 100644 --- a/shared/management/client/mock.go +++ b/shared/management/client/mock.go @@ -11,7 +11,7 @@ import ( // MockClient is a mock implementation of the Client interface for testing. type MockClient struct { CloseFunc func() error - SyncFunc func(ctx context.Context, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error + SyncFunc func(ctx context.Context, getInfo func(ctx context.Context) *system.Info, msgHandler func(msg *proto.SyncResponse) error) error RegisterFunc func(setupKey string, jwtToken string, info *system.Info, sshKey []byte, dnsLabels domain.List) (*proto.LoginResponse, error) LoginFunc func(info *system.Info, sshKey []byte, dnsLabels domain.List) (*proto.LoginResponse, error) ExtendAuthSessionFunc func(info *system.Info, jwtToken string) (*proto.ExtendAuthSessionResponse, error) @@ -38,11 +38,11 @@ func (m *MockClient) Close() error { return m.CloseFunc() } -func (m *MockClient) Sync(ctx context.Context, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error { +func (m *MockClient) Sync(ctx context.Context, getInfo func(ctx context.Context) *system.Info, msgHandler func(msg *proto.SyncResponse) error) error { if m.SyncFunc == nil { return nil } - return m.SyncFunc(ctx, sysInfo, msgHandler) + return m.SyncFunc(ctx, getInfo, msgHandler) } func (m *MockClient) Job(ctx context.Context, msgHandler func(msg *proto.JobRequest) *proto.JobResponse) error { diff --git a/shared/management/http/api/generate.sh b/shared/management/http/api/generate.sh index ba29a6905..8f563e99a 100755 --- a/shared/management/http/api/generate.sh +++ b/shared/management/http/api/generate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if ! which realpath > /dev/null 2>&1 diff --git a/shared/management/http/api/openapi.yml b/shared/management/http/api/openapi.yml index 35b6a8c5f..5ad682c6b 100644 --- a/shared/management/http/api/openapi.yml +++ b/shared/management/http/api/openapi.yml @@ -5357,7 +5357,7 @@ components: upstream_url: type: string description: | - The upstream being configured. Used to reach vendors that serve their listing from the same host as inference, and to read back the region for those whose host embeds one. Ignored when provider_id is supplied. + The upstream being configured. Used to reach vendors that serve their listing from the same host as inference, and to read back the region for those whose host embeds one. Sent alongside provider_id, it overrides the stored upstream, so an edit can be listed against the URL on the form before it is saved. example: "https://bedrock-runtime.eu-central-1.amazonaws.com" api_key: type: string @@ -5365,7 +5365,7 @@ components: example: "sk-..." provider_id: type: string - description: Existing Agent Network provider record whose stored credential and upstream should be used. Lets the form refresh the list without the client holding the key. + description: Existing Agent Network provider record to query with. Its stored credential is used, and its upstream unless upstream_url overrides it, so the form can refresh the list without the client holding the key. example: "ch8i4ug6lnn4g9hqv7m0" required: - catalog_provider_id @@ -6443,6 +6443,44 @@ components: - enable_prompt_collection - redact_pii - access_log_retention_days + AgentNetworkManagedProxy: + type: object + description: A NetBird-managed Agent Network gateway deployment. + properties: + id: + type: string + description: Managed proxy deployment ID. + example: "d1m3kebd9pcs0c1pnu7g" + state: + type: string + description: Derived deployment state. `provisioning` until the gateway is rolled out and connected, `ready` while the gateway actively serves the endpoint, `failed` when the rollout reported a failure. + enum: [ "provisioning", "ready", "failed" ] + example: "ready" + endpoint: + type: string + description: The account's gateway hostname. + example: "brave-otter.gateway.netbird.io" + region: + type: string + description: Region of the cluster hosting the deployment. + example: "us-east" + message: + type: string + description: Failure detail reported by the rollout. Only set when state is `failed`. + required: + - id + - state + - endpoint + AgentNetworkManagedProxyConflict: + type: object + description: Conflict body returned when the account already has an Agent Network endpoint that managed provisioning does not own, naming that endpoint. + properties: + endpoint: + type: string + description: The Agent Network endpoint already assigned to the account. + example: "llm.example.com" + required: + - endpoint AgentNetworkBudgetRule: type: object description: Account-level budget rule. A limit-only rule bound to groups and/or users that applies across all policies as a min-wins ceiling. Empty targets means it applies to every caller. @@ -13531,6 +13569,67 @@ paths: "$ref": "#/components/responses/not_found" '500': "$ref": "#/components/responses/internal_error" + /api/integrations/agent-network/managed-proxy: + post: + summary: Provision a managed Agent Network gateway + description: Starts provisioning of a NetBird-managed Agent Network gateway for the account, allocating its endpoint under the managed zone on the first call. Idempotent — answers 202 when this call started (or, after a failure, restarted) provisioning and 200 when a deployment already exists, reporting current state either way. Returns 409 when the account already has an Agent Network endpoint that managed provisioning does not own, and 503 when endpoint allocation is temporarily exhausted. + tags: [ Agent Network ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + responses: + '200': + description: A managed gateway deployment already exists; reports its current state. + content: + application/json: + schema: + $ref: '#/components/schemas/AgentNetworkManagedProxy' + '202': + description: Provisioning started, or restarted after a reported failure + content: + application/json: + schema: + $ref: '#/components/schemas/AgentNetworkManagedProxy' + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '409': + description: The account already has an Agent Network endpoint not owned by managed provisioning + content: + application/json: + schema: + $ref: '#/components/schemas/AgentNetworkManagedProxyConflict' + '500': + "$ref": "#/components/responses/internal_error" + '503': + description: Endpoint allocation is temporarily exhausted; retry later + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + get: + summary: Retrieve managed Agent Network gateway status + description: Reports the account's managed gateway deployment and its derived state. Returns 404 when the account has no managed deployment. + tags: [ Agent Network ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + responses: + '200': + description: The account's managed gateway deployment + content: + application/json: + schema: + $ref: '#/components/schemas/AgentNetworkManagedProxy' + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '404': + "$ref": "#/components/responses/not_found" + '500': + "$ref": "#/components/responses/internal_error" /api/agent-network/access-logs: get: summary: List Agent Network access logs @@ -14225,7 +14324,14 @@ paths: "$ref": "#/components/responses/internal_error" post: summary: Create an Agent Network Provider - description: Connects a new Agent Network AI provider for the account. + description: | + Connects a new Agent Network AI provider for the account. + + The credential is checked against the vendor's model listing before the provider is stored, so a record the vendor will not accept is refused rather than saved. A rejected credential, a listing endpoint that does not resolve or answer, a vendor outage, and a timeout all block the write and return 422. + + What that proves about the upstream URL is narrower than the URL itself. Only its host is used: the listing is requested over HTTPS at the path the catalog entry declares, so a configured scheme or path is neither used nor validated here. Where the catalog entry has a listing host of its own — Bedrock, whose listing comes from the control plane — even the host is only resolved, never contacted, so a public host that does not answer is still stored. + + Only what cannot be checked at all is exempt and stored unverified: a catalog provider with no listing endpoint, one with no host to derive a listing from, an upstream resolving to a private address the management service will not dial, and a provider configured to skip TLS verification. tags: [ Agent Network ] security: - BearerAuth: [ ] @@ -14251,6 +14357,8 @@ paths: "$ref": "#/components/responses/forbidden" '409': "$ref": "#/components/responses/conflict" + '422': + "$ref": "#/components/responses/validation_failed_simple" '500': "$ref": "#/components/responses/internal_error" /api/agent-network/providers/{providerId}: @@ -14287,7 +14395,10 @@ paths: "$ref": "#/components/responses/internal_error" put: summary: Update an Agent Network Provider - description: Update an existing Agent Network AI provider. + description: | + Update an existing Agent Network AI provider. + + When the upstream URL, the API key or the catalog provider changes, the record is checked against the vendor before the change is stored, and a refusal returns 422 without replacing what was there. Switching TLS verification back on is the fourth trigger: a provider exempt from the check was stored unverified, so the edit that ends the exemption is the first opportunity to check it. Where one of the four does fire, an update that omits the API key is checked against the stored one. Edits touching none of them — a rename, model rows, price edits — are stored without a check, as are the cases the create description lists as unverifiable. tags: [ Agent Network ] security: - BearerAuth: [ ] @@ -14322,6 +14433,8 @@ paths: "$ref": "#/components/responses/not_found" '409': "$ref": "#/components/responses/conflict" + '422': + "$ref": "#/components/responses/validation_failed_simple" '500': "$ref": "#/components/responses/internal_error" delete: diff --git a/shared/management/http/api/types.gen.go b/shared/management/http/api/types.gen.go index 1fa7c5042..b5a7a80ac 100644 --- a/shared/management/http/api/types.gen.go +++ b/shared/management/http/api/types.gen.go @@ -77,6 +77,27 @@ func (e AgentNetworkConsumptionDimensionKind) Valid() bool { } } +// Defines values for AgentNetworkManagedProxyState. +const ( + AgentNetworkManagedProxyStateFailed AgentNetworkManagedProxyState = "failed" + AgentNetworkManagedProxyStateProvisioning AgentNetworkManagedProxyState = "provisioning" + AgentNetworkManagedProxyStateReady AgentNetworkManagedProxyState = "ready" +) + +// Valid indicates whether the value is a known member of the AgentNetworkManagedProxyState enum. +func (e AgentNetworkManagedProxyState) Valid() bool { + switch e { + case AgentNetworkManagedProxyStateFailed: + return true + case AgentNetworkManagedProxyStateProvisioning: + return true + case AgentNetworkManagedProxyStateReady: + return true + default: + return false + } +} + // Defines values for CreateAzureIntegrationRequestHost. const ( CreateAzureIntegrationRequestHostMicrosoftCom CreateAzureIntegrationRequestHost = "microsoft.com" @@ -2224,6 +2245,33 @@ type AgentNetworkGuardrailRequest struct { Name string `json:"name"` } +// AgentNetworkManagedProxy A NetBird-managed Agent Network gateway deployment. +type AgentNetworkManagedProxy struct { + // Endpoint The account's gateway hostname. + Endpoint string `json:"endpoint"` + + // Id Managed proxy deployment ID. + Id string `json:"id"` + + // Message Failure detail reported by the rollout. Only set when state is `failed`. + Message *string `json:"message,omitempty"` + + // Region Region of the cluster hosting the deployment. + Region *string `json:"region,omitempty"` + + // State Derived deployment state. `provisioning` until the gateway is rolled out and connected, `ready` while the gateway actively serves the endpoint, `failed` when the rollout reported a failure. + State AgentNetworkManagedProxyState `json:"state"` +} + +// AgentNetworkManagedProxyState Derived deployment state. `provisioning` until the gateway is rolled out and connected, `ready` while the gateway actively serves the endpoint, `failed` when the rollout reported a failure. +type AgentNetworkManagedProxyState string + +// AgentNetworkManagedProxyConflict Conflict body returned when the account already has an Agent Network endpoint that managed provisioning does not own, naming that endpoint. +type AgentNetworkManagedProxyConflict struct { + // Endpoint The Agent Network endpoint already assigned to the account. + Endpoint string `json:"endpoint"` +} + // AgentNetworkModelDiscoveryRequest defines model for AgentNetworkModelDiscoveryRequest. type AgentNetworkModelDiscoveryRequest struct { // ApiKey Credential to query the vendor with, for a provider that has not been saved yet. Mutually exclusive with provider_id. @@ -2232,10 +2280,10 @@ type AgentNetworkModelDiscoveryRequest struct { // CatalogProviderId Catalog provider to query (AgentNetworkCatalogProvider.id). Determines the listing endpoint, the auth header and the response shape. CatalogProviderId string `json:"catalog_provider_id"` - // ProviderId Existing Agent Network provider record whose stored credential and upstream should be used. Lets the form refresh the list without the client holding the key. + // ProviderId Existing Agent Network provider record to query with. Its stored credential is used, and its upstream unless upstream_url overrides it, so the form can refresh the list without the client holding the key. ProviderId *string `json:"provider_id,omitempty"` - // UpstreamUrl The upstream being configured. Used to reach vendors that serve their listing from the same host as inference, and to read back the region for those whose host embeds one. Ignored when provider_id is supplied. + // UpstreamUrl The upstream being configured. Used to reach vendors that serve their listing from the same host as inference, and to read back the region for those whose host embeds one. Sent alongside provider_id, it overrides the stored upstream, so an edit can be listed against the URL on the form before it is saved. UpstreamUrl *string `json:"upstream_url,omitempty"` } diff --git a/shared/management/proto/generate.sh b/shared/management/proto/generate.sh index 7cb0f75a5..2915b7f0c 100755 --- a/shared/management/proto/generate.sh +++ b/shared/management/proto/generate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if ! which realpath > /dev/null 2>&1 diff --git a/shared/management/proto/proxy_service.pb.go b/shared/management/proto/proxy_service.pb.go index df42d78ff..496774a4b 100644 --- a/shared/management/proto/proxy_service.pb.go +++ b/shared/management/proto/proxy_service.pb.go @@ -903,6 +903,12 @@ type Authentication struct { Pin bool `protobuf:"varint,4,opt,name=pin,proto3" json:"pin,omitempty"` Oidc bool `protobuf:"varint,5,opt,name=oidc,proto3" json:"oidc,omitempty"` HeaderAuths []*HeaderAuth `protobuf:"bytes,6,rep,name=header_auths,json=headerAuths,proto3" json:"header_auths,omitempty"` + // Group ids allowed to reach the service through an OIDC identity. When + // non-empty the proxy requires the session token's groups claim to + // intersect this list before honouring the cookie, so a token minted for + // an identity outside these groups is not a bearer credential for the + // service. Empty means group membership does not restrict access. + AllowedGroupIds []string `protobuf:"bytes,7,rep,name=allowed_group_ids,json=allowedGroupIds,proto3" json:"allowed_group_ids,omitempty"` } func (x *Authentication) Reset() { @@ -979,6 +985,13 @@ func (x *Authentication) GetHeaderAuths() []*HeaderAuth { return nil } +func (x *Authentication) GetAllowedGroupIds() []string { + if x != nil { + return x.AllowedGroupIds + } + return nil +} + type AccessRestrictions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3304,7 +3317,7 @@ var file_proxy_service_proto_rawDesc = []byte{ 0x16, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x68, 0x61, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x68, - 0x61, 0x73, 0x68, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xe5, 0x01, 0x0a, 0x0e, 0x41, + 0x61, 0x73, 0x68, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x91, 0x02, 0x0a, 0x0e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x35, @@ -3319,198 +3332,220 @@ var file_proxy_service_proto_rawDesc = []byte{ 0x72, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, 0x52, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x41, 0x75, 0x74, - 0x68, 0x73, 0x22, 0xdd, 0x01, 0x0a, 0x12, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, - 0x74, 0x72, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x6c, - 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0c, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x43, 0x69, 0x64, 0x72, 0x73, 0x12, 0x23, - 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x43, 0x69, - 0x64, 0x72, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, - 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, - 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x23, 0x0a, - 0x0d, 0x63, 0x72, 0x6f, 0x77, 0x64, 0x73, 0x65, 0x63, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x72, 0x6f, 0x77, 0x64, 0x73, 0x65, 0x63, 0x4d, 0x6f, - 0x64, 0x65, 0x22, 0x80, 0x04, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, - 0x69, 0x6e, 0x67, 0x12, 0x36, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, - 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x12, 0x2b, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x61, - 0x74, 0x68, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, - 0x1d, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2e, - 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, - 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x12, 0x28, - 0x0a, 0x10, 0x70, 0x61, 0x73, 0x73, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x61, 0x73, 0x73, 0x48, 0x6f, - 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x77, 0x72, - 0x69, 0x74, 0x65, 0x5f, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x73, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x0a, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, - 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x4f, 0x0a, 0x13, 0x61, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x74, 0x72, - 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, - 0x65, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, - 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x70, 0x72, - 0x69, 0x76, 0x61, 0x74, 0x65, 0x22, 0x3f, 0x0a, 0x14, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, - 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, - 0x67, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x22, 0x17, 0x0a, 0x15, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0xa9, 0x05, 0x0a, 0x09, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x12, 0x38, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x15, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x6f, 0x67, 0x49, 0x64, 0x12, 0x1d, - 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, - 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x68, 0x6f, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x6d, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x4d, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x23, 0x0a, - 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, - 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x70, 0x18, - 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x70, 0x12, - 0x25, 0x0a, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, - 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x4d, 0x65, 0x63, - 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, - 0x21, 0x0a, 0x0c, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, - 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x75, 0x74, 0x68, 0x53, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x75, 0x70, 0x6c, 0x6f, - 0x61, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x55, - 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x3f, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, - 0x67, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x5f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x1a, 0x3b, - 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf8, 0x01, 0x0a, 0x13, - 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x49, 0x64, 0x12, 0x39, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x48, 0x00, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x2a, 0x0a, - 0x03, 0x70, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x48, 0x00, 0x52, 0x03, 0x70, 0x69, 0x6e, 0x12, 0x40, 0x0a, 0x0b, 0x68, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, - 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x57, 0x0a, 0x11, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x41, 0x75, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x68, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, - 0x2d, 0x0a, 0x0f, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x1e, - 0x0a, 0x0a, 0x50, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, - 0x70, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x70, 0x69, 0x6e, 0x22, 0x55, - 0x0a, 0x14, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xda, 0x02, 0x0a, 0x17, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, - 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, - 0x2f, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, - 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, - 0x69, 0x73, 0x73, 0x75, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x65, - 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x64, 0x12, - 0x28, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x88, 0x01, 0x01, 0x12, 0x50, 0x0a, 0x10, 0x69, 0x6e, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x18, 0x32, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x65, 0x72, 0x48, 0x01, 0x52, 0x0f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x88, 0x01, 0x01, 0x42, 0x10, 0x0a, 0x0e, 0x5f, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x13, 0x0a, - 0x11, 0x5f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x75, - 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, - 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x68, 0x74, 0x74, 0x70, 0x73, - 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x68, 0x74, 0x74, - 0x70, 0x73, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x70, - 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x68, 0x74, 0x74, 0x70, 0x50, - 0x6f, 0x72, 0x74, 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0xb8, 0x01, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, - 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, - 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x30, - 0x0a, 0x14, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x5f, 0x70, 0x75, 0x62, 0x6c, - 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x77, 0x69, - 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, - 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x17, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, - 0x28, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x88, 0x01, 0x01, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x65, 0x0a, 0x11, 0x47, - 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, - 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, - 0x72, 0x6c, 0x22, 0x26, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x55, 0x0a, 0x16, 0x56, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x0a, 0x0d, + 0x68, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x61, + 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x22, 0xdd, + 0x01, 0x0a, 0x12, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x74, 0x72, 0x69, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, + 0x5f, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x6c, + 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x43, 0x69, 0x64, 0x72, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x43, 0x69, 0x64, 0x72, 0x73, 0x12, + 0x2b, 0x0a, 0x11, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x6c, 0x6c, 0x6f, + 0x77, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x11, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x69, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x72, 0x6f, + 0x77, 0x64, 0x73, 0x65, 0x63, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x63, 0x72, 0x6f, 0x77, 0x64, 0x73, 0x65, 0x63, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x80, + 0x04, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, + 0x36, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x2b, + 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x61, + 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2e, 0x0a, 0x04, 0x61, 0x75, + 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x61, 0x75, 0x74, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x61, + 0x73, 0x73, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x61, 0x73, 0x73, 0x48, 0x6f, 0x73, 0x74, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, + 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x10, 0x72, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, + 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6c, 0x69, 0x73, 0x74, + 0x65, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x4f, 0x0a, 0x13, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x5f, 0x72, 0x65, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x74, 0x72, + 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, + 0x74, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, + 0x65, 0x22, 0x3f, 0x0a, 0x14, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, + 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x03, 0x6c, 0x6f, 0x67, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x03, 0x6c, + 0x6f, 0x67, 0x22, 0x17, 0x0a, 0x15, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xa9, 0x05, 0x0a, 0x09, + 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x12, 0x15, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x6f, 0x67, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x73, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, + 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1b, + 0x0a, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x61, + 0x75, 0x74, 0x68, 0x5f, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, + 0x73, 0x6d, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x61, + 0x75, 0x74, 0x68, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0b, 0x61, 0x75, 0x74, 0x68, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x21, + 0x0a, 0x0c, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x0e, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x55, 0x70, 0x6c, 0x6f, 0x61, + 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x3f, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x2e, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf8, 0x01, 0x0a, 0x13, 0x41, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x39, + 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x61, + 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, + 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x2a, 0x0a, 0x03, 0x70, 0x69, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, + 0x52, 0x03, 0x70, 0x69, 0x6e, 0x12, 0x40, 0x0a, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, + 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x41, 0x75, + 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x68, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x22, 0x57, 0x0a, 0x11, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x41, 0x75, 0x74, 0x68, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x2d, 0x0a, 0x0f, 0x50, + 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, + 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x1e, 0x0a, 0x0a, 0x50, 0x69, + 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x70, 0x69, 0x6e, 0x22, 0x55, 0x0a, 0x14, 0x41, 0x75, + 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x22, 0xdc, 0x01, 0x0a, 0x17, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, + 0x6e, 0x22, 0xda, 0x02, 0x0a, 0x17, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2f, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x0a, 0x12, + 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x73, 0x73, 0x75, + 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x64, 0x12, 0x28, 0x0a, 0x0d, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x88, 0x01, 0x01, 0x12, 0x50, 0x0a, 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x18, 0x32, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x48, 0x01, 0x52, 0x0f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, 0x73, 0x74, + 0x65, 0x6e, 0x65, 0x72, 0x88, 0x01, 0x01, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x69, 0x6e, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x22, 0x6f, + 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4c, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x75, 0x6e, 0x6e, 0x65, 0x6c, + 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x75, 0x6e, 0x6e, 0x65, + 0x6c, 0x49, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x68, 0x74, 0x74, 0x70, 0x73, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x68, 0x74, 0x74, 0x70, 0x73, 0x50, 0x6f, + 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x68, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x22, + 0x1a, 0x0a, 0x18, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xb8, 0x01, 0x0a, 0x16, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x30, 0x0a, 0x14, 0x77, 0x69, + 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, + 0x61, 0x72, 0x64, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, + 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x17, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x28, 0x0a, 0x0d, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x88, 0x01, 0x01, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x65, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4f, 0x49, + 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x72, + 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x72, 0x6c, 0x22, 0x26, + 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x55, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xdc, 0x01, + 0x0a, 0x17, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, + 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, + 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x73, + 0x65, 0x72, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6e, 0x69, 0x65, + 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0e, + 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, + 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, + 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x19, + 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x75, 0x6e, + 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x75, + 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x84, + 0x02, 0x0a, 0x1a, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, + 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, @@ -3518,208 +3553,189 @@ var file_proxy_service_proto_rawDesc = []byte{ 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, - 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, - 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, - 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, - 0x22, 0x50, 0x0a, 0x19, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, - 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, - 0x09, 0x74, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x74, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x22, 0x84, 0x02, 0x0a, 0x1a, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, - 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, - 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, - 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x65, 0x65, - 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, - 0x28, 0x0a, 0x10, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x13, 0x53, 0x79, - 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x32, 0x0a, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x48, 0x00, 0x52, - 0x04, 0x69, 0x6e, 0x69, 0x74, 0x12, 0x2f, 0x0a, 0x03, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x48, - 0x00, 0x52, 0x03, 0x61, 0x63, 0x6b, 0x42, 0x05, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xdf, 0x01, - 0x0a, 0x10, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, - 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x64, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x41, 0x0a, 0x0c, - 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, - 0x11, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, - 0x63, 0x6b, 0x22, 0x7e, 0x0a, 0x14, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x32, - 0x0a, 0x15, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, - 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x74, 0x65, 0x22, 0xa9, 0x01, 0x0a, 0x1b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, - 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, - 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, - 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0xff, - 0x01, 0x0a, 0x1c, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x64, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x74, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, - 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x65, 0x6e, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x65, 0x6e, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, - 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x6e, 0x79, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, 0x6e, 0x79, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, - 0x22, 0x91, 0x02, 0x0a, 0x15, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, - 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, - 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, - 0x0e, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, - 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x73, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x73, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, - 0x63, 0x6f, 0x73, 0x74, 0x5f, 0x75, 0x73, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, - 0x63, 0x6f, 0x73, 0x74, 0x55, 0x73, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x5f, 0x69, 0x64, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, - 0x70, 0x49, 0x64, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, - 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x64, - 0x0a, 0x16, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, - 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x4d, 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, - 0x45, 0x44, 0x10, 0x02, 0x2a, 0x46, 0x0a, 0x0f, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x77, 0x72, - 0x69, 0x74, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x50, 0x41, 0x54, 0x48, 0x5f, - 0x52, 0x45, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, - 0x00, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, 0x49, 0x54, - 0x45, 0x5f, 0x50, 0x52, 0x45, 0x53, 0x45, 0x52, 0x56, 0x45, 0x10, 0x01, 0x2a, 0x90, 0x01, 0x0a, - 0x0e, 0x4d, 0x69, 0x64, 0x64, 0x6c, 0x65, 0x77, 0x61, 0x72, 0x65, 0x53, 0x6c, 0x6f, 0x74, 0x12, - 0x1f, 0x0a, 0x1b, 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, - 0x4f, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x1e, 0x0a, 0x1a, 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, - 0x4c, 0x4f, 0x54, 0x5f, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x01, - 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, - 0x4c, 0x4f, 0x54, 0x5f, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, - 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, - 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x4c, 0x10, 0x03, 0x2a, - 0xc8, 0x01, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x18, 0x0a, 0x14, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x50, 0x52, 0x4f, - 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, - 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x54, 0x55, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x52, - 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x50, 0x52, 0x4f, 0x58, 0x59, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, 0x49, 0x43, - 0x41, 0x54, 0x45, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x23, 0x0a, - 0x1f, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, - 0x52, 0x54, 0x49, 0x46, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, - 0x10, 0x04, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x05, 0x32, 0xfc, 0x07, 0x0a, 0x0c, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5f, 0x0a, 0x10, 0x47, - 0x65, 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, - 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, - 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x55, 0x0a, 0x0c, - 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1f, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, - 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, - 0x01, 0x30, 0x01, 0x12, 0x54, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x4c, 0x6f, 0x67, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, - 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, - 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x22, + 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, + 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, + 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, + 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x13, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x32, 0x0a, + 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x48, 0x00, 0x52, 0x04, 0x69, 0x6e, 0x69, + 0x74, 0x12, 0x2f, 0x0a, 0x03, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x03, 0x61, + 0x63, 0x6b, 0x42, 0x05, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xdf, 0x01, 0x0a, 0x10, 0x53, 0x79, + 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x19, + 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x18, + 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x41, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x53, + 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x22, 0x7e, + 0x0a, 0x14, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x52, 0x07, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, + 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e, 0x69, 0x74, 0x69, + 0x61, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x22, 0xa9, + 0x01, 0x0a, 0x1b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, + 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, + 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x49, 0x64, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0xff, 0x01, 0x0a, 0x1c, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, + 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x64, + 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, + 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x6c, + 0x69, 0x63, 0x79, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1b, + 0x0a, 0x09, 0x64, 0x65, 0x6e, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x64, 0x65, 0x6e, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, + 0x65, 0x6e, 0x79, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x64, 0x65, 0x6e, 0x79, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x91, 0x02, 0x0a, + 0x15, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, + 0x0a, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, + 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x6f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x73, 0x74, + 0x5f, 0x75, 0x73, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x63, 0x6f, 0x73, 0x74, + 0x55, 0x73, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, + 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, + 0x22, 0x18, 0x0a, 0x16, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, + 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x64, 0x0a, 0x16, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, + 0x14, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x4f, 0x44, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x02, + 0x2a, 0x46, 0x0a, 0x0f, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x4d, + 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, + 0x49, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, 0x00, 0x12, 0x19, 0x0a, + 0x15, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52, + 0x45, 0x53, 0x45, 0x52, 0x56, 0x45, 0x10, 0x01, 0x2a, 0x90, 0x01, 0x0a, 0x0e, 0x4d, 0x69, 0x64, + 0x64, 0x6c, 0x65, 0x77, 0x61, 0x72, 0x65, 0x53, 0x6c, 0x6f, 0x74, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, + 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, + 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, + 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, + 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, + 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x02, 0x12, 0x1c, 0x0a, + 0x18, 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, + 0x5f, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0xc8, 0x01, 0x0a, 0x0b, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x14, 0x50, + 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, + 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x23, + 0x0a, 0x1f, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, + 0x55, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, + 0x44, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, + 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, 0x4f, + 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, + 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x16, + 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, + 0x52, 0x52, 0x4f, 0x52, 0x10, 0x05, 0x32, 0xfc, 0x07, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x70, + 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, + 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x55, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, + 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, + 0x54, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, + 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, + 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, + 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4f, 0x49, - 0x44, 0x43, 0x55, 0x52, 0x4c, 0x12, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, + 0x4c, 0x12, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, + 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, + 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x5a, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x63, 0x0a, 0x12, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, - 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x12, 0x25, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, - 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x14, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, - 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x27, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, - 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x57, 0x0a, 0x0e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x12, + 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, + 0x65, 0x72, 0x12, 0x25, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, + 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x69, 0x0a, 0x14, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, + 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x27, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, + 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, + 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x63, 0x6f, + 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/shared/management/proto/proxy_service.proto b/shared/management/proto/proxy_service.proto index 89d1f8749..facadc4d5 100644 --- a/shared/management/proto/proxy_service.proto +++ b/shared/management/proto/proxy_service.proto @@ -194,6 +194,12 @@ message Authentication { bool pin = 4; bool oidc = 5; repeated HeaderAuth header_auths = 6; + // Group ids allowed to reach the service through an OIDC identity. When + // non-empty the proxy requires the session token's groups claim to + // intersect this list before honouring the cookie, so a token minted for + // an identity outside these groups is not a bearer credential for the + // service. Empty means group membership does not restrict access. + repeated string allowed_group_ids = 7; } message AccessRestrictions { diff --git a/shared/signal/proto/generate.sh b/shared/signal/proto/generate.sh index 720a5ff66..718eae152 100755 --- a/shared/signal/proto/generate.sh +++ b/shared/signal/proto/generate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if ! which realpath > /dev/null 2>&1 diff --git a/tools/idp-migrate/DEVELOPMENT.md b/tools/idp-migrate/DEVELOPMENT.md index 5697ead40..41b5bc992 100644 --- a/tools/idp-migrate/DEVELOPMENT.md +++ b/tools/idp-migrate/DEVELOPMENT.md @@ -50,6 +50,7 @@ The build requires `CGO_ENABLED=1` because it links the SQLite driver used by `S | `--domain` | string | `""` | Sets both dashboard and API domain (convenience shorthand) | | `--dashboard-domain` | string | *(required)* | Dashboard domain (for redirect URIs) | | `--api-domain` | string | *(required)* | API domain (for Dex issuer and callback URLs) | +| `--single-account-mode-domain` | string | `netbird.selfhosted` | Domain single account mode groups users under. Used only when the account has no domain of its own; passing one that conflicts with the account's existing domain is an error | | `--dry-run` | bool | `false` | Preview changes without writing | | `--force` | bool | `false` | Skip interactive confirmation prompt | | `--skip-config` | bool | `false` | Skip config generation (DB-only migration) | @@ -68,6 +69,7 @@ All flags can be overridden via environment variables. Env vars take precedence | `NETBIRD_CONFIG_PATH` | `--config` | | `NETBIRD_DATA_DIR` | `--datadir` | | `NETBIRD_IDP_SEED_INFO` | `--idp-seed-info` | +| `NETBIRD_SINGLE_ACCOUNT_MODE_DOMAIN` | `--single-account-mode-domain` | | `NETBIRD_DRY_RUN` | `--dry-run` (set to `"true"`) | | `NETBIRD_FORCE` | `--force` (set to `"true"`) | | `NETBIRD_SKIP_CONFIG` | `--skip-config` (set to `"true"`) | diff --git a/tools/idp-migrate/config.go b/tools/idp-migrate/config.go index f4d6b9ea2..6510fd940 100644 --- a/tools/idp-migrate/config.go +++ b/tools/idp-migrate/config.go @@ -6,16 +6,18 @@ import ( "os" "strconv" + "github.com/netbirdio/netbird/management/server/idp/migration" "github.com/netbirdio/netbird/util" ) type migrationConfig struct { // Data - dashboardURL string - apiURL string - configPath string - dataDir string - idpSeedInfo string + dashboardURL string + apiURL string + configPath string + dataDir string + idpSeedInfo string + singleAccountDomain string // Options dryRun bool @@ -51,6 +53,7 @@ func configFromArgs(args []string) (*migrationConfig, error) { fs.StringVar(&cfg.configPath, "config", "", "path to management.json (required)") fs.StringVar(&cfg.dataDir, "datadir", "", "override data directory from config") fs.StringVar(&cfg.idpSeedInfo, "idp-seed-info", "", "base64-encoded connector JSON (overrides auto-detection)") + fs.StringVar(&cfg.singleAccountDomain, "single-account-mode-domain", "", "domain single account mode groups users under, used only when the account has no domain of its own (default "+migration.DefaultSingleAccountDomain+")") fs.BoolVar(&cfg.dryRun, "dry-run", false, "preview changes without writing") fs.BoolVar(&cfg.force, "force", false, "skip confirmation prompt") fs.BoolVar(&cfg.skipConfig, "skip-config", false, "skip config generation (DB migration only)") @@ -118,6 +121,10 @@ func applyOverrides(cfg *migrationConfig, domain string) { cfg.idpSeedInfo = val } + if val, ok := os.LookupEnv("NETBIRD_SINGLE_ACCOUNT_MODE_DOMAIN"); ok { + cfg.singleAccountDomain = val + } + // Enforce dry run if any value is provided if sval, ok := os.LookupEnv("NETBIRD_DRY_RUN"); ok { if val, err := strconv.ParseBool(sval); err == nil { @@ -170,5 +177,9 @@ func validateConfig(cfg *migrationConfig) error { return fmt.Errorf("--dashboard-domain is required") } + if _, err := migration.NormalizeSingleAccountDomain(cfg.singleAccountDomain); err != nil { + return err + } + return nil } diff --git a/tools/idp-migrate/main.go b/tools/idp-migrate/main.go index a8cba0750..652bf3393 100644 --- a/tools/idp-migrate/main.go +++ b/tools/idp-migrate/main.go @@ -71,6 +71,10 @@ func run(cfg *migrationConfig) error { return err } + if err := preflightAccounts(cfg, mgmtConfig); err != nil { + return err + } + if !cfg.skipPopulateUserInfo { err := populateUserInfoFromIDP(cfg, mgmtConfig) if err != nil { @@ -102,6 +106,22 @@ func run(cfg *migrationConfig) error { return generateConfig(cfg, connectorConfig) } +func preflightAccounts(cfg *migrationConfig, mgmtConfig *nbconfig.Config) error { + ctx := context.Background() + migStore, migEventStore, cleanup, err := openStores(ctx, mgmtConfig, cfg.dataDir) + if err != nil { + return err + } + defer cleanup() + + srv := &migrationServer{store: migStore, eventStore: migEventStore} + if err := migration.RequireSingleAccount(srv); err != nil { + return err + } + + return migration.CheckSingleAccountDomain(srv, cfg.singleAccountDomain) +} + // validateSchema opens the store and checks that all required tables and columns // exist. If anything is missing, it returns a descriptive error telling the user // to upgrade their management server. @@ -224,6 +244,8 @@ func migrateDB(cfg *migrationConfig, mgmtConfig *nbconfig.Config, connectorConfi } defer cleanup() + srv := &migrationServer{store: migStore, eventStore: migEventStore} + pending, err := previewUsers(ctx, migStore) if err != nil { return err @@ -243,11 +265,14 @@ func migrateDB(cfg *migrationConfig, mgmtConfig *nbconfig.Config, connectorConfi } } - srv := &migrationServer{store: migStore, eventStore: migEventStore} if err := migration.MigrateUsersToStaticConnectors(srv, connectorConfig); err != nil { return fmt.Errorf("migrate users: %w", err) } + if err := migration.EnsureSingleAccountDomain(srv, cfg.singleAccountDomain); err != nil { + return fmt.Errorf("prepare single account mode: %w", err) + } + if !cfg.dryRun { log.Info("DB migration completed successfully") } diff --git a/tools/idp-migrate/main_test.go b/tools/idp-migrate/main_test.go index 75d0bd7eb..286e15b88 100644 --- a/tools/idp-migrate/main_test.go +++ b/tools/idp-migrate/main_test.go @@ -485,3 +485,77 @@ func TestGenerateConfig(t *testing.T) { assert.True(t, os.IsNotExist(err)) }) } + +func TestValidateConfigRejectsUnusableSingleAccountDomain(t *testing.T) { + base := func() migrationConfig { + return migrationConfig{ + configPath: "/tmp/management.json", + dataDir: "/tmp/datadir", + idpSeedInfo: "seed", + apiURL: "https://api.example.com", + dashboardURL: "https://app.example.com", + singleAccountDomain: migration.DefaultSingleAccountDomain, + } + } + + t.Run("usable domain is accepted", func(t *testing.T) { + cfg := base() + require.NoError(t, validateConfig(&cfg)) + }) + + t.Run("empty falls back to the default", func(t *testing.T) { + cfg := base() + cfg.singleAccountDomain = "" + require.NoError(t, validateConfig(&cfg)) + }) + + // Rejected up front so the migration cannot fail after it has rewritten user IDs. + t.Run("single label domain is rejected", func(t *testing.T) { + cfg := base() + cfg.singleAccountDomain = "corp" + err := validateConfig(&cfg) + require.Error(t, err) + assert.ErrorIs(t, err, migration.ErrUnusableDomain) + }) +} + +func TestApplyOverrides_SingleAccountDomainFromEnv(t *testing.T) { + t.Run("env var overrides the flag", func(t *testing.T) { + t.Setenv("NETBIRD_SINGLE_ACCOUNT_MODE_DOMAIN", "corp.example.com") + + cfg, err := configFromArgs([]string{ + "--config", "/tmp/management.json", + "--datadir", "/tmp/datadir", + "--idp-seed-info", "seed", + "--domain", "example.com", + "--single-account-mode-domain", "flag.example.com", + }) + require.NoError(t, err) + assert.Equal(t, "corp.example.com", cfg.singleAccountDomain) + }) + + t.Run("unset leaves the flag value", func(t *testing.T) { + cfg, err := configFromArgs([]string{ + "--config", "/tmp/management.json", + "--datadir", "/tmp/datadir", + "--idp-seed-info", "seed", + "--domain", "example.com", + "--single-account-mode-domain", "flag.example.com", + }) + require.NoError(t, err) + assert.Equal(t, "flag.example.com", cfg.singleAccountDomain) + }) + + t.Run("unusable env value is rejected", func(t *testing.T) { + t.Setenv("NETBIRD_SINGLE_ACCOUNT_MODE_DOMAIN", "corp") + + _, err := configFromArgs([]string{ + "--config", "/tmp/management.json", + "--datadir", "/tmp/datadir", + "--idp-seed-info", "seed", + "--domain", "example.com", + }) + require.Error(t, err) + assert.ErrorIs(t, err, migration.ErrUnusableDomain) + }) +}