Compare commits

...

3 Commits

19 changed files with 1488 additions and 229 deletions

View File

@@ -42,12 +42,12 @@ type aclManager struct {
optionalEntries map[string][]entry
ipsetStore *ipsetStore
v6 bool
ipsetSupported bool
ipsetSupport *ipsetSupport
stateManager *statemanager.Manager
}
func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*aclManager, error) {
func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, ipsetSupport *ipsetSupport) (*aclManager, error) {
return &aclManager{
iptablesClient: iptablesClient,
wgIface: wgIface,
@@ -55,14 +55,13 @@ func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*acl
optionalEntries: make(map[string][]entry),
ipsetStore: newIpsetStore(),
v6: iptablesClient.Proto() == iptables.ProtocolIPv6,
ipsetSupport: ipsetSupport,
}, nil
}
func (m *aclManager) init(stateManager *statemanager.Manager) error {
m.stateManager = stateManager
m.ipsetSupported = m.probeIPSetSupport()
m.seedInitialEntries()
m.seedInitialOptionalEntries()
@@ -88,18 +87,77 @@ func (m *aclManager) AddPeerFiltering(
action firewall.Action,
ipsetName string,
) ([]firewall.Rule, error) {
chain := chainNameInputRules
ipsetName = m.resolveIPSetName(ipsetName, sPort, dPort, action)
if ipsetName == "" {
return m.addPeerRule(ip, protocol, sPort, dPort, action, "")
}
// A set that is already in the store backs rules installed earlier, so it must
// survive this call's failure.
_, preexisting := m.ipsetStore.ipset(ipsetName)
rules, err := m.addPeerRule(ip, protocol, sPort, dPort, action, ipsetName)
if err == nil {
return rules, nil
}
var unusable *ipsetUnusableError
if !errors.As(err, &unusable) {
return nil, err
}
// The set could not be created or matched. Drop the one this call created and
// retry the rule matching the IP directly; only if that succeeds do we know
// ipset was to blame and latch it off for subsequent rules.
if !preexisting {
m.discardIPSet(ipsetName)
}
rules, retryErr := m.addPeerRule(ip, protocol, sPort, dPort, action, "")
if retryErr != nil {
return nil, fmt.Errorf("add peer rule (ipset: %w): %w", unusable.cause, retryErr)
}
m.ipsetSupport.markUnsupported(unusable.cause)
return rules, nil
}
// resolveIPSetName derives the ipset name for a rule, returning "" when the rule
// must match the IP directly: either no set was requested or ipset is unusable.
func (m *aclManager) resolveIPSetName(ipsetName string, sPort, dPort *firewall.Port, action firewall.Action) string {
ipsetName = transformIPsetName(ipsetName, sPort, dPort, action)
if m.v6 && ipsetName != "" {
if ipsetName == "" || !m.ipsetSupport.supported() {
return ""
}
if m.v6 {
ipsetName += "-v6"
}
// When the kernel lacks the required ipset hash module, fall back to
// per-IP iptables rules (pre-0.68 behavior) so ACLs keep working instead
// of silently leaving the chain empty.
if ipsetName != "" && !m.ipsetSupported {
ipsetName = ""
return ipsetName
}
// discardIPSet removes a set that turned out to be unusable, so a later rule
// does not find it in the store and assume it works.
func (m *aclManager) discardIPSet(ipsetName string) {
m.ipsetStore.deleteIpset(ipsetName)
if err := m.destroyIPSet(ipsetName); err != nil {
log.Debugf("destroy unusable ipset %s: %v", ipsetName, err)
}
}
func (m *aclManager) addPeerRule(
ip net.IP,
protocol firewall.Protocol,
sPort *firewall.Port,
dPort *firewall.Port,
action firewall.Action,
ipsetName string,
) ([]firewall.Rule, error) {
chain := chainNameInputRules
proto := protoForFamily(protocol, m.v6)
specs := filterRuleSpecs(ip, proto, sPort, dPort, action, ipsetName)
@@ -114,7 +172,7 @@ func (m *aclManager) AddPeerFiltering(
if ipsetName != "" {
if ipList, ipsetExists := m.ipsetStore.ipset(ipsetName); ipsetExists {
if err := m.addToIPSet(ipsetName, ip); err != nil {
return nil, fmt.Errorf("add IP to ipset: %w", err)
return nil, ipsetUnusable(fmt.Errorf("add IP to ipset: %w", err))
}
// if ruleset already exists it means we already have the firewall rule
// so we need to update IPs in the ruleset and return new fw.Rule object for ACL manager.
@@ -137,10 +195,10 @@ func (m *aclManager) AddPeerFiltering(
}
}
if err := m.createIPSet(ipsetName); err != nil {
return nil, fmt.Errorf("create ipset: %w", err)
return nil, ipsetUnusable(fmt.Errorf("create ipset: %w", err))
}
if err := m.addToIPSet(ipsetName, ip); err != nil {
return nil, fmt.Errorf("add IP to ipset: %w", err)
return nil, ipsetUnusable(fmt.Errorf("add IP to ipset: %w", err))
}
ipList := newIpList(ip.String())
@@ -149,7 +207,7 @@ func (m *aclManager) AddPeerFiltering(
ok, err := m.iptablesClient.Exists(tableFilter, chain, specs...)
if err != nil {
return nil, fmt.Errorf("failed to check rule: %w", err)
return nil, maybeIPSetUnusable(ipsetName, fmt.Errorf("check rule: %w", err))
}
if ok {
return nil, fmt.Errorf("rule already exists")
@@ -163,7 +221,7 @@ func (m *aclManager) AddPeerFiltering(
err = m.iptablesClient.Append(tableFilter, chain, specs...)
}
if err != nil {
return nil, err
return nil, maybeIPSetUnusable(ipsetName, err)
}
if err := m.iptablesClient.Append(tableMangle, chainRTPRE, mangleSpecs...); err != nil {
@@ -507,40 +565,6 @@ func transformIPsetName(ipsetName string, sPort, dPort *firewall.Port, action fi
}
}
// probeIPSetSupport checks whether the kernel can create the ipset type used for
// ACL rules. On kernels lacking the required ipset hash module, ipset creation
// fails (e.g. "invalid argument"), which would otherwise leave the ACL chain
// empty and silently drop all policy-permitted inbound traffic. When unsupported,
// the manager falls back to per-IP iptables rules.
func (m *aclManager) probeIPSetSupport() bool {
// Use a unique name so concurrent processes don't collide and we only ever
// destroy the set we created ourselves. ipset names are limited to 31 chars,
// so use a short random suffix.
probeName := "nb-probe-" + uuid.New().String()[:8]
opts := ipset.CreateOptions{
Replace: true,
}
if m.v6 {
opts.Family = ipset.FamilyIPV6
}
if err := ipset.Create(probeName, ipset.TypeHashNet, opts); err != nil {
log.Warnf("ipset is not available (failed to create probe set: %v); "+
"falling back to per-IP iptables ACL rules. Ensure the kernel provides "+
"the ipset hash:net module (ip_set_hash_net) for better performance with large rule sets", err)
return false
}
defer func() {
if err := ipset.Destroy(probeName); err != nil {
log.Debugf("destroy ipset probe set %q: %v", probeName, err)
}
}()
return true
}
func (m *aclManager) createIPSet(name string) error {
opts := ipset.CreateOptions{
Replace: true,

View File

@@ -0,0 +1,240 @@
//go:build privileged
package iptables
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
fw "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/iface/wgaddr"
)
func iptRefcountIfaceV4() *iFaceMock {
return &iFaceMock{
NameFunc: func() string { return "wt-refcount" },
AddressFunc: func() wgaddr.Address {
return wgaddr.Address{
IP: netip.MustParseAddr("10.20.0.1"),
Network: netip.MustParsePrefix("10.20.0.0/24"),
}
},
}
}
func iptRefcountIfaceDual() *iFaceMock {
return &iFaceMock{
NameFunc: func() string { return "wt-refcount" },
AddressFunc: func() wgaddr.Address {
return wgaddr.Address{
IP: netip.MustParseAddr("10.20.0.1"),
Network: netip.MustParsePrefix("10.20.0.0/24"),
IPv6: netip.MustParseAddr("fd00::1"),
IPv6Net: netip.MustParsePrefix("fd00::/64"),
}
},
}
}
func newIptRefcountManager(t *testing.T, dual bool) *Manager {
t.Helper()
var ifMock *iFaceMock
if dual {
ifMock = iptRefcountIfaceDual()
} else {
ifMock = iptRefcountIfaceV4()
}
m, err := Create(ifMock, iface.DefaultMTU)
require.NoError(t, err, "create manager")
require.NoError(t, m.Init(nil), "init manager")
t.Cleanup(func() {
require.NoError(t, m.Close(nil), "close manager")
})
return m
}
func iptDnatV4(port uint16) fw.ForwardRule {
return fw.ForwardRule{
Protocol: fw.ProtocolTCP,
DestinationPort: fw.Port{Values: []uint16{port}},
TranslatedAddress: netip.MustParseAddr("10.20.0.2"),
TranslatedPort: fw.Port{Values: []uint16{80}},
}
}
func iptDnatV6(port uint16) fw.ForwardRule {
return fw.ForwardRule{
Protocol: fw.ProtocolTCP,
DestinationPort: fw.Port{Values: []uint16{port}},
TranslatedAddress: netip.MustParseAddr("fd00::2"),
TranslatedPort: fw.Port{Values: []uint16{80}},
}
}
// TestIptablesRouting_RepeatedEnableSingleReference verifies that EnableRouting
// (called on every network-map update) holds at most one reference per family
// and a single DisableRouting drops both back to zero.
func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) {
m := newIptRefcountManager(t, true)
state := m.router.ipFwdState
require.NoError(t, m.EnableRouting(), "first enable")
require.NoError(t, m.EnableRouting(), "second enable")
require.NoError(t, m.EnableRouting(), "third enable")
v4, v6 := state.Counts()
assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference")
assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference")
require.NoError(t, m.DisableRouting(), "disable")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "single disable releases the v4 reference")
assert.Equal(t, 0, v6, "single disable releases the v6 reference")
}
// TestIptablesRouting_DisableKeepsDNATReference verifies that an unpaired
// DisableRouting does not release references held by active DNAT rules.
func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) {
m := newIptRefcountManager(t, true)
state := m.router.ipFwdState
r1, err := m.AddDNATRule(iptDnatV6(9095))
require.NoError(t, err, "add v6 dnat")
require.NoError(t, m.DisableRouting(), "unpaired disable")
_, v6 := state.Counts()
assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting")
require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat")
_, v6 = state.Counts()
assert.Equal(t, 0, v6, "delete releases the DNAT reference")
}
// TestIptablesDNAT_RefcountBalancedV4 covers a Balanced Add/Delete pair on v4.
func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) {
m := newIptRefcountManager(t, false)
state := m.router.ipFwdState
r1, err := m.AddDNATRule(iptDnatV4(7081))
require.NoError(t, err, "add v4 dnat 1")
v4, v6 := state.Counts()
assert.Equal(t, 1, v4, "v4 refcount after first add")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
r2, err := m.AddDNATRule(iptDnatV4(7082))
require.NoError(t, err, "add v4 dnat 2")
v4, v6 = state.Counts()
assert.Equal(t, 2, v4, "v4 refcount after second add")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
require.NoError(t, m.DeleteDNATRule(r1))
v4, v6 = state.Counts()
assert.Equal(t, 1, v4, "v4 refcount after first delete")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
require.NoError(t, m.DeleteDNATRule(r2))
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "v4 refcount after second delete")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
}
// TestIptablesDNAT_RefcountBalancedV6 checks the v6 path increments v6 only and
// decrements back to zero.
func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) {
m := newIptRefcountManager(t, true)
require.NotNil(t, m.router6, "v6 router")
require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state")
state := m.router.ipFwdState
r1, err := m.AddDNATRule(iptDnatV6(9081))
require.NoError(t, err, "add v6 dnat 1")
v4, v6 := state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 1, v6, "v6 refcount after first add")
r2, err := m.AddDNATRule(iptDnatV6(9082))
require.NoError(t, err, "add v6 dnat 2")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "v4 refcount unchanged")
assert.Equal(t, 2, v6, "v6 refcount after second add")
require.NoError(t, m.DeleteDNATRule(r1))
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "v4 refcount unchanged")
assert.Equal(t, 1, v6, "v6 refcount after first delete")
require.NoError(t, m.DeleteDNATRule(r2))
v4, v6 = state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 0, v6, "v6 refcount after second delete")
}
// TestIptablesDNAT_DuplicateAddNoLeak verifies the duplicate-rule path returns
// without bumping the refcount.
func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) {
m := newIptRefcountManager(t, true)
state := m.router.ipFwdState
rule := iptDnatV4(7083)
r1, err := m.AddDNATRule(rule)
require.NoError(t, err)
v4, _ := state.Counts()
assert.Equal(t, 1, v4)
_, err = m.AddDNATRule(rule)
require.NoError(t, err, "duplicate add")
v4, _ = state.Counts()
assert.Equal(t, 1, v4, "duplicate add must not increment")
require.NoError(t, m.DeleteDNATRule(r1))
v4, _ = state.Counts()
assert.Equal(t, 0, v4, "single delete must drop to zero")
}
// TestIptablesDNAT_DeleteMissingNoUnderflow verifies Delete on an unknown rule
// neither errors nor releases the refcount.
func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) {
m := newIptRefcountManager(t, true)
state := m.router.ipFwdState
phantom := iptDnatV4(7099)
require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4")
v4, v6 := state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 0, v6)
phantom6 := iptDnatV6(9099)
require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 0, v6)
r1, err := m.AddDNATRule(iptDnatV4(7100))
require.NoError(t, err)
v4, _ = state.Counts()
assert.Equal(t, 1, v4, "real add still increments after phantom delete")
require.NoError(t, m.DeleteDNATRule(r1))
}
// TestIptablesDNAT_DoubleDeleteNoUnderflow verifies a second Delete on the same
// rule is a no-op.
func TestIptablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) {
m := newIptRefcountManager(t, true)
state := m.router.ipFwdState
r1, err := m.AddDNATRule(iptDnatV6(9083))
require.NoError(t, err)
_, v6 := state.Counts()
assert.Equal(t, 1, v6)
require.NoError(t, m.DeleteDNATRule(r1), "first delete")
_, v6 = state.Counts()
assert.Equal(t, 0, v6)
require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op")
_, v6 = state.Counts()
assert.Equal(t, 0, v6, "double delete must not underflow")
}

View File

@@ -0,0 +1,75 @@
package iptables
import (
"sync"
log "github.com/sirupsen/logrus"
)
// ipsetSupport tracks whether ipset-backed firewall rules can be installed.
//
// It starts optimistic and latches to unsupported the first time the kernel
// proves otherwise: either the hash:net set type is missing (ip_set_hash_net) or
// iptables cannot match against a set (xt_set). Callers then emit per-IP and
// per-prefix rules instead. Without the fallback, a rule referencing an unusable
// set is never installed and the catch-all DROP silently blocks traffic the
// policy permits.
//
// One instance is shared by the ACL managers and routers of both address
// families, because ipset availability is a property of the kernel rather than
// of any single table.
type ipsetSupport struct {
mu sync.RWMutex
unsupported bool
}
func newIPSetSupport() *ipsetSupport {
return &ipsetSupport{}
}
func (s *ipsetSupport) supported() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return !s.unsupported
}
// markUnsupported records that ipset cannot be used, logging the reason once.
func (s *ipsetSupport) markUnsupported(cause error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.unsupported {
return
}
s.unsupported = true
log.Warnf("ipset is unavailable (%v); falling back to per-IP firewall rules. "+
"Ensure the kernel provides ip_set_hash_net and xt_set; without them rule "+
"sets are larger and slower to converge on networks with many peers", cause)
}
// ipsetUnusableError marks a failure attributable to ipset, so the caller can
// retry the same rule in its per-IP form before latching the capability off.
type ipsetUnusableError struct {
cause error
}
func (e *ipsetUnusableError) Error() string { return e.cause.Error() }
func (e *ipsetUnusableError) Unwrap() error { return e.cause }
func ipsetUnusable(cause error) error {
return &ipsetUnusableError{cause: cause}
}
// maybeIPSetUnusable marks an iptables failure as ipset-attributable only when the
// rule actually carried a set match, since the same call can fail for unrelated
// reasons on a rule that matches addresses directly.
func maybeIPSetUnusable(ipsetName string, err error) error {
if ipsetName == "" {
return err
}
return ipsetUnusable(err)
}

View File

@@ -33,6 +33,10 @@ type Manager struct {
router *router
rawSupported bool
// ipsetSupport is shared by the ACL managers and routers of both families,
// so a kernel without usable ipset support degrades them together.
ipsetSupport *ipsetSupport
// IPv6 counterparts, nil when no v6 overlay
ipv6Client *iptables.IPTables
aclMgr6 *aclManager
@@ -53,16 +57,17 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) {
}
m := &Manager{
wgIface: wgIface,
ipv4Client: iptablesClient,
wgIface: wgIface,
ipv4Client: iptablesClient,
ipsetSupport: newIPSetSupport(),
}
m.router, err = newRouter(iptablesClient, wgIface, mtu)
m.router, err = newRouter(iptablesClient, wgIface, mtu, m.ipsetSupport)
if err != nil {
return nil, fmt.Errorf("create router: %w", err)
}
m.aclMgr, err = newAclManager(iptablesClient, wgIface)
m.aclMgr, err = newAclManager(iptablesClient, wgIface, m.ipsetSupport)
if err != nil {
return nil, fmt.Errorf("create acl manager: %w", err)
}
@@ -83,16 +88,16 @@ func (m *Manager) createIPv6Components(wgIface iFaceMapper, mtu uint16) error {
}
m.ipv6Client = ip6Client
m.router6, err = newRouter(ip6Client, wgIface, mtu)
m.router6, err = newRouter(ip6Client, wgIface, mtu, m.ipsetSupport)
if err != nil {
return fmt.Errorf("create v6 router: %w", err)
}
// Share the same IP forwarding state with the v4 router, since
// EnableIPForwarding controls both v4 and v6 sysctls.
// Forwarding refcounter is per-family but shared between v4 and v6 routers.
m.router6.ipFwdState = m.router.ipFwdState
m.aclMgr6, err = newAclManager(ip6Client, wgIface)
m.aclMgr6, err = newAclManager(ip6Client, wgIface, m.ipsetSupport)
if err != nil {
return fmt.Errorf("create v6 acl manager: %w", err)
}
@@ -402,17 +407,12 @@ func (m *Manager) SetLogLevel(log.Level) {
}
func (m *Manager) EnableRouting() error {
if err := m.router.ipFwdState.RequestForwarding(); err != nil {
return fmt.Errorf("enable IP forwarding: %w", err)
}
return nil
// v6 only when the overlay actually has v6.
return m.router.ipFwdState.RequestRouting(m.router6 != nil)
}
func (m *Manager) DisableRouting() error {
if err := m.router.ipFwdState.ReleaseForwarding(); err != nil {
return fmt.Errorf("disable IP forwarding: %w", err)
}
return nil
return m.router.ipFwdState.ReleaseRouting()
}
// AddDNATRule adds a DNAT rule

View File

@@ -292,39 +292,97 @@ func TestIptablesCreatePerformance(t *testing.T) {
}
}
// TestIptablesACLIPSetFallback verifies that when the kernel lacks ipset support,
// the ACL manager falls back to per-IP iptables rules (-s <ip>) instead of
// silently leaving the chain empty. See discussion #6125.
func TestIptablesACLIPSetFallback(t *testing.T) {
ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err)
// newACLTestManager returns a started manager. Create()/Init() is used so the
// router-owned chains (chainRTFWDIN/OUT) exist before the ACL manager's
// createDefaultChains() references them.
func newACLTestManager(t *testing.T) *Manager {
t.Helper()
// Use Create()/Init() so the router-owned chains (chainRTFWDIN/OUT) are
// created before the ACL manager's createDefaultChains() references them.
manager, err := Create(ifaceMock, iface.DefaultMTU)
require.NoError(t, err)
require.NoError(t, manager.Init(nil))
aclMgr := manager.aclMgr
// Simulate a kernel without the ipset hash module.
aclMgr.ipsetSupported = false
defer func() {
t.Cleanup(func() {
require.NoError(t, manager.Close(nil))
}()
})
return manager
}
// TestIptablesACLUsesIPSetOnHealthyKernel guards the default: on a kernel that
// does have ipset, rules must keep matching a set. A regression that reported
// ipset as unusable would silently move every Linux client to per-IP rules.
func TestIptablesACLUsesIPSetOnHealthyKernel(t *testing.T) {
ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err)
manager := newACLTestManager(t)
ip := netip.MustParseAddr("10.20.0.42")
port := &fw.Port{Values: []uint16{22}}
rules, err := aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "nb0000001")
require.NoError(t, err, "AddPeerFiltering should succeed via fallback")
rules, err := manager.aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "nb0000001")
require.NoError(t, err)
require.NotEmpty(t, rules)
rule := rules[0].(*Rule)
require.Equal(t, "nb0000001-dport", rule.ipsetName, "healthy kernel must use an ipset")
require.Contains(t, rule.specs, "--match-set")
require.True(t, manager.ipsetSupport.supported(), "ipset must not be latched off on a healthy kernel")
checkRuleSpecs(t, ipv4Client, rule.chain, true, rule.specs...)
}
// TestIptablesACLFallsBackWhenIPSetUnusable drives the real failure path: an
// oversized set name is rejected by the kernel, which stands in for a kernel
// without ip_set_hash_net or xt_set. The rule must still land in the chain,
// matching the IP directly, and the capability must latch off so later rules skip
// ipset. Before the fallback existed, the rule was dropped and the catch-all DROP
// silently blocked traffic the policy permits.
func TestIptablesACLFallsBackWhenIPSetUnusable(t *testing.T) {
ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err)
manager := newACLTestManager(t)
// ipset names are limited to 31 characters, so creating this set fails.
unusableName := strings.Repeat("a", 40)
ip := netip.MustParseAddr("10.20.0.42")
port := &fw.Port{Values: []uint16{22}}
rules, err := manager.aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, unusableName)
require.NoError(t, err, "AddPeerFiltering must succeed by falling back")
require.NotEmpty(t, rules)
rule := rules[0].(*Rule)
require.Empty(t, rule.ipsetName, "fallback rule must not reference an ipset")
require.Contains(t, strings.Join(rule.specs, " "), "-s 10.20.0.42", "fallback rule must match by source IP")
require.NotContains(t, strings.Join(rule.specs, " "), "--match-set", "fallback rule must not use ipset matching")
require.Contains(t, strings.Join(rule.specs, " "), "-s 10.20.0.42", "fallback rule must match the source IP")
require.NotContains(t, strings.Join(rule.specs, " "), "--match-set")
// The rule must actually be present in the ACL chain (not silently dropped).
// The rule must actually be present, not silently missing.
checkRuleSpecs(t, ipv4Client, rule.chain, true, rule.specs...)
require.False(t, manager.ipsetSupport.supported(), "failure must latch ipset off")
// A subsequent rule with a perfectly valid set name now skips ipset too.
next, err := manager.aclMgr.AddPeerFiltering(nil, netip.MustParseAddr("10.20.0.43").AsSlice(), "tcp", nil, port, fw.ActionAccept, "nb0000001")
require.NoError(t, err)
require.NotEmpty(t, next)
require.Empty(t, next[0].(*Rule).ipsetName, "later rules must skip ipset once latched")
}
// TestIptablesACLLeavesNoIPSetAfterFallback verifies the set created before the
// failure is destroyed, so a later rule does not find a half-built set and assume
// ipset works.
func TestIptablesACLLeavesNoIPSetAfterFallback(t *testing.T) {
manager := newACLTestManager(t)
port := &fw.Port{Values: []uint16{22}}
ip := netip.MustParseAddr("10.20.0.42")
_, err := manager.aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, strings.Repeat("a", 40))
require.NoError(t, err)
_, exists := manager.aclMgr.ipsetStore.ipset(strings.Repeat("a", 40) + "-dport")
require.False(t, exists, "unusable set must not stay in the store")
}

View File

@@ -3,6 +3,7 @@
package iptables
import (
"errors"
"fmt"
"maps"
"net/netip"
@@ -51,6 +52,10 @@ const (
markManglePost = "mark-mangle-post"
matchSet = "--match-set"
// routeSourceSuffix names the extra rules a route ACL needs when ipset is
// unusable and each source prefix has to be matched by its own rule.
routeSourceSuffix = "_src"
dnatSuffix = "_dnat"
snatSuffix = "_snat"
fwdSuffix = "_fwd"
@@ -68,7 +73,6 @@ type ruleInfo struct {
}
type routeFilteringRuleParams struct {
Source firewall.Network
Destination firewall.Network
Proto firewall.Protocol
SPort *firewall.Port
@@ -90,19 +94,21 @@ type router struct {
legacyManagement bool
mtu uint16
v6 bool
ipsetSupport *ipsetSupport
stateManager *statemanager.Manager
ipFwdState *ipfwdstate.IPForwardingState
}
func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16) (*router, error) {
func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16, ipsetSupport *ipsetSupport) (*router, error) {
r := &router{
iptablesClient: iptablesClient,
rules: make(map[string][]string),
wgIface: wgIface,
mtu: mtu,
v6: iptablesClient.Proto() == iptables.ProtocolIPv6,
ipFwdState: ipfwdstate.NewIPForwardingState(),
ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()),
ipsetSupport: ipsetSupport,
}
r.ipsetCounter = refcounter.New(
@@ -151,15 +157,7 @@ func (r *router) AddRouteFiltering(
return ruleKey, nil
}
var source firewall.Network
if len(sources) > 1 {
source.Set = firewall.NewPrefixSet(sources)
} else if len(sources) > 0 {
source.Prefix = sources[0]
}
params := routeFilteringRuleParams{
Source: source,
Destination: destination,
Proto: proto,
SPort: sPort,
@@ -167,30 +165,156 @@ func (r *router) AddRouteFiltering(
Action: action,
}
rule, err := r.genRouteRuleSpec(params, sources)
if err != nil {
return nil, fmt.Errorf("generate route rule spec: %w", err)
}
err := r.installRouteRules(string(ruleKey), params, sources, r.ipsetSupport.supported())
// Insert DROP rules at the beginning, append ACCEPT rules at the end
if action == firewall.ActionDrop {
// after the established rule
err = r.iptablesClient.Insert(tableFilter, chainRTFWDIN, 2, rule...)
} else {
err = r.iptablesClient.Append(tableFilter, chainRTFWDIN, rule...)
var unusable *ipsetUnusableError
if errors.As(err, &unusable) {
// The set could not be created or matched. Retry matching each source
// prefix on its own; only if that works do we know ipset was to blame.
r.removeRouteRules(string(ruleKey))
if retryErr := r.installRouteRules(string(ruleKey), params, sources, false); retryErr != nil {
r.removeRouteRules(string(ruleKey))
return nil, fmt.Errorf("add route rule (ipset: %w): %w", unusable.cause, retryErr)
}
r.ipsetSupport.markUnsupported(unusable.cause)
err = nil
}
if err != nil {
return nil, fmt.Errorf("add route rule: %v", err)
}
// Leave nothing half-installed: a later call finding the rule key would
// report success while some sources were never installed, which for a
// drop rule would leave them unblocked.
r.removeRouteRules(string(ruleKey))
r.rules[string(ruleKey)] = rule
return nil, fmt.Errorf("add route rule: %w", err)
}
r.updateState()
return ruleKey, nil
}
// installRouteRules installs every rule needed for one route ACL and records them
// under ruleKey. It is more than one rule only when useIPSet is false and the
// sources have to be matched one prefix at a time.
func (r *router) installRouteRules(ruleKey string, params routeFilteringRuleParams, sources []netip.Prefix, useIPSet bool) error {
specs, err := r.genRouteRuleSpecs(params, sources, useIPSet)
if err != nil {
return fmt.Errorf("generate route rule spec: %w", err)
}
for i, spec := range specs {
if err := r.insertRouteRule(params.Action, spec); err != nil {
if len(r.findSets(spec)) > 0 {
return ipsetUnusable(err)
}
return err
}
r.rules[routeRuleKey(ruleKey, i)] = spec
}
return nil
}
// genRouteRuleSpecs builds the rules for one route ACL. With ipset available that
// is a single rule matching a set of sources; without it, one rule per source
// prefix, which is the only form a stripped kernel can express.
func (r *router) genRouteRuleSpecs(params routeFilteringRuleParams, sources []netip.Prefix, useIPSet bool) ([][]string, error) {
destExp, err := r.applyNetwork("-d", params.Destination, nil)
if err != nil {
return nil, fmt.Errorf("apply network -d: %w", err)
}
specs, err := r.genSourceRules(params, sources, useIPSet, destExp)
if err != nil {
// The destination match may have taken a set reference already.
if decErr := r.decrementSetCounter(destExp); decErr != nil {
log.Debugf("release destination set after failed rule generation: %v", decErr)
}
return nil, err
}
return specs, nil
}
func (r *router) genSourceRules(params routeFilteringRuleParams, sources []netip.Prefix, useIPSet bool, destExp []string) ([][]string, error) {
if useIPSet || len(sources) <= 1 {
sourceExp, err := r.applyNetwork("-s", sourceNetwork(sources), sources)
if err != nil {
return nil, fmt.Errorf("apply network -s: %w", err)
}
return [][]string{assembleRouteRule(sourceExp, destExp, params, r.v6)}, nil
}
specs := make([][]string, 0, len(sources))
for _, source := range sources {
sourceExp, err := r.applyNetwork("-s", firewall.Network{Prefix: source}, nil)
if err != nil {
return nil, fmt.Errorf("apply network -s: %w", err)
}
specs = append(specs, assembleRouteRule(sourceExp, destExp, params, r.v6))
}
return specs, nil
}
func sourceNetwork(sources []netip.Prefix) firewall.Network {
var source firewall.Network
if len(sources) > 1 {
source.Set = firewall.NewPrefixSet(sources)
} else if len(sources) > 0 {
source.Prefix = sources[0]
}
return source
}
func (r *router) insertRouteRule(action firewall.Action, spec []string) error {
// Insert DROP rules at the beginning, append ACCEPT rules at the end
if action == firewall.ActionDrop {
// after the established rule
return r.iptablesClient.Insert(tableFilter, chainRTFWDIN, 2, spec...)
}
return r.iptablesClient.Append(tableFilter, chainRTFWDIN, spec...)
}
// removeRouteRules deletes the rules recorded for ruleKey, used to undo a partial
// install before retrying without ipset.
func (r *router) removeRouteRules(ruleKey string) {
for i := 0; ; i++ {
key := routeRuleKey(ruleKey, i)
spec, exists := r.rules[key]
if !exists {
return
}
if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWDIN, spec...); err != nil {
log.Debugf("delete partial route rule %s: %v", key, err)
}
delete(r.rules, key)
if err := r.decrementSetCounter(spec); err != nil {
log.Debugf("decrement ipset counter for %s: %v", key, err)
}
}
}
// routeRuleKey names the i-th rule of a route ACL. The first keeps the plain rule
// key so single-rule ACLs, which is every ACL when ipset works, are unaffected.
func routeRuleKey(ruleKey string, i int) string {
if i == 0 {
return ruleKey
}
return fmt.Sprintf("%s%s%d", ruleKey, routeSourceSuffix, i)
}
func (r *router) hasRule(id string) bool {
_, ok := r.rules[id]
return ok
@@ -199,17 +323,29 @@ func (r *router) hasRule(id string) bool {
func (r *router) DeleteRouteRule(rule firewall.Rule) error {
ruleKey := rule.ID()
if rule, exists := r.rules[ruleKey]; exists {
if _, exists := r.rules[ruleKey]; !exists {
log.Debugf("route rule %s not found", ruleKey)
r.updateState()
return nil
}
// In the ipset fallback one ACL is several rules, one per source prefix.
for i := 0; ; i++ {
key := routeRuleKey(ruleKey, i)
rule, exists := r.rules[key]
if !exists {
break
}
if err := r.iptablesClient.Delete(tableFilter, chainRTFWDIN, rule...); err != nil {
return fmt.Errorf("delete route rule: %v", err)
}
delete(r.rules, ruleKey)
delete(r.rules, key)
if err := r.decrementSetCounter(rule); err != nil {
return fmt.Errorf("decrement ipset counter: %w", err)
}
} else {
log.Debugf("route rule %s not found", ruleKey)
}
r.updateState()
@@ -770,10 +906,6 @@ func (r *router) updateState() {
}
func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
if err := r.ipFwdState.RequestForwarding(); err != nil {
return nil, err
}
ruleKey := rule.ID()
if _, exists := r.rules[ruleKey+dnatSuffix]; exists {
return rule, nil
@@ -840,18 +972,34 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
for key, ruleInfo := range rules {
if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil {
if rollbackErr := r.rollbackRules(rules); rollbackErr != nil {
log.Errorf("rollback failed: %v", rollbackErr)
}
r.cleanupFailedDNATAdd(rules)
return nil, fmt.Errorf("add rule %s: %w", key, err)
}
r.rules[key] = ruleInfo.rule
}
if err := r.ipFwdState.RequestForwarding(r.v6); err != nil {
r.cleanupFailedDNATAdd(rules)
return nil, fmt.Errorf("enable forwarding: %w", err)
}
r.updateState()
return rule, nil
}
// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied
// AddDNATRule before rolling back the kernel rules, so no entries remain that
// never got a forwarding refcount. rollbackRules re-adds entries it failed to
// remove from the kernel.
func (r *router) cleanupFailedDNATAdd(rules map[string]ruleInfo) {
for key := range rules {
delete(r.rules, key)
}
if err := r.rollbackRules(rules); err != nil {
log.Errorf("rollback failed: %v", err)
}
}
func (r *router) rollbackRules(rules map[string]ruleInfo) error {
var merr *multierror.Error
for key, ruleInfo := range rules {
@@ -868,63 +1016,70 @@ func (r *router) rollbackRules(rules map[string]ruleInfo) error {
}
func (r *router) DeleteDNATRule(rule firewall.Rule) error {
if err := r.ipFwdState.ReleaseForwarding(); err != nil {
log.Errorf("%v", err)
}
ruleKey := rule.ID()
_, hadDNAT := r.rules[ruleKey+dnatSuffix]
_, hadSNAT := r.rules[ruleKey+snatSuffix]
_, hadFWD := r.rules[ruleKey+fwdSuffix]
if !hadDNAT && !hadSNAT && !hadFWD {
return nil
}
var merr *multierror.Error
if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists {
if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err))
} else {
delete(r.rules, ruleKey+dnatSuffix)
}
delete(r.rules, ruleKey+dnatSuffix)
}
if snatRule, exists := r.rules[ruleKey+snatSuffix]; exists {
if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err))
} else {
delete(r.rules, ruleKey+snatSuffix)
}
delete(r.rules, ruleKey+snatSuffix)
}
if fwdRule, exists := r.rules[ruleKey+fwdSuffix]; exists {
if err := r.iptablesClient.Delete(tableFilter, chainRTFWDOUT, fwdRule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err))
} else {
delete(r.rules, ruleKey+fwdSuffix)
}
}
// Release the refcount only once all rules are gone from the kernel. On
// partial failure the failed entries stay in r.rules so a retry can remove
// them and release then.
if merr == nil {
if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil {
log.Errorf("%v", err)
}
delete(r.rules, ruleKey+fwdSuffix)
}
r.updateState()
return nberrors.FormatErrorOrNil(merr)
}
func (r *router) genRouteRuleSpec(params routeFilteringRuleParams, sources []netip.Prefix) ([]string, error) {
// assembleRouteRule joins the pre-built source and destination matches with the
// protocol, ports and target of a route ACL.
func assembleRouteRule(sourceExp, destExp []string, params routeFilteringRuleParams, v6 bool) []string {
var rule []string
sourceExp, err := r.applyNetwork("-s", params.Source, sources)
if err != nil {
return nil, fmt.Errorf("apply network -s: %w", err)
}
destExp, err := r.applyNetwork("-d", params.Destination, nil)
if err != nil {
return nil, fmt.Errorf("apply network -d: %w", err)
}
rule = append(rule, sourceExp...)
rule = append(rule, destExp...)
if params.Proto != firewall.ProtocolALL {
rule = append(rule, "-p", strings.ToLower(protoForFamily(params.Proto, r.v6)))
rule = append(rule, "-p", strings.ToLower(protoForFamily(params.Proto, v6)))
rule = append(rule, applyPort("--sport", params.SPort)...)
rule = append(rule, applyPort("--dport", params.DPort)...)
}
rule = append(rule, "-j", actionToStr(params.Action))
return rule, nil
return rule
}
func (r *router) applyNetwork(flag string, network firewall.Network, prefixes []netip.Prefix) ([]string, error) {
@@ -934,9 +1089,17 @@ func (r *router) applyNetwork(flag string, network firewall.Network, prefixes []
}
if network.IsSet() {
// A destination set is populated later from DNS results, so unlike a
// source set it cannot be expanded into per-prefix rules here. Without
// ipset such a rule is not expressible; report it instead of installing
// something broader than the policy allows.
if flag == "-d" && !r.ipsetSupport.supported() {
return nil, fmt.Errorf("destination set %s requires ipset (ip_set_hash_net and xt_set)", network.Set.HashedName())
}
name := r.ipsetName(network.Set.HashedName())
if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil {
return nil, fmt.Errorf("create or get ipset: %w", err)
return nil, ipsetUnusable(fmt.Errorf("create or get ipset: %w", err))
}
return []string{"-m", "set", matchSet, name, direction}, nil

View File

@@ -3,9 +3,11 @@
package iptables
import (
"errors"
"fmt"
"net/netip"
"os/exec"
"strings"
"testing"
"github.com/coreos/go-iptables/iptables"
@@ -15,7 +17,9 @@ import (
firewall "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/firewall/test"
"github.com/netbirdio/netbird/client/iface"
nbid "github.com/netbirdio/netbird/client/internal/acl/id"
nbnet "github.com/netbirdio/netbird/client/net"
"github.com/netbirdio/netbird/shared/management/domain"
)
func isIptablesSupported() bool {
@@ -31,7 +35,7 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err, "failed to init iptables client")
manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, newIPSetSupport())
require.NoError(t, err, "should return a valid iptables manager")
require.NoError(t, manager.init(nil))
@@ -84,7 +88,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) {
iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err, "failed to init iptables client")
manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, newIPSetSupport())
require.NoError(t, err, "shouldn't return error")
require.NoError(t, manager.init(nil))
@@ -157,7 +161,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) {
t.Run(testCase.Name, func(t *testing.T) {
iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, newIPSetSupport())
require.NoError(t, err, "shouldn't return error")
require.NoError(t, manager.init(nil))
defer func() {
@@ -219,7 +223,7 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err, "Failed to create iptables client")
r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, newIPSetSupport())
require.NoError(t, err, "Failed to create router manager")
require.NoError(t, r.init(nil))
@@ -337,27 +341,26 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
ruleKey, err := r.AddRouteFiltering(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action)
require.NoError(t, err, "AddRouteFiltering failed")
// Check if the rule is in the internal map
rule, ok := r.rules[ruleKey.ID()]
assert.True(t, ok, "Rule not found in internal map")
// A kernel without usable ipset splits a multi-source ACL into one
// rule per source, so compare against whichever form is in effect.
useIPSet := r.ipsetSupport.supported()
// Log the internal rule
t.Logf("Internal rule: %v", rule)
// Check if the rules are in the internal map
rules := routeRuleSpecs(t, r, ruleKey.ID())
require.NotEmpty(t, rules, "Rule not found in internal map")
// Check if the rule exists in iptables
exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, rule...)
assert.NoError(t, err, "Failed to check rule existence")
assert.True(t, exists, "Rule not found in iptables")
// Log the internal rules
t.Logf("Internal rules: %v", rules)
var source firewall.Network
if len(tt.sources) > 1 {
source.Set = firewall.NewPrefixSet(tt.sources)
} else if len(tt.sources) > 0 {
source.Prefix = tt.sources[0]
// Check if the rules exist in iptables
for _, rule := range rules {
exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, rule...)
assert.NoError(t, err, "Failed to check rule existence")
assert.True(t, exists, "Rule not found in iptables")
}
// Verify rule content
params := routeFilteringRuleParams{
Source: source,
Destination: firewall.Network{Prefix: tt.destination},
Proto: tt.proto,
SPort: tt.sPort,
@@ -365,20 +368,18 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
Action: tt.action,
}
expectedRule, err := r.genRouteRuleSpec(params, nil)
expectedRules, err := r.genRouteRuleSpecs(params, tt.sources, useIPSet)
require.NoError(t, err, "Failed to generate expected rule spec")
if tt.expectSet {
if tt.expectSet && useIPSet {
setName := firewall.NewPrefixSet(tt.sources).HashedName()
expectedRule, err = r.genRouteRuleSpec(params, nil)
require.NoError(t, err, "Failed to generate expected rule spec with set")
// Check if the set was created
_, exists := r.ipsetCounter.Get(setName)
assert.True(t, exists, "IPSet not created")
}
assert.Equal(t, expectedRule, rule, "Rule content mismatch")
assert.Equal(t, expectedRules, rules, "Rule content mismatch")
// Clean up
err = r.DeleteRouteRule(ruleKey)
@@ -445,3 +446,145 @@ func TestFindSetNameInRule(t *testing.T) {
})
}
}
// TestRouter_AddRouteFilteringIPSetFallback covers a kernel that cannot use ipset:
// a multi-source route ACL must become one rule per source prefix, all present in
// the chain, and deleting the ACL must remove every one of them. Without the
// fallback the rule was never installed and the interface-wide DROP in FORWARD
// silently dropped routed traffic.
func TestRouter_AddRouteFilteringIPSetFallback(t *testing.T) {
if !isIptablesSupported() {
t.Skip("iptables not supported on this system")
}
iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err)
support := newIPSetSupport()
support.markUnsupported(errors.New("test: pretend the kernel has no ipset"))
r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, support)
require.NoError(t, err)
require.NoError(t, r.init(nil))
t.Cleanup(func() {
require.NoError(t, r.Reset())
})
sources := []netip.Prefix{
netip.MustParsePrefix("172.16.0.0/16"),
netip.MustParsePrefix("192.168.0.0/16"),
}
destination := firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/8")}
rule, err := r.AddRouteFiltering(nil, sources, destination, firewall.ProtocolTCP, nil,
&firewall.Port{Values: []uint16{443}}, firewall.ActionAccept)
require.NoError(t, err, "route ACL must install without ipset")
specs := routeRuleSpecs(t, r, rule.ID())
require.Len(t, specs, len(sources), "each source prefix needs its own rule")
for i, spec := range specs {
joined := strings.Join(spec, " ")
require.Contains(t, joined, "-s "+sources[i].String(), "rule must match the source prefix directly")
require.NotContains(t, joined, matchSet, "fallback rule must not reference a set")
exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, spec...)
require.NoError(t, err)
require.True(t, exists, "rule %d must be present in %s", i, chainRTFWDIN)
}
require.NoError(t, r.DeleteRouteRule(rule))
for i, spec := range specs {
exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, spec...)
require.NoError(t, err)
require.False(t, exists, "rule %d must be removed", i)
}
require.Empty(t, routeRuleSpecs(t, r, rule.ID()), "no rule may be left recorded")
}
// TestRouter_DestinationSetRequiresIPSet documents that a dynamic (domain)
// destination cannot be expressed without ipset: its prefixes are only known
// after DNS resolution, so there is nothing to expand into per-prefix rules. The
// call must report that rather than install a broader rule than the policy allows.
func TestRouter_DestinationSetRequiresIPSet(t *testing.T) {
if !isIptablesSupported() {
t.Skip("iptables not supported on this system")
}
iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err)
support := newIPSetSupport()
support.markUnsupported(errors.New("test: pretend the kernel has no ipset"))
r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, support)
require.NoError(t, err)
require.NoError(t, r.init(nil))
t.Cleanup(func() {
require.NoError(t, r.Reset())
})
destination := firewall.Network{Set: firewall.NewDomainSet(domain.List{"example.com"})}
_, err = r.AddRouteFiltering(nil, []netip.Prefix{netip.MustParsePrefix("172.16.0.0/16")},
destination, firewall.ProtocolALL, nil, nil, firewall.ActionAccept)
require.Error(t, err, "a domain destination is not expressible without ipset")
require.ErrorContains(t, err, "requires ipset")
}
// TestRouter_RouteFilteringRollsBackPartialInstall covers a fallback ACL whose
// second rule cannot be installed. Nothing may be left behind: if the rule key
// survived, a later call would short-circuit on it and report success while some
// sources were never installed, leaving them unblocked for a drop rule.
func TestRouter_RouteFilteringRollsBackPartialInstall(t *testing.T) {
if !isIptablesSupported() {
t.Skip("iptables not supported on this system")
}
iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
require.NoError(t, err)
support := newIPSetSupport()
support.markUnsupported(errors.New("test: pretend the kernel has no ipset"))
r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU, support)
require.NoError(t, err)
require.NoError(t, r.init(nil))
t.Cleanup(func() {
require.NoError(t, r.Reset())
})
// The v6 prefix is rejected by the v4 iptables binary, so the second rule of
// the expansion fails after the first has been installed.
good := netip.MustParsePrefix("172.16.0.0/16")
sources := []netip.Prefix{good, netip.MustParsePrefix("2001:db8::/32")}
destination := firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/8")}
_, err = r.AddRouteFiltering(nil, sources, destination, firewall.ProtocolALL, nil, nil, firewall.ActionDrop)
require.Error(t, err, "a source that iptables rejects must fail the whole ACL")
ruleKey := nbid.GenerateRouteRuleKey(sources, destination, firewall.ProtocolALL, nil, nil, firewall.ActionDrop)
require.Empty(t, routeRuleSpecs(t, r, string(ruleKey)), "no rule may stay recorded")
// The rule that did get installed must be gone from the chain.
installed := []string{"-s", good.String(), "-d", "10.0.0.0/8", "-j", "DROP"}
exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, installed...)
require.NoError(t, err)
require.False(t, exists, "the already-installed rule must be rolled back")
}
// routeRuleSpecs collects the rules recorded for one route ACL, which is more than
// one when the ipset fallback splits it per source prefix.
func routeRuleSpecs(t *testing.T, r *router, ruleKey string) [][]string {
t.Helper()
var specs [][]string
for i := 0; ; i++ {
spec, exists := r.rules[routeRuleKey(ruleKey, i)]
if !exists {
return specs
}
specs = append(specs, spec)
}
}

View File

@@ -0,0 +1,249 @@
//go:build privileged
package nftables
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
fw "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/iface/wgaddr"
)
func nftRefcountIfaceV4() *iFaceMock {
return &iFaceMock{
NameFunc: func() string { return "wt-refcount" },
AddressFunc: func() wgaddr.Address {
return wgaddr.Address{
IP: netip.MustParseAddr("100.96.0.1"),
Network: netip.MustParsePrefix("100.96.0.0/16"),
}
},
}
}
func nftRefcountIfaceDual() *iFaceMock {
return &iFaceMock{
NameFunc: func() string { return "wt-refcount" },
AddressFunc: func() wgaddr.Address {
return wgaddr.Address{
IP: netip.MustParseAddr("100.96.0.1"),
Network: netip.MustParsePrefix("100.96.0.0/16"),
IPv6: netip.MustParseAddr("fd00::1"),
IPv6Net: netip.MustParsePrefix("fd00::/64"),
}
},
}
}
func newNftRefcountManager(t *testing.T, dual bool) *Manager {
t.Helper()
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
}
var ifMock *iFaceMock
if dual {
ifMock = nftRefcountIfaceDual()
} else {
ifMock = nftRefcountIfaceV4()
}
m, err := Create(ifMock, iface.DefaultMTU)
require.NoError(t, err, "create manager")
require.NoError(t, m.Init(nil), "init manager")
t.Cleanup(func() {
require.NoError(t, m.Close(nil), "close manager")
})
return m
}
func dnatV4(port uint16) fw.ForwardRule {
return fw.ForwardRule{
Protocol: fw.ProtocolTCP,
DestinationPort: fw.Port{Values: []uint16{port}},
TranslatedAddress: netip.MustParseAddr("100.96.0.2"),
TranslatedPort: fw.Port{Values: []uint16{80}},
}
}
func dnatV6(port uint16) fw.ForwardRule {
return fw.ForwardRule{
Protocol: fw.ProtocolTCP,
DestinationPort: fw.Port{Values: []uint16{port}},
TranslatedAddress: netip.MustParseAddr("fd00::2"),
TranslatedPort: fw.Port{Values: []uint16{80}},
}
}
// TestNftablesDNAT_RefcountBalancedV4 verifies that Add/Delete pairs leave the
// v4 refcount at zero.
func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) {
m := newNftRefcountManager(t, false)
state := m.router.ipFwdState
r1, err := m.AddDNATRule(dnatV4(8081))
require.NoError(t, err, "add v4 dnat 1")
v4, v6 := state.Counts()
assert.Equal(t, 1, v4, "v4 refcount after first add")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
r2, err := m.AddDNATRule(dnatV4(8082))
require.NoError(t, err, "add v4 dnat 2")
v4, v6 = state.Counts()
assert.Equal(t, 2, v4, "v4 refcount after second add")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat 1")
v4, v6 = state.Counts()
assert.Equal(t, 1, v4, "v4 refcount after first delete")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
require.NoError(t, m.DeleteDNATRule(r2), "delete v4 dnat 2")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "v4 refcount after second delete")
assert.Equal(t, 0, v6, "v6 refcount unchanged")
}
// TestNftablesDNAT_RefcountBalancedV6 verifies the v6 path increments v6 only
// and decrements back to zero on Delete.
func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) {
m := newNftRefcountManager(t, true)
require.NotNil(t, m.router6, "v6 router")
require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state")
state := m.router.ipFwdState
r1, err := m.AddDNATRule(dnatV6(9091))
require.NoError(t, err, "add v6 dnat 1")
v4, v6 := state.Counts()
assert.Equal(t, 0, v4, "v4 refcount unchanged")
assert.Equal(t, 1, v6, "v6 refcount after first add")
r2, err := m.AddDNATRule(dnatV6(9092))
require.NoError(t, err, "add v6 dnat 2")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 2, v6, "v6 refcount after second add")
require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat 1")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "v4 refcount unchanged")
assert.Equal(t, 1, v6, "v6 refcount after first delete")
require.NoError(t, m.DeleteDNATRule(r2), "delete v6 dnat 2")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 0, v6, "v6 refcount after second delete")
}
// TestNftablesDNAT_DuplicateAddNoLeak verifies that a duplicate Add (same
// ForwardRule) does not double-increment the refcount.
func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
rule := dnatV4(8083)
r1, err := m.AddDNATRule(rule)
require.NoError(t, err, "add v4 dnat")
v4, _ := state.Counts()
assert.Equal(t, 1, v4)
// duplicate add: same rule ID, must be a no-op for the refcount.
_, err = m.AddDNATRule(rule)
require.NoError(t, err, "duplicate add")
v4, _ = state.Counts()
assert.Equal(t, 1, v4, "duplicate add must not increment")
require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat")
v4, _ = state.Counts()
assert.Equal(t, 0, v4, "single delete must drop to zero")
}
// TestNftablesDNAT_DeleteMissingNoUnderflow verifies deleting a rule that was
// never added does not underflow the refcount.
func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
// Construct a Rule reference for something never added. The router stores
// rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing
// entry must be a no-op rather than calling Release.
phantom := dnatV4(8099)
require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4 dnat")
v4, v6 := state.Counts()
assert.Equal(t, 0, v4, "v4 refcount unaffected by missing delete")
assert.Equal(t, 0, v6, "v6 refcount unaffected")
phantom6 := dnatV6(9099)
require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6 dnat")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4)
assert.Equal(t, 0, v6, "v6 refcount unaffected by missing delete")
// And after a phantom delete, a real add still results in count=1.
r1, err := m.AddDNATRule(dnatV4(8100))
require.NoError(t, err, "add v4 dnat after phantom delete")
v4, _ = state.Counts()
assert.Equal(t, 1, v4, "real add still increments after phantom delete")
require.NoError(t, m.DeleteDNATRule(r1))
}
// TestNftablesRouting_RepeatedEnableSingleReference verifies that EnableRouting
// (called on every network-map update) holds at most one reference per family
// and a single DisableRouting drops both back to zero.
func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
require.NoError(t, m.EnableRouting(), "first enable")
require.NoError(t, m.EnableRouting(), "second enable")
require.NoError(t, m.EnableRouting(), "third enable")
v4, v6 := state.Counts()
assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference")
assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference")
require.NoError(t, m.DisableRouting(), "disable")
v4, v6 = state.Counts()
assert.Equal(t, 0, v4, "single disable releases the v4 reference")
assert.Equal(t, 0, v6, "single disable releases the v6 reference")
}
// TestNftablesRouting_DisableKeepsDNATReference verifies that an unpaired
// DisableRouting does not release references held by active DNAT rules.
func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
r1, err := m.AddDNATRule(dnatV6(9095))
require.NoError(t, err, "add v6 dnat")
require.NoError(t, m.DisableRouting(), "unpaired disable")
_, v6 := state.Counts()
assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting")
require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat")
_, v6 = state.Counts()
assert.Equal(t, 0, v6, "delete releases the DNAT reference")
}
// TestNftablesDNAT_DoubleDeleteNoUnderflow verifies that deleting the same rule
// twice does not underflow the refcount (the second delete is a no-op).
func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
r1, err := m.AddDNATRule(dnatV6(9093))
require.NoError(t, err)
_, v6 := state.Counts()
assert.Equal(t, 1, v6)
require.NoError(t, m.DeleteDNATRule(r1), "first delete")
_, v6 = state.Counts()
assert.Equal(t, 0, v6)
require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op")
_, v6 = state.Counts()
assert.Equal(t, 0, v6, "double delete must not underflow")
}

View File

@@ -105,8 +105,8 @@ func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mt
return fmt.Errorf("create v6 router: %w", err)
}
// Share the same IP forwarding state with the v4 router, since
// EnableIPForwarding controls both v4 and v6 sysctls.
// Share the per-family forwarding refcounter with the v4 router so a v4
// rule and a v6 rule against the same state machine cooperate cleanly.
m.router6.ipFwdState = m.router.ipFwdState
m.aclManager6, err = newAclManager(workTable6, wgIface, chainNameRoutingFw)
@@ -530,17 +530,12 @@ func (m *Manager) SetLogLevel(log.Level) {
}
func (m *Manager) EnableRouting() error {
if err := m.router.ipFwdState.RequestForwarding(); err != nil {
return fmt.Errorf("enable IP forwarding: %w", err)
}
return nil
// v6 only when the overlay actually has v6.
return m.router.ipFwdState.RequestRouting(m.router6 != nil)
}
func (m *Manager) DisableRouting() error {
if err := m.router.ipFwdState.ReleaseForwarding(); err != nil {
return fmt.Errorf("disable IP forwarding: %w", err)
}
return nil
return m.router.ipFwdState.ReleaseRouting()
}
// Flush rule/chain/set operations from the buffer

View File

@@ -93,7 +93,7 @@ func newRouter(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) (*rou
rules: make(map[string]*nftables.Rule),
af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4),
wgIface: wgIface,
ipFwdState: ipfwdstate.NewIPForwardingState(),
ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()),
mtu: mtu,
}
@@ -1553,10 +1553,6 @@ func (r *router) refreshRulesMap() error {
}
func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
if err := r.ipFwdState.RequestForwarding(); err != nil {
return nil, err
}
ruleKey := rule.ID()
if _, exists := r.rules[ruleKey+dnatSuffix]; exists {
return rule, nil
@@ -1567,7 +1563,18 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
return nil, fmt.Errorf("convert protocol to number: %w", err)
}
// Request forwarding before queueing rules: addDnatRedirect/addDnatMasq
// buffer netlink messages on r.conn that the next caller's Flush would
// commit if we returned without flushing them ourselves.
v6 := r.af.tableFamily == nftables.TableFamilyIPv6
if err := r.ipFwdState.RequestForwarding(v6); err != nil {
return nil, fmt.Errorf("enable forwarding: %w", err)
}
if err := r.addDnatRedirect(rule, protoNum, ruleKey); err != nil {
if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil {
log.Warnf("rollback forwarding refcount: %v", rerr)
}
return nil, err
}
@@ -1579,6 +1586,11 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
// TODO: find chains with drop policies and add rules there
if err := r.conn.Flush(); err != nil {
if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil {
log.Warnf("rollback forwarding refcount: %v", rerr)
}
delete(r.rules, ruleKey+dnatSuffix)
delete(r.rules, ruleKey+snatSuffix)
return nil, fmt.Errorf("flush rules: %w", err)
}
@@ -1781,16 +1793,18 @@ func (r *router) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleKey
}
func (r *router) DeleteDNATRule(rule firewall.Rule) error {
if err := r.ipFwdState.ReleaseForwarding(); err != nil {
log.Errorf("%v", err)
}
ruleKey := rule.ID()
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
_, hadDNAT := r.rules[ruleKey+dnatSuffix]
_, hadSNAT := r.rules[ruleKey+snatSuffix]
if !hadDNAT && !hadSNAT {
return nil
}
var merr *multierror.Error
var needsFlush bool
@@ -1822,9 +1836,16 @@ func (r *router) DeleteDNATRule(rule firewall.Rule) error {
}
}
// Release the refcount only once the rules are gone from the kernel. On
// failure (including the refreshRulesMap error above) the rules and their
// map entries remain, keeping forwarding on until a retry removes them.
if merr == nil {
delete(r.rules, ruleKey+dnatSuffix)
delete(r.rules, ruleKey+snatSuffix)
if err := r.ipFwdState.ReleaseForwarding(r.af.tableFamily == nftables.TableFamilyIPv6); err != nil {
log.Errorf("%v", err)
}
}
return nberrors.FormatErrorOrNil(merr)

View File

@@ -844,6 +844,10 @@ func collectSysctls() string {
[]string{"net.ipv4.conf.all.src_valid_mark", "net.ipv4.conf.default.src_valid_mark"},
listInterfaceSysctls("ipv4", "src_valid_mark")...,
))
writeSysctlGroup(&builder, "accept_ra", append(
[]string{"net.ipv6.conf.all.accept_ra", "net.ipv6.conf.default.accept_ra"},
listInterfaceSysctls("ipv6", "accept_ra")...,
))
writeSysctlGroup(&builder, "conntrack", []string{
"net.netfilter.nf_conntrack_acct",
"net.netfilter.nf_conntrack_tcp_loose",

View File

@@ -2,54 +2,183 @@ package ipfwdstate
import (
"fmt"
"sync"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
)
// IPForwardingState is a struct that keeps track of the IP forwarding state.
// todo: read initial state of the IP forwarding from the system and reset the state based on it.
// todo: separate v4/v6 forwarding state, since the sysctls are independent
// (net.ipv4.ip_forward vs net.ipv6.conf.all.forwarding). Currently the nftables
// manager shares one instance between both routers, which works only because
// EnableIPForwarding enables both sysctls in a single call.
// IPForwardingState tracks v4 and v6 IP-forwarding sysctl enables with
// independent refcounts so a v4-only routing setup doesn't flip v6 sysctls.
type IPForwardingState struct {
enabledCounter int
mu sync.Mutex
v4Count int
v6Count int
// routingV4/routingV6 track whether the routing path currently holds a
// reference, so repeated EnableRouting calls (one per network-map update)
// hold at most one reference per family and an unpaired DisableRouting
// can't release references held by DNAT rules.
routingV4 bool
routingV6 bool
wgIfaceName string
v6Saved map[string]int
}
func NewIPForwardingState() *IPForwardingState {
return &IPForwardingState{}
// NewIPForwardingState returns a state tracker for the IP-forwarding sysctls.
// wgIfaceName is excluded from the per-interface accept_ra handling.
func NewIPForwardingState(wgIfaceName string) *IPForwardingState {
return &IPForwardingState{wgIfaceName: wgIfaceName}
}
func (f *IPForwardingState) RequestForwarding() error {
if f.enabledCounter != 0 {
f.enabledCounter++
// Counts returns the current v4 and v6 refcounts. Intended for diagnostics
// and tests.
func (f *IPForwardingState) Counts() (v4, v6 int) {
f.mu.Lock()
defer f.mu.Unlock()
return f.v4Count, f.v6Count
}
// RequestRouting takes the forwarding references for the routing path. It is
// idempotent: while routing already holds a reference, further calls don't
// increment the refcounts, and a v4-only request releases a previously held v6
// reference. A v6 sysctl failure is logged and not returned so it can't take
// down v4 routing (the sysctl may be unwritable, e.g. read-only /proc/sys or
// IPv6 disabled on the kernel command line); v6 is retried on the next call.
func (f *IPForwardingState) RequestRouting(v6 bool) error {
f.mu.Lock()
defer f.mu.Unlock()
if !f.routingV4 {
if err := f.requestV4(); err != nil {
return err
}
f.routingV4 = true
}
if !v6 {
if !f.routingV6 {
return nil
}
f.routingV6 = false
return f.releaseV6()
}
if f.routingV6 {
return nil
}
if err := systemops.EnableIPForwarding(); err != nil {
return fmt.Errorf("failed to enable IP forwarding with sysctl: %w", err)
if err := f.requestV6(); err != nil {
log.Warnf("enable IPv6 forwarding for routing: %v", err)
return nil
}
f.enabledCounter = 1
log.Info("IP forwarding enabled")
f.routingV6 = true
return nil
}
func (f *IPForwardingState) ReleaseForwarding() error {
if f.enabledCounter == 0 {
return nil
// ReleaseRouting releases the references RequestRouting holds. Calls without a
// held reference are no-ops.
func (f *IPForwardingState) ReleaseRouting() error {
f.mu.Lock()
defer f.mu.Unlock()
if f.routingV4 {
f.routingV4 = false
f.releaseV4()
}
if f.enabledCounter > 1 {
f.enabledCounter--
return nil
if f.routingV6 {
f.routingV6 = false
return f.releaseV6()
}
// if failed to disable IP forwarding we anyway decrement the counter
f.enabledCounter = 0
// todo call systemops.DisableIPForwarding()
return nil
}
// RequestForwarding enables the family's forwarding sysctl on first request.
func (f *IPForwardingState) RequestForwarding(v6 bool) error {
f.mu.Lock()
defer f.mu.Unlock()
if v6 {
return f.requestV6()
}
return f.requestV4()
}
// ReleaseForwarding decrements the family counter. The last v6 release restores
// what enable captured. v4 stays on: net.ipv4.ip_forward is co-owned by other
// tooling (docker, k8s, libvirt).
func (f *IPForwardingState) ReleaseForwarding(v6 bool) error {
f.mu.Lock()
defer f.mu.Unlock()
if v6 {
return f.releaseV6()
}
f.releaseV4()
return nil
}
func (f *IPForwardingState) requestV4() error {
if f.v4Count == 0 {
if err := systemops.EnableV4IPForwarding(); err != nil {
return fmt.Errorf("enable IPv4 forwarding: %w", err)
}
log.Info("IPv4 forwarding enabled")
}
f.v4Count++
return nil
}
func (f *IPForwardingState) releaseV4() {
if f.v4Count > 0 {
f.v4Count--
}
}
func (f *IPForwardingState) requestV6() error {
if f.v6Count == 0 {
saved, err := systemops.EnableV6IPForwarding(f.wgIfaceName)
if err != nil {
if rerr := systemops.DisableV6IPForwarding(saved); rerr != nil {
log.Warnf("rollback partial v6 sysctls: %v", rerr)
}
return fmt.Errorf("enable IPv6 forwarding: %w", err)
}
// A failed restore on a previous release keeps its saved values; those
// are the true originals, so keep them over what this enable captured.
if f.v6Saved == nil {
f.v6Saved = saved
} else {
for k, v := range saved {
if _, ok := f.v6Saved[k]; !ok {
f.v6Saved[k] = v
}
}
}
log.Info("IPv6 forwarding enabled")
}
f.v6Count++
return nil
}
func (f *IPForwardingState) releaseV6() error {
if f.v6Count == 0 {
return nil
}
f.v6Count--
if f.v6Count > 0 {
return nil
}
// Keep the saved values on failure so a later release or enable/release
// cycle can still restore them; re-restoring an already-restored key is a
// no-op since the sysctl already holds the desired value.
if err := systemops.DisableV6IPForwarding(f.v6Saved); err != nil {
return fmt.Errorf("disable IPv6 forwarding: %w", err)
}
f.v6Saved = nil
log.Info("IPv6 forwarding disabled")
return nil
}

View File

@@ -0,0 +1,39 @@
//go:build privileged
package ipfwdstate
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestRequestRoutingV6ToV4Transition verifies that a v4-only routing request
// releases a previously held routing-owned v6 reference without touching
// references held by DNAT rules.
func TestRequestRoutingV6ToV4Transition(t *testing.T) {
f := NewIPForwardingState("wt-fwd-test")
require.NoError(t, f.RequestRouting(true), "request routing with v6")
v4, v6 := f.Counts()
assert.Equal(t, 1, v4, "v4 reference held")
assert.Equal(t, 1, v6, "v6 reference held")
require.NoError(t, f.RequestRouting(false), "request routing v4-only")
v4, v6 = f.Counts()
assert.Equal(t, 1, v4, "v4 reference kept")
assert.Equal(t, 0, v6, "routing-owned v6 reference released")
// A DNAT-held reference survives a v4-only routing request.
require.NoError(t, f.RequestForwarding(true), "dnat v6 reference")
require.NoError(t, f.RequestRouting(false), "repeat v4-only request")
_, v6 = f.Counts()
assert.Equal(t, 1, v6, "dnat-held v6 reference survives")
require.NoError(t, f.ReleaseForwarding(true), "release dnat v6 reference")
require.NoError(t, f.ReleaseRouting(), "release routing")
v4, v6 = f.Counts()
assert.Equal(t, 0, v4, "all v4 references released")
assert.Equal(t, 0, v6, "all v6 references released")
}

View File

@@ -58,11 +58,7 @@ func Setup(wgIface iface) (map[string]int, error) {
continue
}
// Escape '%' and '.' so they survive the dot-to-slash conversion in Set()
safeName := strings.ReplaceAll(intf.Name, "%", percentEscape)
safeName = strings.ReplaceAll(safeName, ".", dotEscape)
i := fmt.Sprintf(rpFilterInterfacePath, safeName)
i := fmt.Sprintf(rpFilterInterfacePath, EscapeInterfaceName(intf.Name))
oldVal, err := Set(i, 2, true)
if err != nil {
result = multierror.Append(result, err)
@@ -74,6 +70,13 @@ func Setup(wgIface iface) (map[string]int, error) {
return keys, nberrors.FormatErrorOrNil(result)
}
// EscapeInterfaceName escapes '%' and '.' in an interface name (e.g. VLANs
// like eth0.100) so the name survives the dot-to-slash conversion in Set.
func EscapeInterfaceName(name string) string {
safe := strings.ReplaceAll(name, "%", percentEscape)
return strings.ReplaceAll(safe, ".", dotEscape)
}
// Set sets a sysctl configuration, if onlyIfOne is true it will only set the new value if it's set to 1
func Set(key string, desiredValue int, onlyIfOne bool) (int, error) {
path := strings.ReplaceAll(key, ".", "/")

View File

@@ -32,8 +32,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
return nil
}
func EnableIPForwarding() error {
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
func EnableV4IPForwarding() error {
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
return nil
}
func EnableV6IPForwarding(string) (map[string]int, error) {
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
return map[string]int{}, nil
}
func DisableV6IPForwarding(map[string]int) error {
return nil
}

View File

@@ -58,8 +58,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
return nil
}
func EnableIPForwarding() error {
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
func EnableV4IPForwarding() error {
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
return nil
}
func EnableV6IPForwarding(string) (map[string]int, error) {
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
return map[string]int{}, nil
}
func DisableV6IPForwarding(map[string]int) error {
return nil
}

View File

@@ -763,13 +763,10 @@ func flushRoutes(tableID, family int) error {
return nberrors.FormatErrorOrNil(result)
}
func EnableIPForwarding() error {
func EnableV4IPForwarding() error {
if _, err := sysctl.Set(ipv4ForwardingPath, 1, false); err != nil {
return err
}
if _, err := sysctl.Set(ipv6ForwardingPath, 1, false); err != nil {
log.Warnf("failed to enable IPv6 forwarding: %v", err)
}
return nil
}

View File

@@ -43,8 +43,17 @@ func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error
return r.genericRemoveVPNRoute(prefix, intf)
}
func EnableIPForwarding() error {
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
func EnableV4IPForwarding() error {
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
return nil
}
func EnableV6IPForwarding(string) (map[string]int, error) {
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
return map[string]int{}, nil
}
func DisableV6IPForwarding(map[string]int) error {
return nil
}

View File

@@ -0,0 +1,92 @@
//go:build !android
package systemops
import (
"fmt"
"net"
"os"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/internal/routemanager/sysctl"
)
const (
// 1 (default) accepts RAs only while forwarding is off; 2 keeps RA
// acceptance on regardless, so RA-installed host defaults survive our
// v6 forwarding flip.
acceptRAInterfacePath = "net.ipv6.conf.%s.accept_ra"
acceptRADefaultPath = "net.ipv6.conf.default.accept_ra"
acceptRAProcPathFormat = "/proc/sys/net/ipv6/conf/%s/accept_ra"
)
// EnableV6IPForwarding bumps accept_ra=2 on host v6 interfaces before flipping
// forwarding=1, so RA-installed host defaults survive. Returns the prior values
// of sysctls we actually changed; entries already at the target are omitted.
func EnableV6IPForwarding(wgIfaceName string) (map[string]int, error) {
saved := map[string]int{}
bumpAcceptRA(saved, wgIfaceName)
oldVal, err := sysctl.Set(ipv6ForwardingPath, 1, false)
if err != nil {
return saved, err
}
if oldVal != 1 {
saved[ipv6ForwardingPath] = oldVal
}
return saved, nil
}
// DisableV6IPForwarding restores what EnableV6IPForwarding captured.
func DisableV6IPForwarding(saved map[string]int) error {
var result *multierror.Error
for key, value := range saved {
if _, err := sysctl.Set(key, value, false); err != nil {
result = multierror.Append(result, fmt.Errorf("restore %s: %w", key, err))
}
}
return nberrors.FormatErrorOrNil(result)
}
func bumpAcceptRA(saved map[string]int, wgIfaceName string) {
// Also bump conf.default so interfaces created while forwarding is on
// (hotplug, new Wi-Fi/dock) inherit accept_ra=2 and keep accepting RAs.
bumpAcceptRAKey(saved, acceptRADefaultPath)
interfaces, err := net.Interfaces()
if err != nil {
log.Warnf("list interfaces for accept_ra: %v", err)
return
}
for _, intf := range interfaces {
if intf.Name == "lo" || intf.Name == wgIfaceName {
continue
}
bumpAcceptRAForInterface(saved, intf.Name)
}
}
func bumpAcceptRAForInterface(saved map[string]int, name string) {
// Build procfs path from name, not the dotted key: VLAN names like eth0.100.
if _, err := os.Stat(fmt.Sprintf(acceptRAProcPathFormat, name)); err != nil {
return
}
bumpAcceptRAKey(saved, fmt.Sprintf(acceptRAInterfacePath, sysctl.EscapeInterfaceName(name)))
}
func bumpAcceptRAKey(saved map[string]int, key string) {
// onlyIfOne=true: leave admin overrides (0, 2) alone.
oldVal, err := sysctl.Set(key, 2, true)
if err != nil {
log.Warnf("bump %s: %v", key, err)
return
}
// With onlyIfOne, a write only happened when the old value was 1; values
// left untouched (0, 2) must not be recorded for restore.
if oldVal == 1 {
saved[key] = oldVal
}
}