sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-10-02 03:29:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

[client] Unify peer and route ACL filtering with multi-source rules (#6322)

Browse Source 
* Unify peer and route ACL filtering with multi-source peer rules

* Remove partial userspace firewall mode and open foreign chains via a table-less allower

* Snapshot iptables rule maps before persisting state

* Scope userspace firewall wildcard source rules per address family

* Install nftables peer filter and mangle rules in a single transaction

* Share the iptables jump rule spec between install and cleanup

* Fix legacy ACL source wildcard and keep rollback tracking on delete failure

* Fix CI: recognize multi-value port set lookups in tests and correct PeerIP lint suppression

* Fall back to per-prefix filter rules when ipset is unavailable

* Annotate legacy PeerIP usages in ACL tests and fix import formatting

* Keep firewall rule bookkeeping in step with the kernel on replace and teardown

* Release the routing reference when the route manager shuts down

* Keep set references and rule tracking consistent when a routing rule fails
This commit is contained in:
Viktor Liu
2026-09-03 10:02:50 +02:00
committed by GitHub
parent 6aaeed744e
commit 778b3b3264
81 changed files with 9046 additions and 6686 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/firewall/nftables/acl_linux.go
-713
View File
@@ -1,713 +0,0 @@
package nftables
import (
"bytes"
"fmt"
"net"
"slices"
"strconv"
"strings"
"time"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
nbnet "github.com/netbirdio/netbird/client/net"
)
const (
// rules chains contains the effective ACL rules
chainNameInputRules = "netbird-acl-input-rules"
// filter chains contains the rules that jump to the rules chains
chainNameInputFilter = "netbird-acl-input-filter"
chainNameForwardFilter = "netbird-acl-forward-filter"
chainNameManglePrerouting = "netbird-mangle-prerouting"
chainNameManglePostrouting = "netbird-mangle-postrouting"
)
const flushError = "flush: %w"
type AclManager struct {
rConn *nftables.Conn
sConn *nftables.Conn
wgIface iFaceMapper
routingFwChainName string
af addrFamily
workTable *nftables.Table
chainInputRules *nftables.Chain
chainPrerouting *nftables.Chain
ipsetStore *ipsetStore
rules map[string]*Rule
}
func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) {
// sConn is used for creating sets and adding/removing elements from them
// it's differ then rConn (which does create new conn for each flush operation)
// and is permanent. Using same connection for both type of operations
// overloads netlink with high amount of rules ( > 10000)
sConn, err := nftables.New(nftables.AsLasting())
if err != nil {
return nil, fmt.Errorf("create nf conn: %w", err)
}
return &AclManager{
rConn: &nftables.Conn{},
sConn: sConn,
wgIface: wgIface,
workTable: table,
routingFwChainName: routingFwChainName,
af: familyForAddr(table.Family == nftables.TableFamilyIPv4),
ipsetStore: newIpsetStore(),
rules: make(map[string]*Rule),
}, nil
}
func (m *AclManager) init(workTable *nftables.Table) error {
m.workTable = workTable
return m.createDefaultChains()
}
// AddPeerFiltering rule to the firewall
//
// If comment argument is empty firewall manager should set
// rule ID as comment for the rule
func (m *AclManager) AddPeerFiltering(
id []byte,
ip net.IP,
proto firewall.Protocol,
sPort *firewall.Port,
dPort *firewall.Port,
action firewall.Action,
ipsetName string,
) ([]firewall.Rule, error) {
var ipset *nftables.Set
if ipsetName != "" {
var err error
ipset, err = m.addIpToSet(ipsetName, ip)
if err != nil {
return nil, err
}
}
newRules := make([]firewall.Rule, 0, 2)
ioRule, err := m.addIOFiltering(ip, proto, sPort, dPort, action, ipset)
if err != nil {
return nil, err
}
newRules = append(newRules, ioRule)
return newRules, nil
}
// DeletePeerRule from the firewall by rule definition
func (m *AclManager) DeletePeerRule(rule firewall.Rule) error {
r, ok := rule.(*Rule)
if !ok {
return fmt.Errorf("invalid rule type")
}
if r.nftSet == nil {
if err := m.rConn.DelRule(r.nftRule); err != nil {
log.Errorf("failed to delete rule: %v", err)
}
if r.mangleRule != nil {
if err := m.rConn.DelRule(r.mangleRule); err != nil {
log.Errorf("failed to delete mangle rule: %v", err)
}
}
delete(m.rules, r.ID())
return m.rConn.Flush()
}
ips, ok := m.ipsetStore.ips(r.nftSet.Name)
if !ok {
if err := m.rConn.DelRule(r.nftRule); err != nil {
log.Errorf("failed to delete rule: %v", err)
}
if r.mangleRule != nil {
if err := m.rConn.DelRule(r.mangleRule); err != nil {
log.Errorf("failed to delete mangle rule: %v", err)
}
}
delete(m.rules, r.ID())
return m.rConn.Flush()
}
if _, ok := ips[r.ip.String()]; ok {
err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: ipToBytes(r.ip, m.af)}})
if err != nil {
log.Errorf("delete elements for set %q: %v", r.nftSet.Name, err)
}
if err := m.sConn.Flush(); err != nil {
log.Debugf("flush error of set delete element, %s", r.nftSet.Name)
return err
}
m.ipsetStore.DeleteIpFromSet(r.nftSet.Name, r.ip)
}
// if after delete, set still contains other IPs,
// no need to delete firewall rule and we should exit here
if len(ips) > 0 {
return nil
}
if err := m.rConn.DelRule(r.nftRule); err != nil {
log.Errorf("failed to delete rule: %v", err)
}
if r.mangleRule != nil {
if err := m.rConn.DelRule(r.mangleRule); err != nil {
log.Errorf("failed to delete mangle rule: %v", err)
}
}
if err := m.rConn.Flush(); err != nil {
return err
}
delete(m.rules, r.ID())
m.ipsetStore.DeleteReferenceFromIpSet(r.nftSet.Name)
if m.ipsetStore.HasReferenceToSet(r.nftSet.Name) {
return nil
}
// we delete last IP from the set, that means we need to delete
// set itself and associated firewall rule too
m.rConn.FlushSet(r.nftSet)
m.rConn.DelSet(r.nftSet)
m.ipsetStore.deleteIpset(r.nftSet.Name)
return nil
}
// createDefaultAllowRules creates default allow rules for the input and output chains
func (m *AclManager) createDefaultAllowRules() error {
expIn := []expr.Any{
&expr.Verdict{
Kind: expr.VerdictAccept,
},
}
_ = m.rConn.InsertRule(&nftables.Rule{
Table: m.workTable,
Chain: m.chainInputRules,
Position: 0,
Exprs: expIn,
})
if err := m.rConn.Flush(); err != nil {
return fmt.Errorf(flushError, err)
}
return nil
}
// Flush rule/chain/set operations from the buffer
//
// Method also get all rules after flush and refreshes handle values in the rulesets
func (m *AclManager) Flush() error {
if err := m.flushWithBackoff(); err != nil {
return err
}
if err := m.refreshRuleHandles(m.chainInputRules, false); err != nil {
log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err)
}
if err := m.refreshRuleHandles(m.chainPrerouting, true); err != nil {
log.Errorf("failed to refresh rule handles prerouting chain: %v", err)
}
return nil
}
func (m *AclManager) addIOFiltering(
ip net.IP,
proto firewall.Protocol,
sPort *firewall.Port,
dPort *firewall.Port,
action firewall.Action,
ipset *nftables.Set,
) (*Rule, error) {
ruleId := generatePeerRuleId(ip, proto, sPort, dPort, action, ipset)
if r, ok := m.rules[ruleId]; ok {
return &Rule{
nftRule: r.nftRule,
mangleRule: r.mangleRule,
nftSet: r.nftSet,
ruleID: r.ruleID,
ip: ip,
}, nil
}
var expressions []expr.Any
if proto != firewall.ProtocolALL {
expressions = append(expressions, &expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: m.af.protoOffset,
Len: uint32(1),
})
protoData, err := m.af.protoNum(proto)
if err != nil {
return nil, fmt.Errorf("convert protocol to number: %v", err)
}
expressions = append(expressions, &expr.Cmp{
Register: 1,
Op: expr.CmpOpEq,
Data: []byte{protoData},
})
}
rawIP := ipToBytes(ip, m.af)
// check if rawIP contains zeroed IPv4 0.0.0.0 value
// in that case not add IP match expression into the rule definition
if slices.ContainsFunc(rawIP, func(v byte) bool { return v != 0 }) {
expressions = append(expressions,
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: m.af.srcAddrOffset,
Len: m.af.addrLen,
},
)
// add individual IP for match if no ipset defined
if ipset == nil {
expressions = append(expressions,
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: rawIP,
},
)
} else {
expressions = append(expressions,
&expr.Lookup{
SourceRegister: 1,
SetName: ipset.Name,
SetID: ipset.ID,
},
)
}
}
expressions = append(expressions, applyPort(sPort, true)...)
expressions = append(expressions, applyPort(dPort, false)...)
mainExpressions := slices.Clone(expressions)
switch action {
case firewall.ActionAccept:
mainExpressions = append(mainExpressions, &expr.Verdict{Kind: expr.VerdictAccept})
case firewall.ActionDrop:
mainExpressions = append(mainExpressions, &expr.Verdict{Kind: expr.VerdictDrop})
}
userData := []byte(ruleId)
chain := m.chainInputRules
rule := &nftables.Rule{
Table: m.workTable,
Chain: chain,
Exprs: mainExpressions,
UserData: userData,
}
// Insert DROP rules at the beginning, append ACCEPT rules at the end
var nftRule *nftables.Rule
if action == firewall.ActionDrop {
nftRule = m.rConn.InsertRule(rule)
} else {
nftRule = m.rConn.AddRule(rule)
}
if err := m.rConn.Flush(); err != nil {
return nil, fmt.Errorf("flush input rule %s: %v", ruleId, err)
}
ruleStruct := &Rule{
nftRule: nftRule,
// best effort mangle rule
mangleRule: m.createPreroutingRule(expressions, userData),
nftSet: ipset,
ruleID: ruleId,
ip: ip,
}
m.rules[ruleId] = ruleStruct
if ipset != nil {
m.ipsetStore.AddReferenceToIpset(ipset.Name)
}
return ruleStruct, nil
}
func (m *AclManager) createPreroutingRule(expressions []expr.Any, userData []byte) *nftables.Rule {
if m.chainPrerouting == nil {
log.Warn("prerouting chain is not created")
return nil
}
preroutingExprs := slices.Clone(expressions)
// interface
preroutingExprs = append([]expr.Any{
&expr.Meta{
Key: expr.MetaKeyIIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(m.wgIface.Name()),
},
}, preroutingExprs...)
// local destination and mark
preroutingExprs = append(preroutingExprs,
&expr.Fib{
Register: 1,
ResultADDRTYPE: true,
FlagDADDR: true,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL),
},
&expr.Immediate{
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
},
&expr.Meta{
Key: expr.MetaKeyMARK,
Register: 1,
SourceRegister: true,
},
)
nfRule := m.rConn.AddRule(&nftables.Rule{
Table: m.workTable,
Chain: m.chainPrerouting,
Exprs: preroutingExprs,
UserData: userData,
})
if err := m.rConn.Flush(); err != nil {
log.Errorf("failed to flush mangle rule %s: %v", string(userData), err)
return nil
}
return nfRule
}
func (m *AclManager) createDefaultChains() (err error) {
// chainNameInputRules
chain := m.createChain(chainNameInputRules)
err = m.rConn.Flush()
if err != nil {
log.Debugf("failed to create chain (%s): %s", chain.Name, err)
return fmt.Errorf(flushError, err)
}
m.chainInputRules = chain
// netbird-acl-input-filter
// type filter hook input priority filter; policy accept;
chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput)
m.addJumpRule(chain, m.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules
m.addDropExpressions(chain, expr.MetaKeyIIFNAME)
err = m.rConn.Flush()
if err != nil {
log.Debugf("failed to create chain (%s): %s", chain.Name, err)
return err
}
// netbird-acl-forward-filter
chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd
m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME)
err = m.rConn.Flush()
if err != nil {
log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err)
return fmt.Errorf(flushError, err)
}
if err := m.allowRedirectedTraffic(chainFwFilter); err != nil {
log.Errorf("failed to allow redirected traffic: %s", err)
}
return nil
}
// Makes redirected traffic originally destined for the host itself (now subject to the forward filter)
// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the
// netbird peer IP.
func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error {
// Chain is created by route manager
// TODO: move creation to a common place
m.chainPrerouting = &nftables.Chain{
Name: chainNameManglePrerouting,
Table: m.workTable,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityMangle,
}
m.addFwmarkToForward(chainFwFilter)
if err := m.rConn.Flush(); err != nil {
return fmt.Errorf(flushError, err)
}
return nil
}
func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) {
m.rConn.InsertRule(&nftables.Rule{
Table: m.workTable,
Chain: chainFwFilter,
Exprs: []expr.Any{
&expr.Meta{
Key: expr.MetaKeyMARK,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
})
}
func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) {
expressions := []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(m.wgIface.Name()),
},
&expr.Verdict{
Kind: expr.VerdictJump,
Chain: m.routingFwChainName,
},
}
_ = m.rConn.AddRule(&nftables.Rule{
Table: m.workTable,
Chain: chainFwFilter,
Exprs: expressions,
})
}
func (m *AclManager) createChain(name string) *nftables.Chain {
chain := &nftables.Chain{
Name: name,
Table: m.workTable,
}
chain = m.rConn.AddChain(chain)
insertReturnTrafficRule(m.rConn, m.workTable, chain)
return chain
}
func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain {
polAccept := nftables.ChainPolicyAccept
chain := &nftables.Chain{
Name: name,
Table: m.workTable,
Hooknum: hookNum,
Priority: nftables.ChainPriorityFilter,
Type: nftables.ChainTypeFilter,
Policy: &polAccept,
}
return m.rConn.AddChain(chain)
}
func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any {
expressions := []expr.Any{
&expr.Meta{Key: ifaceKey, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(m.wgIface.Name()),
},
&expr.Verdict{Kind: expr.VerdictDrop},
}
_ = m.rConn.AddRule(&nftables.Rule{
Table: m.workTable,
Chain: chain,
Exprs: expressions,
})
return nil
}
func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) {
expressions := []expr.Any{
&expr.Meta{Key: ifaceKey, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(m.wgIface.Name()),
},
&expr.Verdict{
Kind: expr.VerdictJump,
Chain: to,
},
}
_ = m.rConn.AddRule(&nftables.Rule{
Table: chain.Table,
Chain: chain,
Exprs: expressions,
})
}
func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, error) {
ipset, err := m.rConn.GetSetByName(m.workTable, ipsetName)
rawIP := ipToBytes(ip, m.af)
if err != nil {
if ipset, err = m.createSet(m.workTable, ipsetName); err != nil {
return nil, fmt.Errorf("get set name: %v", err)
}
m.ipsetStore.newIpset(ipset.Name)
}
if m.ipsetStore.IsIpInSet(ipset.Name, ip) {
return ipset, nil
}
if err := m.sConn.SetAddElements(ipset, []nftables.SetElement{{Key: rawIP}}); err != nil {
return nil, fmt.Errorf("add set element for the first time: %v", err)
}
m.ipsetStore.AddIpToSet(ipset.Name, ip)
if err := m.sConn.Flush(); err != nil {
return nil, fmt.Errorf("flush add elements: %v", err)
}
return ipset, nil
}
// createSet in given table by name
func (m *AclManager) createSet(table *nftables.Table, name string) (*nftables.Set, error) {
ipset := &nftables.Set{
Name: name,
Table: table,
Dynamic: true,
KeyType: m.af.setKeyType,
}
if err := m.rConn.AddSet(ipset, nil); err != nil {
return nil, fmt.Errorf("create set: %v", err)
}
if err := m.rConn.Flush(); err != nil {
return nil, fmt.Errorf("flush created set: %v", err)
}
return ipset, nil
}
func (m *AclManager) flushWithBackoff() (err error) {
backoff := 4
backoffTime := 1000 * time.Millisecond
for i := 0; ; i++ {
err = m.rConn.Flush()
if err != nil {
log.Debugf("failed to flush nftables: %v", err)
if !strings.Contains(err.Error(), "busy") {
return
}
log.Error("failed to flush nftables, retrying...")
if i == backoff-1 {
return err
}
time.Sleep(backoffTime)
backoffTime *= 2
continue
}
break
}
return
}
func (m *AclManager) refreshRuleHandles(chain *nftables.Chain, mangle bool) error {
if m.workTable == nil || chain == nil {
return nil
}
list, err := m.rConn.GetRules(m.workTable, chain)
if err != nil {
return err
}
for _, rule := range list {
if len(rule.UserData) == 0 {
continue
}
split := bytes.Split(rule.UserData, []byte(" "))
r, ok := m.rules[string(split[0])]
if ok {
if mangle {
*r.mangleRule = *rule
} else {
*r.nftRule = *rule
}
}
}
return nil
}
func generatePeerRuleId(ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action, ipset *nftables.Set) string {
rulesetID := ":" + string(proto) + ":"
if sPort != nil {
rulesetID += sPort.String()
}
rulesetID += ":"
if dPort != nil {
rulesetID += dPort.String()
}
rulesetID += ":"
rulesetID += strconv.Itoa(int(action))
if ipset == nil {
return "ip:" + ip.String() + rulesetID
}
return "set:" + ipset.Name + rulesetID
}
func ifname(n string) []byte {
b := make([]byte, 16)
copy(b, n+"\x00")
return b
}
// ipToBytes converts net.IP to the correct byte length for the address family.
func ipToBytes(ip net.IP, af addrFamily) []byte {
if af.addrLen == 4 {
return ip.To4()
}
return ip.To16()
}
client/firewall/nftables/chains_linux.go
+880
View File
@@ -0,0 +1,880 @@
//go:build !android
package nftables
import (
"bytes"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/coreos/go-iptables/iptables"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/firewall/firewalld"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
nbnet "github.com/netbirdio/netbird/client/net"
)
func (r *family) createContainers() error {
r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{
Name: chainNameRoutingFw,
Table: r.workTable,
})
prio := *nftables.ChainPriorityNATSource - 1
r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
Name: chainNameRoutingNat,
Table: r.workTable,
Hooknum: nftables.ChainHookPostrouting,
Priority: &prio,
Type: nftables.ChainTypeNAT,
})
r.chains[chainNameRoutingRdr] = r.conn.AddChain(&nftables.Chain{
Name: chainNameRoutingRdr,
Table: r.workTable,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityNATDest,
Type: nftables.ChainTypeNAT,
})
r.chains[chainNameManglePostrouting] = r.conn.AddChain(&nftables.Chain{
Name: chainNameManglePostrouting,
Table: r.workTable,
Hooknum: nftables.ChainHookPostrouting,
Priority: nftables.ChainPriorityMangle,
Type: nftables.ChainTypeFilter,
})
r.chains[chainNameManglePrerouting] = r.conn.AddChain(&nftables.Chain{
Name: chainNameManglePrerouting,
Table: r.workTable,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityMangle,
Type: nftables.ChainTypeFilter,
})
r.chains[chainNameMangleForward] = r.conn.AddChain(&nftables.Chain{
Name: chainNameMangleForward,
Table: r.workTable,
Hooknum: nftables.ChainHookForward,
Priority: nftables.ChainPriorityMangle,
Type: nftables.ChainTypeFilter,
})
insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw])
r.addPostroutingRules()
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("initialize tables: %v", err)
}
if err := r.addMSSClampingRules(); err != nil {
log.Errorf("failed to add MSS clamping rules: %s", err)
}
// Kernel routing opens both INPUT and FORWARD.
if err := r.openInterface(true); err != nil {
log.Errorf("failed to open interface in foreign chains: %s", err)
}
if err := firewalld.TrustInterface(r.wgIface.Name()); err != nil {
log.Warnf("failed to trust interface in firewalld: %v", err)
}
if err := r.refreshRulesMap(); err != nil {
log.Errorf("failed to refresh rules: %s", err)
}
return nil
}
// setupDataPlaneMark configures the fwmark for the data plane
func (r *family) setupDataPlaneMark() error {
if r.chains[chainNameManglePrerouting] == nil || r.chains[chainNameManglePostrouting] == nil {
return errors.New("no mangle chains found")
}
ctNew := getCtNewExprs()
preExprs := []expr.Any{
&expr.Meta{
Key: expr.MetaKeyIIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
}
preExprs = append(preExprs, ctNew...)
preExprs = append(preExprs,
&expr.Immediate{
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkIn),
},
&expr.Ct{
Key: expr.CtKeyMARK,
Register: 1,
SourceRegister: true,
},
)
preNftRule := &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameManglePrerouting],
Exprs: preExprs,
}
r.conn.AddRule(preNftRule)
postExprs := []expr.Any{
&expr.Meta{
Key: expr.MetaKeyOIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
}
postExprs = append(postExprs, ctNew...)
postExprs = append(postExprs,
&expr.Immediate{
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkOut),
},
&expr.Ct{
Key: expr.CtKeyMARK,
Register: 1,
SourceRegister: true,
},
)
postNftRule := &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameManglePostrouting],
Exprs: postExprs,
}
r.conn.AddRule(postNftRule)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush: %w", err)
}
return nil
}
// openInterface adds passthrough accept rules for the NetBird interface to the
// kernel's filter table and external chains so they don't drop our traffic.
// includeForward also opens the FORWARD chains (kernel routing); when false only
// INPUT is opened, which is all the userspace router needs since it never
// forwards in the kernel.
func (r *family) openInterface(includeForward bool) error {
var merr *multierror.Error
if err := r.acceptFilterTableRules(includeForward); err != nil {
merr = multierror.Append(merr, err)
}
if err := r.acceptExternalChainsRules(includeForward); err != nil {
merr = multierror.Append(merr, fmt.Errorf("add accept rules to external chains: %w", err))
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) acceptFilterTableRules(includeForward bool) error {
if r.filterTable == nil {
return nil
}
fw := "iptables"
defer func() {
log.Debugf("Used %s to add accept input/forward rules", fw)
}()
// Try iptables first and fallback to nftables if iptables is not available.
// Use the correct protocol (iptables vs ip6tables) for the address family.
ipt, err := iptables.NewWithProtocol(r.iptablesProto())
if err != nil {
log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
fw = "nftables"
return r.acceptFilterRulesNftables(r.filterTable, includeForward)
}
if err := r.acceptFilterRulesIptables(ipt, includeForward); err != nil {
log.Warnf("iptables failed (table may be incompatible), falling back to nftables: %v", err)
fw = "nftables"
return r.acceptFilterRulesNftables(r.filterTable, includeForward)
}
return nil
}
func (r *family) acceptFilterRulesIptables(ipt *iptables.IPTables, includeForward bool) error {
var merr *multierror.Error
if includeForward {
for _, rule := range r.getAcceptForwardRules() {
if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("add iptables forward rule: %v", err))
} else {
log.Debugf("added iptables forward rule: %v", rule)
}
}
}
inputRule := r.getAcceptInputRule()
if err := ipt.Insert("filter", chainNameInput, 1, inputRule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("add iptables input rule: %v", err))
} else {
log.Debugf("added iptables input rule: %v", inputRule)
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) getAcceptForwardRules() [][]string {
intf := r.wgIface.Name()
return [][]string{
{"-i", intf, "-j", "ACCEPT"},
{"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
}
}
func (r *family) getAcceptInputRule() []string {
return []string{"-i", r.wgIface.Name(), "-j", "ACCEPT"}
}
// acceptFilterRulesNftables adds accept rules to the ip filter table using nftables.
// This is used when iptables is not available.
func (r *family) acceptFilterRulesNftables(table *nftables.Table, includeForward bool) error {
intf := ifname(r.wgIface.Name())
if includeForward {
forwardChain := &nftables.Chain{
Name: chainNameForward,
Table: table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookForward,
Priority: nftables.ChainPriorityFilter,
}
r.insertForwardAcceptRules(forwardChain, intf)
}
inputChain := &nftables.Chain{
Name: chainNameInput,
Table: table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookInput,
Priority: nftables.ChainPriorityFilter,
}
r.insertInputAcceptRule(inputChain, intf)
return r.conn.Flush()
}
// acceptExternalChainsRules adds accept rules to external chains (non-netbird, non-iptables tables).
// It dynamically finds chains at call time to handle chains that may have been created after startup.
func (r *family) acceptExternalChainsRules(includeForward bool) error {
chains := r.findExternalChains()
if len(chains) == 0 {
return nil
}
intf := ifname(r.wgIface.Name())
for _, chain := range chains {
r.applyExternalChainAccept(chain, intf, includeForward)
}
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush external chain rules: %w", err)
}
return nil
}
func (r *family) applyExternalChainAccept(chain *nftables.Chain, intf []byte, includeForward bool) {
if chain.Hooknum == nil {
log.Debugf("skipping external chain %s/%s: hooknum is nil", chain.Table.Name, chain.Name)
return
}
log.Debugf("adding accept rules to external %s chain: %s %s/%s",
hookName(chain.Hooknum), familyName(chain.Table.Family), chain.Table.Name, chain.Name)
switch *chain.Hooknum {
case *nftables.ChainHookForward:
if includeForward {
r.insertForwardAcceptRules(chain, intf)
}
case *nftables.ChainHookInput:
r.insertInputAcceptRule(chain, intf)
}
}
func (r *family) insertForwardAcceptRules(chain *nftables.Chain, intf []byte) {
existing, err := r.existingNetbirdRulesInChain(chain)
if err != nil {
log.Warnf("skip forward accept rules in %s/%s: %v", chain.Table.Name, chain.Name, err)
return
}
r.insertForwardIifRule(chain, intf, existing)
r.insertForwardOifEstablishedRule(chain, intf, existing)
}
func (r *family) insertForwardIifRule(chain *nftables.Chain, intf []byte, existing map[string]bool) {
if existing[userDataAcceptForwardRuleIif] {
return
}
r.conn.InsertRule(&nftables.Rule{
Table: chain.Table,
Chain: chain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf},
&expr.Counter{},
&expr.Verdict{Kind: expr.VerdictAccept},
},
UserData: []byte(userDataAcceptForwardRuleIif),
})
}
func (r *family) insertForwardOifEstablishedRule(chain *nftables.Chain, intf []byte, existing map[string]bool) {
if existing[userDataAcceptForwardRuleOif] {
return
}
exprs := []expr.Any{
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf},
}
r.conn.InsertRule(&nftables.Rule{
Table: chain.Table,
Chain: chain,
Exprs: append(exprs, getEstablishedExprs(2)...),
UserData: []byte(userDataAcceptForwardRuleOif),
})
}
func (r *family) insertInputAcceptRule(chain *nftables.Chain, intf []byte) {
existing, err := r.existingNetbirdRulesInChain(chain)
if err != nil {
log.Warnf("skip input accept rule in %s/%s: %v", chain.Table.Name, chain.Name, err)
return
}
if existing[userDataAcceptInputRule] {
return
}
r.conn.InsertRule(&nftables.Rule{
Table: chain.Table,
Chain: chain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf},
&expr.Counter{},
&expr.Verdict{Kind: expr.VerdictAccept},
},
UserData: []byte(userDataAcceptInputRule),
})
}
// existingNetbirdRulesInChain returns the set of netbird-owned UserData tags present in a chain; callers must bail on error since InsertRule is additive.
func (r *family) existingNetbirdRulesInChain(chain *nftables.Chain) (map[string]bool, error) {
rules, err := r.conn.GetRules(chain.Table, chain)
if err != nil {
return nil, fmt.Errorf("list rules: %w", err)
}
present := map[string]bool{}
for _, rule := range rules {
if !isNetbirdAcceptRuleTag(rule.UserData) {
continue
}
present[string(rule.UserData)] = true
}
return present, nil
}
func isNetbirdAcceptRuleTag(userData []byte) bool {
switch string(userData) {
case userDataAcceptForwardRuleIif,
userDataAcceptForwardRuleOif,
userDataAcceptInputRule:
return true
}
return false
}
func (r *family) removeAcceptFilterRules() error {
var merr *multierror.Error
if err := r.removeFilterTableRules(); err != nil {
merr = multierror.Append(merr, err)
}
if err := r.removeExternalChainsRules(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove external chain rules: %w", err))
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) removeFilterTableRules() error {
if r.filterTable == nil {
return nil
}
ipt, err := iptables.NewWithProtocol(r.iptablesProto())
if err != nil {
log.Debugf("iptables not available, using nftables to remove filter rules: %v", err)
return r.removeAcceptRulesFromTable(r.filterTable)
}
if err := r.removeAcceptFilterRulesIptables(ipt); err != nil {
log.Debugf("iptables removal failed (table may be incompatible), falling back to nftables: %v", err)
return r.removeAcceptRulesFromTable(r.filterTable)
}
return nil
}
func (r *family) removeAcceptRulesFromTable(table *nftables.Table) error {
chains, err := r.conn.ListChainsOfTableFamily(table.Family)
if err != nil {
return fmt.Errorf("list chains: %v", err)
}
for _, chain := range chains {
if chain.Table.Name != table.Name {
continue
}
if chain.Name != chainNameForward && chain.Name != chainNameInput {
continue
}
if err := r.removeAcceptRulesFromChain(table, chain); err != nil {
return err
}
}
return r.conn.Flush()
}
func (r *family) removeAcceptRulesFromChain(table *nftables.Table, chain *nftables.Chain) error {
rules, err := r.conn.GetRules(table, chain)
if err != nil {
return fmt.Errorf("get rules from %s/%s: %v", table.Name, chain.Name, err)
}
for _, rule := range rules {
if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) ||
bytes.Equal(rule.UserData, []byte(userDataAcceptInputRule)) {
if err := r.conn.DelRule(rule); err != nil {
return fmt.Errorf("delete rule from %s/%s: %v", table.Name, chain.Name, err)
}
}
}
return nil
}
// removeExternalChainsRules removes our accept rules from all external chains.
// This is deterministic - it scans for chains at removal time rather than relying on saved state,
// ensuring cleanup works even after a crash or if chains changed.
func (r *family) removeExternalChainsRules() error {
chains := r.findExternalChains()
if len(chains) == 0 {
return nil
}
var merr *multierror.Error
for _, chain := range chains {
if err := r.removeAcceptRulesFromChain(chain.Table, chain); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove rules from external chain %s/%s: %w", chain.Table.Name, chain.Name, err))
continue
}
if err := r.conn.Flush(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("flush external chain %s/%s: %w", chain.Table.Name, chain.Name, err))
}
}
return nberrors.FormatErrorOrNil(merr)
}
// findExternalChains scans for chains from non-netbird tables that have FORWARD or INPUT hooks.
// This is used both at startup (to know where to add rules) and at cleanup (to ensure deterministic removal).
func (r *family) findExternalChains() []*nftables.Chain {
var chains []*nftables.Chain
families := []nftables.TableFamily{r.af.tableFamily, nftables.TableFamilyINet}
for _, family := range families {
allChains, err := r.conn.ListChainsOfTableFamily(family)
if err != nil {
log.Debugf("list chains for family %d: %v", family, err)
continue
}
for _, chain := range allChains {
if r.isExternalChain(chain) {
chains = append(chains, chain)
}
}
}
return chains
}
func (r *family) isExternalChain(chain *nftables.Chain) bool {
if r.workTable != nil && chain.Table.Name == r.workTable.Name {
return false
}
// Skip firewalld-owned chains. Firewalld creates its chains with the
// NFT_CHAIN_OWNER flag, so inserting rules into them returns EPERM.
// We delegate acceptance to firewalld by trusting the interface instead.
if chain.Table.Name == firewalldTableName {
return false
}
// Skip iptables/ip6tables-managed tables (adding nft-native rules breaks iptables-save compat)
if (chain.Table.Family == nftables.TableFamilyIPv4 || chain.Table.Family == nftables.TableFamilyIPv6) && isIptablesTable(chain.Table.Name) {
return false
}
if chain.Type != nftables.ChainTypeFilter {
return false
}
if chain.Hooknum == nil {
return false
}
return *chain.Hooknum == *nftables.ChainHookForward || *chain.Hooknum == *nftables.ChainHookInput
}
func isIptablesTable(name string) bool {
switch name {
case tableNameFilter, tableNat, tableMangle, tableRaw, tableSecurity:
return true
}
return false
}
func (r *family) removeAcceptFilterRulesIptables(ipt *iptables.IPTables) error {
var merr *multierror.Error
for _, rule := range r.getAcceptForwardRules() {
if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove iptables forward rule: %v", err))
}
}
inputRule := r.getAcceptInputRule()
if err := ipt.DeleteIfExists("filter", chainNameInput, inputRule...); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove iptables input rule: %v", err))
}
return nberrors.FormatErrorOrNil(merr)
}
// Flush rule/chain/set operations from the buffer
//
// Method also get all rules after flush and refreshes handle values in the rulesets
func (r *family) Flush() error {
if err := r.flushWithBackoff(); err != nil {
return err
}
if err := r.refreshRuleHandles(r.chainInputRules, false); err != nil {
log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err)
}
if err := r.refreshRuleHandles(r.chainPrerouting, true); err != nil {
log.Errorf("failed to refresh rule handles prerouting chain: %v", err)
}
return nil
}
// queuePreroutingRule builds the prerouting mangle rule that marks
// redirected traffic and queues it on the connection without flushing,
// so the caller can commit it in the same transaction as the rule it
// pairs with. Returns nil when the prerouting chain is absent, in which
// case nothing is queued.
func (r *family) queuePreroutingRule(expressions []expr.Any, userData []byte) *nftables.Rule {
if r.chainPrerouting == nil {
log.Warn("prerouting chain is not created")
return nil
}
preroutingExprs := slices.Clone(expressions)
// interface
preroutingExprs = append([]expr.Any{
&expr.Meta{
Key: expr.MetaKeyIIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
}, preroutingExprs...)
// local destination and mark
preroutingExprs = append(preroutingExprs,
&expr.Fib{
Register: 1,
ResultADDRTYPE: true,
FlagDADDR: true,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL),
},
&expr.Immediate{
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
},
&expr.Meta{
Key: expr.MetaKeyMARK,
Register: 1,
SourceRegister: true,
},
)
return r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: r.chainPrerouting,
Exprs: preroutingExprs,
UserData: userData,
})
}
func (r *family) createDefaultChains() (err error) {
// chainNameInputRules
chain := r.createChain(chainNameInputRules)
err = r.conn.Flush()
if err != nil {
log.Debugf("failed to create chain (%s): %s", chain.Name, err)
return fmt.Errorf(flushError, err)
}
r.chainInputRules = chain
// netbird-acl-input-filter
// type filter hook input priority filter; policy accept;
chain = r.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput)
r.addJumpRule(chain, r.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules
r.addDropExpressions(chain, expr.MetaKeyIIFNAME)
err = r.conn.Flush()
if err != nil {
log.Debugf("failed to create chain (%s): %s", chain.Name, err)
return err
}
// netbird-acl-forward-filter
chainFwFilter := r.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
r.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd
r.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME)
err = r.conn.Flush()
if err != nil {
log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err)
return fmt.Errorf(flushError, err)
}
if err := r.allowRedirectedTraffic(chainFwFilter); err != nil {
log.Errorf("failed to allow redirected traffic: %s", err)
}
return nil
}
// Makes redirected traffic originally destined for the host itself (now subject to the forward filter)
// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the
// netbird peer IP.
func (r *family) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error {
r.chainPrerouting = r.chains[chainNameManglePrerouting]
r.addFwmarkToForward(chainFwFilter)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf(flushError, err)
}
return nil
}
func (r *family) addFwmarkToForward(chainFwFilter *nftables.Chain) {
r.conn.InsertRule(&nftables.Rule{
Table: r.workTable,
Chain: chainFwFilter,
Exprs: []expr.Any{
&expr.Meta{
Key: expr.MetaKeyMARK,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
})
}
func (r *family) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) {
expressions := []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Verdict{
Kind: expr.VerdictJump,
Chain: r.routingFwChainName,
},
}
_ = r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: chainFwFilter,
Exprs: expressions,
})
}
func (r *family) createChain(name string) *nftables.Chain {
chain := &nftables.Chain{
Name: name,
Table: r.workTable,
}
chain = r.conn.AddChain(chain)
insertReturnTrafficRule(r.conn, r.workTable, chain)
return chain
}
func (r *family) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain {
polAccept := nftables.ChainPolicyAccept
chain := &nftables.Chain{
Name: name,
Table: r.workTable,
Hooknum: hookNum,
Priority: nftables.ChainPriorityFilter,
Type: nftables.ChainTypeFilter,
Policy: &polAccept,
}
return r.conn.AddChain(chain)
}
func (r *family) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any {
expressions := []expr.Any{
&expr.Meta{Key: ifaceKey, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Verdict{Kind: expr.VerdictDrop},
}
_ = r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: chain,
Exprs: expressions,
})
return nil
}
func (r *family) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) {
expressions := []expr.Any{
&expr.Meta{Key: ifaceKey, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Verdict{
Kind: expr.VerdictJump,
Chain: to,
},
}
_ = r.conn.AddRule(&nftables.Rule{
Table: chain.Table,
Chain: chain,
Exprs: expressions,
})
}
func (r *family) flushWithBackoff() (err error) {
backoff := 4
backoffTime := 1000 * time.Millisecond
for i := 0; ; i++ {
err = r.conn.Flush()
if err != nil {
log.Debugf("failed to flush nftables: %v", err)
if !strings.Contains(err.Error(), "busy") {
return
}
log.Error("failed to flush nftables, retrying...")
if i == backoff-1 {
return err
}
time.Sleep(backoffTime)
backoffTime *= 2
continue
}
break
}
return
}
func (r *family) refreshRuleHandles(chain *nftables.Chain, mangle bool) error {
if r.workTable == nil || chain == nil {
return nil
}
list, err := r.conn.GetRules(r.workTable, chain)
if err != nil {
return err
}
for _, rule := range list {
if len(rule.UserData) == 0 {
continue
}
pr, ok := r.filters[firewall.RuleID(rule.UserData)]
if !ok {
continue
}
if mangle {
if pr.mangleRule != nil {
*pr.mangleRule = *rule
}
} else {
*pr.nftRule = *rule
}
}
return nil
}
client/firewall/nftables/dnat_linux.go
+573
View File
@@ -0,0 +1,573 @@
//go:build !android
package nftables
import (
"fmt"
"net/netip"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"github.com/google/nftables/xt"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
nberrors "github.com/netbirdio/netbird/client/errors"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
)
func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) {
ruleID := rule.ID()
if _, exists := r.rules[ruleID+dnatSuffix]; exists {
return rule, nil
}
protoNum, err := r.af.protoNum(rule.Protocol)
if err != nil {
return nil, fmt.Errorf("convert protocol to number: %w", err)
}
// Request forwarding before queueing rules: addDnatRedirect/addDnatMasq
// buffer netlink messages on r.conn that the next caller's Flush would
// commit if we returned without flushing them ourselves.
if err := r.ipFwdState.RequestForwarding(r.isV6()); err != nil {
return nil, fmt.Errorf("enable forwarding: %w", err)
}
if err := r.addDnatRedirect(rule, protoNum, ruleID); err != nil {
r.releaseForwarding()
return nil, err
}
if err := r.addDnatMasq(rule, protoNum, ruleID); err != nil {
r.releaseForwarding()
delete(r.rules, ruleID+dnatSuffix)
return nil, err
}
// Unlike iptables, there's no point in adding "out" rules in the forward chain here as our policy is ACCEPT.
// To overcome DROP policies in other chains, we'd have to add rules to the chains there.
// We also cannot just add "oif <iface> accept" there and filter in our own table as we don't know what is supposed to be allowed.
// TODO: find chains with drop policies and add rules there
if err := r.conn.Flush(); err != nil {
r.releaseForwarding()
delete(r.rules, ruleID+dnatSuffix)
delete(r.rules, ruleID+snatSuffix)
return nil, fmt.Errorf("flush rules: %w", err)
}
return &rule, nil
}
func (r *family) addDnatRedirect(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error {
dnatExprs := []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{protoNum},
},
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
}
portExprs, err := r.applyPort(&rule.DestinationPort, false)
if err != nil {
return fmt.Errorf("apply destination port: %w", err)
}
dnatExprs = append(dnatExprs, portExprs...)
// shifted translated port is not supported in nftables, so we hand this over to xtables
if rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2 {
if rule.TranslatedPort.Values[0] != rule.DestinationPort.Values[0] ||
rule.TranslatedPort.Values[1] != rule.DestinationPort.Values[1] {
return r.addXTablesRedirect(dnatExprs, ruleID, rule)
}
}
additionalExprs, regProtoMin, regProtoMax, err := r.handleTranslatedPort(rule)
if err != nil {
return err
}
dnatExprs = append(dnatExprs, additionalExprs...)
dnatExprs = append(dnatExprs,
&expr.NAT{
Type: expr.NATTypeDestNAT,
Family: uint32(r.af.tableFamily),
RegAddrMin: 1,
RegProtoMin: regProtoMin,
RegProtoMax: regProtoMax,
},
)
dnatRule := &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingRdr],
Exprs: dnatExprs,
UserData: []byte(ruleID + dnatSuffix),
}
r.conn.AddRule(dnatRule)
r.rules[ruleID+dnatSuffix] = dnatRule
return nil
}
func (r *family) handleTranslatedPort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) {
switch {
case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2:
return r.handlePortRange(rule)
case len(rule.TranslatedPort.Values) == 0:
return r.handleAddressOnly(rule)
case len(rule.TranslatedPort.Values) == 1:
return r.handleSinglePort(rule)
default:
return nil, 0, 0, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort)
}
}
func (r *family) handlePortRange(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) {
exprs := []expr.Any{
&expr.Immediate{
Register: 1,
Data: rule.TranslatedAddress.AsSlice(),
},
&expr.Immediate{
Register: 2,
Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]),
},
&expr.Immediate{
Register: 3,
Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[1]),
},
}
return exprs, 2, 3, nil
}
func (r *family) handleAddressOnly(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) {
exprs := []expr.Any{
&expr.Immediate{
Register: 1,
Data: rule.TranslatedAddress.AsSlice(),
},
}
return exprs, 0, 0, nil
}
func (r *family) handleSinglePort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) {
exprs := []expr.Any{
&expr.Immediate{
Register: 1,
Data: rule.TranslatedAddress.AsSlice(),
},
&expr.Immediate{
Register: 2,
Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]),
},
}
return exprs, 2, 0, nil
}
func (r *family) addXTablesRedirect(dnatExprs []expr.Any, ruleID firewall.RuleID, rule firewall.ForwardRule) error {
dnatExprs = append(dnatExprs,
&expr.Counter{},
&expr.Target{
Name: "DNAT",
Rev: 2,
Info: &xt.NatRange2{
NatRange: xt.NatRange{
Flags: uint(xt.NatRangeMapIPs | xt.NatRangeProtoSpecified | xt.NatRangeProtoOffset),
MinIP: rule.TranslatedAddress.AsSlice(),
MaxIP: rule.TranslatedAddress.AsSlice(),
MinPort: rule.TranslatedPort.Values[0],
MaxPort: rule.TranslatedPort.Values[1],
},
BasePort: rule.DestinationPort.Values[0],
},
},
)
natTable := &nftables.Table{
Name: tableNat,
Family: r.af.tableFamily,
}
dnatRule := &nftables.Rule{
Table: natTable,
Chain: &nftables.Chain{
Name: chainNameNatPrerouting,
Table: natTable,
Type: nftables.ChainTypeNAT,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityNATDest,
},
Exprs: dnatExprs,
UserData: []byte(ruleID + dnatSuffix),
}
r.conn.AddRule(dnatRule)
r.rules[ruleID+dnatSuffix] = dnatRule
return nil
}
func (r *family) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error {
portExprs, err := r.applyPort(&rule.TranslatedPort, false)
if err != nil {
return fmt.Errorf("apply translated port: %w", err)
}
masqExprs := []expr.Any{
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{protoNum},
},
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: r.af.dstAddrOffset,
Len: r.af.addrLen,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: rule.TranslatedAddress.AsSlice(),
},
}
masqExprs = append(masqExprs, portExprs...)
masqExprs = append(masqExprs, &expr.Masq{})
masqRule := &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingNat],
Exprs: masqExprs,
UserData: []byte(ruleID + snatSuffix),
}
r.conn.AddRule(masqRule)
r.rules[ruleID+snatSuffix] = masqRule
return nil
}
func (r *family) DeleteDNATRule(rule firewall.Rule) error {
ruleID := rule.ID()
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
var merr *multierror.Error
var needsFlush bool
var found bool
if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists {
found = true
if dnatRule.Handle == 0 {
log.Warnf("dnat rule %s has no handle, removing stale entry", ruleID+dnatSuffix)
delete(r.rules, ruleID+dnatSuffix)
} else if err := r.conn.DelRule(dnatRule); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete dnat rule: %w", err))
} else {
needsFlush = true
}
}
if masqRule, exists := r.rules[ruleID+snatSuffix]; exists {
found = true
if masqRule.Handle == 0 {
log.Warnf("snat rule %s has no handle, removing stale entry", ruleID+snatSuffix)
delete(r.rules, ruleID+snatSuffix)
} else if err := r.conn.DelRule(masqRule); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete snat rule: %w", err))
} else {
needsFlush = true
}
}
if needsFlush {
if err := r.conn.Flush(); err != nil {
merr = multierror.Append(merr, fmt.Errorf(flushError, err))
}
}
if merr != nil {
return nberrors.FormatErrorOrNil(merr)
}
delete(r.rules, ruleID+dnatSuffix)
delete(r.rules, ruleID+snatSuffix)
// Release once, only if the rule was present and removed.
if found {
r.releaseForwarding()
}
return nil
}
// releaseForwarding drops one IP forwarding reference, logging any error.
func (r *family) releaseForwarding() {
if err := r.ipFwdState.ReleaseForwarding(r.isV6()); err != nil {
log.Errorf("release IP forwarding: %v", err)
}
}
// isV6 reports whether this family handles the IPv6 table.
func (r *family) isV6() bool {
return r.af.tableFamily == nftables.TableFamilyIPv6
}
func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error {
ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort))
if _, exists := r.rules[ruleID]; exists {
return nil
}
protoNum, err := r.af.protoNum(protocol)
if err != nil {
return fmt.Errorf("convert protocol to number: %w", err)
}
exprs := []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 2},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 2,
Data: []byte{protoNum},
},
&expr.Payload{
DestRegister: 3,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 3,
Data: binaryutil.BigEndian.PutUint16(originalPort),
},
}
bits := 32
if localAddr.Is6() {
bits = 128
}
exprs = append(exprs, prefixMatchExprs(r.af, netip.PrefixFrom(localAddr, bits), false)...)
exprs = append(exprs,
&expr.Immediate{
Register: 1,
Data: localAddr.AsSlice(),
},
&expr.Immediate{
Register: 2,
Data: binaryutil.BigEndian.PutUint16(translatedPort),
},
&expr.NAT{
Type: expr.NATTypeDestNAT,
Family: uint32(r.af.tableFamily),
RegAddrMin: 1,
RegProtoMin: 2,
RegProtoMax: 0,
},
)
dnatRule := &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingRdr],
Exprs: exprs,
UserData: []byte(ruleID),
}
r.conn.AddRule(dnatRule)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("add inbound DNAT rule: %w", err)
}
r.rules[ruleID] = dnatRule
return nil
}
// RemoveInboundDNAT removes an inbound DNAT rule.
func (r *family) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error {
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort))
rule, exists := r.rules[ruleID]
if !exists {
return nil
}
if rule.Handle == 0 {
log.Warnf("inbound DNAT rule %s has no handle, removing stale entry", ruleID)
delete(r.rules, ruleID)
return nil
}
if err := r.conn.DelRule(rule); err != nil {
return fmt.Errorf("delete inbound DNAT rule %s: %w", ruleID, err)
}
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush delete inbound DNAT rule: %w", err)
}
delete(r.rules, ruleID)
return nil
}
// ensureNATOutputChain lazily creates the OUTPUT NAT chain on first use.
func (r *family) ensureNATOutputChain() error {
if _, exists := r.chains[chainNameNATOutput]; exists {
return nil
}
r.chains[chainNameNATOutput] = r.conn.AddChain(&nftables.Chain{
Name: chainNameNATOutput,
Table: r.workTable,
Hooknum: nftables.ChainHookOutput,
Priority: nftables.ChainPriorityNATDest,
Type: nftables.ChainTypeNAT,
})
if err := r.conn.Flush(); err != nil {
delete(r.chains, chainNameNATOutput)
return fmt.Errorf("create NAT output chain: %w", err)
}
return nil
}
// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic.
func (r *family) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error {
ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort))
if _, exists := r.rules[ruleID]; exists {
return nil
}
if err := r.ensureNATOutputChain(); err != nil {
return err
}
protoNum, err := r.af.protoNum(protocol)
if err != nil {
return fmt.Errorf("convert protocol to number: %w", err)
}
exprs := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{protoNum},
},
&expr.Payload{
DestRegister: 2,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 2,
Data: binaryutil.BigEndian.PutUint16(originalPort),
},
}
bits := 32
if localAddr.Is6() {
bits = 128
}
exprs = append(exprs, prefixMatchExprs(r.af, netip.PrefixFrom(localAddr, bits), false)...)
exprs = append(exprs,
&expr.Immediate{
Register: 1,
Data: localAddr.AsSlice(),
},
&expr.Immediate{
Register: 2,
Data: binaryutil.BigEndian.PutUint16(translatedPort),
},
&expr.NAT{
Type: expr.NATTypeDestNAT,
Family: uint32(r.af.tableFamily),
RegAddrMin: 1,
RegProtoMin: 2,
},
)
dnatRule := &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameNATOutput],
Exprs: exprs,
UserData: []byte(ruleID),
}
r.conn.AddRule(dnatRule)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("add output DNAT rule: %w", err)
}
r.rules[ruleID] = dnatRule
return nil
}
// RemoveOutputDNAT removes an OUTPUT chain DNAT rule.
func (r *family) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error {
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort))
rule, exists := r.rules[ruleID]
if !exists {
return nil
}
if rule.Handle == 0 {
log.Warnf("output DNAT rule %s has no handle, removing stale entry", ruleID)
delete(r.rules, ruleID)
return nil
}
if err := r.conn.DelRule(rule); err != nil {
return fmt.Errorf("delete output DNAT rule %s: %w", ruleID, err)
}
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush delete output DNAT rule: %w", err)
}
delete(r.rules, ruleID)
return nil
}
client/firewall/nftables/dnat_refcount_linux_test.go
+9 -9
View File
@@ -82,7 +82,7 @@ func dnatV6(port uint16) fw.ForwardRule {
// v4 refcount at zero.
func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) {
m := newNftRefcountManager(t, false)
state := m.router.ipFwdState
state := m.family4.ipFwdState
r1, err := m.AddDNATRule(dnatV4(8081))
require.NoError(t, err, "add v4 dnat 1")
@@ -111,9 +111,9 @@ func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) {
// and decrements back to zero on Delete.
func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) {
m := newNftRefcountManager(t, true)
require.NotNil(t, m.router6, "v6 router")
require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state")
state := m.router.ipFwdState
require.NotNil(t, m.family6, "v6 family")
require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state")
state := m.family4.ipFwdState
r1, err := m.AddDNATRule(dnatV6(9091))
require.NoError(t, err, "add v6 dnat 1")
@@ -142,7 +142,7 @@ func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) {
// ForwardRule) does not double-increment the refcount.
func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
state := m.family4.ipFwdState
rule := dnatV4(8083)
r1, err := m.AddDNATRule(rule)
@@ -165,7 +165,7 @@ func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) {
// never added does not underflow the refcount.
func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
state := m.family4.ipFwdState
// Construct a Rule reference for something never added. The router stores
// rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing
@@ -195,7 +195,7 @@ func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) {
// and a single DisableRouting drops both back to zero.
func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
state := m.family4.ipFwdState
require.NoError(t, m.EnableRouting(), "first enable")
require.NoError(t, m.EnableRouting(), "second enable")
@@ -214,7 +214,7 @@ func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) {
// DisableRouting does not release references held by active DNAT rules.
func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
state := m.family4.ipFwdState
r1, err := m.AddDNATRule(dnatV6(9095))
require.NoError(t, err, "add v6 dnat")
@@ -232,7 +232,7 @@ func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) {
// twice does not underflow the refcount (the second delete is a no-op).
func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) {
m := newNftRefcountManager(t, true)
state := m.router.ipFwdState
state := m.family4.ipFwdState
r1, err := m.AddDNATRule(dnatV6(9093))
require.NoError(t, err)
client/firewall/nftables/family_linux.go
+249
View File
@@ -0,0 +1,249 @@
//go:build !android
package nftables
import (
"fmt"
"net/netip"
"github.com/coreos/go-iptables/iptables"
"github.com/google/nftables"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/firewall/firewalld"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate"
"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
)
const (
tableNat = "nat"
tableMangle = "mangle"
tableRaw = "raw"
tableSecurity = "security"
chainNameNatPrerouting = "PREROUTING"
chainNameRoutingFw = "netbird-rt-fwd"
chainNameRoutingNat = "netbird-rt-postrouting"
chainNameRoutingRdr = "netbird-rt-redirect"
chainNameNATOutput = "netbird-nat-output"
chainNameForward = "FORWARD"
chainNameMangleForward = "netbird-mangle-forward"
// Peer ACL chain names.
chainNameInputRules = "netbird-acl-input-rules"
chainNameInputFilter = "netbird-acl-input-filter"
chainNameForwardFilter = "netbird-acl-forward-filter"
chainNameManglePrerouting = "netbird-mangle-prerouting"
chainNameManglePostrouting = "netbird-mangle-postrouting"
flushError = "flush: %w"
firewalldTableName = "firewalld"
userDataAcceptForwardRuleIif = "frwacceptiif"
userDataAcceptForwardRuleOif = "frwacceptoif"
userDataAcceptInputRule = "inputaccept"
dnatSuffix firewall.RuleID = "_dnat"
snatSuffix firewall.RuleID = "_snat"
// ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation.
ipv4TCPHeaderSize = 40
// ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation.
ipv6TCPHeaderSize = 60
// maxPrefixesSet 1638 prefixes start to fail, taking some margin
maxPrefixesSet = 1500
refreshRulesMapError = "refresh rules map: %w"
)
var (
errFilterTableNotFound = fmt.Errorf("'filter' table not found")
)
type setInput struct {
set firewall.Set
prefixes []netip.Prefix
}
// family holds the per-address-family nftables state. One instance
// handles route ACLs, peer ACLs, NAT, DNAT, and MSS clamping for a
// single family; the top-level Manager owns one for v4 and another
// for v6. The name predates the peer-ACL absorption; it's effectively
// the per-family backend now.
type family struct {
conn *nftables.Conn
workTable *nftables.Table
filterTable *nftables.Table
chains map[string]*nftables.Chain
// filters holds peer + route filter rules keyed by content hash.
// AddFilterRule writes here; DeleteFilterRule looks up by id.
filters map[firewall.RuleID]*Rule
// rules holds NAT, DNAT, and external accept rules (auxiliary
// plumbing that isn't a filter rule).
rules map[firewall.RuleID]*nftables.Rule
// Peer ACL chain handles.
chainInputRules *nftables.Chain
chainPrerouting *nftables.Chain
routingFwChainName string
ipsetCounter *refcounter.Counter[string, setInput, *nftables.Set]
af addrFamily
wgIface iFaceMapper
ipFwdState *ipfwdstate.IPForwardingState
legacyManagement bool
mtu uint16
}
func newFamily(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) *family {
r := &family{
conn: &nftables.Conn{},
workTable: workTable,
chains: make(map[string]*nftables.Chain),
filters: make(map[firewall.RuleID]*Rule),
rules: make(map[firewall.RuleID]*nftables.Rule),
routingFwChainName: chainNameRoutingFw,
af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4),
wgIface: wgIface,
ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()),
mtu: mtu,
}
r.ipsetCounter = refcounter.New(
r.createIpSet,
r.deleteIpSet,
)
var err error
r.filterTable, err = r.loadFilterTable()
if err != nil {
log.Debugf("ip filter table not found: %v", err)
}
return r
}
func (r *family) init(workTable *nftables.Table) error {
r.workTable = workTable
if err := r.removeAcceptFilterRules(); err != nil {
log.Errorf("failed to clean up rules from filter table: %s", err)
}
if err := r.createContainers(); err != nil {
return fmt.Errorf("create containers: %w", err)
}
if err := r.setupDataPlaneMark(); err != nil {
log.Errorf("failed to set up data plane mark: %v", err)
}
if err := r.createDefaultChains(); err != nil {
return fmt.Errorf("create default acl chains: %w", err)
}
return nil
}
// Reset cleans existing nftables filter table rules from the system
func (r *family) Reset() error {
// clear without deleting the ipsets, the nf table will be deleted by the caller
r.ipsetCounter.Clear()
var merr *multierror.Error
if err := r.removeAcceptFilterRules(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove accept filter rules: %w", err))
}
if err := firewalld.UntrustInterface(r.wgIface.Name()); err != nil {
merr = multierror.Append(merr, err)
}
if err := r.removeNatPreroutingRules(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove filter prerouting rules: %w", err))
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) loadFilterTable() (*nftables.Table, error) {
tables, err := r.conn.ListTablesOfFamily(r.af.tableFamily)
if err != nil {
return nil, fmt.Errorf("list tables: %w", err)
}
for _, table := range tables {
if table.Name == "filter" {
return table, nil
}
}
return nil, errFilterTableNotFound
}
func hookName(hook *nftables.ChainHook) string {
if hook == nil {
return "unknown"
}
switch *hook {
case *nftables.ChainHookForward:
return chainNameForward
case *nftables.ChainHookInput:
return chainNameInput
default:
return fmt.Sprintf("hook(%d)", *hook)
}
}
func familyName(family nftables.TableFamily) string {
switch family {
case nftables.TableFamilyIPv4:
return "ip"
case nftables.TableFamilyIPv6:
return "ip6"
case nftables.TableFamilyINet:
return "inet"
default:
return fmt.Sprintf("family(%d)", family)
}
}
func (r *family) iptablesProto() iptables.Protocol {
if r.af.tableFamily == nftables.TableFamilyIPv6 {
return iptables.ProtocolIPv6
}
return iptables.ProtocolIPv4
}
func (r *family) refreshRulesMap() error {
var merr *multierror.Error
newRules := make(map[firewall.RuleID]*nftables.Rule)
for _, chain := range r.chains {
rules, err := r.conn.GetRules(chain.Table, chain)
if err != nil {
merr = multierror.Append(merr, fmt.Errorf("list rules for chain %s: %w", chain.Name, err))
// preserve existing entries for this chain since we can't verify their state
for k, v := range r.rules {
if v.Chain != nil && v.Chain.Name == chain.Name {
newRules[k] = v
}
}
continue
}
for _, rule := range rules {
if len(rule.UserData) > 0 {
newRules[firewall.RuleID(rule.UserData)] = rule
}
}
}
r.rules = newRules
return nberrors.FormatErrorOrNil(merr)
}
client/firewall/nftables/filter_linux.go
+540
View File
@@ -0,0 +1,540 @@
//go:build !android
package nftables
import (
"fmt"
"net"
"net/netip"
"slices"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
nberrors "github.com/netbirdio/netbird/client/errors"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
nbid "github.com/netbirdio/netbird/client/internal/acl/id"
)
// AddFilterRule installs one nftables packet-filter rule. With
// destination empty the rule goes to the peer ACL input chain plus a
// paired prerouting mangle rule for the redirect mark. With
// destination set (prefix or named set) it goes to the route ACL
// forward chain. Multi-source rules collapse to one nftables rule
// backed by the shared refcounted hash:net set.
func (r *family) AddFilterRule(
id []byte,
sources []netip.Prefix,
destination firewall.Network,
proto firewall.Protocol,
sPort *firewall.Port,
dPort *firewall.Port,
action firewall.Action,
) (firewall.Rule, error) {
isRoute := !destination.IsZero()
ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action)
if existing, ok := r.filters[ruleID]; ok {
return existing, nil
}
srcExprs, err := r.applyNetwork(sourceNetwork(sources), sources, true)
if err != nil {
return nil, fmt.Errorf("apply source: %w", err)
}
var exprs []expr.Any
if isRoute {
exprs, err = r.buildRouteFilterExprs(srcExprs, destination, proto, sPort, dPort)
} else {
exprs, err = r.buildPeerFilterExprs(srcExprs, proto, sPort, dPort)
}
if err != nil {
r.dropNetworkMatch(srcExprs)
return nil, err
}
mainExprs := slices.Clone(exprs)
verdict := expr.VerdictAccept
if action == firewall.ActionDrop {
verdict = expr.VerdictDrop
}
mainExprs = append(mainExprs, &expr.Verdict{Kind: verdict})
chain := r.chainInputRules
if isRoute {
chain = r.chains[chainNameRoutingFw]
}
userData := []byte(ruleID)
// Build the paired prerouting mangle rule before flushing so both
// rules commit in one transaction. An anonymous port set binds to
// exactly one rule, so the mangle rule needs its own expression list
// with fresh sets, not a clone of the main rule's. Guard on the
// prerouting chain first: building the expressions queues the port
// set, so skipping the build when there is no chain to bind it to
// keeps an unbound set out of the connection batch.
var mangleRule *nftables.Rule
if !isRoute && r.chainPrerouting != nil {
mangleExprs, err := r.buildPeerFilterExprs(srcExprs, proto, sPort, dPort)
if err != nil {
r.dropNetworkMatch(exprs)
return nil, fmt.Errorf("build mangle rule: %w", err)
}
mangleRule = r.queuePreroutingRule(mangleExprs, userData)
}
nftRule := &nftables.Rule{
Table: r.workTable,
Chain: chain,
Exprs: mainExprs,
UserData: userData,
}
if action == firewall.ActionDrop {
nftRule = r.conn.InsertRule(nftRule)
} else {
nftRule = r.conn.AddRule(nftRule)
}
if err := r.conn.Flush(); err != nil {
r.dropNetworkMatch(exprs)
return nil, fmt.Errorf(flushError, err)
}
rule := &Rule{
nftRule: nftRule,
mangleRule: mangleRule,
sources: sources,
id: ruleID,
}
r.filters[ruleID] = rule
log.Debugf("added filter rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v",
sources, destination, proto, sPort, dPort, action)
return rule, nil
}
// buildPeerFilterExprs assembles the input-chain (peer ACL) match: the
// IP-header protocol byte read via Payload, then source, then ports
// (no counter), matching the historical peer shape so per-rule kernel
// state is identical to pre-unification.
func (r *family) buildPeerFilterExprs(
srcExprs []expr.Any,
proto firewall.Protocol,
sPort, dPort *firewall.Port,
) ([]expr.Any, error) {
var exprs []expr.Any
if proto != firewall.ProtocolALL {
protoNum, err := r.af.protoNum(proto)
if err != nil {
return nil, fmt.Errorf("convert protocol to number: %w", err)
}
exprs = append(exprs,
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: r.af.protoOffset,
Len: 1,
},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoNum}},
)
}
exprs = append(exprs, srcExprs...)
portExprs, err := r.applyPorts(sPort, dPort)
if err != nil {
return nil, err
}
exprs = append(exprs, portExprs...)
return exprs, nil
}
// buildRouteFilterExprs assembles the forward-chain (route ACL) match:
// source, then destination, then optional proto/ports, then a counter.
func (r *family) buildRouteFilterExprs(
srcExprs []expr.Any,
destination firewall.Network,
proto firewall.Protocol,
sPort, dPort *firewall.Port,
) ([]expr.Any, error) {
exprs := append([]expr.Any{}, srcExprs...)
destExprs, err := r.applyNetwork(destination, nil, false)
if err != nil {
return nil, fmt.Errorf("apply destination: %w", err)
}
exprs = append(exprs, destExprs...)
if proto != firewall.ProtocolALL {
protoNum, err := r.af.protoNum(proto)
if err != nil {
r.dropNetworkMatch(destExprs)
return nil, fmt.Errorf("convert protocol to number: %w", err)
}
exprs = append(exprs,
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoNum}},
)
portExprs, err := r.applyPorts(sPort, dPort)
if err != nil {
r.dropNetworkMatch(destExprs)
return nil, err
}
exprs = append(exprs, portExprs...)
}
exprs = append(exprs, &expr.Counter{})
return exprs, nil
}
func (r *family) hasRule(id firewall.RuleID) bool {
_, ok := r.filters[id]
return ok
}
func (r *family) hasDNATRule(id firewall.RuleID) bool {
_, ok := r.rules[id+dnatSuffix]
return ok
}
// DeleteFilterRule removes a previously installed filter rule. Source
// set references are recovered from the stored rule's expressions via
// findSets and dropped from the shared refcounter.
func (r *family) DeleteFilterRule(rule firewall.Rule) error {
ruleID := rule.ID()
pr, ok := r.filters[ruleID]
if !ok {
log.Debugf("filter rule %s not found", ruleID)
return nil
}
// A freshly added rule carries no handle until it is read back from
// the kernel, and Flush only refreshes the peer chains. Pull live
// handles for this rule's chain before deciding it is stale so route
// rules (which Flush never refreshes) can actually be deleted. A
// refresh failure aborts the delete without touching tracking state,
// so the caller can retry while the rule may still exist in the kernel.
if pr.nftRule.Handle == 0 {
if err := r.refreshRuleHandles(pr.nftRule.Chain, false); err != nil {
return fmt.Errorf("refresh handles for chain %s: %w", pr.nftRule.Chain.Name, err)
}
}
// Refresh the mangle handle independently: the main rule's handle can
// be populated while the prerouting refresh during Flush failed, and
// gating the mangle refresh on the main handle would leak the mangle
// rule on delete.
if pr.mangleRule != nil && pr.mangleRule.Handle == 0 {
if err := r.refreshRuleHandles(r.chainPrerouting, true); err != nil {
return fmt.Errorf("refresh mangle handles: %w", err)
}
}
if pr.nftRule.Handle == 0 {
log.Warnf("filter rule %s has no handle, removing stale entry", ruleID)
// The paired mangle rule can still be in the kernel with a live
// handle. Dropping the tracking entry without removing it would
// leave a prerouting rule that nothing can find again.
if err := r.deleteMangleRule(pr, ruleID); err != nil {
return err
}
r.dropNetworkMatch(pr.nftRule.Exprs)
delete(r.filters, ruleID)
return nil
}
if err := r.conn.DelRule(pr.nftRule); err != nil {
log.Errorf("queue rule delete: %v", err)
}
r.queueMangleDelete(pr)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush delete %s: %w", ruleID, err)
}
r.dropNetworkMatch(pr.nftRule.Exprs)
delete(r.filters, ruleID)
return nil
}
// deleteMangleRule removes the prerouting rule paired with a filter rule on
// its own, for the paths that drop the filter rule's tracking without queueing
// a delete for it.
func (r *family) deleteMangleRule(pr *Rule, ruleID firewall.RuleID) error {
if pr.mangleRule == nil || pr.mangleRule.Handle == 0 {
return nil
}
r.queueMangleDelete(pr)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush mangle delete %s: %w", ruleID, err)
}
return nil
}
// queueMangleDelete queues the delete of the rule's prerouting counterpart, if
// it has one. The caller commits it.
func (r *family) queueMangleDelete(pr *Rule) {
if pr.mangleRule == nil {
return
}
if err := r.conn.DelRule(pr.mangleRule); err != nil {
log.Errorf("queue mangle rule delete: %v", err)
}
}
func (r *family) decrementSetCounter(rule *nftables.Rule) error {
if r.ipsetCounter == nil {
return nil
}
sets := findSets(rule)
var merr *multierror.Error
for _, setName := range sets {
if _, err := r.ipsetCounter.Decrement(setName); err != nil {
merr = multierror.Append(merr, fmt.Errorf("decrement set counter: %w", err))
}
}
return nberrors.FormatErrorOrNil(merr)
}
// dropNetworkMatch undoes whatever the source/destination match
// reserved. Safe to call when the spec is empty or holds only inline
// matchers.
func (r *family) dropNetworkMatch(exprs []expr.Any) {
if r.ipsetCounter == nil {
return
}
for _, e := range exprs {
lookup, ok := e.(*expr.Lookup)
if !ok {
continue
}
if _, err := r.ipsetCounter.Decrement(lookup.SetName); err != nil {
log.Errorf("rollback ipset decrement %s: %v", lookup.SetName, err)
}
}
}
func (r *family) applyNetwork(
network firewall.Network,
setPrefixes []netip.Prefix,
isSource bool,
) ([]expr.Any, error) {
if network.IsSet() {
exprs, err := r.getIpSet(network.Set, setPrefixes, isSource)
if err != nil {
side := "destination"
if isSource {
side = "source"
}
return nil, fmt.Errorf("%s set: %w", side, err)
}
return exprs, nil
}
if network.IsPrefix() {
return prefixMatchExprs(r.af, network.Prefix, isSource), nil
}
return nil, nil
}
// applyPort builds the transport-header port match. A single value
// compares directly, a range uses a range expression, and multiple
// values go through an anonymous constant set: consecutive cmp
// expressions AND together, so chained equality comparisons could
// never match more than one port. The set is queued on the
// connection and committed by the caller's flush together with the
// rule that binds it.
func (r *family) applyPort(port *firewall.Port, isSource bool) ([]expr.Any, error) {
if port == nil || len(port.Values) == 0 {
return nil, nil
}
// dst port
offset := uint32(2)
if isSource {
// src port
offset = 0
}
exprs := []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: offset,
Len: 2,
},
}
switch {
case port.IsRange && len(port.Values) == 2:
exprs = append(exprs, &expr.Range{
Op: expr.CmpOpEq,
Register: 1,
FromData: binaryutil.BigEndian.PutUint16(port.Values[0]),
ToData: binaryutil.BigEndian.PutUint16(port.Values[1]),
})
case len(port.Values) == 1:
exprs = append(exprs, &expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.BigEndian.PutUint16(port.Values[0]),
})
default:
lookup, err := r.anonymousPortSet(port.Values)
if err != nil {
return nil, err
}
exprs = append(exprs, lookup)
}
return exprs, nil
}
// anonymousPortSet queues an anonymous constant set holding the given
// ports on the connection and returns a lookup against it. The set is
// committed by the caller's flush together with the rule that binds it.
func (r *family) anonymousPortSet(values []uint16) (*expr.Lookup, error) {
set := &nftables.Set{
Anonymous: true,
Constant: true,
Table: r.workTable,
KeyType: nftables.TypeInetService,
}
elements := make([]nftables.SetElement, 0, len(values))
for _, p := range values {
elements = append(elements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(p)})
}
if err := r.conn.AddSet(set, elements); err != nil {
return nil, fmt.Errorf("add anonymous port set: %w", err)
}
return &expr.Lookup{
SourceRegister: 1,
SetID: set.ID,
SetName: set.Name,
}, nil
}
// applyPorts builds the source then destination port matches.
func (r *family) applyPorts(sPort, dPort *firewall.Port) ([]expr.Any, error) {
sPortExprs, err := r.applyPort(sPort, true)
if err != nil {
return nil, fmt.Errorf("apply source port: %w", err)
}
dPortExprs, err := r.applyPort(dPort, false)
if err != nil {
return nil, fmt.Errorf("apply destination port: %w", err)
}
return append(sPortExprs, dPortExprs...), nil
}
// prefixMatchExprs is the family-aware match sequence for a CIDR
// prefix. /0 returns nil; a host prefix (full bit length for the
// family) skips the bitwise step since the mask is all-ones. Shared
// between family and aclManager so both treat single prefixes
// identically.
func prefixMatchExprs(af addrFamily, prefix netip.Prefix, isSource bool) []expr.Any {
offset := af.dstAddrOffset
if isSource {
offset = af.srcAddrOffset
}
ones := prefix.Bits()
if ones == 0 {
return nil
}
payload := &expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: offset,
Len: af.addrLen,
}
cmp := &expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: prefix.Masked().Addr().AsSlice(),
}
if ones == af.totalBits {
return []expr.Any{payload, cmp}
}
mask := net.CIDRMask(ones, af.totalBits)
xor := make([]byte, af.addrLen)
return []expr.Any{
payload,
&expr.Bitwise{
DestRegister: 1,
SourceRegister: 1,
Len: af.addrLen,
Mask: mask,
Xor: xor,
},
cmp,
}
}
func getCtNewExprs() []expr.Any {
return []expr.Any{
&expr.Ct{
Key: expr.CtKeySTATE,
Register: 1,
},
&expr.Bitwise{
SourceRegister: 1,
DestRegister: 1,
Len: 4,
Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
Xor: binaryutil.NativeEndian.PutUint32(0),
},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: []byte{0, 0, 0, 0},
},
}
}
// sourceNetwork classifies a source-prefix list into the firewall.Network
// shape the rest of the spec-builder consumes: empty for match-any, a
// single prefix inline, or an ipset for multiple sources.
func sourceNetwork(sources []netip.Prefix) firewall.Network {
switch {
case len(sources) == 0:
return firewall.Network{}
case len(sources) == 1 && sources[0].Bits() == 0:
return firewall.Network{}
case len(sources) == 1:
return firewall.Network{Prefix: sources[0]}
default:
return firewall.Network{Set: firewall.NewPrefixSet(sources)}
}
}
func ifname(n string) []byte {
b := make([]byte, 16)
copy(b, n+"\x00")
return b
}
// findSets scans an nftables rule's expressions for expr.Lookup and
// returns the named sets in occurrence order. Used at delete time to
// drop ipsetCounter references; peer and route ACLs go through it.
func findSets(rule *nftables.Rule) []string {
var sets []string
for _, e := range rule.Exprs {
if lookup, ok := e.(*expr.Lookup); ok {
sets = append(sets, lookup.SetName)
}
}
return sets
}
client/firewall/nftables/interface_allower_integration_linux_test.go
+90
View File
@@ -0,0 +1,90 @@
//go:build privileged
package nftables
import (
"bytes"
"os"
"testing"
"github.com/google/nftables"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/client/iface"
)
// TestInterfaceAllowerInputOnly verifies the userspace-mode allower opens the
// interface on the INPUT hook of foreign chains only (not FORWARD, since the
// userspace router never forwards in the kernel), creates no netbird work
// table, and removes its rules on Close.
func TestInterfaceAllowerInputOnly(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("root required")
}
require.False(t, ipTableExists(t, getTableName()), "precondition: no stale netbird table")
conn := &nftables.Conn{}
extTable := conn.AddTable(&nftables.Table{Name: "nbtest_extchains", Family: nftables.TableFamilyINet})
inputChain := conn.AddChain(&nftables.Chain{
Name: "ext_input", Table: extTable,
Hooknum: nftables.ChainHookInput, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter,
})
forwardChain := conn.AddChain(&nftables.Chain{
Name: "ext_forward", Table: extTable,
Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter,
})
require.NoError(t, conn.Flush(), "create external table and chains")
t.Cleanup(func() {
c := &nftables.Conn{}
c.DelTable(extTable)
_ = c.Flush()
})
allower, err := NewInterfaceAllower(ifaceMock, iface.DefaultMTU)
require.NoError(t, err, "create allower")
require.NoError(t, allower.Apply(), "apply")
require.True(t, chainHasUserData(t, extTable, inputChain, userDataAcceptInputRule),
"external INPUT chain should get the accept rule")
require.Len(t, listRules(t, extTable, forwardChain), 0,
"external FORWARD chain must not be opened in userspace mode")
require.False(t, ipTableExists(t, getTableName()),
"allower must not create a netbird work table")
require.NoError(t, allower.Close(), "close")
require.False(t, chainHasUserData(t, extTable, inputChain, userDataAcceptInputRule),
"accept rule should be removed on close")
}
func listRules(t *testing.T, table *nftables.Table, chain *nftables.Chain) []*nftables.Rule {
t.Helper()
c := &nftables.Conn{}
rules, err := c.GetRules(table, chain)
require.NoError(t, err)
return rules
}
func chainHasUserData(t *testing.T, table *nftables.Table, chain *nftables.Chain, ud string) bool {
for _, r := range listRules(t, table, chain) {
if bytes.Equal(r.UserData, []byte(ud)) {
return true
}
}
return false
}
func ipTableExists(t *testing.T, name string) bool {
t.Helper()
c := &nftables.Conn{}
for _, fam := range []nftables.TableFamily{nftables.TableFamilyIPv4, nftables.TableFamilyIPv6} {
tbls, err := c.ListTablesOfFamily(fam)
require.NoError(t, err)
for _, tb := range tbls {
if tb.Name == name {
return true
}
}
}
return false
}
client/firewall/nftables/interface_allower_linux.go
+107
View File
@@ -0,0 +1,107 @@
package nftables
import (
"fmt"
"github.com/google/nftables"
"github.com/hashicorp/go-multierror"
nberrors "github.com/netbirdio/netbird/client/errors"
)
// InterfaceAllower opens the NetBird interface in the kernel's filter table and
// external chains and keeps them reconciled via a netlink monitor, so the host
// firewall doesn't drop traffic the NetBird firewall handles. It is used by the
// userspace firewall, where routing happens in the forwarder, so only INPUT is
// opened (the userspace router never forwards in the kernel).
//
// It owns its own families/connection and never creates a netbird work table.
// firewalld trust is handled by the caller, not here. Its operations are serial
// (Apply before the monitor starts; reconciles run on the single monitor
// goroutine; Close stops the monitor before removing), so it needs no locking.
//
// TODO: this opens nftables and the iptables-nft filter table (detected via
// nft), but not a legacy-iptables ruleset running in parallel with nftables.
// Such a host would keep its legacy filter chains closed for the interface.
type InterfaceAllower struct {
family4 *family
family6 *family
extMonitor *externalChainMonitor
}
// NewInterfaceAllower builds an allower for the given interface. It returns an
// error when nftables is unavailable (e.g. an iptables-legacy host), so the
// caller can fall back to firewalld trust.
func NewInterfaceAllower(wgIface iFaceMapper, mtu uint16) (*InterfaceAllower, error) {
tableName := getTableName()
family4 := newFamily(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4}, wgIface, mtu)
// Probe nftables availability before committing to this backend.
if _, err := family4.conn.ListChainsOfTableFamily(nftables.TableFamilyINet); err != nil {
return nil, fmt.Errorf("nftables not available: %w", err)
}
a := &InterfaceAllower{family4: family4}
if wgIface.Address().HasIPv6() {
a.family6 = newFamily(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6}, wgIface, mtu)
}
a.extMonitor = newExternalChainMonitor(a)
return a, nil
}
// Apply opens the interface (INPUT only) in the foreign filter chains and starts
// reconciling them on nftables changes.
func (a *InterfaceAllower) Apply() error {
var merr *multierror.Error
for _, f := range a.families() {
// Remove any stale accepts first so a prior unclean exit (e.g. SIGKILL,
// where Close never ran) is recovered deterministically rather than
// accumulating duplicate rules on the iptables filter table.
if err := f.removeAcceptFilterRules(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("clean stale accept rules: %w", err))
}
if err := f.openInterface(false); err != nil {
merr = multierror.Append(merr, err)
}
}
a.extMonitor.start()
return nberrors.FormatErrorOrNil(merr)
}
// families returns the configured address families (v4, and v6 when present).
func (a *InterfaceAllower) families() []*family {
families := []*family{a.family4}
if a.family6 != nil {
families = append(families, a.family6)
}
return families
}
// reconcileExternalChains re-applies the INPUT accepts to external chains. It
// implements externalChainReconciler for the monitor.
func (a *InterfaceAllower) reconcileExternalChains() error {
var merr *multierror.Error
for _, f := range a.families() {
if err := f.acceptExternalChainsRules(false); err != nil {
merr = multierror.Append(merr, err)
}
}
return nberrors.FormatErrorOrNil(merr)
}
// Close stops the monitor and removes the accept rules.
func (a *InterfaceAllower) Close() error {
a.extMonitor.stop()
var merr *multierror.Error
for _, f := range a.families() {
if err := f.removeAcceptFilterRules(); err != nil {
merr = multierror.Append(merr, err)
}
}
return nberrors.FormatErrorOrNil(merr)
}
client/firewall/nftables/ipset_linux.go
+210
View File
@@ -0,0 +1,210 @@
//go:build !android
package nftables
import (
"encoding/binary"
"fmt"
"net/netip"
"github.com/google/nftables"
"github.com/google/nftables/expr"
log "github.com/sirupsen/logrus"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
)
func (r *family) getIpSet(set firewall.Set, prefixes []netip.Prefix, isSource bool) ([]expr.Any, error) {
ref, err := r.ipsetCounter.Increment(set.HashedName(), setInput{
set: set,
prefixes: prefixes,
})
if err != nil {
return nil, fmt.Errorf("create or get ipset: %w", err)
}
return r.getIpSetExprs(ref, isSource)
}
func (r *family) createIpSet(setName string, input setInput) (*nftables.Set, error) {
// overlapping prefixes will result in an error, so we need to merge them
prefixes := firewall.MergeIPRanges(input.prefixes)
nfset := &nftables.Set{
Name: setName,
Comment: input.set.Comment(),
Table: r.workTable,
// required for prefixes
Interval: true,
KeyType: r.af.setKeyType,
}
elements := r.convertPrefixesToSet(prefixes)
nElements := len(elements)
maxElements := maxPrefixesSet * 2
initialElements := elements[:min(maxElements, nElements)]
if err := r.conn.AddSet(nfset, initialElements); err != nil {
return nil, fmt.Errorf("error adding set %s: %w", setName, err)
}
if err := r.conn.Flush(); err != nil {
return nil, fmt.Errorf("flush error: %w", err)
}
log.Debugf("Created new ipset: %s with %d initial prefixes (total prefixes %d)", setName, len(initialElements)/2, len(prefixes))
// The set is committed now. If a later batch fails, destroy it: the
// refcounter records nothing on a create-callback error, so it would
// otherwise leak, and a partial source set fails-open for deny rules.
if err := r.addRemainingElements(nfset, elements, maxElements); err != nil {
if derr := r.deleteIpSet(setName, nfset); derr != nil {
log.Warnf("rollback ipset %s after add failure: %v", setName, derr)
}
return nil, err
}
log.Infof("Created new ipset: %s with %d prefixes", setName, len(prefixes))
return nfset, nil
}
// addRemainingElements adds element batches beyond the initial one in
// maxElements-sized chunks, flushing each. Called after the set has been
// created with its first batch.
func (r *family) addRemainingElements(nfset *nftables.Set, elements []nftables.SetElement, maxElements int) error {
nElements := len(elements)
for subStart := maxElements; subStart < nElements; subStart += maxElements {
subEnd := min(subStart+maxElements, nElements)
subElement := elements[subStart:subEnd]
nSubPrefixes := len(subElement) / 2
log.Tracef("Adding new prefixes (%d) in ipset: %s", nSubPrefixes, nfset.Name)
if err := r.conn.SetAddElements(nfset, subElement); err != nil {
return fmt.Errorf("error adding prefixes (%d) to set %s: %w", nSubPrefixes, nfset.Name, err)
}
if err := r.conn.Flush(); err != nil {
return fmt.Errorf("flush error: %w", err)
}
log.Debugf("Added new prefixes (%d) in ipset: %s", nSubPrefixes, nfset.Name)
}
return nil
}
func (r *family) convertPrefixesToSet(prefixes []netip.Prefix) []nftables.SetElement {
var elements []nftables.SetElement
for _, prefix := range prefixes {
// nftables needs half-open intervals [firstIP, lastIP) for prefixes
// e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc
firstIP := prefix.Addr()
// For a /0 the last address is the broadcast and its Next() overflows
// to an invalid Addr with an empty key, so wrap to the zero address,
// which nftables reads as the open end of a full-range interval.
var lastKey []byte
if prefix.Bits() == 0 {
lastKey = make([]byte, r.af.addrLen)
} else {
lastKey = calculateLastIP(prefix).Next().AsSlice()
}
// the nft tool also adds a zero-address IntervalEnd element, see https://github.com/google/nftables/issues/247
// nftables.SetElement{Key: make([]byte, r.af.addrLen), IntervalEnd: true},
elements = append(elements,
nftables.SetElement{Key: firstIP.AsSlice()},
nftables.SetElement{Key: lastKey, IntervalEnd: true},
)
}
return elements
}
// calculateLastIP determines the last IP in a given prefix.
func calculateLastIP(prefix netip.Prefix) netip.Addr {
masked := prefix.Masked()
if masked.Addr().Is4() {
hostMask := ^uint32(0) >> masked.Bits()
lastIP := uint32FromNetipAddr(masked.Addr()) | hostMask
return netip.AddrFrom4(uint32ToBytes(lastIP))
}
// IPv6: set host bits to all 1s
b := masked.Addr().As16()
bits := masked.Bits()
for i := bits; i < 128; i++ {
b[i/8] |= 1 << (7 - i%8)
}
return netip.AddrFrom16(b)
}
// Utility function to convert netip.Addr to uint32.
func uint32FromNetipAddr(addr netip.Addr) uint32 {
b := addr.As4()
return binary.BigEndian.Uint32(b[:])
}
// Utility function to convert uint32 to a netip-compatible byte slice.
func uint32ToBytes(ip uint32) [4]byte {
var b [4]byte
binary.BigEndian.PutUint32(b[:], ip)
return b
}
func (r *family) deleteIpSet(setName string, nfset *nftables.Set) error {
r.conn.DelSet(nfset)
if err := r.conn.Flush(); err != nil {
return fmt.Errorf(flushError, err)
}
log.Debugf("Deleted unused ipset %s", setName)
return nil
}
func (r *family) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error {
nfset, err := r.conn.GetSetByName(r.workTable, set.HashedName())
if err != nil {
return fmt.Errorf("get set %s: %w", set.HashedName(), err)
}
// Overlapping prefixes (e.g. duplicate resolved addresses) make the
// interval set reject the batch, so merge them as createIpSet does.
prefixes = firewall.MergeIPRanges(prefixes)
elements := r.convertPrefixesToSet(prefixes)
// Add in batches sized like createIpSet so a large update does not
// exceed the netlink message size limit.
maxElements := maxPrefixesSet * 2
for start := 0; start < len(elements); start += maxElements {
end := min(start+maxElements, len(elements))
if err := r.conn.SetAddElements(nfset, elements[start:end]); err != nil {
return fmt.Errorf("add elements to set %s: %w", set.HashedName(), err)
}
if err := r.conn.Flush(); err != nil {
return fmt.Errorf(flushError, err)
}
}
log.Debugf("updated set %s with %d prefixes", set.HashedName(), len(prefixes))
return nil
}
func (r *family) getIpSetExprs(ref refcounter.Ref[*nftables.Set], isSource bool) ([]expr.Any, error) {
// dst offset by default
offset := r.af.dstAddrOffset
if isSource {
// src offset
offset = r.af.srcAddrOffset
}
return []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: offset,
Len: r.af.addrLen,
},
&expr.Lookup{
SourceRegister: 1,
SetName: ref.Out.Name,
SetID: ref.Out.ID,
},
}, nil
}
client/firewall/nftables/ipset_linux_test.go
+36
View File
@@ -0,0 +1,36 @@
package nftables
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestConvertPrefixesToSetWildcard verifies that a /0 prefix produces a
// usable interval. The last address of a /0 is the broadcast, whose Next()
// overflows to an invalid Addr with an empty key; the IntervalEnd must wrap
// to the zero address instead so nftables sees a full-range interval.
func TestConvertPrefixesToSetWildcard(t *testing.T) {
tests := []struct {
name string
af addrFamily
prefix string
}{
{"IPv4 /0", afIPv4, "0.0.0.0/0"},
{"IPv6 /0", afIPv6, "::/0"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &family{af: tt.af}
elements := r.convertPrefixesToSet([]netip.Prefix{netip.MustParsePrefix(tt.prefix)})
require.Len(t, elements, 2, "expected start and interval-end element")
assert.False(t, elements[0].IntervalEnd, "first element is the interval start")
assert.True(t, elements[1].IntervalEnd, "second element is the interval end")
assert.Len(t, elements[1].Key, int(tt.af.addrLen), "interval-end key must be a zero address, not empty")
})
}
}
client/firewall/nftables/ipsetstore_linux.go
-85
View File
@@ -1,85 +0,0 @@
package nftables
import (
"net"
)
type ipsetStore struct {
ipsetReference map[string]int
ipsets map[string]map[string]struct{} // ipsetName -> list of ips
}
func newIpsetStore() *ipsetStore {
return &ipsetStore{
ipsetReference: make(map[string]int),
ipsets: make(map[string]map[string]struct{}),
}
}
func (s *ipsetStore) ips(ipsetName string) (map[string]struct{}, bool) {
r, ok := s.ipsets[ipsetName]
return r, ok
}
func (s *ipsetStore) newIpset(ipsetName string) map[string]struct{} {
s.ipsetReference[ipsetName] = 0
ipList := make(map[string]struct{})
s.ipsets[ipsetName] = ipList
return ipList
}
func (s *ipsetStore) deleteIpset(ipsetName string) {
delete(s.ipsetReference, ipsetName)
delete(s.ipsets, ipsetName)
}
func (s *ipsetStore) DeleteIpFromSet(ipsetName string, ip net.IP) {
ipList, ok := s.ipsets[ipsetName]
if !ok {
return
}
delete(ipList, ip.String())
}
func (s *ipsetStore) AddIpToSet(ipsetName string, ip net.IP) {
ipList, ok := s.ipsets[ipsetName]
if !ok {
return
}
ipList[ip.String()] = struct{}{}
}
func (s *ipsetStore) IsIpInSet(ipsetName string, ip net.IP) bool {
ipList, ok := s.ipsets[ipsetName]
if !ok {
return false
}
_, ok = ipList[ip.String()]
return ok
}
func (s *ipsetStore) AddReferenceToIpset(ipsetName string) {
s.ipsetReference[ipsetName]++
}
func (s *ipsetStore) DeleteReferenceFromIpSet(ipsetName string) {
r, ok := s.ipsetReference[ipsetName]
if !ok {
return
}
if r == 0 {
return
}
s.ipsetReference[ipsetName]--
}
func (s *ipsetStore) HasReferenceToSet(ipsetName string) bool {
if _, ok := s.ipsetReference[ipsetName]; !ok {
return false
}
if s.ipsetReference[ipsetName] == 0 {
return false
}
return true
}
client/firewall/nftables/manager_linux.go
+113 -203
View File
@@ -3,7 +3,6 @@ package nftables
import (
"context"
"fmt"
"net"
"net/netip"
"os"
"sync"
@@ -16,7 +15,6 @@ import (
"golang.org/x/sys/unix"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/firewall/firewalld"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/statemanager"
@@ -45,18 +43,17 @@ type iFaceMapper interface {
Address() wgaddr.Address
}
// Manager of iptables firewall
// Manager of nftables firewall. Per-family state (peer ACLs, route
// ACLs, NAT, DNAT, MSS clamping) lives on family; Manager dispatches
// by family and provides the public firewall.Manager surface.
type Manager struct {
mutex sync.Mutex
rConn *nftables.Conn
wgIface iFaceMapper
router *router
aclManager *AclManager
// IPv6 counterparts, nil when no v6 overlay
router6 *router
aclManager6 *AclManager
family4 *family
// IPv6 counterpart, nil when no v6 overlay.
family6 *family
notrackOutputChain *nftables.Chain
notrackPreroutingChain *nftables.Chain
@@ -74,21 +71,10 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) {
tableName := getTableName()
workTable := &nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4}
var err error
m.router, err = newRouter(workTable, wgIface, mtu)
if err != nil {
return nil, fmt.Errorf("create router: %w", err)
}
m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw)
if err != nil {
return nil, fmt.Errorf("create acl manager: %w", err)
}
m.family4 = newFamily(workTable, wgIface, mtu)
if wgIface.Address().HasIPv6() {
if err := m.createIPv6Components(tableName, wgIface, mtu); err != nil {
return nil, fmt.Errorf("create IPv6 firewall: %w", err)
}
m.createIPv6Components(tableName, wgIface, mtu)
}
m.extMonitor = newExternalChainMonitor(m)
@@ -96,30 +82,19 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) {
return m, nil
}
func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mtu uint16) error {
func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mtu uint16) {
workTable6 := &nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6}
var err error
m.router6, err = newRouter(workTable6, wgIface, mtu)
if err != nil {
return fmt.Errorf("create v6 router: %w", err)
}
m.family6 = newFamily(workTable6, wgIface, mtu)
// Share the per-family forwarding refcounter with the v4 router so a v4
// Share the per-family forwarding refcounter with the v4 family so a v4
// rule and a v6 rule against the same state machine cooperate cleanly.
m.router6.ipFwdState = m.router.ipFwdState
m.aclManager6, err = newAclManager(workTable6, wgIface, chainNameRoutingFw)
if err != nil {
return fmt.Errorf("create v6 acl manager: %w", err)
}
return nil
m.family6.ipFwdState = m.family4.ipFwdState
}
// hasIPv6 reports whether the manager has IPv6 components initialized.
func (m *Manager) hasIPv6() bool {
return m.router6 != nil
return m.family6 != nil
}
func (m *Manager) initIPv6() error {
@@ -128,12 +103,8 @@ func (m *Manager) initIPv6() error {
return fmt.Errorf("create v6 work table: %w", err)
}
if err := m.router6.init(workTable6); err != nil {
return fmt.Errorf("v6 router init: %w", err)
}
if err := m.aclManager6.init(workTable6); err != nil {
return fmt.Errorf("v6 acl manager init: %w", err)
if err := m.family6.init(workTable6); err != nil {
return fmt.Errorf("v6 family init: %w", err)
}
return nil
@@ -156,19 +127,20 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error {
// reconcileExternalChains re-applies passthrough accept rules to external
// filter chains for both IPv4 and IPv6 routers. Called by the monitor when
// tables or chains appear (e.g. after firewalld reloads).
// tables or chains appear (e.g. after firewalld reloads). Kernel routing opens
// both INPUT and FORWARD.
func (m *Manager) reconcileExternalChains() error {
m.mutex.Lock()
defer m.mutex.Unlock()
var merr *multierror.Error
if m.router != nil {
if err := m.router.acceptExternalChainsRules(); err != nil {
if m.family4 != nil {
if err := m.family4.acceptExternalChainsRules(true); err != nil {
merr = multierror.Append(merr, fmt.Errorf("v4: %w", err))
}
}
if m.hasIPv6() {
if err := m.router6.acceptExternalChainsRules(); err != nil {
if err := m.family6.acceptExternalChainsRules(true); err != nil {
merr = multierror.Append(merr, fmt.Errorf("v6: %w", err))
}
}
@@ -187,12 +159,8 @@ func (m *Manager) initFirewall() (err error) {
}
}()
if err := m.router.init(workTable); err != nil {
return fmt.Errorf("router init: %w", err)
}
if err := m.aclManager.init(workTable); err != nil {
return fmt.Errorf("acl manager init: %w", err)
if err := m.family4.init(workTable); err != nil {
return fmt.Errorf("family init: %w", err)
}
if m.hasIPv6() {
@@ -220,7 +188,7 @@ func (m *Manager) persistState(stateManager *statemanager.Manager) {
InterfaceState: &InterfaceState{
NameStr: m.wgIface.Name(),
WGAddress: m.wgIface.Address(),
MTU: m.router.mtu,
MTU: m.family4.mtu,
},
}); err != nil {
log.Errorf("failed to update state: %v", err)
@@ -235,12 +203,12 @@ func (m *Manager) persistState(stateManager *statemanager.Manager) {
// rollbackInit performs best-effort cleanup of already-initialized state when Init fails partway through.
func (m *Manager) rollbackInit() {
if err := m.router.Reset(); err != nil {
log.Warnf("rollback router: %v", err)
if err := m.family4.Reset(); err != nil {
log.Warnf("rollback family: %v", err)
}
if m.hasIPv6() {
if err := m.router6.Reset(); err != nil {
log.Warnf("rollback v6 router: %v", err)
if err := m.family6.Reset(); err != nil {
log.Warnf("rollback v6 family: %v", err)
}
}
if err := m.cleanupNetbirdTables(); err != nil {
@@ -251,118 +219,82 @@ func (m *Manager) rollbackInit() {
}
}
// AddPeerFiltering rule to the firewall
// AddFilterRule installs a packet-filtering rule.
//
// If comment argument is empty firewall manager should set
// rule ID as comment for the rule
func (m *Manager) AddPeerFiltering(
id []byte,
ip net.IP,
proto firewall.Protocol,
sPort *firewall.Port,
dPort *firewall.Port,
action firewall.Action,
ipsetName string,
) ([]firewall.Rule, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
if ip.To4() != nil {
return m.aclManager.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName)
}
if !m.hasIPv6() {
return nil, fmt.Errorf("add peer filtering for %s: %w", ip, firewall.ErrIPv6NotInitialized)
}
return m.aclManager6.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName)
}
func (m *Manager) AddRouteFiltering(
// Destination semantics: zero Network → input chain (peer ACL);
// set Network → forward chain (route ACL).
//
// Sources are a single address family; the rule is dispatched to the
// matching per-family backend.
func (m *Manager) AddFilterRule(
id []byte,
sources []netip.Prefix,
destination firewall.Network,
proto firewall.Protocol,
sPort, dPort *firewall.Port,
sPort *firewall.Port,
dPort *firewall.Port,
action firewall.Action,
) (firewall.Rule, error) {
if len(sources) == 0 {
return nil, firewall.ErrNoSources
}
m.mutex.Lock()
defer m.mutex.Unlock()
if isIPv6RouteRule(sources, destination) {
fam := m.family4
if isIPv6Rule(sources, destination) {
if !m.hasIPv6() {
return nil, fmt.Errorf("add route filtering: %w", firewall.ErrIPv6NotInitialized)
return nil, fmt.Errorf("add filtering: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action)
fam = m.family6
}
return m.router.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action)
return fam.AddFilterRule(id, sources, destination, proto, sPort, dPort, action)
}
// DeletePeerRule from the firewall by rule definition
func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
// DeleteFilterRule removes a filtering rule. The owning family is found
// by id in the in-memory filter maps, which are the only tracking for
// filter rules. family.DeleteFilterRule is idempotent when the id is
// absent.
func (m *Manager) DeleteFilterRule(rule firewall.Rule) error {
m.mutex.Lock()
defer m.mutex.Unlock()
if m.hasIPv6() && isIPv6Rule(rule) {
return m.aclManager6.DeletePeerRule(rule)
}
return m.aclManager.DeletePeerRule(rule)
}
func isIPv6Rule(rule firewall.Rule) bool {
r, ok := rule.(*Rule)
return ok && r.nftRule != nil && r.nftRule.Table != nil && r.nftRule.Table.Family == nftables.TableFamilyIPv6
}
// isIPv6RouteRule determines whether a route rule belongs to the v6 table.
// For static routes, the destination prefix determines the family. For dynamic
// routes (DomainSet), the sources determine the family since management
// duplicates dynamic rules per family.
func isIPv6RouteRule(sources []netip.Prefix, destination firewall.Network) bool {
if destination.IsPrefix() {
return destination.Prefix.Addr().Is6()
}
return len(sources) > 0 && sources[0].Addr().Is6()
}
// DeleteRouteRule deletes a routing rule. Route rules live in exactly one
// router; the cached maps are normally authoritative, so the kernel is only
// consulted when neither map knows about the rule.
func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
m.mutex.Lock()
defer m.mutex.Unlock()
id := rule.ID()
r, err := m.routerForRuleID(id, (*router).hasRule)
fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule, false)
if err != nil {
return err
}
return r.DeleteRouteRule(rule)
return fam.DeleteFilterRule(rule)
}
// routerForRuleID picks the router holding the rule with the given id, using
// the supplied lookup. If the cached maps disagree (or both miss), it refreshes
// from the kernel once and re-checks before falling back to the v4 router.
func (m *Manager) routerForRuleID(id string, has func(*router, string) bool) (*router, error) {
if has(m.router, id) {
return m.router, nil
}
if m.hasIPv6() && has(m.router6, id) {
return m.router6, nil
// familyForRuleID picks the family holding the rule with the given id, using
// the supplied lookup. With refresh set, a miss in both cached maps reloads
// the NAT/DNAT rule maps from the kernel once and re-checks before falling
// back to the v4 family. Filter rules are tracked only in memory and have no
// kernel-backed reload, so their callers pass refresh as false.
func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool, refresh bool) (*family, error) {
if has(m.family4, id) {
return m.family4, nil
}
if !m.hasIPv6() {
return m.router, nil
return m.family4, nil
}
if err := m.router.refreshRulesMap(); err != nil {
if has(m.family6, id) {
return m.family6, nil
}
if !refresh {
return m.family4, nil
}
if err := m.family4.refreshRulesMap(); err != nil {
return nil, fmt.Errorf("refresh v4 rules: %w", err)
}
if err := m.router6.refreshRulesMap(); err != nil {
if err := m.family6.refreshRulesMap(); err != nil {
return nil, fmt.Errorf("refresh v6 rules: %w", err)
}
if has(m.router6, id) && !has(m.router, id) {
return m.router6, nil
if has(m.family6, id) && !has(m.family4, id) {
return m.family6, nil
}
return m.router, nil
return m.family4, nil
}
func (m *Manager) IsServerRouteSupported() bool {
@@ -381,10 +313,10 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
if !m.hasIPv6() {
return fmt.Errorf("add NAT rule: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.AddNatRule(pair)
return m.family6.AddNatRule(pair)
}
if err := m.router.AddNatRule(pair); err != nil {
if err := m.family4.AddNatRule(pair); err != nil {
return err
}
@@ -396,7 +328,7 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
// so the eventual cleanup still works.
if m.hasIPv6() && pair.Dynamic {
v6Pair := firewall.ToV6NatPair(pair)
if err := m.router6.AddNatRule(v6Pair); err != nil {
if err := m.family6.AddNatRule(v6Pair); err != nil {
return fmt.Errorf("add v6 NAT rule: %w", err)
}
}
@@ -412,18 +344,18 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
if !m.hasIPv6() {
return nil
}
return m.router6.RemoveNatRule(pair)
return m.family6.RemoveNatRule(pair)
}
var merr *multierror.Error
if err := m.router.RemoveNatRule(pair); err != nil {
if err := m.family4.RemoveNatRule(pair); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove v4 NAT rule: %w", err))
}
if m.hasIPv6() && pair.Dynamic {
v6Pair := firewall.ToV6NatPair(pair)
if err := m.router6.RemoveNatRule(v6Pair); err != nil {
if err := m.family6.RemoveNatRule(v6Pair); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove v6 NAT rule: %w", err))
}
}
@@ -431,46 +363,13 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
return nberrors.FormatErrorOrNil(merr)
}
// AllowNetbird allows netbird interface traffic.
// This is called when USPFilter wraps the native firewall, adding blanket accept
// rules so that packet filtering is handled in userspace instead of by netfilter.
//
// TODO: In USP mode this only adds ACCEPT to the netbird table's own chains,
// which doesn't override DROP rules in external tables (e.g. firewalld).
// Should add passthrough rules to external chains (like the native mode router's
// addExternalChainsRules does) for both the netbird table family and inet tables.
// The netbird table itself is fine (routing chains already exist there), but
// non-netbird tables with INPUT/FORWARD hooks can still DROP our WG traffic.
func (m *Manager) AllowNetbird() error {
m.mutex.Lock()
defer m.mutex.Unlock()
if err := m.aclManager.createDefaultAllowRules(); err != nil {
return fmt.Errorf("create default allow rules: %w", err)
}
if m.hasIPv6() {
if err := m.aclManager6.createDefaultAllowRules(); err != nil {
return fmt.Errorf("create v6 default allow rules: %w", err)
}
}
if err := m.rConn.Flush(); err != nil {
return fmt.Errorf("flush allow input netbird rules: %w", err)
}
if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil {
log.Warnf("failed to trust interface in firewalld: %v", err)
}
return nil
}
// SetLegacyManagement sets the route manager to use legacy management
func (m *Manager) SetLegacyManagement(isLegacy bool) error {
if err := firewall.SetLegacyManagement(m.router, isLegacy); err != nil {
if err := firewall.SetLegacyManagement(m.family4, isLegacy); err != nil {
return err
}
if m.hasIPv6() {
return firewall.SetLegacyManagement(m.router6, isLegacy)
return firewall.SetLegacyManagement(m.family6, isLegacy)
}
return nil
}
@@ -484,13 +383,13 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error {
var merr *multierror.Error
if err := m.router.Reset(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("reset router: %v", err))
if err := m.family4.Reset(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("reset family: %w", err))
}
if m.hasIPv6() {
if err := m.router6.Reset(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("reset v6 router: %v", err))
if err := m.family6.Reset(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err))
}
}
@@ -531,11 +430,11 @@ func (m *Manager) SetLogLevel(log.Level) {
func (m *Manager) EnableRouting() error {
// v6 only when the overlay actually has v6.
return m.router.ipFwdState.RequestRouting(m.router6 != nil)
return m.family4.ipFwdState.RequestRouting(m.hasIPv6())
}
func (m *Manager) DisableRouting() error {
return m.router.ipFwdState.ReleaseRouting()
return m.family4.ipFwdState.ReleaseRouting()
}
// Flush rule/chain/set operations from the buffer
@@ -546,13 +445,13 @@ func (m *Manager) Flush() error {
m.mutex.Lock()
defer m.mutex.Unlock()
if err := m.aclManager.Flush(); err != nil {
if err := m.family4.Flush(); err != nil {
return err
}
if m.hasIPv6() {
if err := m.aclManager6.Flush(); err != nil {
return fmt.Errorf("flush v6 acl: %w", err)
if err := m.family6.Flush(); err != nil {
return fmt.Errorf("flush v6 family: %w", err)
}
}
@@ -572,9 +471,9 @@ func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error)
if !m.hasIPv6() {
return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.AddDNATRule(rule)
return m.family6.AddDNATRule(rule)
}
return m.router.AddDNATRule(rule)
return m.family4.AddDNATRule(rule)
}
// DeleteDNATRule deletes a DNAT rule
@@ -582,7 +481,7 @@ func (m *Manager) DeleteDNATRule(rule firewall.Rule) error {
m.mutex.Lock()
defer m.mutex.Unlock()
r, err := m.routerForRuleID(rule.ID(), (*router).hasDNATRule)
r, err := m.familyForRuleID(rule.ID(), (*family).hasDNATRule, true)
if err != nil {
return err
}
@@ -603,12 +502,12 @@ func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error {
}
}
if err := m.router.UpdateSet(set, v4Prefixes); err != nil {
if err := m.family4.UpdateSet(set, v4Prefixes); err != nil {
return err
}
if m.hasIPv6() && len(v6Prefixes) > 0 {
if err := m.router6.UpdateSet(set, v6Prefixes); err != nil {
if err := m.family6.UpdateSet(set, v6Prefixes); err != nil {
return fmt.Errorf("update v6 set: %w", err)
}
}
@@ -625,9 +524,9 @@ func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protoco
if !m.hasIPv6() {
return fmt.Errorf("add inbound DNAT: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort)
}
return m.router.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family4.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort)
}
// RemoveInboundDNAT removes an inbound DNAT rule.
@@ -639,9 +538,9 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot
if !m.hasIPv6() {
return fmt.Errorf("remove inbound DNAT: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort)
}
return m.router.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family4.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort)
}
// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic.
@@ -653,9 +552,9 @@ func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol
if !m.hasIPv6() {
return fmt.Errorf("add output DNAT: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort)
}
return m.router.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family4.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort)
}
// RemoveOutputDNAT removes an OUTPUT chain DNAT rule.
@@ -667,9 +566,9 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto
if !m.hasIPv6() {
return fmt.Errorf("remove output DNAT: %w", firewall.ErrIPv6NotInitialized)
}
return m.router6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
}
return m.router.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
}
const (
@@ -898,3 +797,14 @@ func getEstablishedExprs(register uint32) []expr.Any {
},
}
}
// isIPv6Rule reports whether the rule belongs to the v6 table. For a
// prefix destination the destination family decides; otherwise the
// (single-family) sources do, since management duplicates rules per
// family.
func isIPv6Rule(sources []netip.Prefix, destination firewall.Network) bool {
if destination.IsPrefix() {
return destination.Prefix.Addr().Is6()
}
return len(sources) > 0 && sources[0].Addr().Is6()
}
client/firewall/nftables/manager_linux_test.go
+101 -41
View File
@@ -72,13 +72,13 @@ func TestNftablesManager(t *testing.T) {
testClient := &nftables.Conn{}
rule, err := manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop, "")
rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop)
require.NoError(t, err, "failed to add rule")
err = manager.Flush()
require.NoError(t, err, "failed to flush")
rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules)
require.NoError(t, err, "failed to get rules")
require.Len(t, rules, 2, "expected 2 rules")
@@ -149,15 +149,12 @@ func TestNftablesManager(t *testing.T) {
// Compare connection tracking rule at position 1 (pushed down by DROP rule insertion)
compareExprsIgnoringCounters(t, rules[1].Exprs, expectedExprs1)
for _, r := range rule {
err = manager.DeletePeerRule(r)
require.NoError(t, err, "failed to delete rule")
}
require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete rule")
err = manager.Flush()
require.NoError(t, err, "failed to flush")
rules, err = testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
rules, err = testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules)
require.NoError(t, err, "failed to get rules")
// established rule remains
require.Len(t, rules, 1, "expected 1 rules after deletion")
@@ -182,47 +179,39 @@ func TestNftablesManagerRuleOrder(t *testing.T) {
testClient := &nftables.Conn{}
// Add accept rule first
_, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "accept-http")
_, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept)
require.NoError(t, err, "failed to add accept rule")
// Add deny rule second for the same traffic
_, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop, "deny-http")
_, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop)
require.NoError(t, err, "failed to add deny rule")
err = manager.Flush()
require.NoError(t, err, "failed to flush")
rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules)
require.NoError(t, err, "failed to get rules")
t.Logf("Found %d rules in nftables chain", len(rules))
// Find the accept and deny rules and verify deny comes before accept
// Single-source rules emit a direct payload+cmp on the source IP
// (no set lookup). Match by source-IP + port + verdict instead of
// the legacy per-(action,port) set names ("deny-http"/"accept-http")
// that this test predates.
wantSrc := ip.AsSlice()
var acceptRuleIndex, denyRuleIndex = -1, -1
for i, rule := range rules {
hasAcceptHTTPSet := false
hasDenyHTTPSet := false
hasPort80 := false
var hasSrc, hasPort80 bool
var action string
for _, e := range rule.Exprs {
// Check for set lookup
if lookup, ok := e.(*expr.Lookup); ok {
switch lookup.SetName {
case "accept-http":
hasAcceptHTTPSet = true
case "deny-http":
hasDenyHTTPSet = true
if cmp, ok := e.(*expr.Cmp); ok && cmp.Op == expr.CmpOpEq {
if bytes.Equal(cmp.Data, wantSrc) {
hasSrc = true
}
}
// Check for port 80
if cmp, ok := e.(*expr.Cmp); ok {
if cmp.Op == expr.CmpOpEq && len(cmp.Data) == 2 && binary.BigEndian.Uint16(cmp.Data) == 80 {
if len(cmp.Data) == 2 && binary.BigEndian.Uint16(cmp.Data) == 80 {
hasPort80 = true
}
}
// Check for verdict
if verdict, ok := e.(*expr.Verdict); ok {
switch verdict.Kind {
case expr.VerdictAccept:
@@ -233,11 +222,15 @@ func TestNftablesManagerRuleOrder(t *testing.T) {
}
}
if hasAcceptHTTPSet && hasPort80 && action == "ACCEPT" {
t.Logf("Rule [%d]: accept-http set + Port 80 + ACCEPT", i)
if !hasSrc || !hasPort80 {
continue
}
switch action {
case "ACCEPT":
t.Logf("Rule [%d]: src=%s port=80 ACCEPT", i, ip)
acceptRuleIndex = i
} else if hasDenyHTTPSet && hasPort80 && action == "DROP" {
t.Logf("Rule [%d]: deny-http set + Port 80 + DROP", i)
case "DROP":
t.Logf("Rule [%d]: src=%s port=80 DROP", i, ip)
denyRuleIndex = i
}
}
@@ -281,7 +274,7 @@ func TestNFtablesCreatePerformance(t *testing.T) {
start := time.Now()
for i := 0; i < testMax; i++ {
port := &fw.Port{Values: []uint16{uint16(1000 + i)}}
_, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "")
_, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept)
require.NoError(t, err, "failed to add rule")
if i%100 == 0 {
@@ -363,10 +356,10 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) {
})
ip := netip.MustParseAddr("100.96.0.1")
_, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "")
_, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept)
require.NoError(t, err, "failed to add peer filtering rule")
_, err = manager.AddRouteFiltering(
_, err = manager.AddFilterRule(
nil,
[]netip.Prefix{netip.MustParsePrefix("192.168.2.0/24")},
fw.Network{Prefix: netip.MustParsePrefix("10.1.0.0/24")},
@@ -439,10 +432,10 @@ func TestNftablesManagerIPv6CompatibilityWithIp6tables(t *testing.T) {
})
ip := netip.MustParseAddr("fd00::2")
_, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "")
_, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept)
require.NoError(t, err, "add v6 peer filtering rule")
_, err = manager.AddRouteFiltering(
_, err = manager.AddFilterRule(
nil,
[]netip.Prefix{netip.MustParsePrefix("fd00:1::/64")},
fw.Network{Prefix: netip.MustParsePrefix("2001:db8::/48")},
@@ -552,7 +545,7 @@ func TestNftablesManagerCompatibilityWithIptablesFor6kPrefixes(t *testing.T) {
prefixes = append(prefixes, netip.PrefixFrom(addr, 24))
}
}
_, err = manager.AddRouteFiltering(
_, err = manager.AddFilterRule(
nil,
prefixes,
fw.Network{Prefix: netip.MustParsePrefix("10.2.0.0/24")},
@@ -567,7 +560,7 @@ func TestNftablesManagerCompatibilityWithIptablesFor6kPrefixes(t *testing.T) {
verifyIptablesOutput(t, stdout, stderr)
}
func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) {
func TestNftablesManagerCompatibilityWithIptablesForWildcardSource(t *testing.T) {
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
}
@@ -593,9 +586,9 @@ func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T)
verifyIptablesOutput(t, stdout, stderr)
})
_, err = manager.AddRouteFiltering(
_, err = manager.AddFilterRule(
nil,
[]netip.Prefix{},
[]netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")},
fw.Network{Prefix: netip.MustParsePrefix("10.2.0.0/24")},
fw.ProtocolTCP,
nil,
@@ -608,6 +601,73 @@ func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T)
verifyIptablesOutput(t, stdout, stderr)
}
func TestNftablesManagerMultiPortFilter(t *testing.T) {
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
}
manager, err := Create(ifaceMock, iface.DefaultMTU)
require.NoError(t, err)
require.NoError(t, manager.Init(nil))
t.Cleanup(func() {
require.NoError(t, manager.Close(nil), "failed to reset manager state")
})
ip := netip.MustParseAddr("100.96.0.1")
rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80, 443}}, fw.ActionAccept)
require.NoError(t, err, "failed to add multi-port rule")
testClient := &nftables.Conn{}
rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules)
require.NoError(t, err, "failed to get rules")
var lookup *expr.Lookup
for _, kernelRule := range rules {
if string(kernelRule.UserData) != string(rule.ID()) {
continue
}
for _, e := range kernelRule.Exprs {
if l, ok := e.(*expr.Lookup); ok {
lookup = l
}
}
}
require.NotNil(t, lookup, "multi-port rule must match ports via a set lookup")
sets, err := testClient.GetSets(manager.family4.workTable)
require.NoError(t, err, "failed to get sets")
var portSet *nftables.Set
for _, s := range sets {
if s.Name == lookup.SetName {
portSet = s
}
}
require.NotNil(t, portSet, "anonymous port set not found in kernel")
portSet.Table = manager.family4.workTable
elements, err := testClient.GetSetElements(portSet)
require.NoError(t, err, "failed to get set elements")
ports := make(map[uint16]bool)
for _, e := range elements {
require.Len(t, e.Key, 2, "port set element key should be 2 bytes")
ports[binary.BigEndian.Uint16(e.Key)] = true
}
require.True(t, ports[80], "port set should contain port 80")
require.True(t, ports[443], "port set should contain port 443")
require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete rule")
rules, err = testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules)
require.NoError(t, err, "failed to get rules after delete")
for _, kernelRule := range rules {
require.NotEqual(t, string(rule.ID()), string(kernelRule.UserData), "rule should be removed from kernel")
}
}
func compareExprsIgnoringCounters(t *testing.T, got, want []expr.Any) {
t.Helper()
require.Equal(t, len(got), len(want), "expression count mismatch")
client/firewall/nftables/router_linux.go
-2268
View File
File diff suppressed because it is too large Load Diff
client/firewall/nftables/router_linux_test.go
+156 -49
View File
@@ -37,7 +37,7 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
for _, testCase := range test.InsertRuleTestCases {
t.Run(testCase.Name, func(t *testing.T) {
// need fw manager to init both acl mgr and router for all chains to be present
// need fw manager to init both acl mgr and family for all chains to be present
manager, err := Create(ifaceMock, iface.DefaultMTU)
t.Cleanup(func() {
require.NoError(t, manager.Close(nil))
@@ -47,7 +47,7 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
nftablesTestingClient := &nftables.Conn{}
rtr := manager.router
rtr := manager.family4
err = rtr.AddNatRule(testCase.InputPair)
require.NoError(t, err, "pair should be inserted")
@@ -90,9 +90,9 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
}
// Build CIDR matching expressions
testRouter := &router{af: afIPv4}
sourceExp := testRouter.applyPrefix(testCase.InputPair.Source.Prefix, true)
destExp := testRouter.applyPrefix(testCase.InputPair.Destination.Prefix, false)
testRouter := &family{af: afIPv4}
sourceExp := prefixMatchExprs(testRouter.af, testCase.InputPair.Source.Prefix, true)
destExp := prefixMatchExprs(testRouter.af, testCase.InputPair.Destination.Prefix, false)
// Combine all expressions in the correct order
// nolint:gocritic
@@ -100,14 +100,14 @@ func TestNftablesManager_AddNatRule(t *testing.T) {
testingExpression = append(testingExpression, sourceExp...)
testingExpression = append(testingExpression, destExp...)
natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
natRuleKey := testCase.InputPair.GenKey(firewall.PreroutingFormat)
found := 0
for _, chain := range rtr.chains {
if chain.Name == chainNameManglePrerouting {
rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
for _, rule := range rules {
if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey {
// Compare expressions up to the mark setting expressions
require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "prerouting nat rule elements should match")
found = 1
@@ -135,19 +135,19 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) {
require.NoError(t, err)
require.NoError(t, manager.Init(nil))
rtr := manager.router
rtr := manager.family4
// First add the NAT rule using the router's method
// First add the NAT rule using the family's method
err = rtr.AddNatRule(testCase.InputPair)
require.NoError(t, err, "should add NAT rule")
// Verify the rule was added
natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
natRuleKey := testCase.InputPair.GenKey(firewall.PreroutingFormat)
found := false
rules, err := rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameManglePrerouting])
require.NoError(t, err, "should list rules")
for _, rule := range rules {
if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey {
found = true
break
}
@@ -163,7 +163,7 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) {
rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameManglePrerouting])
require.NoError(t, err, "should list rules after removal")
for _, rule := range rules {
if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey {
found = true
break
}
@@ -200,11 +200,10 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
defer deleteWorkTable()
r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, err, "Failed to create router")
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func(r *router) {
defer func(r *family) {
require.NoError(t, r.Reset(), "Failed to reset rules")
}(r)
@@ -314,16 +313,16 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ruleKey, err := r.AddRouteFiltering(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action)
require.NoError(t, err, "AddRouteFiltering failed")
ruleKey, err := r.AddFilterRule(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action)
require.NoError(t, err, "AddFilterRule failed")
t.Cleanup(func() {
require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule")
require.NoError(t, r.DeleteFilterRule(ruleKey), "Failed to delete rule")
})
// Check if the rule is in the internal map
rule, ok := r.rules[ruleKey.ID()]
assert.True(t, ok, "Rule not found in internal map")
stored, ok := r.filters[id.RuleID(ruleKey.ID())]
require.True(t, ok, "Rule not found in filters map")
rule := stored.nftRule
t.Log("Internal rule expressions:")
for i, expr := range rule.Exprs {
@@ -339,7 +338,7 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
var nftRule *nftables.Rule
for _, rule := range rules {
if string(rule.UserData) == ruleKey.ID() {
if firewall.RuleID(rule.UserData) == ruleKey.ID() {
nftRule = rule
break
}
@@ -367,12 +366,11 @@ func TestNftablesCreateIpSet(t *testing.T) {
defer deleteWorkTable()
r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, err, "Failed to create router")
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func() {
require.NoError(t, r.Reset(), "Failed to reset router")
require.NoError(t, r.Reset(), "Failed to reset family")
}()
tests := []struct {
@@ -509,6 +507,58 @@ func TestNftablesCreateIpSet(t *testing.T) {
}
}
// TestNftablesUpdateSetMergesOverlapping verifies that UpdateSet merges
// overlapping prefixes before adding them. An interval set rejects
// overlapping elements, so without the merge a batch holding a /32 already
// covered by a /24, or a duplicate address as DNS resolution can produce,
// would fail.
func TestNftablesUpdateSetMergesOverlapping(t *testing.T) {
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
}
workTable, err := createWorkTable()
require.NoError(t, err, "create work table")
defer deleteWorkTable()
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func() {
require.NoError(t, r.Reset(), "reset family")
}()
initial := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}
set := firewall.NewPrefixSet(initial)
created, err := r.createIpSet(set.HashedName(), setInput{prefixes: initial})
require.NoError(t, err, "create ip set")
require.NotNil(t, created)
overlapping := []netip.Prefix{
netip.MustParsePrefix("192.168.1.0/24"),
netip.MustParsePrefix("192.168.1.1/32"),
netip.MustParsePrefix("192.168.1.1/32"),
}
require.NoError(t, r.UpdateSet(set, overlapping), "UpdateSet must merge overlapping prefixes")
fetchedSet, err := r.conn.GetSetByName(r.workTable, set.HashedName())
require.NoError(t, err, "fetch updated set")
elements, err := r.conn.GetSetElements(fetchedSet)
require.NoError(t, err, "get set elements")
starts := make(map[string]bool)
for _, elem := range elements {
if elem.IntervalEnd {
continue
}
starts[netip.AddrFrom4(*(*[4]byte)(elem.Key)).String()] = true
}
// The /32s are covered by the /24, so the update adds one interval and
// leaves the one created earlier in place.
assert.Equal(t, map[string]bool{"10.0.0.0": true, "192.168.1.0": true}, starts,
"merged set must hold the original and the merged interval")
}
func TestNftablesCreateIpSet_IPv6(t *testing.T) {
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
@@ -518,11 +568,10 @@ func TestNftablesCreateIpSet_IPv6(t *testing.T) {
require.NoError(t, err, "Failed to create v6 work table")
defer deleteWorkTableIPv6()
r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, err, "Failed to create router")
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func() {
require.NoError(t, r.Reset(), "Failed to reset router")
require.NoError(t, r.Reset(), "Failed to reset family")
}()
tests := []struct {
@@ -748,6 +797,14 @@ func containsPort(exprs []expr.Any, port *firewall.Port, isSource bool) bool {
}
}
}
case *expr.Lookup:
// Multiple discrete ports compile to an anonymous set lookup
// rather than a chain of comparisons. The set's id and name are
// assigned dynamically, so matching the lookup is enough here;
// the set elements are verified separately.
if !port.IsRange && len(port.Values) > 1 {
portMatchFound = true
}
}
if payloadFound && portMatchFound {
return true
@@ -861,13 +918,12 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) {
require.NoError(t, err)
defer deleteWorkTable()
r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, err)
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func() { require.NoError(t, r.Reset()) }()
// Add a real rule to the kernel
ruleKey, err := r.AddRouteFiltering(
ruleKey, err := r.AddFilterRule(
nil,
[]netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")},
firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")},
@@ -878,11 +934,11 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) {
)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, r.DeleteRouteRule(ruleKey))
require.NoError(t, r.DeleteFilterRule(ruleKey))
})
// Inject a stale entry with Handle=0 (simulates store-before-flush failure)
staleKey := "stale-rule-that-does-not-exist"
staleKey := firewall.RuleID("stale-rule-that-does-not-exist")
r.rules[staleKey] = &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingFw],
@@ -902,6 +958,54 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) {
assert.NotZero(t, realRule.Handle, "real rule should have a valid handle")
}
// TestRouter_DeleteRouteRule_RemovesKernelRule verifies a route filter
// rule is actually removed from the kernel on delete. The route chain is
// not refreshed by Flush, so the stored rule carries a zero handle;
// DeleteFilterRule must pull live handles itself before issuing the
// delete or the kernel rule leaks. Regression test for that path.
func TestRouter_DeleteRouteRule_RemovesKernelRule(t *testing.T) {
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
}
workTable, err := createWorkTable()
require.NoError(t, err)
defer deleteWorkTable()
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func() { require.NoError(t, r.Reset()) }()
ruleKey, err := r.AddFilterRule(
nil,
[]netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")},
firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")},
firewall.ProtocolTCP,
nil,
&firewall.Port{Values: []uint16{80}},
firewall.ActionAccept,
)
require.NoError(t, err)
countKernelRules := func() int {
list, err := r.conn.GetRules(r.workTable, r.chains[chainNameRoutingFw])
require.NoError(t, err)
n := 0
for _, rule := range list {
if string(rule.UserData) == string(ruleKey.ID()) {
n++
}
}
return n
}
require.Equal(t, 1, countKernelRules(), "rule should be present in the kernel after add")
require.NoError(t, r.DeleteFilterRule(ruleKey))
assert.Equal(t, 0, countKernelRules(), "rule must be removed from the kernel after delete")
assert.NotContains(t, r.filters, ruleKey.ID(), "filters map entry should be cleared")
}
func TestRouter_DeleteRouteRule_StaleHandle(t *testing.T) {
if check() != NFTABLES {
t.Skip("nftables not supported on this system")
@@ -911,24 +1015,27 @@ func TestRouter_DeleteRouteRule_StaleHandle(t *testing.T) {
require.NoError(t, err)
defer deleteWorkTable()
r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, err)
r := newFamily(workTable, ifaceMock, iface.DefaultMTU)
require.NoError(t, r.init(workTable))
defer func() { require.NoError(t, r.Reset()) }()
// Inject a stale entry with Handle=0
staleKey := "stale-route-rule"
r.rules[staleKey] = &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingFw],
Handle: 0,
UserData: []byte(staleKey),
staleKey := id.RuleID("stale-route-rule")
staleRule := &Rule{
nftRule: &nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingFw],
Handle: 0,
UserData: []byte(staleKey),
},
id: staleKey,
}
r.filters[staleKey] = staleRule
// DeleteRouteRule should not return an error for stale handles
err = r.DeleteRouteRule(id.RuleID(staleKey))
// DeleteFilterRule should not return an error for stale handles
err = r.DeleteFilterRule(staleRule)
assert.NoError(t, err, "deleting a stale rule should not error")
assert.NotContains(t, r.rules, staleKey, "stale entry should be cleaned up")
assert.NotContains(t, r.filters, staleKey, "stale entry should be cleaned up")
}
func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) {
@@ -950,7 +1057,7 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) {
Masquerade: true,
}
rtr := manager.router
rtr := manager.family4
// First add succeeds
err = rtr.AddNatRule(pair)
@@ -960,11 +1067,11 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) {
})
// Corrupt the handle to simulate stale state
natRuleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
natRuleKey := pair.GenKey(firewall.PreroutingFormat)
if rule, exists := rtr.rules[natRuleKey]; exists {
rule.Handle = 0
}
inverseKey := firewall.GenKey(firewall.PreroutingFormat, firewall.GetInversePair(pair))
inverseKey := firewall.GetInversePair(pair).GenKey(firewall.PreroutingFormat)
if rule, exists := rtr.rules[inverseKey]; exists {
rule.Handle = 0
}
@@ -979,7 +1086,7 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) {
found := 0
for _, rule := range rules {
if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey {
found++
}
}
@@ -1010,7 +1117,7 @@ func TestCalculateLastIP(t *testing.T) {
}
func TestConvertPrefixesToSet_IPv6(t *testing.T) {
r := &router{af: afIPv6}
r := &family{af: afIPv6}
prefixes := []netip.Prefix{
netip.MustParsePrefix("fd00::/64"),
netip.MustParsePrefix("2001:db8::1/128"),
client/firewall/nftables/routing_linux.go
+558
View File
@@ -0,0 +1,558 @@
//go:build !android
package nftables
import (
"fmt"
"strings"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
nberrors "github.com/netbirdio/netbird/client/errors"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
nbnet "github.com/netbirdio/netbird/client/net"
)
func (r *family) AddNatRule(pair firewall.RouterPair) error {
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
// Resolve every rule's match expressions before queueing any of them: a
// message buffered on the shared connection cannot be un-queued, so
// returning an error after queueing would leave the next caller's Flush
// to commit a rule nothing tracks.
var legacyExprs []expr.Any
if r.legacyManagement {
log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination)
var err error
legacyExprs, err = r.legacyRouteRuleExprs(pair)
if err != nil {
return fmt.Errorf("build legacy routing rule: %w", err)
}
}
inverse := firewall.GetInversePair(pair)
var natExprs, inverseExprs []expr.Any
if pair.Masquerade {
var err error
natExprs, err = r.natRuleExprs(pair)
if err != nil {
r.dropNetworkMatch(legacyExprs)
return fmt.Errorf("build nat rule: %w", err)
}
inverseExprs, err = r.natRuleExprs(inverse)
if err != nil {
r.dropNetworkMatch(legacyExprs)
r.dropNetworkMatch(natExprs)
return fmt.Errorf("build inverse nat rule: %w", err)
}
}
if legacyExprs != nil {
r.queueLegacyRouteRule(pair, legacyExprs)
}
if pair.Masquerade {
r.queueNatRule(pair, natExprs)
r.queueNatRule(inverse, inverseExprs)
}
if err := r.conn.Flush(); err != nil {
r.rollbackRules(pair)
return fmt.Errorf("insert rules for %s: %w", pair.Destination, err)
}
return nil
}
// rollbackRules cleans up unflushed rules and their set counters after a flush failure.
func (r *family) rollbackRules(pair firewall.RouterPair) {
keys := []firewall.RuleID{
pair.GenKey(firewall.ForwardingFormat),
pair.GenKey(firewall.PreroutingFormat),
firewall.GetInversePair(pair).GenKey(firewall.PreroutingFormat),
}
for _, key := range keys {
rule, ok := r.rules[key]
if !ok {
continue
}
if err := r.decrementSetCounter(rule); err != nil {
log.Warnf("rollback set counter for %s: %v", key, err)
}
delete(r.rules, key)
}
}
// natRuleExprs resolves the match expressions of the pair's prerouting
// marking rule. It reserves the ipset references the matches need but queues
// nothing on the connection, so its error paths leave the connection clean.
func (r *family) natRuleExprs(pair firewall.RouterPair) ([]expr.Any, error) {
sourceExp, err := r.applyNetwork(pair.Source, nil, true)
if err != nil {
return nil, fmt.Errorf("apply source: %w", err)
}
destExp, err := r.applyNetwork(pair.Destination, nil, false)
if err != nil {
r.dropNetworkMatch(sourceExp)
return nil, fmt.Errorf("apply destination: %w", err)
}
op := expr.CmpOpEq
if pair.Inverse {
op = expr.CmpOpNeq
}
exprs := []expr.Any{
&expr.Meta{
Key: expr.MetaKeyIIFNAME,
Register: 1,
},
&expr.Cmp{
Op: op,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
}
// We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading.
// Masquerading will take care of the conntrack state, which means we won't need to mark established connections.
exprs = append(exprs, getCtNewExprs()...)
exprs = append(exprs, sourceExp...)
exprs = append(exprs, destExp...)
markValue := nbnet.PreroutingFwmarkMasquerade
if pair.Inverse {
markValue = nbnet.PreroutingFwmarkMasqueradeReturn
}
exprs = append(exprs,
&expr.Immediate{
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(markValue),
},
&expr.Meta{
Key: expr.MetaKeyMARK,
SourceRegister: true,
Register: 1,
},
)
return exprs, nil
}
// queueNatRule replaces any tracked rule for the pair and queues the new
// prerouting marking rule on the connection. Failures are logged rather than
// returned: the caller has already queued messages that only a Flush can
// commit, so it must not return early.
func (r *family) queueNatRule(pair firewall.RouterPair, exprs []expr.Any) {
ruleID := pair.GenKey(firewall.PreroutingFormat)
if _, exists := r.rules[ruleID]; exists {
if err := r.removeNatRule(pair); err != nil {
// The rule this replaces may still be in the kernel. Keep tracking
// it and skip the new one: overwriting the entry would leave the old
// rule installed with nothing that can find it again, while keeping
// it lets the next update retry the whole replacement.
log.Errorf("replace prerouting rule %s: %v", ruleID, err)
r.dropNetworkMatch(exprs)
return
}
}
// Ensure nat rules come first, so the mark can be overwritten.
// Currently overwritten by the dst-type LOCAL rules for redirected traffic.
r.rules[ruleID] = r.conn.InsertRule(&nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameManglePrerouting],
Exprs: exprs,
UserData: []byte(ruleID),
})
}
func (r *family) addPostroutingRules() {
// First masquerade rule for traffic coming in from WireGuard interface
exprs := []expr.Any{
// Match on the first fwmark
&expr.Meta{
Key: expr.MetaKeyMARK,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade),
},
// We need to exclude the loopback interface as this changes the ebpf proxy port
&expr.Meta{
Key: expr.MetaKeyOIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: ifname("lo"),
},
&expr.Counter{},
&expr.Masq{},
}
r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingNat],
Exprs: exprs,
})
// Second masquerade rule for traffic going out through WireGuard interface
exprs2 := []expr.Any{
// Match on the second fwmark
&expr.Meta{
Key: expr.MetaKeyMARK,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn),
},
// Match WireGuard interface
&expr.Meta{
Key: expr.MetaKeyOIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Counter{},
&expr.Masq{},
}
r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingNat],
Exprs: exprs2,
})
}
// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic.
func (r *family) addMSSClampingRules() error {
overhead := uint16(ipv4TCPHeaderSize)
if r.af.tableFamily == nftables.TableFamilyIPv6 {
overhead = ipv6TCPHeaderSize
}
if r.mtu <= overhead {
log.Debugf("MTU %d too small for MSS clamping (overhead %d), skipping", r.mtu, overhead)
return nil
}
mss := r.mtu - overhead
exprsOut := []expr.Any{
&expr.Meta{
Key: expr.MetaKeyOIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname(r.wgIface.Name()),
},
&expr.Meta{
Key: expr.MetaKeyL4PROTO,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_TCP},
},
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 13,
Len: 1,
},
&expr.Bitwise{
DestRegister: 1,
SourceRegister: 1,
Len: 1,
Mask: []byte{0x02},
Xor: []byte{0x00},
},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: []byte{0x00},
},
&expr.Counter{},
&expr.Exthdr{
DestRegister: 1,
Type: 2,
Offset: 2,
Len: 2,
Op: expr.ExthdrOpTcpopt,
},
&expr.Cmp{
Op: expr.CmpOpGt,
Register: 1,
Data: binaryutil.BigEndian.PutUint16(uint16(mss)),
},
&expr.Immediate{
Register: 1,
Data: binaryutil.BigEndian.PutUint16(uint16(mss)),
},
&expr.Exthdr{
SourceRegister: 1,
Type: 2,
Offset: 2,
Len: 2,
Op: expr.ExthdrOpTcpopt,
},
}
r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameMangleForward],
Exprs: exprsOut,
})
return r.conn.Flush()
}
func buildLegacyRouteRuleExpressions(sourceExp, destExp []expr.Any) []expr.Any {
exprs := make([]expr.Any, 0, len(sourceExp)+len(destExp)+2)
exprs = append(exprs, sourceExp...)
exprs = append(exprs, destExp...)
exprs = append(exprs,
&expr.Counter{},
&expr.Verdict{Kind: expr.VerdictAccept},
)
return exprs
}
// legacyRouteRuleExprs resolves the match expressions of the pair's legacy
// forwarding rule, queueing nothing on the connection.
func (r *family) legacyRouteRuleExprs(pair firewall.RouterPair) ([]expr.Any, error) {
sourceExp, err := r.applyNetwork(pair.Source, nil, true)
if err != nil {
return nil, fmt.Errorf("apply source: %w", err)
}
destExp, err := r.applyNetwork(pair.Destination, nil, false)
if err != nil {
r.dropNetworkMatch(sourceExp)
return nil, fmt.Errorf("apply destination: %w", err)
}
return buildLegacyRouteRuleExpressions(sourceExp, destExp), nil
}
// queueLegacyRouteRule replaces any tracked rule for the pair and queues the
// new legacy forwarding rule. Failures are logged for the same reason as in
// queueNatRule.
func (r *family) queueLegacyRouteRule(pair firewall.RouterPair, exprs []expr.Any) {
ruleID := pair.GenKey(firewall.ForwardingFormat)
if _, exists := r.rules[ruleID]; exists {
if err := r.removeLegacyRouteRule(pair); err != nil {
// Keep the old rule tracked instead of losing it, as in queueNatRule.
log.Errorf("replace legacy forwarding rule %s: %v", ruleID, err)
r.dropNetworkMatch(exprs)
return
}
}
r.rules[ruleID] = r.conn.AddRule(&nftables.Rule{
Table: r.workTable,
Chain: r.chains[chainNameRoutingFw],
Exprs: exprs,
UserData: []byte(ruleID),
})
}
// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls
func (r *family) removeLegacyRouteRule(pair firewall.RouterPair) error {
ruleID := pair.GenKey(firewall.ForwardingFormat)
rule, exists := r.rules[ruleID]
if !exists {
return nil
}
return r.deleteLegacyRuleEntry(ruleID, rule)
}
// deleteLegacyRuleEntry removes one legacy forwarding rule and drops its
// ipset references. It also clears stale entries that never got a handle.
func (r *family) deleteLegacyRuleEntry(ruleID firewall.RuleID, rule *nftables.Rule) error {
if rule.Handle == 0 {
log.Warnf("legacy forwarding rule %s has no handle, removing stale entry", ruleID)
if err := r.decrementSetCounter(rule); err != nil {
log.Warnf("decrement set counter for stale rule %s: %v", ruleID, err)
}
delete(r.rules, ruleID)
return nil
}
if err := r.conn.DelRule(rule); err != nil {
return fmt.Errorf("remove legacy forwarding rule %s: %w", ruleID, err)
}
delete(r.rules, ruleID)
if err := r.decrementSetCounter(rule); err != nil {
return fmt.Errorf("decrement set counter: %w", err)
}
return nil
}
// GetLegacyManagement returns the route manager's legacy management mode
func (r *family) GetLegacyManagement() bool {
return r.legacyManagement
}
// SetLegacyManagement sets the route manager to use legacy management mode
func (r *family) SetLegacyManagement(isLegacy bool) {
r.legacyManagement = isLegacy
}
// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls
func (r *family) RemoveAllLegacyRouteRules() error {
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
var merr *multierror.Error
var found bool
for k, rule := range r.rules {
if !strings.HasPrefix(string(k), firewall.ForwardingFormatPrefix) {
continue
}
found = true
if err := r.deleteLegacyRuleEntry(k, rule); err != nil {
merr = multierror.Append(merr, err)
}
}
// Commit the queued deletes here instead of leaving them for whichever
// caller flushes next: the tracking entries are already gone, so an
// uncommitted delete would leave a rule in the kernel that nothing can
// find again.
if found {
if err := r.conn.Flush(); err != nil {
merr = multierror.Append(merr, fmt.Errorf(flushError, err))
}
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) removeNatPreroutingRules() error {
table := &nftables.Table{
Name: tableNat,
Family: r.af.tableFamily,
}
chain := &nftables.Chain{
Name: chainNameNatPrerouting,
Table: table,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityNATDest,
Type: nftables.ChainTypeNAT,
}
rules, err := r.conn.GetRules(table, chain)
if err != nil {
return fmt.Errorf("get rules from nat table: %w", err)
}
var merr *multierror.Error
// Delete rules that have our UserData suffix
for _, rule := range rules {
if len(rule.UserData) == 0 || !strings.HasSuffix(string(rule.UserData), string(dnatSuffix)) {
continue
}
if err := r.conn.DelRule(rule); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete rule %s: %w", rule.UserData, err))
}
}
if err := r.conn.Flush(); err != nil {
merr = multierror.Append(merr, fmt.Errorf(flushError, err))
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) RemoveNatRule(pair firewall.RouterPair) error {
if err := r.refreshRulesMap(); err != nil {
return fmt.Errorf(refreshRulesMapError, err)
}
var merr *multierror.Error
if pair.Masquerade {
if err := r.removeNatRule(pair); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove prerouting rule: %w", err))
}
if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove inverse prerouting rule: %w", err))
}
}
if err := r.removeLegacyRouteRule(pair); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove legacy routing rule: %w", err))
}
// Set counters are decremented in the sub-methods above before flush. If flush fails,
// counters will be off until the next successful removal or refresh cycle.
if err := r.conn.Flush(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("flush remove nat rules %s: %w", pair.Destination, err))
}
return nberrors.FormatErrorOrNil(merr)
}
func (r *family) removeNatRule(pair firewall.RouterPair) error {
ruleID := pair.GenKey(firewall.PreroutingFormat)
rule, exists := r.rules[ruleID]
if !exists {
log.Debugf("prerouting rule %s not found", ruleID)
return nil
}
if rule.Handle == 0 {
log.Warnf("prerouting rule %s has no handle, removing stale entry", ruleID)
if err := r.decrementSetCounter(rule); err != nil {
log.Warnf("decrement set counter for stale rule %s: %v", ruleID, err)
}
delete(r.rules, ruleID)
return nil
}
if err := r.conn.DelRule(rule); err != nil {
return fmt.Errorf("remove prerouting rule %s -> %s: %w", pair.Source, pair.Destination, err)
}
log.Debugf("removed prerouting rule %s -> %s", pair.Source, pair.Destination)
delete(r.rules, ruleID)
if err := r.decrementSetCounter(rule); err != nil {
return fmt.Errorf("decrement set counter: %w", err)
}
return nil
}
client/firewall/nftables/rule_linux.go
+13 -8
View File
@@ -1,21 +1,26 @@
package nftables
import (
"net"
"net/netip"
"github.com/google/nftables"
"github.com/netbirdio/netbird/client/firewall/manager"
)
// Rule to handle management of rules
// Rule wraps an installed filter rule (peer or route). Source set
// membership is encoded in the rule's expressions; DeleteFilterRule
// recovers the set name via findSets so the refcounter can drop the
// right reference. mangleRule is set only for peer rules.
type Rule struct {
nftRule *nftables.Rule
mangleRule *nftables.Rule
nftSet *nftables.Set
ruleID string
ip net.IP
// sources is the canonical source list this rule was created for.
sources []netip.Prefix
id manager.RuleID
}
// GetRuleID returns the rule id
func (r *Rule) ID() string {
return r.ruleID
// ID returns the rule id
func (r *Rule) ID() manager.RuleID {
return r.id
}
client/firewall/nftables/testhelpers_linux_test.go
+27
View File
@@ -0,0 +1,27 @@
//go:build privileged
package nftables
import (
"fmt"
"net"
"net/netip"
)
func pfx(ip net.IP) []netip.Prefix {
if ip == nil {
return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)}
}
if ip.IsUnspecified() {
if ip.To4() != nil {
return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)}
}
return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)}
}
a, ok := netip.AddrFromSlice(ip)
if !ok {
panic(fmt.Sprintf("invalid IP length: %d", len(ip)))
}
a = a.Unmap()
return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())}
}