package nftables import ( "bytes" "encoding/binary" "fmt" "net" "net/netip" "strconv" "strings" "time" "github.com/google/nftables" "github.com/google/nftables/expr" log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/iface" ) const ( // rules chains contains the effective ACL rules chainNameInputRules = "netbird-acl-input-rules" chainNameOutputRules = "netbird-acl-output-rules" // filter chains contains the rules that jump to the rules chains chainNameInputFilter = "netbird-acl-input-filter" chainNameOutputFilter = "netbird-acl-output-filter" chainNameForwardFilter = "netbird-acl-forward-filter" allowNetbirdInputRuleID = "allow Netbird incoming traffic" ) var ( anyIP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} nullAddress4 = []byte{0x0, 0x0, 0x0, 0x0} nullAddress6 = []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0} postroutingMark = []byte{0xe4, 0x7, 0x0, 0x00} ) type AclManager struct { rConn *nftables.Conn sConn *nftables.Conn wgIface iFaceMapper routeingFwChainName string workTable *nftables.Table workTable6 *nftables.Table v6Active bool chainInputRules *nftables.Chain chainOutputRules *nftables.Chain chainFwFilter *nftables.Chain chainPrerouting *nftables.Chain chainInputRules6 *nftables.Chain chainOutputRules6 *nftables.Chain chainFwFilter6 *nftables.Chain chainPrerouting6 *nftables.Chain ipsetStore *ipsetStore ipsetStore6 *ipsetStore rules map[string]*Rule } // iFaceMapper defines subset methods of interface required for manager type iFaceMapper interface { Name() string Address() iface.WGAddress Address6() *iface.WGAddress IsUserspaceBind() bool } func newAclManager(table *nftables.Table, table6 *nftables.Table, wgIface iFaceMapper, routeingFwChainName string) (*AclManager, error) { // sConn is used for creating sets and adding/removing elements from them // it's differ then rConn (which does create new conn for each flush operation) // and is permanent. Using same connection for booth type of operations // overloads netlink with high amount of rules ( > 10000) sConn, err := nftables.New(nftables.AsLasting()) if err != nil { return nil, err } m := &AclManager{ rConn: &nftables.Conn{}, sConn: sConn, wgIface: wgIface, workTable: table, workTable6: table6, v6Active: wgIface.Address6() != nil, routeingFwChainName: routeingFwChainName, ipsetStore: newIpsetStore(), ipsetStore6: newIpsetStore(), rules: make(map[string]*Rule), } err = m.createDefaultChains(false) if err != nil { return nil, err } if m.v6Active { err = m.createDefaultChains(true) if err != nil { return nil, err } } return m, nil } // PrepareV6Reset prepares the ACL manager for a full V6 table reset (necessary if IPv6 address changes). // Deletes all IPv6 rules and sets, and sets the V6 active status to false. func (m *AclManager) PrepareV6Reset() (*nftables.Table, error) { if m.workTable6 != nil { for k, r := range m.rules { if r.ip.To4() == nil { err := m.DeleteRule(r) if err != nil { return nil, err } delete(m.rules, k) } } sets, err := m.rConn.GetSets(m.workTable6) if err != nil { for _, set := range sets { m.rConn.DelSet(set) } } m.ipsetStore6 = newIpsetStore() } // Set to false just in case of concurrent accesses (will be set to actual values in ReinitAfterV6Reset()). m.v6Active = false // return the current work table. return m.workTable6, nil } // ReinitAfterV6Reset reinitializes the IPv6 table after an IPv6 address change. func (m *AclManager) ReinitAfterV6Reset(workTable6 *nftables.Table) error { // If we have an IPv6 address after the address update, initialize firewall table, else not. if m.wgIface.Address6() != nil { m.workTable6 = workTable6 err := m.createDefaultChains(true) if err != nil { return err } m.v6Active = true } else { m.workTable6 = nil m.v6Active = false } return nil } // AddFiltering rule to the firewall // // If comment argument is empty firewall manager should set // rule ID as comment for the rule func (m *AclManager) AddFiltering( ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, ipsetName string, comment string, ) ([]firewall.Rule, error) { var ipset *nftables.Set if ipsetName != "" { var err error ipset, err = m.addIpToSet(ipsetName, ip) if err != nil { return nil, err } } if !m.v6Active && ip.To4() == nil { return nil, fmt.Errorf("attempted to configure filtering for IPv6 address even though IPv6 is not active") } newRules := make([]firewall.Rule, 0, 2) ioRule, err := m.addIOFiltering(ip, proto, sPort, dPort, direction, action, ipset, comment) if err != nil { return nil, err } newRules = append(newRules, ioRule) if !shouldAddToPrerouting(proto, dPort, direction) { return newRules, nil } preroutingRule, err := m.addPreroutingFiltering(ipset, proto, dPort, ip) if err != nil { return newRules, err } newRules = append(newRules, preroutingRule) return newRules, nil } // DeleteRule from the firewall by rule definition func (m *AclManager) DeleteRule(rule firewall.Rule) error { r, ok := rule.(*Rule) if !ok { return fmt.Errorf("invalid rule type") } ipsetStorage := m.ipsetStore if r.ip.To4() == nil { ipsetStorage = m.ipsetStore6 } if r.nftSet == nil { err := m.rConn.DelRule(r.nftRule) if err != nil { log.Errorf("failed to delete rule: %v", err) } delete(m.rules, r.GetRuleID()) return m.rConn.Flush() } ips, ok := ipsetStorage.ips(r.nftSet.Name) if !ok { err := m.rConn.DelRule(r.nftRule) if err != nil { log.Errorf("failed to delete rule: %v", err) } delete(m.rules, r.GetRuleID()) return m.rConn.Flush() } if _, ok := ips[r.ip.String()]; ok { rawIP := r.ip.To4() if rawIP == nil { rawIP = r.ip.To16() } err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: rawIP}}) if err != nil { log.Errorf("delete elements for set %q: %v", r.nftSet.Name, err) } if err := m.sConn.Flush(); err != nil { log.Debugf("flush error of set delete element, %s", r.nftSet.Name) return err } ipsetStorage.DeleteIpFromSet(r.nftSet.Name, r.ip) } // if after delete, set still contains other IPs, // no need to delete firewall rule and we should exit here if len(ips) > 0 { return nil } err := m.rConn.DelRule(r.nftRule) if err != nil { log.Errorf("failed to delete rule: %v", err) } err = m.rConn.Flush() if err != nil { return err } delete(m.rules, r.GetRuleID()) ipsetStorage.DeleteReferenceFromIpSet(r.nftSet.Name) if ipsetStorage.HasReferenceToSet(r.nftSet.Name) { return nil } // we delete last IP from the set, that means we need to delete // set itself and associated firewall rule too m.rConn.FlushSet(r.nftSet) m.rConn.DelSet(r.nftSet) ipsetStorage.deleteIpset(r.nftSet.Name) return nil } // createDefaultAllowRules In case if the USP firewall manager can use the native firewall manager we must create allow rules for // input and output chains func (m *AclManager) createDefaultAllowRules() error { expIn := []expr.Any{ &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4, }, // mask &expr.Bitwise{ SourceRegister: 1, DestRegister: 1, Len: 4, Mask: []byte{0x00, 0x00, 0x00, 0x00}, Xor: zeroXor, }, // net address &expr.Cmp{ Register: 1, Data: []byte{0x00, 0x00, 0x00, 0x00}, }, &expr.Verdict{ Kind: expr.VerdictAccept, }, } _ = m.rConn.InsertRule(&nftables.Rule{ Table: m.workTable, Chain: m.chainInputRules, Position: 0, Exprs: expIn, }) expOut := []expr.Any{ &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, }, // mask &expr.Bitwise{ SourceRegister: 1, DestRegister: 1, Len: 4, Mask: []byte{0x00, 0x00, 0x00, 0x00}, Xor: zeroXor, }, // net address &expr.Cmp{ Register: 1, Data: []byte{0x00, 0x00, 0x00, 0x00}, }, &expr.Verdict{ Kind: expr.VerdictAccept, }, } _ = m.rConn.InsertRule(&nftables.Rule{ Table: m.workTable, Chain: m.chainOutputRules, Position: 0, Exprs: expOut, }) err := m.rConn.Flush() if err != nil { log.Debugf("failed to create default allow rules: %s", err) return err } return nil } // Flush rule/chain/set operations from the buffer // // Method also get all rules after flush and refreshes handle values in the rulesets func (m *AclManager) Flush() error { if err := m.flushWithBackoff(); err != nil { return err } if err := m.refreshRuleHandles(m.workTable, m.chainInputRules); err != nil { log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err) } if err := m.refreshRuleHandles(m.workTable, m.chainOutputRules); err != nil { log.Errorf("failed to refresh rule handles IPv4 output chain: %v", err) } if err := m.refreshRuleHandles(m.workTable, m.chainPrerouting); err != nil { log.Errorf("failed to refresh rule handles IPv4 prerouting chain: %v", err) } if err := m.refreshRuleHandles(m.workTable6, m.chainInputRules6); err != nil { log.Errorf("failed to refresh rule handles IPv6 input chain: %v", err) } if err := m.refreshRuleHandles(m.workTable6, m.chainOutputRules6); err != nil { log.Errorf("failed to refresh rule handles IPv6 output chain: %v", err) } if err := m.refreshRuleHandles(m.workTable6, m.chainPrerouting6); err != nil { log.Errorf("failed to refresh rule handles IPv6 prerouting chain: %v", err) } return nil } func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, ipset *nftables.Set, comment string) (*Rule, error) { ruleId := generateRuleId(ip, sPort, dPort, direction, action, ipset) if r, ok := m.rules[ruleId]; ok { return &Rule{ r.nftRule, r.nftSet, r.ruleID, ip, }, nil } ifaceKey := expr.MetaKeyIIFNAME if direction == firewall.RuleDirectionOUT { ifaceKey = expr.MetaKeyOIFNAME } expressions := []expr.Any{ &expr.Meta{Key: ifaceKey, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, } if proto != firewall.ProtocolALL { expressions = append(expressions, &expr.Meta{ Key: expr.MetaKeyL4PROTO, Register: 1, }) var protoData []byte switch proto { case firewall.ProtocolTCP: protoData = []byte{unix.IPPROTO_TCP} case firewall.ProtocolUDP: protoData = []byte{unix.IPPROTO_UDP} case firewall.ProtocolICMP: protoData = []byte{unix.IPPROTO_ICMP} default: return nil, fmt.Errorf("unsupported protocol: %s", proto) } expressions = append(expressions, &expr.Cmp{ Register: 1, Op: expr.CmpOpEq, Data: protoData, }) } workTable := m.workTable // Raw bytes of IP to match (if IPv4). rawIP := ip.To4() // source address position (in IPv4) srcAddrOffset := uint32(12) // destination address position (in IPv4) dstAddrOffset := uint32(16) // address length addrLen := uint32(4) // IP set storage for IPv4. ipsetStorage := m.ipsetStore // If rawIP == nil, we have an IPv6 address, replace previously defined values with IPv6 counterparts. if rawIP == nil { rawIP = ip.To16() srcAddrOffset = uint32(8) dstAddrOffset = uint32(24) addrLen = 16 workTable = m.workTable6 ipsetStorage = m.ipsetStore6 } // check if rawIP contains zeroed IP address value // in that case not add IP match expression into the rule definition if !bytes.HasPrefix(anyIP, rawIP) { addrOffset := srcAddrOffset if direction == firewall.RuleDirectionOUT { addrOffset = dstAddrOffset } expressions = append(expressions, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: addrOffset, Len: addrLen, }, ) // add individual IP for match if no ipset defined if ipset == nil { expressions = append(expressions, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: rawIP, }, ) } else { expressions = append(expressions, &expr.Lookup{ SourceRegister: 1, SetName: ipset.Name, SetID: ipset.ID, }, ) } } if sPort != nil && len(sPort.Values) != 0 { expressions = append(expressions, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: encodePort(*sPort), }, ) } if dPort != nil && len(dPort.Values) != 0 { expressions = append(expressions, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: encodePort(*dPort), }, ) } switch action { case firewall.ActionAccept: expressions = append(expressions, &expr.Verdict{Kind: expr.VerdictAccept}) case firewall.ActionDrop: expressions = append(expressions, &expr.Verdict{Kind: expr.VerdictDrop}) } userData := []byte(strings.Join([]string{ruleId, comment}, " ")) var chain *nftables.Chain if direction == firewall.RuleDirectionIN { chain = m.chainInputRules } else { chain = m.chainOutputRules } nftRule := m.rConn.InsertRule(&nftables.Rule{ Table: workTable, Chain: chain, Position: 0, Exprs: expressions, UserData: userData, }) rule := &Rule{ nftRule: nftRule, nftSet: ipset, ruleID: ruleId, ip: ip, } m.rules[ruleId] = rule if ipset != nil { ipsetStorage.AddReferenceToIpset(ipset.Name) } return rule, nil } func (m *AclManager) addPreroutingFiltering(ipset *nftables.Set, proto firewall.Protocol, port *firewall.Port, ip net.IP) (*Rule, error) { var protoData []byte switch proto { case firewall.ProtocolTCP: protoData = []byte{unix.IPPROTO_TCP} case firewall.ProtocolUDP: protoData = []byte{unix.IPPROTO_UDP} case firewall.ProtocolICMP: protoData = []byte{unix.IPPROTO_ICMP} default: return nil, fmt.Errorf("unsupported protocol: %s", proto) } // Raw bytes of IP to match (if IPv4). rawIP := ip.To4() // source address position (in IPv4) srcAddrOffset := uint32(12) // destination address position (in IPv4) dstAddrOffset := uint32(16) // address length addrLen := uint32(4) // Raw bytes of the wireguard interface's IPv4 address. ifaceRawIP := m.wgIface.Address().IP.To4() // table to insert rule in workTable := m.workTable // chain to insert rule in preroutingChain := m.chainPrerouting // IP set store to use ipsetStorage := m.ipsetStore // If rawIP == nil, we have an IPv6 address, replace previously defined values with IPv6 counterparts. if rawIP == nil { rawIP = ip.To16() srcAddrOffset = uint32(8) dstAddrOffset = uint32(24) addrLen = 16 ifaceRawIP = m.wgIface.Address6().IP.To16() workTable = m.workTable6 preroutingChain = m.chainPrerouting6 ipsetStorage = m.ipsetStore6 } ruleId := generateRuleIdForMangle(ipset, ip, proto, port) if r, ok := m.rules[ruleId]; ok { return &Rule{ r.nftRule, r.nftSet, r.ruleID, ip, }, nil } var ipExpression expr.Any // add individual IP for match if no ipset defined if ipset == nil { ipExpression = &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: rawIP, } } else { ipExpression = &expr.Lookup{ SourceRegister: 1, SetName: ipset.Name, SetID: ipset.ID, } } expressions := []expr.Any{ &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: srcAddrOffset, Len: addrLen, }, ipExpression, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: dstAddrOffset, Len: addrLen, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifaceRawIP, }, &expr.Meta{ Key: expr.MetaKeyL4PROTO, Register: 1, }, &expr.Cmp{ Register: 1, Op: expr.CmpOpEq, Data: protoData, }, } if port != nil { expressions = append(expressions, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: encodePort(*port), }, ) } expressions = append(expressions, &expr.Immediate{ Register: 1, Data: postroutingMark, }, &expr.Meta{ Key: expr.MetaKeyMARK, SourceRegister: true, Register: 1, }, ) nftRule := m.rConn.InsertRule(&nftables.Rule{ Table: workTable, Chain: preroutingChain, Position: 0, Exprs: expressions, UserData: []byte(ruleId), }) if err := m.rConn.Flush(); err != nil { return nil, fmt.Errorf("flush insert rule: %v", err) } rule := &Rule{ nftRule: nftRule, nftSet: ipset, ruleID: ruleId, ip: ip, } m.rules[ruleId] = rule if ipset != nil { ipsetStorage.AddReferenceToIpset(ipset.Name) } return rule, nil } func (m *AclManager) createDefaultChains(forV6 bool) (err error) { workTable := m.workTable if forV6 { workTable = m.workTable6 } // chainNameInputRules chain := m.createChain(chainNameInputRules, workTable) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chain.Name, err) return err } chainInputRules := chain if forV6 { m.chainInputRules6 = chainInputRules } else { m.chainInputRules = chainInputRules } // chainNameOutputRules chain = m.createChain(chainNameOutputRules, workTable) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chainNameOutputRules, err) return err } chainOutputRules := chain if forV6 { m.chainOutputRules6 = chainOutputRules } else { m.chainOutputRules = chainOutputRules } // netbird-acl-input-filter // type filter hook input priority filter; policy accept; chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput, workTable) //netbird-acl-input-filter iifname "wt0" ip saddr 100.72.0.0/16 ip daddr != 100.72.0.0/16 accept m.addRouteAllowRule(chain, expr.MetaKeyIIFNAME) m.addFwdAllow(chain, expr.MetaKeyIIFNAME) m.addJumpRule(chain, chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules m.addDropExpressions(chain, expr.MetaKeyIIFNAME) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chainNameInputFilter, err) return err } // netbird-acl-output-filter // type filter hook output priority filter; policy accept; chain = m.createFilterChainWithHook(chainNameOutputFilter, nftables.ChainHookOutput, workTable) m.addRouteAllowRule(chain, expr.MetaKeyOIFNAME) m.addFwdAllow(chain, expr.MetaKeyOIFNAME) m.addJumpRule(chain, chainOutputRules.Name, expr.MetaKeyOIFNAME) // to netbird-acl-output-rules m.addDropExpressions(chain, expr.MetaKeyOIFNAME) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chainNameOutputFilter, err) return err } // netbird-acl-forward-filter chain = m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward, workTable) m.addJumpRulesToRtForward(workTable, chain) // to m.addMarkAccept(workTable, chain) m.addJumpRuleToInputChain(workTable, chain, chainInputRules) // to netbird-acl-input-rules m.addDropExpressions(chain, expr.MetaKeyIIFNAME) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err) return err } if forV6 { m.chainFwFilter6 = chain } else { m.chainFwFilter = chain } // netbird-acl-output-filter // type filter hook output priority filter; policy accept; chain = m.createPreroutingMangle(forV6) err = m.rConn.Flush() if err != nil { log.Debugf("failed to create chain (%s): %s", chain.Name, err) return err } if forV6 { m.chainPrerouting6 = chain } else { m.chainPrerouting = chain } return nil } func (m *AclManager) addJumpRulesToRtForward(table *nftables.Table, chain *nftables.Chain) { expressions := []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Verdict{ Kind: expr.VerdictJump, Chain: m.routeingFwChainName, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: table, Chain: chain, Exprs: expressions, }) expressions = []expr.Any{ &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Verdict{ Kind: expr.VerdictJump, Chain: m.routeingFwChainName, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: table, Chain: chain, Exprs: expressions, }) } func (m *AclManager) addMarkAccept(table *nftables.Table, chain *nftables.Chain) { // oifname "wt0" meta mark 0x000007e4 accept // iifname "wt0" meta mark 0x000007e4 accept ifaces := []expr.MetaKey{expr.MetaKeyIIFNAME, expr.MetaKeyOIFNAME} for _, iface := range ifaces { expressions := []expr.Any{ &expr.Meta{Key: iface, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Meta{ Key: expr.MetaKeyMARK, Register: 1, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: postroutingMark, }, &expr.Verdict{ Kind: expr.VerdictAccept, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: table, Chain: chain, Exprs: expressions, }) } } func (m *AclManager) createChain(name string, table *nftables.Table) *nftables.Chain { chain := &nftables.Chain{ Name: name, Table: table, } chain = m.rConn.AddChain(chain) return chain } func (m *AclManager) createFilterChainWithHook(name string, hookNum nftables.ChainHook, table *nftables.Table) *nftables.Chain { polAccept := nftables.ChainPolicyAccept chain := &nftables.Chain{ Name: name, Table: table, Hooknum: hookNum, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter, Policy: &polAccept, } return m.rConn.AddChain(chain) } func (m *AclManager) createPreroutingMangle(forV6 bool) *nftables.Chain { workTable := m.workTable // Raw bytes of the wireguard interface's IPv4 address. rawIP := m.wgIface.Address().Network.IP.To4() // Subnet mask of the wireguard interface's network. mask := m.wgIface.Address().Network.Mask // Length of an IPv4 address addrLen := uint32(4) // source address position srcAddrOffset := uint32(12) // destination address position dstAddrOffset := uint32(16) // An array representing a null address in IPv4 (0.0.0.0) nullAddressArray := nullAddress4 // If prerouting mangle should be created for IPv6 table, replace previously defined values with IPv6 counterparts. if forV6 { workTable = m.workTable6 rawIP = m.wgIface.Address6().Network.IP.To16() mask = m.wgIface.Address6().Network.Mask addrLen = 16 srcAddrOffset = uint32(8) dstAddrOffset = uint32(24) nullAddressArray = nullAddress6 // corresponds to :: } polAccept := nftables.ChainPolicyAccept chain := &nftables.Chain{ Name: "netbird-acl-prerouting-filter", Table: workTable, Hooknum: nftables.ChainHookPrerouting, Priority: nftables.ChainPriorityMangle, Type: nftables.ChainTypeFilter, Policy: &polAccept, } chain = m.rConn.AddChain(chain) ip, _ := netip.AddrFromSlice(rawIP) expressions := []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: srcAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: expr.CmpOpNeq, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: dstAddrOffset, Len: addrLen, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: rawIP, }, &expr.Immediate{ Register: 1, Data: postroutingMark, }, &expr.Meta{ Key: expr.MetaKeyMARK, SourceRegister: true, Register: 1, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: workTable, Chain: chain, Exprs: expressions, }) return chain } func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any { expressions := []expr.Any{ &expr.Meta{Key: ifaceKey, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Verdict{Kind: expr.VerdictDrop}, } _ = m.rConn.AddRule(&nftables.Rule{ Table: chain.Table, Chain: chain, Exprs: expressions, }) return nil } func (m *AclManager) addJumpRuleToInputChain(table *nftables.Table, chain *nftables.Chain, inputChain *nftables.Chain) { expressions := []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Verdict{ Kind: expr.VerdictJump, Chain: inputChain.Name, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: table, Chain: chain, Exprs: expressions, }) } func (m *AclManager) addRouteAllowRule(chain *nftables.Chain, netIfName expr.MetaKey) { // Raw bytes of the wireguard interface's IPv4 address. rawIP := m.wgIface.Address().Network.IP.To4() // Subnet mask of the wireguard interface's network. mask := m.wgIface.Address().Network.Mask // Length of an IPv4 address addrLen := uint32(4) // source address position srcAddrOffset := uint32(12) // destination address position dstAddrOffset := uint32(16) // An array representing a null address in IPv4 (0.0.0.0) nullAddressArray := nullAddress4 // If route allow rule should be created for IPv6 table, replace previously defined values with IPv6 counterparts. if chain.Table.Family == nftables.TableFamilyIPv6 { rawIP = m.wgIface.Address6().Network.IP.To16() mask = m.wgIface.Address6().Network.Mask addrLen = 16 srcAddrOffset = 8 dstAddrOffset = 24 nullAddressArray = nullAddress6 // corresponds to :: } ip, _ := netip.AddrFromSlice(rawIP) var srcOp, dstOp expr.CmpOp if netIfName == expr.MetaKeyIIFNAME { srcOp = expr.CmpOpEq dstOp = expr.CmpOpNeq } else { srcOp = expr.CmpOpNeq dstOp = expr.CmpOpEq } expressions := []expr.Any{ &expr.Meta{Key: netIfName, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: srcAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: srcOp, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: dstAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: dstOp, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Verdict{ Kind: expr.VerdictAccept, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: chain.Table, Chain: chain, Exprs: expressions, }) } func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) { // Raw bytes of the wireguard interface's IPv4 address. rawIP := m.wgIface.Address().Network.IP.To4() // Subnet mask of the wireguard interface's network. mask := m.wgIface.Address().Network.Mask // Length of an IPv4 address addrLen := uint32(4) // source address position srcAddrOffset := uint32(12) // destination address position dstAddrOffset := uint32(16) // An array representing a null address in IPv4 (0.0.0.0) nullAddressArray := nullAddress4 // If forward allow rule should be created for IPv6 table, replace previously defined values with IPv6 counterparts. if chain.Table.Family == nftables.TableFamilyIPv6 { rawIP = m.wgIface.Address6().Network.IP.To16() mask = m.wgIface.Address6().Network.Mask addrLen = 16 srcAddrOffset = 8 dstAddrOffset = 24 nullAddressArray = nullAddress6 // corresponds to :: } ip, _ := netip.AddrFromSlice(rawIP) var srcOp, dstOp expr.CmpOp if iifname == expr.MetaKeyIIFNAME { srcOp = expr.CmpOpNeq dstOp = expr.CmpOpEq } else { srcOp = expr.CmpOpEq dstOp = expr.CmpOpNeq } expressions := []expr.Any{ &expr.Meta{Key: iifname, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: srcAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: srcOp, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: dstAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: dstOp, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Verdict{ Kind: expr.VerdictAccept, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: chain.Table, Chain: chain, Exprs: expressions, }) } func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) { // Raw bytes of the wireguard interface's IPv4 address. rawIP := m.wgIface.Address().Network.IP.To4() // Subnet mask of the wireguard interface's network. mask := m.wgIface.Address().Network.Mask // Length of an IPv4 address addrLen := uint32(4) // source address position srcAddrOffset := uint32(12) // destination address position dstAddrOffset := uint32(16) // An array representing a null address in IPv4 (0.0.0.0) nullAddressArray := nullAddress4 // If jump rule should be created for IPv6 table, replace previously defined values with IPv6 counterparts. if chain.Table.Family == nftables.TableFamilyIPv6 { rawIP = m.wgIface.Address6().Network.IP.To16() mask = m.wgIface.Address6().Network.Mask addrLen = 16 srcAddrOffset = 8 dstAddrOffset = 24 nullAddressArray = nullAddress6 // corresponds to :: } ip, _ := netip.AddrFromSlice(rawIP) expressions := []expr.Any{ &expr.Meta{Key: ifaceKey, Register: 1}, &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, Data: ifname(m.wgIface.Name()), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: srcAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Payload{ DestRegister: 2, Base: expr.PayloadBaseNetworkHeader, Offset: dstAddrOffset, Len: addrLen, }, &expr.Bitwise{ SourceRegister: 2, DestRegister: 2, Len: addrLen, Xor: nullAddressArray, Mask: mask, }, &expr.Cmp{ Op: expr.CmpOpEq, Register: 2, Data: ip.Unmap().AsSlice(), }, &expr.Verdict{ Kind: expr.VerdictJump, Chain: to, }, } _ = m.rConn.AddRule(&nftables.Rule{ Table: chain.Table, Chain: chain, Exprs: expressions, }) } func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, error) { workTable := m.workTable // Raw bytes of the IPv4 address to add rawIP := ip.To4() // Type of set to add to ipsetType := nftables.TypeIPAddr // IP set store to use ipsetStorage := m.ipsetStore // If rawIP == nil, we have an IPv6 address, replace previously defined values with IPv6 counterparts. if rawIP == nil { workTable = m.workTable6 rawIP = ip.To16() ipsetType = nftables.TypeIP6Addr ipsetStorage = m.ipsetStore6 } ipset, err := m.rConn.GetSetByName(workTable, ipsetName) if err != nil { if ipset, err = m.createSet(workTable, ipsetName, ipsetType); err != nil { return nil, fmt.Errorf("get set name: %v", err) } ipsetStorage.newIpset(ipset.Name) } if ipsetStorage.IsIpInSet(ipset.Name, ip) { return ipset, nil } if err := m.sConn.SetAddElements(ipset, []nftables.SetElement{{Key: rawIP}}); err != nil { return nil, fmt.Errorf("add set element for the first time: %v", err) } ipsetStorage.AddIpToSet(ipset.Name, ip) if err := m.sConn.Flush(); err != nil { return nil, fmt.Errorf("flush add elements: %v", err) } return ipset, nil } // createSet in given table by name func (m *AclManager) createSet(table *nftables.Table, name string, ipsetType nftables.SetDatatype) (*nftables.Set, error) { ipset := &nftables.Set{ Name: name, Table: table, Dynamic: true, KeyType: ipsetType, } if err := m.rConn.AddSet(ipset, nil); err != nil { return nil, fmt.Errorf("create set: %v", err) } if err := m.rConn.Flush(); err != nil { return nil, fmt.Errorf("flush created set: %v", err) } return ipset, nil } func (m *AclManager) flushWithBackoff() (err error) { backoff := 4 backoffTime := 1000 * time.Millisecond for i := 0; ; i++ { err = m.rConn.Flush() if err != nil { if !strings.Contains(err.Error(), "busy") { return } log.Error("failed to flush nftables, retrying...") if i == backoff-1 { return err } time.Sleep(backoffTime) backoffTime *= 2 continue } break } return } func (m *AclManager) refreshRuleHandles(table *nftables.Table, chain *nftables.Chain) error { if table == nil || chain == nil { return nil } list, err := m.rConn.GetRules(table, chain) if err != nil { return err } for _, rule := range list { if len(rule.UserData) == 0 { continue } split := bytes.Split(rule.UserData, []byte(" ")) r, ok := m.rules[string(split[0])] if ok { *r.nftRule = *rule } } return nil } func generateRuleId( ip net.IP, sPort *firewall.Port, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, ipset *nftables.Set, ) string { ipver := "v4" if ip.To4() == nil { ipver = "v6" } rulesetID := ":" + strconv.Itoa(int(direction)) + ":" if sPort != nil { rulesetID += sPort.String() } rulesetID += ":" if dPort != nil { rulesetID += dPort.String() } rulesetID += ":" rulesetID += strconv.Itoa(int(action)) if ipset == nil { return "ip:" + ip.String() + rulesetID } return "set:" + ipver + ":" + ipset.Name + rulesetID } func generateRuleIdForMangle(ipset *nftables.Set, ip net.IP, proto firewall.Protocol, port *firewall.Port) string { // case of icmp port is empty var p string if port != nil { p = port.String() } ipver := "v4" if (ipset != nil && ipset.Table.Family == nftables.TableFamilyIPv6) || ip.To4() == nil { ipver = "v6" } if ipset != nil { return fmt.Sprintf("p:set:%s:%s:%s:%v", ipver, ipset.Name, proto, p) } else { return fmt.Sprintf("p:ip:%s:%s:%s:%v", ipver, ip.String(), proto, p) } } func shouldAddToPrerouting(proto firewall.Protocol, dPort *firewall.Port, direction firewall.RuleDirection) bool { if proto == "all" { return false } if direction != firewall.RuleDirectionIN { return false } if dPort == nil && proto != firewall.ProtocolICMP { return false } return true } func encodePort(port firewall.Port) []byte { bs := make([]byte, 2) binary.BigEndian.PutUint16(bs, uint16(port.Values[0])) return bs } func ifname(n string) []byte { b := make([]byte, 16) copy(b, []byte(n+"\x00")) return b }