Fix/acl for forward (#1305)

Fix ACL on routed traffic and code refactor

client/firewall/nftables/ipsetstore_linux.go

@@ -0,0 +1,85 @@
+package nftables
+
+import (
+	"net"
+)
+
+type ipsetStore struct {
+	ipsetReference map[string]int
+	ipsets         map[string]map[string]struct{} // ipsetName -> list of ips
+}
+
+func newIpsetStore() *ipsetStore {
+	return &ipsetStore{
+		ipsetReference: make(map[string]int),
+		ipsets:         make(map[string]map[string]struct{}),
+	}
+}
+
+func (s *ipsetStore) ips(ipsetName string) (map[string]struct{}, bool) {
+	r, ok := s.ipsets[ipsetName]
+	return r, ok
+}
+
+func (s *ipsetStore) newIpset(ipsetName string) map[string]struct{} {
+	s.ipsetReference[ipsetName] = 0
+	ipList := make(map[string]struct{})
+	s.ipsets[ipsetName] = ipList
+	return ipList
+}
+
+func (s *ipsetStore) deleteIpset(ipsetName string) {
+	delete(s.ipsetReference, ipsetName)
+	delete(s.ipsets, ipsetName)
+}
+
+func (s *ipsetStore) DeleteIpFromSet(ipsetName string, ip net.IP) {
+	ipList, ok := s.ipsets[ipsetName]
+	if !ok {
+		return
+	}
+	delete(ipList, ip.String())
+}
+
+func (s *ipsetStore) AddIpToSet(ipsetName string, ip net.IP) {
+	ipList, ok := s.ipsets[ipsetName]
+	if !ok {
+		return
+	}
+	ipList[ip.String()] = struct{}{}
+}
+
+func (s *ipsetStore) IsIpInSet(ipsetName string, ip net.IP) bool {
+	ipList, ok := s.ipsets[ipsetName]
+	if !ok {
+		return false
+	}
+	_, ok = ipList[ip.String()]
+	return ok
+}
+
+func (s *ipsetStore) AddReferenceToIpset(ipsetName string) {
+	s.ipsetReference[ipsetName]++
+}
+
+func (s *ipsetStore) DeleteReferenceFromIpSet(ipsetName string) {
+	r, ok := s.ipsetReference[ipsetName]
+	if !ok {
+		return
+	}
+	if r == 0 {
+		return
+	}
+	s.ipsetReference[ipsetName]--
+}
+
+func (s *ipsetStore) HasReferenceToSet(ipsetName string) bool {
+	if _, ok := s.ipsetReference[ipsetName]; !ok {
+		return false
+	}
+	if s.ipsetReference[ipsetName] == 0 {
+		return false
+	}
+
+	return true
+}

client/firewall/nftables/manager_linux.go

@@ -2,90 +2,52 @@ package nftables
 
 import (
 	"bytes"
-	"encoding/binary"
+	"context"
 	"fmt"
 	"net"
-	"net/netip"
-	"strconv"
-	"strings"
 	"sync"
-	"time"
 
 	"github.com/google/nftables"
 	"github.com/google/nftables/expr"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
 
-	fw "github.com/netbirdio/netbird/client/firewall"
-	"github.com/netbirdio/netbird/iface"
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 )
 
 const (
-	// FilterTableName is the name of the table that is used for filtering by the Netbird client
-	FilterTableName = "netbird-acl"
-
-	// FilterInputChainName is the name of the chain that is used for filtering incoming packets
-	FilterInputChainName = "netbird-acl-input-filter"
-
-	// FilterOutputChainName is the name of the chain that is used for filtering outgoing packets
-	FilterOutputChainName = "netbird-acl-output-filter"
-
-	AllowNetbirdInputRuleID = "allow Netbird incoming traffic"
+	// tableName is the name of the table that is used for filtering by the Netbird client
+	tableName = "netbird"
 )
 
-var anyIP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-
 // Manager of iptables firewall
 type Manager struct {
-	mutex sync.Mutex
-
-	rConn     *nftables.Conn
-	sConn     *nftables.Conn
-	tableIPv4 *nftables.Table
-	tableIPv6 *nftables.Table
-
-	filterInputChainIPv4  *nftables.Chain
-	filterOutputChainIPv4 *nftables.Chain
-
-	filterInputChainIPv6  *nftables.Chain
-	filterOutputChainIPv6 *nftables.Chain
-
-	rulesetManager *rulesetManager
-	setRemovedIPs  map[string]struct{}
-	setRemoved     map[string]*nftables.Set
-
+	mutex   sync.Mutex
+	rConn   *nftables.Conn
 	wgIface iFaceMapper
-}
 
-// iFaceMapper defines subset methods of interface required for manager
-type iFaceMapper interface {
-	Name() string
-	Address() iface.WGAddress
+	router     *router
+	aclManager *AclManager
 }
 
 // Create nftables firewall manager
-func Create(wgIface iFaceMapper) (*Manager, error) {
-	// sConn is used for creating sets and adding/removing elements from them
-	// it's differ then rConn (which does create new conn for each flush operation)
-	// and is permanent. Using same connection for booth type of operations
-	// overloads netlink with high amount of rules ( > 10000)
-	sConn, err := nftables.New(nftables.AsLasting())
+func Create(context context.Context, wgIface iFaceMapper) (*Manager, error) {
+	m := &Manager{
+		rConn:   &nftables.Conn{},
+		wgIface: wgIface,
+	}
+
+	workTable, err := m.createWorkTable()
 	if err != nil {
 		return nil, err
 	}
 
-	m := &Manager{
-		rConn: &nftables.Conn{},
-		sConn: sConn,
-
-		rulesetManager: newRuleManager(),
-		setRemovedIPs:  map[string]struct{}{},
-		setRemoved:     map[string]*nftables.Set{},
-
-		wgIface: wgIface,
+	m.router, err = newRouter(context, workTable)
+	if err != nil {
+		return nil, err
 	}
 
-	if err := m.Reset(); err != nil {
+	m.aclManager, err = newAclManager(workTable, wgIface, m.router.RouteingFwChainName())
+	if err != nil {
 		return nil, err
 	}
 
@@ -98,649 +60,58 @@ func Create(wgIface iFaceMapper) (*Manager, error) {
 // rule ID as comment for the rule
 func (m *Manager) AddFiltering(
 	ip net.IP,
-	proto fw.Protocol,
-	sPort *fw.Port,
-	dPort *fw.Port,
-	direction fw.RuleDirection,
-	action fw.Action,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
 	ipsetName string,
 	comment string,
-) (fw.Rule, error) {
+) ([]firewall.Rule, error) {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	var (
-		err   error
-		ipset *nftables.Set
-		table *nftables.Table
-		chain *nftables.Chain
-	)
-
-	if direction == fw.RuleDirectionOUT {
-		table, chain, err = m.chain(
-			ip,
-			FilterOutputChainName,
-			nftables.ChainHookOutput,
-			nftables.ChainPriorityFilter,
-			nftables.ChainTypeFilter)
-	} else {
-		table, chain, err = m.chain(
-			ip,
-			FilterInputChainName,
-			nftables.ChainHookInput,
-			nftables.ChainPriorityFilter,
-			nftables.ChainTypeFilter)
-	}
-	if err != nil {
-		return nil, err
-	}
-
 	rawIP := ip.To4()
 	if rawIP == nil {
-		rawIP = ip.To16()
+		return nil, fmt.Errorf("unsupported IP version: %s", ip.String())
 	}
 
-	rulesetID := m.getRulesetID(ip, proto, sPort, dPort, direction, action, ipsetName)
-
-	if ipsetName != "" {
-		// if we already have set with given name, just add ip to the set
-		// and return rule with new ID in other case let's create rule
-		// with fresh created set and set element
-
-		var isSetNew bool
-		ipset, err = m.rConn.GetSetByName(table, ipsetName)
-		if err != nil {
-			if ipset, err = m.createSet(table, rawIP, ipsetName); err != nil {
-				return nil, fmt.Errorf("get set name: %v", err)
-			}
-			isSetNew = true
-		}
-
-		if err := m.sConn.SetAddElements(ipset, []nftables.SetElement{{Key: rawIP}}); err != nil {
-			return nil, fmt.Errorf("add set element for the first time: %v", err)
-		}
-		if err := m.sConn.Flush(); err != nil {
-			return nil, fmt.Errorf("flush add elements: %v", err)
-		}
-
-		if !isSetNew {
-			// if we already have nftables rules with set for given direction
-			// just add new rule to the ruleset and return new fw.Rule object
-
-			if ruleset, ok := m.rulesetManager.getRuleset(rulesetID); ok {
-				return m.rulesetManager.addRule(ruleset, rawIP)
-			}
-			// if ipset exists but it is not linked to rule for given direction
-			// create new rule for direction and bind ipset to it later
-		}
-	}
-
-	ifaceKey := expr.MetaKeyIIFNAME
-	if direction == fw.RuleDirectionOUT {
-		ifaceKey = expr.MetaKeyOIFNAME
-	}
-	expressions := []expr.Any{
-		&expr.Meta{Key: ifaceKey, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-	}
-
-	if proto != "all" {
-		expressions = append(expressions, &expr.Payload{
-			DestRegister: 1,
-			Base:         expr.PayloadBaseNetworkHeader,
-			Offset:       uint32(9),
-			Len:          uint32(1),
-		})
-
-		var protoData []byte
-		switch proto {
-		case fw.ProtocolTCP:
-			protoData = []byte{unix.IPPROTO_TCP}
-		case fw.ProtocolUDP:
-			protoData = []byte{unix.IPPROTO_UDP}
-		case fw.ProtocolICMP:
-			protoData = []byte{unix.IPPROTO_ICMP}
-		default:
-			return nil, fmt.Errorf("unsupported protocol: %s", proto)
-		}
-		expressions = append(expressions, &expr.Cmp{
-			Register: 1,
-			Op:       expr.CmpOpEq,
-			Data:     protoData,
-		})
-	}
-
-	// check if rawIP contains zeroed IPv4 0.0.0.0 or same IPv6 value
-	// in that case not add IP match expression into the rule definition
-	if !bytes.HasPrefix(anyIP, rawIP) {
-		// source address position
-		addrLen := uint32(len(rawIP))
-		addrOffset := uint32(12)
-		if addrLen == 16 {
-			addrOffset = 8
-		}
-
-		// change to destination address position if need
-		if direction == fw.RuleDirectionOUT {
-			addrOffset += addrLen
-		}
-
-		expressions = append(expressions,
-			&expr.Payload{
-				DestRegister: 1,
-				Base:         expr.PayloadBaseNetworkHeader,
-				Offset:       addrOffset,
-				Len:          addrLen,
-			},
-		)
-		// add individual IP for match if no ipset defined
-		if ipset == nil {
-			expressions = append(expressions,
-				&expr.Cmp{
-					Op:       expr.CmpOpEq,
-					Register: 1,
-					Data:     rawIP,
-				},
-			)
-		} else {
-			expressions = append(expressions,
-				&expr.Lookup{
-					SourceRegister: 1,
-					SetName:        ipsetName,
-					SetID:          ipset.ID,
-				},
-			)
-		}
-	}
-
-	if sPort != nil && len(sPort.Values) != 0 {
-		expressions = append(expressions,
-			&expr.Payload{
-				DestRegister: 1,
-				Base:         expr.PayloadBaseTransportHeader,
-				Offset:       0,
-				Len:          2,
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     encodePort(*sPort),
-			},
-		)
-	}
-
-	if dPort != nil && len(dPort.Values) != 0 {
-		expressions = append(expressions,
-			&expr.Payload{
-				DestRegister: 1,
-				Base:         expr.PayloadBaseTransportHeader,
-				Offset:       2,
-				Len:          2,
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpEq,
-				Register: 1,
-				Data:     encodePort(*dPort),
-			},
-		)
-	}
-
-	if action == fw.ActionAccept {
-		expressions = append(expressions, &expr.Verdict{Kind: expr.VerdictAccept})
-	} else {
-		expressions = append(expressions, &expr.Verdict{Kind: expr.VerdictDrop})
-	}
-
-	userData := []byte(strings.Join([]string{rulesetID, comment}, " "))
-
-	rule := m.rConn.InsertRule(&nftables.Rule{
-		Table:    table,
-		Chain:    chain,
-		Position: 0,
-		Exprs:    expressions,
-		UserData: userData,
-	})
-	if err := m.rConn.Flush(); err != nil {
-		return nil, fmt.Errorf("flush insert rule: %v", err)
-	}
-
-	ruleset := m.rulesetManager.createRuleset(rulesetID, rule, ipset)
-	return m.rulesetManager.addRule(ruleset, rawIP)
-}
-
-// getRulesetID returns ruleset ID based on given parameters
-func (m *Manager) getRulesetID(
-	ip net.IP,
-	proto fw.Protocol,
-	sPort *fw.Port,
-	dPort *fw.Port,
-	direction fw.RuleDirection,
-	action fw.Action,
-	ipsetName string,
-) string {
-	rulesetID := ":" + strconv.Itoa(int(direction)) + ":"
-	if sPort != nil {
-		rulesetID += sPort.String()
-	}
-	rulesetID += ":"
-	if dPort != nil {
-		rulesetID += dPort.String()
-	}
-	rulesetID += ":"
-	rulesetID += strconv.Itoa(int(action))
-	if ipsetName == "" {
-		return "ip:" + ip.String() + rulesetID
-	}
-	return "set:" + ipsetName + rulesetID
-}
-
-// createSet in given table by name
-func (m *Manager) createSet(
-	table *nftables.Table,
-	rawIP []byte,
-	name string,
-) (*nftables.Set, error) {
-	keyType := nftables.TypeIPAddr
-	if len(rawIP) == 16 {
-		keyType = nftables.TypeIP6Addr
-	}
-	// else we create new ipset and continue creating rule
-	ipset := &nftables.Set{
-		Name:    name,
-		Table:   table,
-		Dynamic: true,
-		KeyType: keyType,
-	}
-
-	if err := m.rConn.AddSet(ipset, nil); err != nil {
-		return nil, fmt.Errorf("create set: %v", err)
-	}
-
-	if err := m.rConn.Flush(); err != nil {
-		return nil, fmt.Errorf("flush created set: %v", err)
-	}
-
-	return ipset, nil
-}
-
-// chain returns the chain for the given IP address with specific settings
-func (m *Manager) chain(
-	ip net.IP,
-	name string,
-	hook nftables.ChainHook,
-	priority nftables.ChainPriority,
-	cType nftables.ChainType,
-) (*nftables.Table, *nftables.Chain, error) {
-	var err error
-
-	getChain := func(c *nftables.Chain, tf nftables.TableFamily) (*nftables.Chain, error) {
-		if c != nil {
-			return c, nil
-		}
-		return m.createChainIfNotExists(tf, FilterTableName, name, hook, priority, cType)
-	}
-
-	if ip.To4() != nil {
-		if name == FilterInputChainName {
-			m.filterInputChainIPv4, err = getChain(m.filterInputChainIPv4, nftables.TableFamilyIPv4)
-			return m.tableIPv4, m.filterInputChainIPv4, err
-		}
-		m.filterOutputChainIPv4, err = getChain(m.filterOutputChainIPv4, nftables.TableFamilyIPv4)
-		return m.tableIPv4, m.filterOutputChainIPv4, err
-	}
-	if name == FilterInputChainName {
-		m.filterInputChainIPv6, err = getChain(m.filterInputChainIPv6, nftables.TableFamilyIPv6)
-		return m.tableIPv4, m.filterInputChainIPv6, err
-	}
-	m.filterOutputChainIPv6, err = getChain(m.filterOutputChainIPv6, nftables.TableFamilyIPv6)
-	return m.tableIPv4, m.filterOutputChainIPv6, err
-}
-
-// table returns the table for the given family of the IP address
-func (m *Manager) table(
-	family nftables.TableFamily, tableName string,
-) (*nftables.Table, error) {
-	// we cache access to Netbird ACL table only
-	if tableName != FilterTableName {
-		return m.createTableIfNotExists(nftables.TableFamilyIPv4, tableName)
-	}
-
-	if family == nftables.TableFamilyIPv4 {
-		if m.tableIPv4 != nil {
-			return m.tableIPv4, nil
-		}
-
-		table, err := m.createTableIfNotExists(nftables.TableFamilyIPv4, tableName)
-		if err != nil {
-			return nil, err
-		}
-		m.tableIPv4 = table
-		return m.tableIPv4, nil
-	}
-
-	if m.tableIPv6 != nil {
-		return m.tableIPv6, nil
-	}
-
-	table, err := m.createTableIfNotExists(nftables.TableFamilyIPv6, tableName)
-	if err != nil {
-		return nil, err
-	}
-	m.tableIPv6 = table
-	return m.tableIPv6, nil
-}
-
-func (m *Manager) createTableIfNotExists(
-	family nftables.TableFamily, tableName string,
-) (*nftables.Table, error) {
-	tables, err := m.rConn.ListTablesOfFamily(family)
-	if err != nil {
-		return nil, fmt.Errorf("list of tables: %w", err)
-	}
-
-	for _, t := range tables {
-		if t.Name == tableName {
-			return t, nil
-		}
-	}
-
-	table := m.rConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4})
-	if err := m.rConn.Flush(); err != nil {
-		return nil, err
-	}
-	return table, nil
-}
-
-func (m *Manager) createChainIfNotExists(
-	family nftables.TableFamily,
-	tableName string,
-	name string,
-	hooknum nftables.ChainHook,
-	priority nftables.ChainPriority,
-	chainType nftables.ChainType,
-) (*nftables.Chain, error) {
-	table, err := m.table(family, tableName)
-	if err != nil {
-		return nil, err
-	}
-
-	chains, err := m.rConn.ListChainsOfTableFamily(family)
-	if err != nil {
-		return nil, fmt.Errorf("list of chains: %w", err)
-	}
-
-	for _, c := range chains {
-		if c.Name == name && c.Table.Name == table.Name {
-			return c, nil
-		}
-	}
-
-	polAccept := nftables.ChainPolicyAccept
-	chain := &nftables.Chain{
-		Name:     name,
-		Table:    table,
-		Hooknum:  hooknum,
-		Priority: priority,
-		Type:     chainType,
-		Policy:   &polAccept,
-	}
-
-	chain = m.rConn.AddChain(chain)
-
-	ifaceKey := expr.MetaKeyIIFNAME
-	shiftDSTAddr := 0
-	if name == FilterOutputChainName {
-		ifaceKey = expr.MetaKeyOIFNAME
-		shiftDSTAddr = 1
-	}
-
-	expressions := []expr.Any{
-		&expr.Meta{Key: ifaceKey, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-	}
-
-	mask, _ := netip.AddrFromSlice(m.wgIface.Address().Network.Mask)
-	if m.wgIface.Address().IP.To4() == nil {
-		ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To16())
-		expressions = append(expressions,
-			&expr.Payload{
-				DestRegister: 2,
-				Base:         expr.PayloadBaseNetworkHeader,
-				Offset:       uint32(8 + (16 * shiftDSTAddr)),
-				Len:          16,
-			},
-			&expr.Bitwise{
-				SourceRegister: 2,
-				DestRegister:   2,
-				Len:            16,
-				Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-				Mask:           mask.Unmap().AsSlice(),
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpNeq,
-				Register: 2,
-				Data:     ip.Unmap().AsSlice(),
-			},
-			&expr.Verdict{Kind: expr.VerdictAccept},
-		)
-	} else {
-		ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4())
-		expressions = append(expressions,
-			&expr.Payload{
-				DestRegister: 2,
-				Base:         expr.PayloadBaseNetworkHeader,
-				Offset:       uint32(12 + (4 * shiftDSTAddr)),
-				Len:          4,
-			},
-			&expr.Bitwise{
-				SourceRegister: 2,
-				DestRegister:   2,
-				Len:            4,
-				Xor:            []byte{0x0, 0x0, 0x0, 0x0},
-				Mask:           m.wgIface.Address().Network.Mask,
-			},
-			&expr.Cmp{
-				Op:       expr.CmpOpNeq,
-				Register: 2,
-				Data:     ip.Unmap().AsSlice(),
-			},
-			&expr.Verdict{Kind: expr.VerdictAccept},
-		)
-	}
-
-	_ = m.rConn.AddRule(&nftables.Rule{
-		Table: table,
-		Chain: chain,
-		Exprs: expressions,
-	})
-
-	expressions = []expr.Any{
-		&expr.Meta{Key: ifaceKey, Register: 1},
-		&expr.Cmp{
-			Op:       expr.CmpOpEq,
-			Register: 1,
-			Data:     ifname(m.wgIface.Name()),
-		},
-		&expr.Verdict{Kind: expr.VerdictDrop},
-	}
-	_ = m.rConn.AddRule(&nftables.Rule{
-		Table: table,
-		Chain: chain,
-		Exprs: expressions,
-	})
-
-	if err := m.rConn.Flush(); err != nil {
-		return nil, err
-	}
-
-	return chain, nil
+	return m.aclManager.AddFiltering(ip, proto, sPort, dPort, direction, action, ipsetName, comment)
 }
 
 // DeleteRule from the firewall by rule definition
-func (m *Manager) DeleteRule(rule fw.Rule) error {
+func (m *Manager) DeleteRule(rule firewall.Rule) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	nativeRule, ok := rule.(*Rule)
-	if !ok {
-		return fmt.Errorf("invalid rule type")
-	}
-
-	if nativeRule.nftRule == nil {
-		return nil
-	}
-
-	if nativeRule.nftSet != nil {
-		// call twice of delete set element raises error
-		// so we need to check if element is already removed
-		key := fmt.Sprintf("%s:%v", nativeRule.nftSet.Name, nativeRule.ip)
-		if _, ok := m.setRemovedIPs[key]; !ok {
-			err := m.sConn.SetDeleteElements(nativeRule.nftSet, []nftables.SetElement{{Key: nativeRule.ip}})
-			if err != nil {
-				log.Errorf("delete elements for set %q: %v", nativeRule.nftSet.Name, err)
-			}
-			if err := m.sConn.Flush(); err != nil {
-				return err
-			}
-			m.setRemovedIPs[key] = struct{}{}
-		}
-	}
-
-	if m.rulesetManager.deleteRule(nativeRule) {
-		// deleteRule indicates that we still have IP in the ruleset
-		// it means we should not remove the nftables rule but need to update set
-		// so we prepare IP to be removed from set on the next flush call
-		return nil
-	}
-
-	// ruleset doesn't contain IP anymore (or contains only one), remove nft rule
-	if err := m.rConn.DelRule(nativeRule.nftRule); err != nil {
-		log.Errorf("failed to delete rule: %v", err)
-	}
-	if err := m.rConn.Flush(); err != nil {
-		return err
-	}
-	nativeRule.nftRule = nil
-
-	if nativeRule.nftSet != nil {
-		if _, ok := m.setRemoved[nativeRule.nftSet.Name]; !ok {
-			m.setRemoved[nativeRule.nftSet.Name] = nativeRule.nftSet
-		}
-		nativeRule.nftSet = nil
-	}
-
-	return nil
+	return m.aclManager.DeleteRule(rule)
 }
 
-// Reset firewall to the default state
-func (m *Manager) Reset() error {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-
-	chains, err := m.rConn.ListChains()
-	if err != nil {
-		return fmt.Errorf("list of chains: %w", err)
-	}
-	for _, c := range chains {
-		// delete Netbird allow input traffic rule if it exists
-		if c.Table.Name == "filter" && c.Name == "INPUT" {
-			rules, err := m.rConn.GetRules(c.Table, c)
-			if err != nil {
-				log.Errorf("get rules for chain %q: %v", c.Name, err)
-				continue
-			}
-			for _, r := range rules {
-				if bytes.Equal(r.UserData, []byte(AllowNetbirdInputRuleID)) {
-					if err := m.rConn.DelRule(r); err != nil {
-						log.Errorf("delete rule: %v", err)
-					}
-				}
-			}
-		}
-
-		if c.Name == FilterInputChainName || c.Name == FilterOutputChainName {
-			m.rConn.DelChain(c)
-		}
-	}
-
-	tables, err := m.rConn.ListTables()
-	if err != nil {
-		return fmt.Errorf("list of tables: %w", err)
-	}
-	for _, t := range tables {
-		if t.Name == FilterTableName {
-			m.rConn.DelTable(t)
-		}
-	}
-
-	return m.rConn.Flush()
+func (m *Manager) IsServerRouteSupported() bool {
+	return true
 }
 
-// Flush rule/chain/set operations from the buffer
-//
-// Method also get all rules after flush and refreshes handle values in the rulesets
-func (m *Manager) Flush() error {
+func (m *Manager) InsertRoutingRules(pair firewall.RouterPair) error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	if err := m.flushWithBackoff(); err != nil {
-		return err
-	}
+	return m.router.InsertRoutingRules(pair)
+}
 
-	// set must be removed after flush rule changes
-	// otherwise we will get error
-	for _, s := range m.setRemoved {
-		m.rConn.FlushSet(s)
-		m.rConn.DelSet(s)
-	}
+func (m *Manager) RemoveRoutingRules(pair firewall.RouterPair) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
 
-	if len(m.setRemoved) > 0 {
-		if err := m.flushWithBackoff(); err != nil {
-			return err
-		}
-	}
-
-	m.setRemovedIPs = map[string]struct{}{}
-	m.setRemoved = map[string]*nftables.Set{}
-
-	if err := m.refreshRuleHandles(m.tableIPv4, m.filterInputChainIPv4); err != nil {
-		log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err)
-	}
-
-	if err := m.refreshRuleHandles(m.tableIPv4, m.filterOutputChainIPv4); err != nil {
-		log.Errorf("failed to refresh rule handles IPv4 output chain: %v", err)
-	}
-
-	if err := m.refreshRuleHandles(m.tableIPv6, m.filterInputChainIPv6); err != nil {
-		log.Errorf("failed to refresh rule handles IPv6 input chain: %v", err)
-	}
-
-	if err := m.refreshRuleHandles(m.tableIPv6, m.filterOutputChainIPv6); err != nil {
-		log.Errorf("failed to refresh rule handles IPv6 output chain: %v", err)
-	}
-
-	return nil
+	return m.router.RemoveRoutingRules(pair)
 }
 
 // AllowNetbird allows netbird interface traffic
+// todo review this method usage
 func (m *Manager) AllowNetbird() error {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	tf := nftables.TableFamilyIPv4
-	if m.wgIface.Address().IP.To4() == nil {
-		tf = nftables.TableFamilyIPv6
-	}
-
-	chains, err := m.rConn.ListChainsOfTableFamily(tf)
+	chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
 	if err != nil {
 		return fmt.Errorf("list of chains: %w", err)
 	}
@@ -777,47 +148,75 @@ func (m *Manager) AllowNetbird() error {
 	return nil
 }
 
-func (m *Manager) flushWithBackoff() (err error) {
-	backoff := 4
-	backoffTime := 1000 * time.Millisecond
-	for i := 0; ; i++ {
-		err = m.rConn.Flush()
-		if err != nil {
-			if !strings.Contains(err.Error(), "busy") {
-				return
-			}
-			log.Error("failed to flush nftables, retrying...")
-			if i == backoff-1 {
-				return err
-			}
-			time.Sleep(backoffTime)
-			backoffTime *= 2
-			continue
-		}
-		break
+// Reset firewall to the default state
+func (m *Manager) Reset() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	chains, err := m.rConn.ListChains()
+	if err != nil {
+		return fmt.Errorf("list of chains: %w", err)
 	}
-	return
+
+	for _, c := range chains {
+		// delete Netbird allow input traffic rule if it exists
+		if c.Table.Name == "filter" && c.Name == "INPUT" {
+			rules, err := m.rConn.GetRules(c.Table, c)
+			if err != nil {
+				log.Errorf("get rules for chain %q: %v", c.Name, err)
+				continue
+			}
+			for _, r := range rules {
+				if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
+					if err := m.rConn.DelRule(r); err != nil {
+						log.Errorf("delete rule: %v", err)
+					}
+				}
+			}
+		}
+	}
+
+	m.router.ResetForwardRules()
+
+	tables, err := m.rConn.ListTables()
+	if err != nil {
+		return fmt.Errorf("list of tables: %w", err)
+	}
+	for _, t := range tables {
+		if t.Name == tableName {
+			m.rConn.DelTable(t)
+		}
+	}
+
+	return m.rConn.Flush()
 }
 
-func (m *Manager) refreshRuleHandles(table *nftables.Table, chain *nftables.Chain) error {
-	if table == nil || chain == nil {
-		return nil
-	}
+// Flush rule/chain/set operations from the buffer
+//
+// Method also get all rules after flush and refreshes handle values in the rulesets
+// todo review this method usage
+func (m *Manager) Flush() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
 
-	list, err := m.rConn.GetRules(table, chain)
+	return m.aclManager.Flush()
+}
+
+func (m *Manager) createWorkTable() (*nftables.Table, error) {
+	tables, err := m.rConn.ListTablesOfFamily(nftables.TableFamilyIPv4)
 	if err != nil {
-		return err
+		return nil, fmt.Errorf("list of tables: %w", err)
 	}
 
-	for _, rule := range list {
-		if len(rule.UserData) != 0 {
-			if err := m.rulesetManager.setNftRuleHandle(rule); err != nil {
-				log.Errorf("failed to set rule handle: %v", err)
-			}
+	for _, t := range tables {
+		if t.Name == tableName {
+			m.rConn.DelTable(t)
 		}
 	}
 
-	return nil
+	table := m.rConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4})
+	err = m.rConn.Flush()
+	return table, err
 }
 
 func (m *Manager) applyAllowNetbirdRules(chain *nftables.Chain) {
@@ -835,7 +234,7 @@ func (m *Manager) applyAllowNetbirdRules(chain *nftables.Chain) {
 				Kind: expr.VerdictAccept,
 			},
 		},
-		UserData: []byte(AllowNetbirdInputRuleID),
+		UserData: []byte(allowNetbirdInputRuleID),
 	}
 	_ = m.rConn.InsertRule(rule)
 }
@@ -857,15 +256,3 @@ func (m *Manager) detectAllowNetbirdRule(existedRules []*nftables.Rule) *nftable
 	}
 	return nil
 }
-
-func encodePort(port fw.Port) []byte {
-	bs := make([]byte, 2)
-	binary.BigEndian.PutUint16(bs, uint16(port.Values[0]))
-	return bs
-}
-
-func ifname(n string) []byte {
-	b := make([]byte, 16)
-	copy(b, []byte(n+"\x00"))
-	return b
-}

client/firewall/nftables/manager_linux_test.go

@@ -1,6 +1,7 @@
 package nftables
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/netip"
@@ -12,7 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/unix"
 
-	fw "github.com/netbirdio/netbird/client/firewall"
+	fw "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/iface"
 )
 
@@ -53,7 +54,7 @@ func TestNftablesManager(t *testing.T) {
 	}
 
 	// just check on the local interface
-	manager, err := Create(mock)
+	manager, err := Create(context.Background(), mock)
 	require.NoError(t, err)
 	time.Sleep(time.Second * 3)
 
@@ -82,14 +83,10 @@ func TestNftablesManager(t *testing.T) {
 	err = manager.Flush()
 	require.NoError(t, err, "failed to flush")
 
-	rules, err := testClient.GetRules(manager.tableIPv4, manager.filterInputChainIPv4)
+	rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
 	require.NoError(t, err, "failed to get rules")
 
-	// test expectations:
-	// 1) regular rule
-	// 2) "accept extra routed traffic rule" for the interface
-	// 3) "drop all rule" for the interface
-	require.Len(t, rules, 3, "expected 3 rules")
+	require.Len(t, rules, 1, "expected 1 rules")
 
 	ipToAdd, _ := netip.AddrFromSlice(ip)
 	add := ipToAdd.Unmap()
@@ -137,18 +134,17 @@ func TestNftablesManager(t *testing.T) {
 	}
 	require.ElementsMatch(t, rules[0].Exprs, expectedExprs, "expected the same expressions")
 
-	err = manager.DeleteRule(rule)
-	require.NoError(t, err, "failed to delete rule")
+	for _, r := range rule {
+		err = manager.DeleteRule(r)
+		require.NoError(t, err, "failed to delete rule")
+	}
 
 	err = manager.Flush()
 	require.NoError(t, err, "failed to flush")
 
-	rules, err = testClient.GetRules(manager.tableIPv4, manager.filterInputChainIPv4)
+	rules, err = testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
 	require.NoError(t, err, "failed to get rules")
-	// test expectations:
-	// 1) "accept extra routed traffic rule" for the interface
-	// 2) "drop all rule" for the interface
-	require.Len(t, rules, 2, "expected 2 rules after deletion")
+	require.Len(t, rules, 0, "expected 0 rules after deletion")
 
 	err = manager.Reset()
 	require.NoError(t, err, "failed to reset")
@@ -173,7 +169,7 @@ func TestNFtablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(mock)
+			manager, err := Create(context.Background(), mock)
 			require.NoError(t, err)
 			time.Sleep(time.Second * 3)
 

client/firewall/nftables/route_linux.go

@@ -0,0 +1,413 @@
+package nftables
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/firewall/manager"
+)
+
+const (
+	chainNameRouteingFw = "netbird-rt-fwd"
+	chainNameRoutingNat = "netbird-rt-nat"
+
+	userDataAcceptForwardRuleSrc = "frwacceptsrc"
+	userDataAcceptForwardRuleDst = "frwacceptdst"
+)
+
+// some presets for building nftable rules
+var (
+	zeroXor = binaryutil.NativeEndian.PutUint32(0)
+
+	exprCounterAccept = []expr.Any{
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+
+	errFilterTableNotFound = fmt.Errorf("nftables: 'filter' table not found")
+)
+
+type router struct {
+	ctx         context.Context
+	stop        context.CancelFunc
+	conn        *nftables.Conn
+	workTable   *nftables.Table
+	filterTable *nftables.Table
+	chains      map[string]*nftables.Chain
+	// rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules
+	rules                    map[string]*nftables.Rule
+	isDefaultFwdRulesEnabled bool
+}
+
+func newRouter(parentCtx context.Context, workTable *nftables.Table) (*router, error) {
+	ctx, cancel := context.WithCancel(parentCtx)
+
+	r := &router{
+		ctx:       ctx,
+		stop:      cancel,
+		conn:      &nftables.Conn{},
+		workTable: workTable,
+		chains:    make(map[string]*nftables.Chain),
+		rules:     make(map[string]*nftables.Rule),
+	}
+
+	var err error
+	r.filterTable, err = r.loadFilterTable()
+	if err != nil {
+		if errors.Is(err, errFilterTableNotFound) {
+			log.Warnf("table 'filter' not found for forward rules")
+		} else {
+			return nil, err
+		}
+	}
+
+	err = r.cleanUpDefaultForwardRules()
+	if err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	err = r.createContainers()
+	if err != nil {
+		log.Errorf("failed to create containers for route: %s", err)
+	}
+	return r, err
+}
+
+func (r *router) RouteingFwChainName() string {
+	return chainNameRouteingFw
+}
+
+// ResetForwardRules cleans existing nftables default forward rules from the system
+func (r *router) ResetForwardRules() {
+	err := r.cleanUpDefaultForwardRules()
+	if err != nil {
+		log.Errorf("failed to reset forward rules: %s", err)
+	}
+}
+
+func (r *router) loadFilterTable() (*nftables.Table, error) {
+	tables, err := r.conn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return nil, fmt.Errorf("nftables: unable to list tables: %v", err)
+	}
+
+	for _, table := range tables {
+		if table.Name == "filter" {
+			return table, nil
+		}
+	}
+
+	return nil, errFilterTableNotFound
+}
+
+func (r *router) createContainers() error {
+
+	r.chains[chainNameRouteingFw] = r.conn.AddChain(&nftables.Chain{
+		Name:  chainNameRouteingFw,
+		Table: r.workTable,
+	})
+
+	r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
+		Name:     chainNameRoutingNat,
+		Table:    r.workTable,
+		Hooknum:  nftables.ChainHookPostrouting,
+		Priority: nftables.ChainPriorityNATSource - 1,
+		Type:     nftables.ChainTypeNAT,
+	})
+
+	err := r.refreshRulesMap()
+	if err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	err = r.conn.Flush()
+	if err != nil {
+		return fmt.Errorf("nftables: unable to initialize table: %v", err)
+	}
+	return nil
+}
+
+// InsertRoutingRules inserts a nftable rule pair to the forwarding chain and if enabled, to the nat chain
+func (r *router) InsertRoutingRules(pair manager.RouterPair) error {
+	err := r.refreshRulesMap()
+	if err != nil {
+		return err
+	}
+
+	err = r.insertRoutingRule(manager.ForwardingFormat, chainNameRouteingFw, pair, false)
+	if err != nil {
+		return err
+	}
+	err = r.insertRoutingRule(manager.InForwardingFormat, chainNameRouteingFw, manager.GetInPair(pair), false)
+	if err != nil {
+		return err
+	}
+
+	if pair.Masquerade {
+		err = r.insertRoutingRule(manager.NatFormat, chainNameRoutingNat, pair, true)
+		if err != nil {
+			return err
+		}
+		err = r.insertRoutingRule(manager.InNatFormat, chainNameRoutingNat, manager.GetInPair(pair), true)
+		if err != nil {
+			return err
+		}
+	}
+
+	if r.filterTable != nil && !r.isDefaultFwdRulesEnabled {
+		log.Debugf("add default accept forward rule")
+		r.acceptForwardRule(pair.Source)
+	}
+
+	err = r.conn.Flush()
+	if err != nil {
+		return fmt.Errorf("nftables: unable to insert rules for %s: %v", pair.Destination, err)
+	}
+	return nil
+}
+
+// insertRoutingRule inserts a nftable rule to the conn client flush queue
+func (r *router) insertRoutingRule(format, chainName string, pair manager.RouterPair, isNat bool) error {
+	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
+	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
+
+	var expression []expr.Any
+	if isNat {
+		expression = append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) // nolint:gocritic
+	} else {
+		expression = append(sourceExp, append(destExp, exprCounterAccept...)...) // nolint:gocritic
+	}
+
+	ruleKey := manager.GenKey(format, pair.ID)
+
+	_, exists := r.rules[ruleKey]
+	if exists {
+		err := r.removeRoutingRule(format, pair)
+		if err != nil {
+			return err
+		}
+	}
+
+	r.rules[ruleKey] = r.conn.InsertRule(&nftables.Rule{
+		Table:    r.workTable,
+		Chain:    r.chains[chainName],
+		Exprs:    expression,
+		UserData: []byte(ruleKey),
+	})
+	return nil
+}
+
+func (r *router) acceptForwardRule(sourceNetwork string) {
+	src := generateCIDRMatcherExpressions(true, sourceNetwork)
+	dst := generateCIDRMatcherExpressions(false, "0.0.0.0/0")
+
+	var exprs []expr.Any
+	exprs = append(src, append(dst, &expr.Verdict{ // nolint:gocritic
+		Kind: expr.VerdictAccept,
+	})...)
+
+	rule := &nftables.Rule{
+		Table: r.filterTable,
+		Chain: &nftables.Chain{
+			Name:     "FORWARD",
+			Table:    r.filterTable,
+			Type:     nftables.ChainTypeFilter,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityFilter,
+		},
+		Exprs:    exprs,
+		UserData: []byte(userDataAcceptForwardRuleSrc),
+	}
+
+	r.conn.AddRule(rule)
+
+	src = generateCIDRMatcherExpressions(true, "0.0.0.0/0")
+	dst = generateCIDRMatcherExpressions(false, sourceNetwork)
+
+	exprs = append(src, append(dst, &expr.Verdict{ //nolint:gocritic
+		Kind: expr.VerdictAccept,
+	})...)
+
+	rule = &nftables.Rule{
+		Table: r.filterTable,
+		Chain: &nftables.Chain{
+			Name:     "FORWARD",
+			Table:    r.filterTable,
+			Type:     nftables.ChainTypeFilter,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityFilter,
+		},
+		Exprs:    exprs,
+		UserData: []byte(userDataAcceptForwardRuleDst),
+	}
+	r.conn.AddRule(rule)
+	r.isDefaultFwdRulesEnabled = true
+}
+
+// RemoveRoutingRules removes a nftable rule pair from forwarding and nat chains
+func (r *router) RemoveRoutingRules(pair manager.RouterPair) error {
+	err := r.refreshRulesMap()
+	if err != nil {
+		return err
+	}
+
+	err = r.removeRoutingRule(manager.ForwardingFormat, pair)
+	if err != nil {
+		return err
+	}
+
+	err = r.removeRoutingRule(manager.InForwardingFormat, manager.GetInPair(pair))
+	if err != nil {
+		return err
+	}
+
+	err = r.removeRoutingRule(manager.NatFormat, pair)
+	if err != nil {
+		return err
+	}
+
+	err = r.removeRoutingRule(manager.InNatFormat, manager.GetInPair(pair))
+	if err != nil {
+		return err
+	}
+
+	if len(r.rules) == 0 {
+		err := r.cleanUpDefaultForwardRules()
+		if err != nil {
+			log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+		}
+	}
+
+	err = r.conn.Flush()
+	if err != nil {
+		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
+	}
+	log.Debugf("nftables: removed rules for %s", pair.Destination)
+	return nil
+}
+
+// removeRoutingRule add a nftable rule to the removal queue and delete from rules map
+func (r *router) removeRoutingRule(format string, pair manager.RouterPair) error {
+	ruleKey := manager.GenKey(format, pair.ID)
+
+	rule, found := r.rules[ruleKey]
+	if found {
+		ruleType := "forwarding"
+		if rule.Chain.Type == nftables.ChainTypeNAT {
+			ruleType = "nat"
+		}
+
+		err := r.conn.DelRule(rule)
+		if err != nil {
+			return fmt.Errorf("nftables: unable to remove %s rule for %s: %v", ruleType, pair.Destination, err)
+		}
+
+		log.Debugf("nftables: removing %s rule for %s", ruleType, pair.Destination)
+
+		delete(r.rules, ruleKey)
+	}
+	return nil
+}
+
+// refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid
+// duplicates and to get missing attributes that we don't have when adding new rules
+func (r *router) refreshRulesMap() error {
+	for _, chain := range r.chains {
+		rules, err := r.conn.GetRules(chain.Table, chain)
+		if err != nil {
+			return fmt.Errorf("nftables: unable to list rules: %v", err)
+		}
+		for _, rule := range rules {
+			if len(rule.UserData) > 0 {
+				r.rules[string(rule.UserData)] = rule
+			}
+		}
+	}
+	return nil
+}
+
+func (r *router) cleanUpDefaultForwardRules() error {
+	if r.filterTable == nil {
+		r.isDefaultFwdRulesEnabled = false
+		return nil
+	}
+
+	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return err
+	}
+
+	var rules []*nftables.Rule
+	for _, chain := range chains {
+		if chain.Table.Name != r.filterTable.Name {
+			continue
+		}
+		if chain.Name != "FORWARD" {
+			continue
+		}
+
+		rules, err = r.conn.GetRules(r.filterTable, chain)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, rule := range rules {
+		if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleSrc)) || bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleDst)) {
+			err := r.conn.DelRule(rule)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	r.isDefaultFwdRulesEnabled = false
+	return r.conn.Flush()
+}
+
+// generateCIDRMatcherExpressions generates nftables expressions that matches a CIDR
+func generateCIDRMatcherExpressions(source bool, cidr string) []expr.Any {
+	ip, network, _ := net.ParseCIDR(cidr)
+	ipToAdd, _ := netip.AddrFromSlice(ip)
+	add := ipToAdd.Unmap()
+
+	var offSet uint32
+	if source {
+		offSet = 12 // src offset
+	} else {
+		offSet = 16 // dst offset
+	}
+
+	return []expr.Any{
+		// fetch src add
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       offSet,
+			Len:          4,
+		},
+		// net mask
+		&expr.Bitwise{
+			DestRegister:   1,
+			SourceRegister: 1,
+			Len:            4,
+			Mask:           network.Mask,
+			Xor:            zeroXor,
+		},
+		// net address
+		&expr.Cmp{
+			Register: 1,
+			Data:     add.AsSlice(),
+		},
+	}
+}

client/firewall/nftables/router_linux_test.go

@@ -0,0 +1,280 @@
+//go:build !android
+
+package nftables
+
+import (
+	"context"
+	"testing"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/google/nftables"
+	"github.com/google/nftables/expr"
+	"github.com/stretchr/testify/require"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/firewall/test"
+)
+
+const (
+	// UNKNOWN is the default value for the firewall type for unknown firewall type
+	UNKNOWN = iota
+	// IPTABLES is the value for the iptables firewall type
+	IPTABLES
+	// NFTABLES is the value for the nftables firewall type
+	NFTABLES
+)
+
+func TestNftablesManager_InsertRoutingRules(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this OS")
+	}
+
+	table, err := createWorkTable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer deleteWorkTable()
+
+	for _, testCase := range test.InsertRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			manager, err := newRouter(context.TODO(), table)
+			require.NoError(t, err, "failed to create router")
+
+			nftablesTestingClient := &nftables.Conn{}
+
+			defer manager.ResetForwardRules()
+
+			require.NoError(t, err, "shouldn't return error")
+
+			err = manager.InsertRoutingRules(testCase.InputPair)
+			defer func() {
+				_ = manager.RemoveRoutingRules(testCase.InputPair)
+			}()
+			require.NoError(t, err, "forwarding pair should be inserted")
+
+			sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
+			destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
+			testingExpression := append(sourceExp, destExp...) //nolint:gocritic
+			fwdRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID)
+
+			found := 0
+			for _, chain := range manager.chains {
+				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+				for _, rule := range rules {
+					if len(rule.UserData) > 0 && string(rule.UserData) == fwdRuleKey {
+						require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "forwarding rule elements should match")
+						found = 1
+					}
+				}
+			}
+
+			require.Equal(t, 1, found, "should find at least 1 rule to test")
+
+			if testCase.InputPair.Masquerade {
+				natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID)
+				found := 0
+				for _, chain := range manager.chains {
+					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+					for _, rule := range rules {
+						if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "nat rule elements should match")
+							found = 1
+						}
+					}
+				}
+				require.Equal(t, 1, found, "should find at least 1 rule to test")
+			}
+
+			sourceExp = generateCIDRMatcherExpressions(true, firewall.GetInPair(testCase.InputPair).Source)
+			destExp = generateCIDRMatcherExpressions(false, firewall.GetInPair(testCase.InputPair).Destination)
+			testingExpression = append(sourceExp, destExp...) //nolint:gocritic
+			inFwdRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID)
+
+			found = 0
+			for _, chain := range manager.chains {
+				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+				for _, rule := range rules {
+					if len(rule.UserData) > 0 && string(rule.UserData) == inFwdRuleKey {
+						require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "income forwarding rule elements should match")
+						found = 1
+					}
+				}
+			}
+
+			require.Equal(t, 1, found, "should find at least 1 rule to test")
+
+			if testCase.InputPair.Masquerade {
+				inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID)
+				found := 0
+				for _, chain := range manager.chains {
+					rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+					require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+					for _, rule := range rules {
+						if len(rule.UserData) > 0 && string(rule.UserData) == inNatRuleKey {
+							require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "income nat rule elements should match")
+							found = 1
+						}
+					}
+				}
+				require.Equal(t, 1, found, "should find at least 1 rule to test")
+			}
+		})
+	}
+}
+
+func TestNftablesManager_RemoveRoutingRules(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this OS")
+	}
+
+	table, err := createWorkTable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer deleteWorkTable()
+
+	for _, testCase := range test.RemoveRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			manager, err := newRouter(context.TODO(), table)
+			require.NoError(t, err, "failed to create router")
+
+			nftablesTestingClient := &nftables.Conn{}
+
+			defer manager.ResetForwardRules()
+
+			sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
+			destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
+
+			forwardExp := append(sourceExp, append(destExp, exprCounterAccept...)...) //nolint:gocritic
+			forwardRuleKey := firewall.GenKey(firewall.ForwardingFormat, testCase.InputPair.ID)
+			insertedForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
+				Table:    manager.workTable,
+				Chain:    manager.chains[chainNameRouteingFw],
+				Exprs:    forwardExp,
+				UserData: []byte(forwardRuleKey),
+			})
+
+			natExp := append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) //nolint:gocritic
+			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair.ID)
+
+			insertedNat := nftablesTestingClient.InsertRule(&nftables.Rule{
+				Table:    manager.workTable,
+				Chain:    manager.chains[chainNameRoutingNat],
+				Exprs:    natExp,
+				UserData: []byte(natRuleKey),
+			})
+
+			sourceExp = generateCIDRMatcherExpressions(true, firewall.GetInPair(testCase.InputPair).Source)
+			destExp = generateCIDRMatcherExpressions(false, firewall.GetInPair(testCase.InputPair).Destination)
+
+			forwardExp = append(sourceExp, append(destExp, exprCounterAccept...)...) //nolint:gocritic
+			inForwardRuleKey := firewall.GenKey(firewall.InForwardingFormat, testCase.InputPair.ID)
+			insertedInForwarding := nftablesTestingClient.InsertRule(&nftables.Rule{
+				Table:    manager.workTable,
+				Chain:    manager.chains[chainNameRouteingFw],
+				Exprs:    forwardExp,
+				UserData: []byte(inForwardRuleKey),
+			})
+
+			natExp = append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) //nolint:gocritic
+			inNatRuleKey := firewall.GenKey(firewall.InNatFormat, testCase.InputPair.ID)
+
+			insertedInNat := nftablesTestingClient.InsertRule(&nftables.Rule{
+				Table:    manager.workTable,
+				Chain:    manager.chains[chainNameRoutingNat],
+				Exprs:    natExp,
+				UserData: []byte(inNatRuleKey),
+			})
+
+			err = nftablesTestingClient.Flush()
+			require.NoError(t, err, "shouldn't return error")
+
+			manager.ResetForwardRules()
+
+			err = manager.RemoveRoutingRules(testCase.InputPair)
+			require.NoError(t, err, "shouldn't return error")
+
+			for _, chain := range manager.chains {
+				rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+				require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+				for _, rule := range rules {
+					if len(rule.UserData) > 0 {
+						require.NotEqual(t, insertedForwarding.UserData, rule.UserData, "forwarding rule should not exist")
+						require.NotEqual(t, insertedNat.UserData, rule.UserData, "nat rule should not exist")
+						require.NotEqual(t, insertedInForwarding.UserData, rule.UserData, "income forwarding rule should not exist")
+						require.NotEqual(t, insertedInNat.UserData, rule.UserData, "income nat rule should not exist")
+					}
+				}
+			}
+		})
+	}
+}
+
+// check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found.
+func check() int {
+	nf := nftables.Conn{}
+	if _, err := nf.ListChains(); err == nil {
+		return NFTABLES
+	}
+
+	ip, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	if err != nil {
+		return UNKNOWN
+	}
+	if isIptablesClientAvailable(ip) {
+		return IPTABLES
+	}
+
+	return UNKNOWN
+}
+
+func isIptablesClientAvailable(client *iptables.IPTables) bool {
+	_, err := client.ListChains("filter")
+	return err == nil
+}
+
+func createWorkTable() (*nftables.Table, error) {
+	sConn, err := nftables.New(nftables.AsLasting())
+	if err != nil {
+		return nil, err
+	}
+
+	tables, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, t := range tables {
+		if t.Name == tableName {
+			sConn.DelTable(t)
+		}
+	}
+
+	table := sConn.AddTable(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4})
+	err = sConn.Flush()
+
+	return table, err
+}
+
+func deleteWorkTable() {
+	sConn, err := nftables.New(nftables.AsLasting())
+	if err != nil {
+		return
+	}
+
+	tables, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return
+	}
+
+	for _, t := range tables {
+		if t.Name == tableName {
+			sConn.DelTable(t)
+		}
+	}
+}

client/firewall/nftables/rule_linux.go

@@ -1,6 +1,8 @@
 package nftables
 
 import (
+	"net"
+
 	"github.com/google/nftables"
 )
 
@@ -8,9 +10,8 @@ import (
 type Rule struct {
 	nftRule *nftables.Rule
 	nftSet  *nftables.Set
-
-	ruleID string
-	ip     []byte
+	ruleID  string
+	ip      net.IP
 }
 
 // GetRuleID returns the rule id

client/firewall/nftables/ruleset_linux.go

@@ -1,115 +0,0 @@
-package nftables
-
-import (
-	"bytes"
-	"fmt"
-
-	"github.com/google/nftables"
-	"github.com/rs/xid"
-)
-
-// nftRuleset links native firewall rule and ipset to ACL generated rules
-type nftRuleset struct {
-	nftRule     *nftables.Rule
-	nftSet      *nftables.Set
-	issuedRules map[string]*Rule
-	rulesetID   string
-}
-
-type rulesetManager struct {
-	rulesets map[string]*nftRuleset
-
-	nftSetName2rulesetID   map[string]string
-	issuedRuleID2rulesetID map[string]string
-}
-
-func newRuleManager() *rulesetManager {
-	return &rulesetManager{
-		rulesets: map[string]*nftRuleset{},
-
-		nftSetName2rulesetID:   map[string]string{},
-		issuedRuleID2rulesetID: map[string]string{},
-	}
-}
-
-func (r *rulesetManager) getRuleset(rulesetID string) (*nftRuleset, bool) {
-	ruleset, ok := r.rulesets[rulesetID]
-	return ruleset, ok
-}
-
-func (r *rulesetManager) createRuleset(
-	rulesetID string,
-	nftRule *nftables.Rule,
-	nftSet *nftables.Set,
-) *nftRuleset {
-	ruleset := nftRuleset{
-		rulesetID:   rulesetID,
-		nftRule:     nftRule,
-		nftSet:      nftSet,
-		issuedRules: map[string]*Rule{},
-	}
-	r.rulesets[ruleset.rulesetID] = &ruleset
-	if nftSet != nil {
-		r.nftSetName2rulesetID[nftSet.Name] = ruleset.rulesetID
-	}
-	return &ruleset
-}
-
-func (r *rulesetManager) addRule(
-	ruleset *nftRuleset,
-	ip []byte,
-) (*Rule, error) {
-	if _, ok := r.rulesets[ruleset.rulesetID]; !ok {
-		return nil, fmt.Errorf("ruleset not found")
-	}
-
-	rule := Rule{
-		nftRule: ruleset.nftRule,
-		nftSet:  ruleset.nftSet,
-		ruleID:  xid.New().String(),
-		ip:      ip,
-	}
-
-	ruleset.issuedRules[rule.ruleID] = &rule
-	r.issuedRuleID2rulesetID[rule.ruleID] = ruleset.rulesetID
-
-	return &rule, nil
-}
-
-// deleteRule from ruleset and returns true if contains other rules
-func (r *rulesetManager) deleteRule(rule *Rule) bool {
-	rulesetID, ok := r.issuedRuleID2rulesetID[rule.ruleID]
-	if !ok {
-		return false
-	}
-
-	ruleset := r.rulesets[rulesetID]
-	if ruleset.nftRule == nil {
-		return false
-	}
-	delete(r.issuedRuleID2rulesetID, rule.ruleID)
-	delete(ruleset.issuedRules, rule.ruleID)
-
-	if len(ruleset.issuedRules) == 0 {
-		delete(r.rulesets, ruleset.rulesetID)
-		if rule.nftSet != nil {
-			delete(r.nftSetName2rulesetID, rule.nftSet.Name)
-		}
-		return false
-	}
-	return true
-}
-
-// setNftRuleHandle finds rule by userdata which contains rulesetID and updates it's handle number
-//
-// This is important to do, because after we add rule to the nftables we can't update it until
-// we set correct handle value to it.
-func (r *rulesetManager) setNftRuleHandle(nftRule *nftables.Rule) error {
-	split := bytes.Split(nftRule.UserData, []byte(" "))
-	ruleset, ok := r.rulesets[string(split[0])]
-	if !ok {
-		return fmt.Errorf("ruleset not found")
-	}
-	*ruleset.nftRule = *nftRule
-	return nil
-}

client/firewall/nftables/ruleset_linux_test.go

@@ -1,122 +0,0 @@
-package nftables
-
-import (
-	"testing"
-
-	"github.com/google/nftables"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRulesetManager_createRuleset(t *testing.T) {
-	// Create a ruleset manager.
-	rulesetManager := newRuleManager()
-
-	// Create a ruleset.
-	rulesetID := "ruleset-1"
-	nftRule := nftables.Rule{
-		UserData: []byte(rulesetID),
-	}
-	ruleset := rulesetManager.createRuleset(rulesetID, &nftRule, nil)
-	require.NotNil(t, ruleset, "createRuleset() failed")
-	require.Equal(t, ruleset.rulesetID, rulesetID, "rulesetID is incorrect")
-	require.Equal(t, ruleset.nftRule, &nftRule, "nftRule is incorrect")
-}
-
-func TestRulesetManager_addRule(t *testing.T) {
-	// Create a ruleset manager.
-	rulesetManager := newRuleManager()
-
-	// Create a ruleset.
-	rulesetID := "ruleset-1"
-	nftRule := nftables.Rule{}
-	ruleset := rulesetManager.createRuleset(rulesetID, &nftRule, nil)
-
-	// Add a rule to the ruleset.
-	ip := []byte("192.168.1.1")
-	rule, err := rulesetManager.addRule(ruleset, ip)
-	require.NoError(t, err, "addRule() failed")
-	require.NotNil(t, rule, "rule should not be nil")
-	require.NotEqual(t, rule.ruleID, "ruleID is empty")
-	require.EqualValues(t, rule.ip, ip, "ip is incorrect")
-	require.Contains(t, ruleset.issuedRules, rule.ruleID, "ruleID already exists in ruleset")
-	require.Contains(t, rulesetManager.issuedRuleID2rulesetID, rule.ruleID, "ruleID already exists in ruleset manager")
-
-	ruleset2 := &nftRuleset{
-		rulesetID: "ruleset-2",
-	}
-	_, err = rulesetManager.addRule(ruleset2, ip)
-	require.Error(t, err, "addRule() should have failed")
-}
-
-func TestRulesetManager_deleteRule(t *testing.T) {
-	// Create a ruleset manager.
-	rulesetManager := newRuleManager()
-
-	// Create a ruleset.
-	rulesetID := "ruleset-1"
-	nftRule := nftables.Rule{}
-	ruleset := rulesetManager.createRuleset(rulesetID, &nftRule, nil)
-
-	// Add a rule to the ruleset.
-	ip := []byte("192.168.1.1")
-	rule, err := rulesetManager.addRule(ruleset, ip)
-	require.NoError(t, err, "addRule() failed")
-	require.NotNil(t, rule, "rule should not be nil")
-
-	ip2 := []byte("192.168.1.1")
-	rule2, err := rulesetManager.addRule(ruleset, ip2)
-	require.NoError(t, err, "addRule() failed")
-	require.NotNil(t, rule2, "rule should not be nil")
-
-	hasNext := rulesetManager.deleteRule(rule)
-	require.True(t, hasNext, "deleteRule() should have returned true")
-
-	// Check that the rule is no longer in the manager.
-	require.NotContains(t, rulesetManager.issuedRuleID2rulesetID, rule.ruleID, "rule should have been deleted")
-
-	hasNext = rulesetManager.deleteRule(rule2)
-	require.False(t, hasNext, "deleteRule() should have returned false")
-}
-
-func TestRulesetManager_setNftRuleHandle(t *testing.T) {
-	// Create a ruleset manager.
-	rulesetManager := newRuleManager()
-	// Create a ruleset.
-	rulesetID := "ruleset-1"
-	nftRule := nftables.Rule{}
-	ruleset := rulesetManager.createRuleset(rulesetID, &nftRule, nil)
-	// Add a rule to the ruleset.
-	ip := []byte("192.168.0.1")
-
-	rule, err := rulesetManager.addRule(ruleset, ip)
-	require.NoError(t, err, "addRule() failed")
-	require.NotNil(t, rule, "rule should not be nil")
-
-	nftRuleCopy := nftRule
-	nftRuleCopy.Handle = 2
-	nftRuleCopy.UserData = []byte(rulesetID)
-	err = rulesetManager.setNftRuleHandle(&nftRuleCopy)
-	require.NoError(t, err, "setNftRuleHandle() failed")
-	// check correct work with references
-	require.Equal(t, nftRule.Handle, uint64(2), "nftRule.Handle is incorrect")
-}
-
-func TestRulesetManager_getRuleset(t *testing.T) {
-	// Create a ruleset manager.
-	rulesetManager := newRuleManager()
-	// Create a ruleset.
-	rulesetID := "ruleset-1"
-	nftRule := nftables.Rule{}
-	nftSet := nftables.Set{
-		ID: 2,
-	}
-	ruleset := rulesetManager.createRuleset(rulesetID, &nftRule, &nftSet)
-	require.NotNil(t, ruleset, "createRuleset() failed")
-
-	find, ok := rulesetManager.getRuleset(rulesetID)
-	require.True(t, ok, "getRuleset() failed")
-	require.Equal(t, ruleset, find, "getRulesetBySetID() failed")
-
-	_, ok = rulesetManager.getRuleset("does-not-exist")
-	require.False(t, ok, "getRuleset() failed")
-}