netbird/client/firewall/nftables/route_linux.go

package nftables

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net"
	"net/netip"

	"github.com/google/nftables"
	"github.com/google/nftables/binaryutil"
	"github.com/google/nftables/expr"
	log "github.com/sirupsen/logrus"

	"github.com/netbirdio/netbird/client/firewall/manager"
)

const (
	chainNameRouteingFw = "netbird-rt-fwd"
	chainNameRoutingNat = "netbird-rt-nat"

	userDataAcceptForwardRuleSrc = "frwacceptsrc"
	userDataAcceptForwardRuleDst = "frwacceptdst"
)

// some presets for building nftable rules
var (
	zeroXor  = binaryutil.NativeEndian.PutUint32(0)
	zeroXor6 = []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}

	exprCounterAccept = []expr.Any{
		&expr.Counter{},
		&expr.Verdict{
			Kind: expr.VerdictAccept,
		},
	}

	errFilterTableNotFound = fmt.Errorf("nftables: 'filter' table not found")
)

type router struct {
	ctx          context.Context
	stop         context.CancelFunc
	conn         *nftables.Conn
	workTable    *nftables.Table
	workTable6   *nftables.Table
	filterTable  *nftables.Table
	filterTable6 *nftables.Table
	chains       map[string]*nftables.Chain
	chains6      map[string]*nftables.Chain
	// rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules
	rules                     map[string]*nftables.Rule
	rules6                    map[string]*nftables.Rule
	isDefaultFwdRulesEnabled  bool
	isDefaultFwdRulesEnabled6 bool
}

func newRouter(parentCtx context.Context, workTable *nftables.Table, workTable6 *nftables.Table) (*router, error) {
	ctx, cancel := context.WithCancel(parentCtx)

	r := &router{
		ctx:        ctx,
		stop:       cancel,
		conn:       &nftables.Conn{},
		workTable:  workTable,
		workTable6: workTable6,
		chains:     make(map[string]*nftables.Chain),
		chains6:    make(map[string]*nftables.Chain),
		rules:      make(map[string]*nftables.Rule),
		rules6:     make(map[string]*nftables.Rule),
	}

	var err error
	r.filterTable, r.filterTable6, err = r.loadFilterTables()
	if err != nil {
		if errors.Is(err, errFilterTableNotFound) {
			log.Warnf("table 'filter' not found for forward rules for one of the supported address families-")
		} else {
			return nil, err
		}
	}

	err = r.cleanUpDefaultForwardRules(false)
	if err != nil {
		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
	}

	err = r.cleanUpDefaultForwardRules(true)
	if err != nil {
		log.Errorf("failed to clean up rules from IPv6 FORWARD chain: %s", err)
	}

	err = r.createContainers(false)
	if err != nil {
		log.Errorf("failed to create containers for route: %s", err)
	}

	if r.workTable6 != nil {
		err = r.createContainers(true)
		if err != nil {
			log.Errorf("failed to create v6 containers for route: %s", err)
		}
	}

	return r, err
}

func (r *router) RouteingFwChainName() string {
	return chainNameRouteingFw
}

// ResetForwardRules cleans existing nftables default forward rules from the system
func (r *router) ResetForwardRules() {
	err := r.cleanUpDefaultForwardRules(false)
	if err != nil {
		log.Errorf("failed to reset forward rules: %s", err)
	}
	err = r.cleanUpDefaultForwardRules(true)
	if err != nil {
		log.Errorf("failed to reset forward rules: %s", err)
	}
}

func (r *router) RestoreAfterV6Reset(newWorktable6 *nftables.Table) error {
	r.workTable6 = newWorktable6
	if newWorktable6 != nil {

		err := r.cleanUpDefaultForwardRules(true)
		if err != nil {
			log.Errorf("failed to clean up rules from IPv6 FORWARD chain: %s", err)
		}

		err = r.createContainers(true)
		if err != nil {
			return err
		}

		for name, rule := range r.rules6 {
			rule = &nftables.Rule{
				Table:    r.workTable6,
				Chain:    r.chains6[rule.Chain.Name],
				Exprs:    rule.Exprs,
				UserData: rule.UserData,
			}
			r.rules6[name] = r.conn.AddRule(rule)
		}
	}
	return r.conn.Flush()
}

func (r *router) loadFilterTables() (*nftables.Table, *nftables.Table, error) {
	tables, err := r.conn.ListTablesOfFamily(nftables.TableFamilyIPv4)
	if err != nil {
		return nil, nil, fmt.Errorf("nftables: unable to list tables: %v", err)
	}

	var table4 *nftables.Table = nil
	for _, table := range tables {
		if table.Name == "filter" {
			table4 = table
			break
		}
	}

	var table6 *nftables.Table = nil
	tables, err = r.conn.ListTablesOfFamily(nftables.TableFamilyIPv6)
	if err != nil {
		return nil, nil, fmt.Errorf("nftables: unable to list tables: %v", err)
	}
	for _, table := range tables {
		if table.Name == "filter" {
			table6 = table
			break
		}
	}

	err = nil
	if table4 == nil || table6 == nil {
		err = errFilterTableNotFound
	}

	return table4, table6, err
}

func (r *router) createContainers(forV6 bool) error {
	workTable := r.workTable
	chainStorage := r.chains
	if forV6 {
		workTable = r.workTable6
		chainStorage = r.chains6
	}

	chainStorage[chainNameRouteingFw] = r.conn.AddChain(&nftables.Chain{
		Name:  chainNameRouteingFw,
		Table: workTable,
	})

	chainStorage[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
		Name:     chainNameRoutingNat,
		Table:    workTable,
		Hooknum:  nftables.ChainHookPostrouting,
		Priority: nftables.ChainPriorityNATSource - 1,
		Type:     nftables.ChainTypeNAT,
	})

	err := r.refreshRulesMap(forV6)
	if err != nil {
		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
	}

	err = r.conn.Flush()
	if err != nil {
		return fmt.Errorf("nftables: unable to initialize table: %v", err)
	}
	return nil
}

// InsertRoutingRules inserts a nftable rule pair to the forwarding chain and if enabled, to the nat chain
func (r *router) InsertRoutingRules(pair manager.RouterPair) error {
	parsedIp, _, _ := net.ParseCIDR(pair.Source)

	if parsedIp.To4() == nil && r.workTable6 == nil {
		return fmt.Errorf("nftables: attempted to add IPv6 routing rule even though IPv6 is not enabled for this host")
	}

	err := r.refreshRulesMap(parsedIp.To4() == nil)
	if err != nil {
		return err
	}

	err = r.insertRoutingRule(manager.ForwardingFormat, chainNameRouteingFw, pair, false)
	if err != nil {
		return err
	}
	err = r.insertRoutingRule(manager.InForwardingFormat, chainNameRouteingFw, manager.GetInPair(pair), false)
	if err != nil {
		return err
	}

	if pair.Masquerade {
		err = r.insertRoutingRule(manager.NatFormat, chainNameRoutingNat, pair, true)
		if err != nil {
			return err
		}
		err = r.insertRoutingRule(manager.InNatFormat, chainNameRoutingNat, manager.GetInPair(pair), true)
		if err != nil {
			return err
		}
	}

	filterTable := r.filterTable
	if parsedIp.To4() == nil {
		filterTable = r.filterTable6
	}
	if filterTable != nil && !r.isDefaultFwdRulesEnabled {
		log.Debugf("add default accept forward rule")
		r.acceptForwardRule(pair.Source)
	}

	err = r.conn.Flush()
	if err != nil {
		return fmt.Errorf("nftables: unable to insert rules for %s: %v", pair.Destination, err)
	}
	return nil
}

// insertRoutingRule inserts a nftable rule to the conn client flush queue
func (r *router) insertRoutingRule(format, chainName string, pair manager.RouterPair, isNat bool) error {
	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
	destExp := generateCIDRMatcherExpressions(false, pair.Destination)

	var expression []expr.Any
	if isNat {
		expression = append(sourceExp, append(destExp, &expr.Counter{}, &expr.Masq{})...) // nolint:gocritic
	} else {
		expression = append(sourceExp, append(destExp, exprCounterAccept...)...) // nolint:gocritic
	}

	ruleKey := manager.GenKey(format, pair.ID)

	parsedIp, _, _ := net.ParseCIDR(pair.Source)
	rules := r.rules
	if parsedIp.To4() == nil {
		rules = r.rules6
	}

	_, exists := rules[ruleKey]
	if exists {
		err := r.removeRoutingRule(format, pair)
		if err != nil {
			return err
		}
	}

	table, chain := r.workTable, r.chains[chainName]
	if parsedIp.To4() == nil {
		table, chain = r.workTable6, r.chains6[chainName]
	}

	newRule := r.conn.InsertRule(&nftables.Rule{
		Table:    table,
		Chain:    chain,
		Exprs:    expression,
		UserData: []byte(ruleKey),
	})

	if parsedIp.To4() == nil {
		r.rules[ruleKey] = newRule
	} else {
		r.rules6[ruleKey] = newRule
	}
	return nil
}

func (r *router) acceptForwardRule(sourceNetwork string) {
	src := generateCIDRMatcherExpressions(true, sourceNetwork)
	dst := generateCIDRMatcherExpressions(false, "0.0.0.0/0")
	table := r.filterTable
	parsedIp, _, _ := net.ParseCIDR(sourceNetwork)
	if parsedIp.To4() == nil {
		dst = generateCIDRMatcherExpressions(false, "::/0")
		table = r.filterTable6
	}

	var exprs []expr.Any
	exprs = append(src, append(dst, &expr.Verdict{ // nolint:gocritic
		Kind: expr.VerdictAccept,
	})...)

	rule := &nftables.Rule{
		Table: table,
		Chain: &nftables.Chain{
			Name:     "FORWARD",
			Table:    table,
			Type:     nftables.ChainTypeFilter,
			Hooknum:  nftables.ChainHookForward,
			Priority: nftables.ChainPriorityFilter,
		},
		Exprs:    exprs,
		UserData: []byte(userDataAcceptForwardRuleSrc),
	}

	r.conn.AddRule(rule)

	src = generateCIDRMatcherExpressions(true, "0.0.0.0/0")
	if parsedIp.To4() == nil {
		src = generateCIDRMatcherExpressions(true, "::/0")
	}
	dst = generateCIDRMatcherExpressions(false, sourceNetwork)

	exprs = append(src, append(dst, &expr.Verdict{ //nolint:gocritic
		Kind: expr.VerdictAccept,
	})...)

	rule = &nftables.Rule{
		Table: table,
		Chain: &nftables.Chain{
			Name:     "FORWARD",
			Table:    table,
			Type:     nftables.ChainTypeFilter,
			Hooknum:  nftables.ChainHookForward,
			Priority: nftables.ChainPriorityFilter,
		},
		Exprs:    exprs,
		UserData: []byte(userDataAcceptForwardRuleDst),
	}
	r.conn.AddRule(rule)
	if parsedIp.To4() == nil {
		r.isDefaultFwdRulesEnabled6 = true
	} else {
		r.isDefaultFwdRulesEnabled = true
	}
}

// RemoveRoutingRules removes a nftable rule pair from forwarding and nat chains
func (r *router) RemoveRoutingRules(pair manager.RouterPair) error {
	parsedIp, _, _ := net.ParseCIDR(pair.Source)
	if parsedIp.To4() == nil && r.workTable6 == nil {
		return fmt.Errorf("nftables: attempted to remove IPv6 routing rule even though IPv6 is not enabled for this host")
	}

	err := r.refreshRulesMap(parsedIp.To4() == nil)
	if err != nil {
		return err
	}

	err = r.removeRoutingRule(manager.ForwardingFormat, pair)
	if err != nil {
		return err
	}

	err = r.removeRoutingRule(manager.InForwardingFormat, manager.GetInPair(pair))
	if err != nil {
		return err
	}

	err = r.removeRoutingRule(manager.NatFormat, pair)
	if err != nil {
		return err
	}

	err = r.removeRoutingRule(manager.InNatFormat, manager.GetInPair(pair))
	if err != nil {
		return err
	}

	rulesList := r.rules
	if parsedIp.To4() == nil {
		rulesList = r.rules6
	}
	if len(rulesList) == 0 {
		err := r.cleanUpDefaultForwardRules(parsedIp.To4() == nil)
		if err != nil {
			log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
		}
	}

	err = r.conn.Flush()
	if err != nil {
		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
	}
	log.Debugf("nftables: removed rules for %s", pair.Destination)
	return nil
}

// removeRoutingRule add a nftable rule to the removal queue and delete from rules map
func (r *router) removeRoutingRule(format string, pair manager.RouterPair) error {
	ruleKey := manager.GenKey(format, pair.ID)

	parsedIp, _, _ := net.ParseCIDR(pair.Source)
	rules := r.rules
	if parsedIp.To4() == nil {
		rules = r.rules6
	}

	rule, found := rules[ruleKey]
	if found {
		ruleType := "forwarding"
		if rule.Chain.Type == nftables.ChainTypeNAT {
			ruleType = "nat"
		}

		err := r.conn.DelRule(rule)
		if err != nil {
			return fmt.Errorf("nftables: unable to remove %s rule for %s: %v", ruleType, pair.Destination, err)
		}

		log.Debugf("nftables: removing %s rule for %s", ruleType, pair.Destination)

		delete(rules, ruleKey)
	}
	return nil
}

// refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid
// duplicates and to get missing attributes that we don't have when adding new rules
func (r *router) refreshRulesMap(forV6 bool) error {
	chainList := r.chains
	if forV6 {
		chainList = r.chains6
	}
	for _, chain := range chainList {
		rules, err := r.conn.GetRules(chain.Table, chain)
		if err != nil {
			return fmt.Errorf("nftables: unable to list rules: %v", err)
		}
		for _, rule := range rules {
			if len(rule.UserData) > 0 {
				if forV6 {
					r.rules6[string(rule.UserData)] = rule
				} else {
					r.rules[string(rule.UserData)] = rule
				}
			}
		}
	}
	return nil
}

func (r *router) cleanUpDefaultForwardRules(forV6 bool) error {
	tableFamily := nftables.TableFamilyIPv4
	filterTable := r.filterTable
	if forV6 {
		tableFamily = nftables.TableFamilyIPv6
		filterTable = r.filterTable6
	}

	if filterTable == nil {
		if forV6 {
			r.isDefaultFwdRulesEnabled6 = false
		} else {
			r.isDefaultFwdRulesEnabled = false
		}
		return nil
	}

	chains, err := r.conn.ListChainsOfTableFamily(tableFamily)
	if err != nil {
		return err
	}

	var rules []*nftables.Rule
	for _, chain := range chains {
		if chain.Table.Name != filterTable.Name {
			continue
		}
		if chain.Name != "FORWARD" {
			continue
		}

		rules, err = r.conn.GetRules(filterTable, chain)
		if err != nil {
			return err
		}
	}

	for _, rule := range rules {
		if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleSrc)) || bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleDst)) {
			err := r.conn.DelRule(rule)
			if err != nil {
				return err
			}
		}
	}

	if forV6 {
		r.isDefaultFwdRulesEnabled6 = false
	} else {
		r.isDefaultFwdRulesEnabled = false
	}
	return r.conn.Flush()
}

// generateCIDRMatcherExpressions generates nftables expressions that matches a CIDR
func generateCIDRMatcherExpressions(source bool, cidr string) []expr.Any {
	ip, network, _ := net.ParseCIDR(cidr)
	ipToAdd, _ := netip.AddrFromSlice(ip)
	add := ipToAdd.Unmap()

	var offSet uint32
	if source {
		offSet = 12 // src offset
	} else {
		offSet = 16 // dst offset
	}
	addrLen := uint32(4)
	zeroXor := zeroXor

	if ip.To4() == nil {
		if source {
			offSet = 8 // src offset
		} else {
			offSet = 24 // dst offset
		}
		addrLen = 16
		zeroXor = zeroXor6
	}

	return []expr.Any{
		// fetch src add
		&expr.Payload{
			DestRegister: 1,
			Base:         expr.PayloadBaseNetworkHeader,
			Offset:       offSet,
			Len:          addrLen,
		},
		// net mask
		&expr.Bitwise{
			DestRegister:   1,
			SourceRegister: 1,
			Len:            addrLen,
			Mask:           network.Mask,
			Xor:            zeroXor,
		},
		// net address
		&expr.Cmp{
			Register: 1,
			Data:     add.AsSlice(),
		},
	}
}