mirror of
https://github.com/netbirdio/netbird.git
synced 2026-04-18 16:26:38 +00:00
[client] Add detailed routes and resolved IPs to debug bundle (#4141)
This commit is contained in:
@@ -2,9 +2,12 @@
|
||||
|
||||
package systemops
|
||||
|
||||
import "syscall"
|
||||
import (
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// filterRoutesByFlags - return true if need to ignore such route message because it consists specific flags.
|
||||
// filterRoutesByFlags returns true if the route message should be ignored based on its flags.
|
||||
func filterRoutesByFlags(routeMessageFlags int) bool {
|
||||
if routeMessageFlags&syscall.RTF_UP == 0 {
|
||||
return true
|
||||
@@ -16,3 +19,50 @@ func filterRoutesByFlags(routeMessageFlags int) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// formatBSDFlags formats route flags for BSD systems (excludes FreeBSD-specific handling)
|
||||
func formatBSDFlags(flags int) string {
|
||||
var flagStrs []string
|
||||
|
||||
if flags&syscall.RTF_UP != 0 {
|
||||
flagStrs = append(flagStrs, "U")
|
||||
}
|
||||
if flags&syscall.RTF_GATEWAY != 0 {
|
||||
flagStrs = append(flagStrs, "G")
|
||||
}
|
||||
if flags&syscall.RTF_HOST != 0 {
|
||||
flagStrs = append(flagStrs, "H")
|
||||
}
|
||||
if flags&syscall.RTF_REJECT != 0 {
|
||||
flagStrs = append(flagStrs, "R")
|
||||
}
|
||||
if flags&syscall.RTF_DYNAMIC != 0 {
|
||||
flagStrs = append(flagStrs, "D")
|
||||
}
|
||||
if flags&syscall.RTF_MODIFIED != 0 {
|
||||
flagStrs = append(flagStrs, "M")
|
||||
}
|
||||
if flags&syscall.RTF_STATIC != 0 {
|
||||
flagStrs = append(flagStrs, "S")
|
||||
}
|
||||
if flags&syscall.RTF_LLINFO != 0 {
|
||||
flagStrs = append(flagStrs, "L")
|
||||
}
|
||||
if flags&syscall.RTF_LOCAL != 0 {
|
||||
flagStrs = append(flagStrs, "l")
|
||||
}
|
||||
if flags&syscall.RTF_BLACKHOLE != 0 {
|
||||
flagStrs = append(flagStrs, "B")
|
||||
}
|
||||
if flags&syscall.RTF_CLONING != 0 {
|
||||
flagStrs = append(flagStrs, "C")
|
||||
}
|
||||
if flags&syscall.RTF_WASCLONED != 0 {
|
||||
flagStrs = append(flagStrs, "W")
|
||||
}
|
||||
|
||||
if len(flagStrs) == 0 {
|
||||
return "-"
|
||||
}
|
||||
return strings.Join(flagStrs, "")
|
||||
}
|
||||
|
||||
@@ -1,19 +1,64 @@
|
||||
//go:build: freebsd
|
||||
//go:build freebsd
|
||||
|
||||
package systemops
|
||||
|
||||
import "syscall"
|
||||
import (
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// filterRoutesByFlags - return true if need to ignore such route message because it consists specific flags.
|
||||
// filterRoutesByFlags returns true if the route message should be ignored based on its flags.
|
||||
func filterRoutesByFlags(routeMessageFlags int) bool {
|
||||
if routeMessageFlags&syscall.RTF_UP == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// NOTE: syscall.RTF_WASCLONED deprecated in FreeBSD 8.0 (https://www.freebsd.org/releases/8.0R/relnotes-detailed/)
|
||||
// a concept of cloned route (a route generated by an entry with RTF_CLONING flag) is deprecated.
|
||||
// NOTE: syscall.RTF_WASCLONED deprecated in FreeBSD 8.0
|
||||
if routeMessageFlags&(syscall.RTF_REJECT|syscall.RTF_BLACKHOLE) != 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// formatBSDFlags formats route flags for FreeBSD (excludes deprecated RTF_CLONING and RTF_WASCLONED)
|
||||
func formatBSDFlags(flags int) string {
|
||||
var flagStrs []string
|
||||
|
||||
if flags&syscall.RTF_UP != 0 {
|
||||
flagStrs = append(flagStrs, "U")
|
||||
}
|
||||
if flags&syscall.RTF_GATEWAY != 0 {
|
||||
flagStrs = append(flagStrs, "G")
|
||||
}
|
||||
if flags&syscall.RTF_HOST != 0 {
|
||||
flagStrs = append(flagStrs, "H")
|
||||
}
|
||||
if flags&syscall.RTF_REJECT != 0 {
|
||||
flagStrs = append(flagStrs, "R")
|
||||
}
|
||||
if flags&syscall.RTF_DYNAMIC != 0 {
|
||||
flagStrs = append(flagStrs, "D")
|
||||
}
|
||||
if flags&syscall.RTF_MODIFIED != 0 {
|
||||
flagStrs = append(flagStrs, "M")
|
||||
}
|
||||
if flags&syscall.RTF_STATIC != 0 {
|
||||
flagStrs = append(flagStrs, "S")
|
||||
}
|
||||
if flags&syscall.RTF_LLINFO != 0 {
|
||||
flagStrs = append(flagStrs, "L")
|
||||
}
|
||||
if flags&syscall.RTF_LOCAL != 0 {
|
||||
flagStrs = append(flagStrs, "l")
|
||||
}
|
||||
if flags&syscall.RTF_BLACKHOLE != 0 {
|
||||
flagStrs = append(flagStrs, "B")
|
||||
}
|
||||
// Note: RTF_CLONING and RTF_WASCLONED deprecated in FreeBSD 8.0
|
||||
|
||||
if len(flagStrs) == 0 {
|
||||
return "-"
|
||||
}
|
||||
return strings.Join(flagStrs, "")
|
||||
}
|
||||
|
||||
@@ -19,6 +19,26 @@ type Nexthop struct {
|
||||
Intf *net.Interface
|
||||
}
|
||||
|
||||
// Route represents a basic network route with core routing information
|
||||
type Route struct {
|
||||
Dst netip.Prefix
|
||||
Gw netip.Addr
|
||||
Interface *net.Interface
|
||||
}
|
||||
|
||||
// DetailedRoute extends Route with additional metadata for display and debugging
|
||||
type DetailedRoute struct {
|
||||
Route
|
||||
Metric int
|
||||
InterfaceMetric int
|
||||
InterfaceIndex int
|
||||
Protocol string
|
||||
Scope string
|
||||
Type string
|
||||
Table string
|
||||
Flags string
|
||||
}
|
||||
|
||||
// Equal checks if two nexthops are equal.
|
||||
func (n Nexthop) Equal(other Nexthop) bool {
|
||||
return n.IP == other.IP && (n.Intf == nil && other.Intf == nil ||
|
||||
|
||||
@@ -16,12 +16,6 @@ import (
|
||||
"golang.org/x/net/route"
|
||||
)
|
||||
|
||||
type Route struct {
|
||||
Dst netip.Prefix
|
||||
Gw netip.Addr
|
||||
Interface *net.Interface
|
||||
}
|
||||
|
||||
func GetRoutesFromTable() ([]netip.Prefix, error) {
|
||||
tab, err := retryFetchRIB()
|
||||
if err != nil {
|
||||
@@ -47,25 +41,134 @@ func GetRoutesFromTable() ([]netip.Prefix, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
route, err := MsgToRoute(m)
|
||||
r, err := MsgToRoute(m)
|
||||
if err != nil {
|
||||
log.Warnf("Failed to parse route message: %v", err)
|
||||
continue
|
||||
}
|
||||
if route.Dst.IsValid() {
|
||||
prefixList = append(prefixList, route.Dst)
|
||||
if r.Dst.IsValid() {
|
||||
prefixList = append(prefixList, r.Dst)
|
||||
}
|
||||
}
|
||||
return prefixList, nil
|
||||
}
|
||||
|
||||
func GetDetailedRoutesFromTable() ([]DetailedRoute, error) {
|
||||
tab, err := retryFetchRIB()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch RIB: %v", err)
|
||||
}
|
||||
|
||||
msgs, err := route.ParseRIB(route.RIBTypeRoute, tab)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse RIB: %v", err)
|
||||
}
|
||||
|
||||
return processRouteMessages(msgs)
|
||||
}
|
||||
|
||||
func processRouteMessages(msgs []route.Message) ([]DetailedRoute, error) {
|
||||
var detailedRoutes []DetailedRoute
|
||||
|
||||
for _, msg := range msgs {
|
||||
m := msg.(*route.RouteMessage)
|
||||
|
||||
if !isValidRouteMessage(m) {
|
||||
continue
|
||||
}
|
||||
|
||||
if filterRoutesByFlags(m.Flags) {
|
||||
continue
|
||||
}
|
||||
|
||||
detailed, err := buildDetailedRouteFromMessage(m)
|
||||
if err != nil {
|
||||
log.Warnf("Failed to parse route message: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if detailed != nil {
|
||||
detailedRoutes = append(detailedRoutes, *detailed)
|
||||
}
|
||||
}
|
||||
|
||||
return detailedRoutes, nil
|
||||
}
|
||||
|
||||
func isValidRouteMessage(m *route.RouteMessage) bool {
|
||||
if m.Version < 3 || m.Version > 5 {
|
||||
log.Warnf("Unexpected RIB message version: %d", m.Version)
|
||||
return false
|
||||
}
|
||||
if m.Type != syscall.RTM_GET {
|
||||
log.Warnf("Unexpected RIB message type: %d", m.Type)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func buildDetailedRouteFromMessage(m *route.RouteMessage) (*DetailedRoute, error) {
|
||||
routeMsg, err := MsgToRoute(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !routeMsg.Dst.IsValid() {
|
||||
return nil, errors.New("invalid destination")
|
||||
}
|
||||
|
||||
detailed := DetailedRoute{
|
||||
Route: Route{
|
||||
Dst: routeMsg.Dst,
|
||||
Gw: routeMsg.Gw,
|
||||
Interface: routeMsg.Interface,
|
||||
},
|
||||
Metric: extractBSDMetric(m),
|
||||
Protocol: extractBSDProtocol(m.Flags),
|
||||
Scope: "global",
|
||||
Type: "unicast",
|
||||
Table: "main",
|
||||
Flags: formatBSDFlags(m.Flags),
|
||||
}
|
||||
|
||||
return &detailed, nil
|
||||
}
|
||||
|
||||
func buildLinkInterface(t *route.LinkAddr) *net.Interface {
|
||||
interfaceName := fmt.Sprintf("link#%d", t.Index)
|
||||
if t.Name != "" {
|
||||
interfaceName = t.Name
|
||||
}
|
||||
return &net.Interface{
|
||||
Index: t.Index,
|
||||
Name: interfaceName,
|
||||
}
|
||||
}
|
||||
|
||||
func extractBSDMetric(m *route.RouteMessage) int {
|
||||
return -1
|
||||
}
|
||||
|
||||
func extractBSDProtocol(flags int) string {
|
||||
if flags&syscall.RTF_STATIC != 0 {
|
||||
return "static"
|
||||
}
|
||||
if flags&syscall.RTF_DYNAMIC != 0 {
|
||||
return "dynamic"
|
||||
}
|
||||
if flags&syscall.RTF_LOCAL != 0 {
|
||||
return "local"
|
||||
}
|
||||
return "kernel"
|
||||
}
|
||||
|
||||
func retryFetchRIB() ([]byte, error) {
|
||||
var out []byte
|
||||
operation := func() error {
|
||||
var err error
|
||||
out, err = route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0)
|
||||
if errors.Is(err, syscall.ENOMEM) {
|
||||
log.Debug("~etrying fetchRIB due to 'cannot allocate memory' error")
|
||||
log.Debug("Retrying fetchRIB due to 'cannot allocate memory' error")
|
||||
return err
|
||||
} else if err != nil {
|
||||
return backoff.Permanent(err)
|
||||
@@ -100,7 +203,6 @@ func toNetIP(a route.Addr) netip.Addr {
|
||||
}
|
||||
}
|
||||
|
||||
// ones returns the number of leading ones in the mask.
|
||||
func ones(a route.Addr) (int, error) {
|
||||
switch t := a.(type) {
|
||||
case *route.Inet4Addr:
|
||||
@@ -114,7 +216,6 @@ func ones(a route.Addr) (int, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// MsgToRoute converts a route message to a Route.
|
||||
func MsgToRoute(msg *route.RouteMessage) (*Route, error) {
|
||||
dstIP, nexthop, dstMask := msg.Addrs[0], msg.Addrs[1], msg.Addrs[2]
|
||||
|
||||
@@ -127,10 +228,7 @@ func MsgToRoute(msg *route.RouteMessage) (*Route, error) {
|
||||
case *route.Inet4Addr, *route.Inet6Addr:
|
||||
nexthopAddr = toNetIP(t)
|
||||
case *route.LinkAddr:
|
||||
nexthopIntf = &net.Interface{
|
||||
Index: t.Index,
|
||||
Name: t.Name,
|
||||
}
|
||||
nexthopIntf = buildLinkInterface(t)
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected next hop type: %T", t)
|
||||
}
|
||||
@@ -156,5 +254,4 @@ func MsgToRoute(msg *route.RouteMessage) (*Route, error) {
|
||||
Gw: nexthopAddr,
|
||||
Interface: nexthopIntf,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/hashicorp/go-multierror"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
nberrors "github.com/netbirdio/netbird/client/errors"
|
||||
"github.com/netbirdio/netbird/client/internal/routemanager/sysctl"
|
||||
@@ -22,6 +23,25 @@ import (
|
||||
nbnet "github.com/netbirdio/netbird/util/net"
|
||||
)
|
||||
|
||||
// IPRule contains IP rule information for debugging
|
||||
type IPRule struct {
|
||||
Priority int
|
||||
From netip.Prefix
|
||||
To netip.Prefix
|
||||
IIF string
|
||||
OIF string
|
||||
Table string
|
||||
Action string
|
||||
Mark uint32
|
||||
Mask uint32
|
||||
TunID uint32
|
||||
Goto uint32
|
||||
Flow uint32
|
||||
SuppressPlen int
|
||||
SuppressIFL int
|
||||
Invert bool
|
||||
}
|
||||
|
||||
const (
|
||||
// NetbirdVPNTableID is the ID of the custom routing table used by Netbird.
|
||||
NetbirdVPNTableID = 0x1BD0
|
||||
@@ -37,6 +57,8 @@ const (
|
||||
|
||||
var ErrTableIDExists = errors.New("ID exists with different name")
|
||||
|
||||
const errParsePrefixMsg = "failed to parse prefix %s: %w"
|
||||
|
||||
// originalSysctl stores the original sysctl values before they are modified
|
||||
var originalSysctl map[string]int
|
||||
|
||||
@@ -209,6 +231,277 @@ func GetRoutesFromTable() ([]netip.Prefix, error) {
|
||||
return append(v4Routes, v6Routes...), nil
|
||||
}
|
||||
|
||||
// GetDetailedRoutesFromTable returns detailed route information from all routing tables
|
||||
func GetDetailedRoutesFromTable() ([]DetailedRoute, error) {
|
||||
tables := discoverRoutingTables()
|
||||
return collectRoutesFromTables(tables), nil
|
||||
}
|
||||
|
||||
func discoverRoutingTables() []int {
|
||||
tables, err := getAllRoutingTables()
|
||||
if err != nil {
|
||||
log.Warnf("Failed to get all routing tables, using fallback list: %v", err)
|
||||
return []int{
|
||||
syscall.RT_TABLE_MAIN,
|
||||
syscall.RT_TABLE_LOCAL,
|
||||
NetbirdVPNTableID,
|
||||
}
|
||||
}
|
||||
return tables
|
||||
}
|
||||
|
||||
func collectRoutesFromTables(tables []int) []DetailedRoute {
|
||||
var allRoutes []DetailedRoute
|
||||
|
||||
for _, tableID := range tables {
|
||||
routes := collectRoutesFromTable(tableID)
|
||||
allRoutes = append(allRoutes, routes...)
|
||||
}
|
||||
|
||||
return allRoutes
|
||||
}
|
||||
|
||||
func collectRoutesFromTable(tableID int) []DetailedRoute {
|
||||
var routes []DetailedRoute
|
||||
|
||||
if v4Routes := getRoutesForFamily(tableID, netlink.FAMILY_V4); len(v4Routes) > 0 {
|
||||
routes = append(routes, v4Routes...)
|
||||
}
|
||||
|
||||
if v6Routes := getRoutesForFamily(tableID, netlink.FAMILY_V6); len(v6Routes) > 0 {
|
||||
routes = append(routes, v6Routes...)
|
||||
}
|
||||
|
||||
return routes
|
||||
}
|
||||
|
||||
func getRoutesForFamily(tableID, family int) []DetailedRoute {
|
||||
routes, err := getDetailedRoutes(tableID, family)
|
||||
if err != nil {
|
||||
log.Debugf("Failed to get routes from table %d family %d: %v", tableID, family, err)
|
||||
return nil
|
||||
}
|
||||
return routes
|
||||
}
|
||||
|
||||
func getAllRoutingTables() ([]int, error) {
|
||||
tablesMap := make(map[int]bool)
|
||||
families := []int{netlink.FAMILY_V4, netlink.FAMILY_V6}
|
||||
|
||||
// Use table 0 (RT_TABLE_UNSPEC) to discover all tables
|
||||
for _, family := range families {
|
||||
routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: 0}, netlink.RT_FILTER_TABLE)
|
||||
if err != nil {
|
||||
log.Debugf("Failed to list routes from table 0 for family %d: %v", family, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract unique table IDs from all routes
|
||||
for _, route := range routes {
|
||||
if route.Table > 0 {
|
||||
tablesMap[route.Table] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var tables []int
|
||||
for tableID := range tablesMap {
|
||||
tables = append(tables, tableID)
|
||||
}
|
||||
|
||||
standardTables := []int{syscall.RT_TABLE_MAIN, syscall.RT_TABLE_LOCAL, NetbirdVPNTableID}
|
||||
for _, table := range standardTables {
|
||||
if !tablesMap[table] {
|
||||
tables = append(tables, table)
|
||||
}
|
||||
}
|
||||
|
||||
return tables, nil
|
||||
}
|
||||
|
||||
// getDetailedRoutes fetches detailed routes from a specific routing table
|
||||
func getDetailedRoutes(tableID, family int) ([]DetailedRoute, error) {
|
||||
var detailedRoutes []DetailedRoute
|
||||
|
||||
routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list routes from table %d: %v", tableID, err)
|
||||
}
|
||||
|
||||
for _, route := range routes {
|
||||
detailed := buildDetailedRoute(route, tableID, family)
|
||||
if detailed != nil {
|
||||
detailedRoutes = append(detailedRoutes, *detailed)
|
||||
}
|
||||
}
|
||||
|
||||
return detailedRoutes, nil
|
||||
}
|
||||
|
||||
func buildDetailedRoute(route netlink.Route, tableID, family int) *DetailedRoute {
|
||||
detailed := DetailedRoute{
|
||||
Route: Route{},
|
||||
Metric: route.Priority,
|
||||
InterfaceMetric: -1, // Interface metrics not typically used on Linux
|
||||
InterfaceIndex: route.LinkIndex,
|
||||
Protocol: routeProtocolToString(int(route.Protocol)),
|
||||
Scope: routeScopeToString(route.Scope),
|
||||
Type: routeTypeToString(route.Type),
|
||||
Table: routeTableToString(tableID),
|
||||
Flags: "-",
|
||||
}
|
||||
|
||||
if !processRouteDestination(&detailed, route, family) {
|
||||
return nil
|
||||
}
|
||||
|
||||
processRouteGateway(&detailed, route)
|
||||
|
||||
processRouteInterface(&detailed, route)
|
||||
|
||||
return &detailed
|
||||
}
|
||||
|
||||
func processRouteDestination(detailed *DetailedRoute, route netlink.Route, family int) bool {
|
||||
if route.Dst != nil {
|
||||
addr, ok := netip.AddrFromSlice(route.Dst.IP)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
ones, _ := route.Dst.Mask.Size()
|
||||
prefix := netip.PrefixFrom(addr.Unmap(), ones)
|
||||
if prefix.IsValid() {
|
||||
detailed.Route.Dst = prefix
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
if family == netlink.FAMILY_V4 {
|
||||
detailed.Route.Dst = netip.MustParsePrefix("0.0.0.0/0")
|
||||
} else {
|
||||
detailed.Route.Dst = netip.MustParsePrefix("::/0")
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func processRouteGateway(detailed *DetailedRoute, route netlink.Route) {
|
||||
if route.Gw != nil {
|
||||
if gateway, ok := netip.AddrFromSlice(route.Gw); ok {
|
||||
detailed.Route.Gw = gateway.Unmap()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processRouteInterface(detailed *DetailedRoute, route netlink.Route) {
|
||||
if route.LinkIndex > 0 {
|
||||
if link, err := netlink.LinkByIndex(route.LinkIndex); err == nil {
|
||||
detailed.Route.Interface = &net.Interface{
|
||||
Index: link.Attrs().Index,
|
||||
Name: link.Attrs().Name,
|
||||
}
|
||||
} else {
|
||||
detailed.Route.Interface = &net.Interface{
|
||||
Index: route.LinkIndex,
|
||||
Name: fmt.Sprintf("index-%d", route.LinkIndex),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper functions to convert netlink constants to strings
|
||||
func routeProtocolToString(protocol int) string {
|
||||
switch protocol {
|
||||
case syscall.RTPROT_UNSPEC:
|
||||
return "unspec"
|
||||
case syscall.RTPROT_REDIRECT:
|
||||
return "redirect"
|
||||
case syscall.RTPROT_KERNEL:
|
||||
return "kernel"
|
||||
case syscall.RTPROT_BOOT:
|
||||
return "boot"
|
||||
case syscall.RTPROT_STATIC:
|
||||
return "static"
|
||||
case syscall.RTPROT_DHCP:
|
||||
return "dhcp"
|
||||
case unix.RTPROT_RA:
|
||||
return "ra"
|
||||
case unix.RTPROT_ZEBRA:
|
||||
return "zebra"
|
||||
case unix.RTPROT_BIRD:
|
||||
return "bird"
|
||||
case unix.RTPROT_DNROUTED:
|
||||
return "dnrouted"
|
||||
case unix.RTPROT_XORP:
|
||||
return "xorp"
|
||||
case unix.RTPROT_NTK:
|
||||
return "ntk"
|
||||
default:
|
||||
return fmt.Sprintf("%d", protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func routeScopeToString(scope netlink.Scope) string {
|
||||
switch scope {
|
||||
case netlink.SCOPE_UNIVERSE:
|
||||
return "global"
|
||||
case netlink.SCOPE_SITE:
|
||||
return "site"
|
||||
case netlink.SCOPE_LINK:
|
||||
return "link"
|
||||
case netlink.SCOPE_HOST:
|
||||
return "host"
|
||||
case netlink.SCOPE_NOWHERE:
|
||||
return "nowhere"
|
||||
default:
|
||||
return fmt.Sprintf("%d", scope)
|
||||
}
|
||||
}
|
||||
|
||||
func routeTypeToString(routeType int) string {
|
||||
switch routeType {
|
||||
case syscall.RTN_UNSPEC:
|
||||
return "unspec"
|
||||
case syscall.RTN_UNICAST:
|
||||
return "unicast"
|
||||
case syscall.RTN_LOCAL:
|
||||
return "local"
|
||||
case syscall.RTN_BROADCAST:
|
||||
return "broadcast"
|
||||
case syscall.RTN_ANYCAST:
|
||||
return "anycast"
|
||||
case syscall.RTN_MULTICAST:
|
||||
return "multicast"
|
||||
case syscall.RTN_BLACKHOLE:
|
||||
return "blackhole"
|
||||
case syscall.RTN_UNREACHABLE:
|
||||
return "unreachable"
|
||||
case syscall.RTN_PROHIBIT:
|
||||
return "prohibit"
|
||||
case syscall.RTN_THROW:
|
||||
return "throw"
|
||||
case syscall.RTN_NAT:
|
||||
return "nat"
|
||||
case syscall.RTN_XRESOLVE:
|
||||
return "xresolve"
|
||||
default:
|
||||
return fmt.Sprintf("%d", routeType)
|
||||
}
|
||||
}
|
||||
|
||||
func routeTableToString(tableID int) string {
|
||||
switch tableID {
|
||||
case syscall.RT_TABLE_MAIN:
|
||||
return "main"
|
||||
case syscall.RT_TABLE_LOCAL:
|
||||
return "local"
|
||||
case NetbirdVPNTableID:
|
||||
return "netbird"
|
||||
default:
|
||||
return fmt.Sprintf("%d", tableID)
|
||||
}
|
||||
}
|
||||
|
||||
// getRoutes fetches routes from a specific routing table identified by tableID.
|
||||
func getRoutes(tableID, family int) ([]netip.Prefix, error) {
|
||||
var prefixList []netip.Prefix
|
||||
@@ -237,6 +530,115 @@ func getRoutes(tableID, family int) ([]netip.Prefix, error) {
|
||||
return prefixList, nil
|
||||
}
|
||||
|
||||
// GetIPRules returns IP rules for debugging
|
||||
func GetIPRules() ([]IPRule, error) {
|
||||
v4Rules, err := getIPRules(netlink.FAMILY_V4)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get v4 rules: %w", err)
|
||||
}
|
||||
v6Rules, err := getIPRules(netlink.FAMILY_V6)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get v6 rules: %w", err)
|
||||
}
|
||||
return append(v4Rules, v6Rules...), nil
|
||||
}
|
||||
|
||||
// getIPRules fetches IP rules for the specified address family
|
||||
func getIPRules(family int) ([]IPRule, error) {
|
||||
rules, err := netlink.RuleList(family)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list rules for family %d: %w", family, err)
|
||||
}
|
||||
|
||||
var ipRules []IPRule
|
||||
for _, rule := range rules {
|
||||
ipRule := buildIPRule(rule)
|
||||
ipRules = append(ipRules, ipRule)
|
||||
}
|
||||
|
||||
return ipRules, nil
|
||||
}
|
||||
|
||||
func buildIPRule(rule netlink.Rule) IPRule {
|
||||
var mask uint32
|
||||
if rule.Mask != nil {
|
||||
mask = *rule.Mask
|
||||
}
|
||||
|
||||
ipRule := IPRule{
|
||||
Priority: rule.Priority,
|
||||
IIF: rule.IifName,
|
||||
OIF: rule.OifName,
|
||||
Table: ruleTableToString(rule.Table),
|
||||
Action: ruleActionToString(int(rule.Type)),
|
||||
Mark: rule.Mark,
|
||||
Mask: mask,
|
||||
TunID: uint32(rule.TunID),
|
||||
Goto: uint32(rule.Goto),
|
||||
Flow: uint32(rule.Flow),
|
||||
SuppressPlen: rule.SuppressPrefixlen,
|
||||
SuppressIFL: rule.SuppressIfgroup,
|
||||
Invert: rule.Invert,
|
||||
}
|
||||
|
||||
if rule.Src != nil {
|
||||
ipRule.From = parseRulePrefix(rule.Src)
|
||||
}
|
||||
|
||||
if rule.Dst != nil {
|
||||
ipRule.To = parseRulePrefix(rule.Dst)
|
||||
}
|
||||
|
||||
return ipRule
|
||||
}
|
||||
|
||||
func parseRulePrefix(ipNet *net.IPNet) netip.Prefix {
|
||||
if addr, ok := netip.AddrFromSlice(ipNet.IP); ok {
|
||||
ones, _ := ipNet.Mask.Size()
|
||||
prefix := netip.PrefixFrom(addr.Unmap(), ones)
|
||||
if prefix.IsValid() {
|
||||
return prefix
|
||||
}
|
||||
}
|
||||
return netip.Prefix{}
|
||||
}
|
||||
|
||||
func ruleTableToString(table int) string {
|
||||
switch table {
|
||||
case syscall.RT_TABLE_MAIN:
|
||||
return "main"
|
||||
case syscall.RT_TABLE_LOCAL:
|
||||
return "local"
|
||||
case syscall.RT_TABLE_DEFAULT:
|
||||
return "default"
|
||||
case NetbirdVPNTableID:
|
||||
return "netbird"
|
||||
default:
|
||||
return fmt.Sprintf("%d", table)
|
||||
}
|
||||
}
|
||||
|
||||
func ruleActionToString(action int) string {
|
||||
switch action {
|
||||
case unix.FR_ACT_UNSPEC:
|
||||
return "unspec"
|
||||
case unix.FR_ACT_TO_TBL:
|
||||
return "lookup"
|
||||
case unix.FR_ACT_GOTO:
|
||||
return "goto"
|
||||
case unix.FR_ACT_NOP:
|
||||
return "nop"
|
||||
case unix.FR_ACT_BLACKHOLE:
|
||||
return "blackhole"
|
||||
case unix.FR_ACT_UNREACHABLE:
|
||||
return "unreachable"
|
||||
case unix.FR_ACT_PROHIBIT:
|
||||
return "prohibit"
|
||||
default:
|
||||
return fmt.Sprintf("%d", action)
|
||||
}
|
||||
}
|
||||
|
||||
// addRoute adds a route to a specific routing table identified by tableID.
|
||||
func addRoute(prefix netip.Prefix, nexthop Nexthop, tableID int) error {
|
||||
route := &netlink.Route{
|
||||
@@ -247,7 +649,7 @@ func addRoute(prefix netip.Prefix, nexthop Nexthop, tableID int) error {
|
||||
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
return fmt.Errorf(errParsePrefixMsg, prefix, err)
|
||||
}
|
||||
route.Dst = ipNet
|
||||
|
||||
@@ -268,7 +670,7 @@ func addRoute(prefix netip.Prefix, nexthop Nexthop, tableID int) error {
|
||||
func addUnreachableRoute(prefix netip.Prefix, tableID int) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
return fmt.Errorf(errParsePrefixMsg, prefix, err)
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
@@ -288,7 +690,7 @@ func addUnreachableRoute(prefix netip.Prefix, tableID int) error {
|
||||
func removeUnreachableRoute(prefix netip.Prefix, tableID int) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
return fmt.Errorf(errParsePrefixMsg, prefix, err)
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
@@ -313,7 +715,7 @@ func removeUnreachableRoute(prefix netip.Prefix, tableID int) error {
|
||||
func removeRoute(prefix netip.Prefix, nexthop Nexthop, tableID int) error {
|
||||
_, ipNet, err := net.ParseCIDR(prefix.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse prefix %s: %w", prefix, err)
|
||||
return fmt.Errorf(errParsePrefixMsg, prefix, err)
|
||||
}
|
||||
|
||||
route := &netlink.Route{
|
||||
|
||||
@@ -10,6 +10,25 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// IPRule contains IP rule information for debugging
|
||||
type IPRule struct {
|
||||
Priority int
|
||||
From netip.Prefix
|
||||
To netip.Prefix
|
||||
IIF string
|
||||
OIF string
|
||||
Table string
|
||||
Action string
|
||||
Mark uint32
|
||||
Mask uint32
|
||||
TunID uint32
|
||||
Goto uint32
|
||||
Flow uint32
|
||||
SuppressPlen int
|
||||
SuppressIFL int
|
||||
Invert bool
|
||||
}
|
||||
|
||||
func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error {
|
||||
if err := r.validateRoute(prefix); err != nil {
|
||||
return err
|
||||
@@ -32,3 +51,9 @@ func EnableIPForwarding() error {
|
||||
func hasSeparateRouting() ([]netip.Prefix, error) {
|
||||
return GetRoutesFromTable()
|
||||
}
|
||||
|
||||
// GetIPRules returns IP rules for debugging (not supported on non-Linux platforms)
|
||||
func GetIPRules() ([]IPRule, error) {
|
||||
log.Infof("IP rules collection is not supported on %s", runtime.GOOS)
|
||||
return []IPRule{}, nil
|
||||
}
|
||||
|
||||
@@ -40,13 +40,6 @@ type RouteMonitor struct {
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// Route represents a single routing table entry.
|
||||
type Route struct {
|
||||
Destination netip.Prefix
|
||||
Nexthop netip.Addr
|
||||
Interface *net.Interface
|
||||
}
|
||||
|
||||
type MSFT_NetRoute struct {
|
||||
DestinationPrefix string
|
||||
NextHop string
|
||||
@@ -78,6 +71,12 @@ type MIB_IPFORWARD_ROW2 struct {
|
||||
Origin uint32
|
||||
}
|
||||
|
||||
// MIB_IPFORWARD_TABLE2 represents a table of IP forward entries
|
||||
type MIB_IPFORWARD_TABLE2 struct {
|
||||
NumEntries uint32
|
||||
Table [1]MIB_IPFORWARD_ROW2 // Flexible array member
|
||||
}
|
||||
|
||||
// IP_ADDRESS_PREFIX is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-ip_address_prefix
|
||||
type IP_ADDRESS_PREFIX struct {
|
||||
Prefix SOCKADDR_INET
|
||||
@@ -108,6 +107,45 @@ type SOCKADDR_INET_NEXTHOP struct {
|
||||
// MIB_NOTIFICATION_TYPE is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ne-netioapi-mib_notification_type
|
||||
type MIB_NOTIFICATION_TYPE int32
|
||||
|
||||
// MIB_IPINTERFACE_ROW is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_ipinterface_row
|
||||
type MIB_IPINTERFACE_ROW struct {
|
||||
Family uint16
|
||||
InterfaceLuid luid
|
||||
InterfaceIndex uint32
|
||||
MaxReassemblySize uint32
|
||||
InterfaceIdentifier uint64
|
||||
MinRouterAdvertisementInterval uint32
|
||||
MaxRouterAdvertisementInterval uint32
|
||||
AdvertisingEnabled uint8
|
||||
ForwardingEnabled uint8
|
||||
WeakHostSend uint8
|
||||
WeakHostReceive uint8
|
||||
UseAutomaticMetric uint8
|
||||
UseNeighborUnreachabilityDetection uint8
|
||||
ManagedAddressConfigurationSupported uint8
|
||||
OtherStatefulConfigurationSupported uint8
|
||||
AdvertiseDefaultRoute uint8
|
||||
RouterDiscoveryBehavior uint32
|
||||
DadTransmits uint32
|
||||
BaseReachableTime uint32
|
||||
RetransmitTime uint32
|
||||
PathMtuDiscoveryTimeout uint32
|
||||
LinkLocalAddressBehavior uint32
|
||||
LinkLocalAddressTimeout uint32
|
||||
ZoneIndices [16]uint32
|
||||
SitePrefixLength uint32
|
||||
Metric uint32
|
||||
NlMtu uint32
|
||||
Connected uint8
|
||||
SupportsWakeUpPatterns uint8
|
||||
SupportsNeighborDiscovery uint8
|
||||
SupportsRouterDiscovery uint8
|
||||
ReachableTime uint32
|
||||
TransmitOffload uint32
|
||||
ReceiveOffload uint32
|
||||
DisableDefaultRoutes uint8
|
||||
}
|
||||
|
||||
var (
|
||||
modiphlpapi = windows.NewLazyDLL("iphlpapi.dll")
|
||||
procNotifyRouteChange2 = modiphlpapi.NewProc("NotifyRouteChange2")
|
||||
@@ -115,8 +153,11 @@ var (
|
||||
procCreateIpForwardEntry2 = modiphlpapi.NewProc("CreateIpForwardEntry2")
|
||||
procDeleteIpForwardEntry2 = modiphlpapi.NewProc("DeleteIpForwardEntry2")
|
||||
procGetIpForwardEntry2 = modiphlpapi.NewProc("GetIpForwardEntry2")
|
||||
procGetIpForwardTable2 = modiphlpapi.NewProc("GetIpForwardTable2")
|
||||
procInitializeIpForwardEntry = modiphlpapi.NewProc("InitializeIpForwardEntry")
|
||||
procConvertInterfaceIndexToLuid = modiphlpapi.NewProc("ConvertInterfaceIndexToLuid")
|
||||
procGetIpInterfaceEntry = modiphlpapi.NewProc("GetIpInterfaceEntry")
|
||||
procFreeMibTable = modiphlpapi.NewProc("FreeMibTable")
|
||||
|
||||
prefixList []netip.Prefix
|
||||
lastUpdate time.Time
|
||||
@@ -429,6 +470,8 @@ func (rm *RouteMonitor) parseUpdate(row *MIB_IPFORWARD_ROW2, notificationType MI
|
||||
updateType = RouteAdded
|
||||
case MibDeleteInstance:
|
||||
updateType = RouteDeleted
|
||||
case MibInitialNotification:
|
||||
updateType = RouteAdded // Treat initial notifications as additions
|
||||
}
|
||||
|
||||
update.Type = updateType
|
||||
@@ -508,7 +551,7 @@ func GetRoutesFromTable() ([]netip.Prefix, error) {
|
||||
|
||||
prefixList = nil
|
||||
for _, route := range routes {
|
||||
prefixList = append(prefixList, route.Destination)
|
||||
prefixList = append(prefixList, route.Dst)
|
||||
}
|
||||
|
||||
lastUpdate = time.Now()
|
||||
@@ -551,15 +594,159 @@ func GetRoutes() ([]Route, error) {
|
||||
}
|
||||
|
||||
routes = append(routes, Route{
|
||||
Destination: dest,
|
||||
Nexthop: nexthop,
|
||||
Interface: intf,
|
||||
Dst: dest,
|
||||
Gw: nexthop,
|
||||
Interface: intf,
|
||||
})
|
||||
}
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
// GetDetailedRoutesFromTable returns detailed route information using Windows syscalls
|
||||
func GetDetailedRoutesFromTable() ([]DetailedRoute, error) {
|
||||
table, err := getWindowsRoutingTable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer freeWindowsRoutingTable(table)
|
||||
|
||||
return parseWindowsRoutingTable(table), nil
|
||||
}
|
||||
|
||||
func getWindowsRoutingTable() (*MIB_IPFORWARD_TABLE2, error) {
|
||||
var table *MIB_IPFORWARD_TABLE2
|
||||
|
||||
ret, _, err := procGetIpForwardTable2.Call(
|
||||
uintptr(windows.AF_UNSPEC),
|
||||
uintptr(unsafe.Pointer(&table)),
|
||||
)
|
||||
if ret != 0 {
|
||||
return nil, fmt.Errorf("GetIpForwardTable2 failed: %w", err)
|
||||
}
|
||||
|
||||
if table == nil {
|
||||
return nil, fmt.Errorf("received nil routing table")
|
||||
}
|
||||
|
||||
return table, nil
|
||||
}
|
||||
|
||||
func freeWindowsRoutingTable(table *MIB_IPFORWARD_TABLE2) {
|
||||
if table != nil {
|
||||
ret, _, _ := procFreeMibTable.Call(uintptr(unsafe.Pointer(table)))
|
||||
if ret != 0 {
|
||||
log.Warnf("FreeMibTable failed with return code: %d", ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseWindowsRoutingTable(table *MIB_IPFORWARD_TABLE2) []DetailedRoute {
|
||||
var detailedRoutes []DetailedRoute
|
||||
|
||||
entrySize := unsafe.Sizeof(MIB_IPFORWARD_ROW2{})
|
||||
basePtr := uintptr(unsafe.Pointer(&table.Table[0]))
|
||||
|
||||
for i := uint32(0); i < table.NumEntries; i++ {
|
||||
entryPtr := basePtr + uintptr(i)*entrySize
|
||||
entry := (*MIB_IPFORWARD_ROW2)(unsafe.Pointer(entryPtr))
|
||||
|
||||
detailed := buildWindowsDetailedRoute(entry)
|
||||
if detailed != nil {
|
||||
detailedRoutes = append(detailedRoutes, *detailed)
|
||||
}
|
||||
}
|
||||
|
||||
return detailedRoutes
|
||||
}
|
||||
|
||||
func buildWindowsDetailedRoute(entry *MIB_IPFORWARD_ROW2) *DetailedRoute {
|
||||
dest := parseIPPrefix(entry.DestinationPrefix, int(entry.InterfaceIndex))
|
||||
if !dest.IsValid() {
|
||||
return nil
|
||||
}
|
||||
|
||||
gateway := parseIPNexthop(entry.NextHop, int(entry.InterfaceIndex))
|
||||
|
||||
var intf *net.Interface
|
||||
if entry.InterfaceIndex != 0 {
|
||||
if netIntf, err := net.InterfaceByIndex(int(entry.InterfaceIndex)); err == nil {
|
||||
intf = netIntf
|
||||
} else {
|
||||
// Create a synthetic interface for display when we can't resolve the name
|
||||
intf = &net.Interface{
|
||||
Index: int(entry.InterfaceIndex),
|
||||
Name: fmt.Sprintf("index-%d", entry.InterfaceIndex),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
detailed := DetailedRoute{
|
||||
Route: Route{
|
||||
Dst: dest,
|
||||
Gw: gateway,
|
||||
Interface: intf,
|
||||
},
|
||||
|
||||
Metric: int(entry.Metric),
|
||||
InterfaceMetric: getInterfaceMetric(entry.InterfaceIndex, entry.DestinationPrefix.Prefix.sin6_family),
|
||||
InterfaceIndex: int(entry.InterfaceIndex),
|
||||
Protocol: windowsProtocolToString(entry.Protocol),
|
||||
Scope: formatRouteAge(entry.Age),
|
||||
Type: windowsOriginToString(entry.Origin),
|
||||
Table: "main",
|
||||
Flags: "-",
|
||||
}
|
||||
|
||||
return &detailed
|
||||
}
|
||||
|
||||
func windowsProtocolToString(protocol uint32) string {
|
||||
switch protocol {
|
||||
case 1:
|
||||
return "other"
|
||||
case 2:
|
||||
return "local"
|
||||
case 3:
|
||||
return "netmgmt"
|
||||
case 4:
|
||||
return "icmp"
|
||||
case 5:
|
||||
return "egp"
|
||||
case 6:
|
||||
return "ggp"
|
||||
case 7:
|
||||
return "hello"
|
||||
case 8:
|
||||
return "rip"
|
||||
case 9:
|
||||
return "isis"
|
||||
case 10:
|
||||
return "esis"
|
||||
case 11:
|
||||
return "cisco"
|
||||
case 12:
|
||||
return "bbn"
|
||||
case 13:
|
||||
return "ospf"
|
||||
case 14:
|
||||
return "bgp"
|
||||
case 15:
|
||||
return "idpr"
|
||||
case 16:
|
||||
return "eigrp"
|
||||
case 17:
|
||||
return "dvmrp"
|
||||
case 18:
|
||||
return "rpl"
|
||||
case 19:
|
||||
return "dhcp"
|
||||
default:
|
||||
return fmt.Sprintf("unknown-%d", protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func isCacheDisabled() bool {
|
||||
return os.Getenv("NB_DISABLE_ROUTE_CACHE") == "true"
|
||||
}
|
||||
@@ -614,3 +801,59 @@ func addZone(ip netip.Addr, interfaceIndex int) netip.Addr {
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
// getInterfaceMetric retrieves the interface metric for a given interface and address family
|
||||
func getInterfaceMetric(interfaceIndex uint32, family int16) int {
|
||||
if interfaceIndex == 0 {
|
||||
return -1
|
||||
}
|
||||
|
||||
var ipInterfaceRow MIB_IPINTERFACE_ROW
|
||||
ipInterfaceRow.Family = uint16(family)
|
||||
ipInterfaceRow.InterfaceIndex = interfaceIndex
|
||||
|
||||
ret, _, _ := procGetIpInterfaceEntry.Call(uintptr(unsafe.Pointer(&ipInterfaceRow)))
|
||||
if ret != 0 {
|
||||
log.Debugf("GetIpInterfaceEntry failed for interface %d: %d", interfaceIndex, ret)
|
||||
return -1
|
||||
}
|
||||
|
||||
return int(ipInterfaceRow.Metric)
|
||||
}
|
||||
|
||||
// formatRouteAge formats the route age in seconds to a human-readable string
|
||||
func formatRouteAge(ageSeconds uint32) string {
|
||||
if ageSeconds == 0 {
|
||||
return "0s"
|
||||
}
|
||||
|
||||
age := time.Duration(ageSeconds) * time.Second
|
||||
switch {
|
||||
case age < time.Minute:
|
||||
return fmt.Sprintf("%ds", int(age.Seconds()))
|
||||
case age < time.Hour:
|
||||
return fmt.Sprintf("%dm", int(age.Minutes()))
|
||||
case age < 24*time.Hour:
|
||||
return fmt.Sprintf("%dh", int(age.Hours()))
|
||||
default:
|
||||
return fmt.Sprintf("%dd", int(age.Hours()/24))
|
||||
}
|
||||
}
|
||||
|
||||
// windowsOriginToString converts Windows route origin to string
|
||||
func windowsOriginToString(origin uint32) string {
|
||||
switch origin {
|
||||
case 0:
|
||||
return "manual"
|
||||
case 1:
|
||||
return "wellknown"
|
||||
case 2:
|
||||
return "dhcp"
|
||||
case 3:
|
||||
return "routeradvert"
|
||||
case 4:
|
||||
return "6to4"
|
||||
default:
|
||||
return fmt.Sprintf("unknown-%d", origin)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user