From a0482ebc7b38b2672160a03f65ca49ba6ca314a5 Mon Sep 17 00:00:00 2001 From: Pascal Fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Fri, 23 May 2025 14:04:12 +0200 Subject: [PATCH 01/37] [client] avoid overwriting state manager on iOS (#3870) --- client/internal/engine.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 7c501e5aa..d6bcc66f6 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -241,6 +241,8 @@ func NewEngine( checks: checks, connSemaphore: semaphoregroup.NewSemaphoreGroup(connInitLimit), } + + path := statemanager.GetDefaultStatePath() if runtime.GOOS == "ios" { if !fileExists(mobileDep.StateFilePath) { err := createFile(mobileDep.StateFilePath) @@ -250,11 +252,9 @@ func NewEngine( } } - engine.stateManager = statemanager.New(mobileDep.StateFilePath) - } - if path := statemanager.GetDefaultStatePath(); path != "" { - engine.stateManager = statemanager.New(path) + path = mobileDep.StateFilePath } + engine.stateManager = statemanager.New(path) return engine } From 5bed6777d568f1e0e96a4c3dbd290e175502eb81 Mon Sep 17 00:00:00 2001 From: Pedro Maia Costa <550684+pnmcosta@users.noreply.github.com> Date: Fri, 23 May 2025 14:42:42 +0100 Subject: [PATCH 02/37] [management] force account id on save groups update (#3850) --- management/server/account.go | 4 ++-- management/server/group.go | 2 +- management/server/posture_checks_test.go | 2 +- management/server/store/sql_store.go | 12 ++++++++++-- management/server/store/sql_store_test.go | 6 +++--- management/server/store/store.go | 2 +- management/server/user.go | 2 +- 7 files changed, 19 insertions(+), 11 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index 6dc449c1e..033ec5fa1 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -1248,7 +1248,7 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth return nil } - if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, newGroupsToCreate); err != nil { + if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, userAuth.AccountId, newGroupsToCreate); err != nil { return fmt.Errorf("error saving groups: %w", err) } @@ -1282,7 +1282,7 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth return fmt.Errorf("error modifying user peers in groups: %w", err) } - if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, updatedGroups); err != nil { + if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, userAuth.AccountId, updatedGroups); err != nil { return fmt.Errorf("error saving groups: %w", err) } diff --git a/management/server/group.go b/management/server/group.go index 87d649228..c26a0cfc1 100644 --- a/management/server/group.go +++ b/management/server/group.go @@ -116,7 +116,7 @@ func (am *DefaultAccountManager) SaveGroups(ctx context.Context, accountID, user return err } - return transaction.SaveGroups(ctx, store.LockingStrengthUpdate, groupsToSave) + return transaction.SaveGroups(ctx, store.LockingStrengthUpdate, accountID, groupsToSave) }) if err != nil { return err diff --git a/management/server/posture_checks_test.go b/management/server/posture_checks_test.go index 232955f7d..8bd2fab66 100644 --- a/management/server/posture_checks_test.go +++ b/management/server/posture_checks_test.go @@ -455,7 +455,7 @@ func TestArePostureCheckChangesAffectPeers(t *testing.T) { AccountID: account.Id, Peers: []string{}, } - err = manager.Store.SaveGroups(context.Background(), store.LockingStrengthUpdate, []*types.Group{groupA, groupB}) + err = manager.Store.SaveGroups(context.Background(), store.LockingStrengthUpdate, account.Id, []*types.Group{groupA, groupB}) require.NoError(t, err, "failed to save groups") postureCheckA := &posture.Checks{ diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index eb194ca9b..6c3104ef0 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -448,12 +448,20 @@ func (s *SqlStore) SaveUser(ctx context.Context, lockStrength LockingStrength, u } // SaveGroups saves the given list of groups to the database. -func (s *SqlStore) SaveGroups(ctx context.Context, lockStrength LockingStrength, groups []*types.Group) error { +func (s *SqlStore) SaveGroups(ctx context.Context, lockStrength LockingStrength, accountID string, groups []*types.Group) error { if len(groups) == 0 { return nil } - result := s.db.Clauses(clause.Locking{Strength: string(lockStrength)}, clause.OnConflict{UpdateAll: true}).Create(&groups) + result := s.db. + Clauses( + clause.Locking{Strength: string(lockStrength)}, + clause.OnConflict{ + Where: clause.Where{Exprs: []clause.Expression{clause.Eq{Column: "groups.account_id", Value: accountID}}}, + UpdateAll: true, + }, + ). + Create(&groups) if result.Error != nil { return status.Errorf(status.Internal, "failed to save groups to store: %v", result.Error) } diff --git a/management/server/store/sql_store_test.go b/management/server/store/sql_store_test.go index 8e99b34e1..2c1f5f8e6 100644 --- a/management/server/store/sql_store_test.go +++ b/management/server/store/sql_store_test.go @@ -1324,11 +1324,11 @@ func TestSqlStore_SaveGroups(t *testing.T) { Peers: []string{"peer3", "peer4"}, }, } - err = store.SaveGroups(context.Background(), LockingStrengthUpdate, groups) + err = store.SaveGroups(context.Background(), LockingStrengthUpdate, accountID, groups) require.NoError(t, err) groups[1].Peers = []string{} - err = store.SaveGroups(context.Background(), LockingStrengthUpdate, groups) + err = store.SaveGroups(context.Background(), LockingStrengthUpdate, accountID, groups) require.NoError(t, err) group, err := store.GetGroupByID(context.Background(), LockingStrengthShare, accountID, groups[1].ID) @@ -3240,7 +3240,7 @@ func TestSqlStore_SaveGroups_LargeBatch(t *testing.T) { }) } - err = store.SaveGroups(context.Background(), LockingStrengthUpdate, groupsToSave) + err = store.SaveGroups(context.Background(), LockingStrengthUpdate, accountID, groupsToSave) require.NoError(t, err) accountGroups, err = store.GetAccountGroups(context.Background(), LockingStrengthShare, accountID) diff --git a/management/server/store/store.go b/management/server/store/store.go index 3d529ceb5..b3c2fceff 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -98,7 +98,7 @@ type Store interface { GetGroupByID(ctx context.Context, lockStrength LockingStrength, accountID, groupID string) (*types.Group, error) GetGroupByName(ctx context.Context, lockStrength LockingStrength, groupName, accountID string) (*types.Group, error) GetGroupsByIDs(ctx context.Context, lockStrength LockingStrength, accountID string, groupIDs []string) (map[string]*types.Group, error) - SaveGroups(ctx context.Context, lockStrength LockingStrength, groups []*types.Group) error + SaveGroups(ctx context.Context, lockStrength LockingStrength, accountID string, groups []*types.Group) error SaveGroup(ctx context.Context, lockStrength LockingStrength, group *types.Group) error DeleteGroup(ctx context.Context, lockStrength LockingStrength, accountID, groupID string) error DeleteGroups(ctx context.Context, strength LockingStrength, accountID string, groupIDs []string) error diff --git a/management/server/user.go b/management/server/user.go index 44ad3b68f..2c762a8eb 100644 --- a/management/server/user.go +++ b/management/server/user.go @@ -676,7 +676,7 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact return false, nil, nil, nil, fmt.Errorf("error modifying user peers in groups: %w", err) } - if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, updatedGroups); err != nil { + if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, update.AccountID, updatedGroups); err != nil { return false, nil, nil, nil, fmt.Errorf("error saving groups: %w", err) } } From 670446d42e385397b8be87b13c5fd504c303ce86 Mon Sep 17 00:00:00 2001 From: "M. Essam" Date: Sun, 25 May 2025 17:57:34 +0300 Subject: [PATCH 03/37] [management/client/rest] Fix panic on unknown errors (#3865) --- management/client/rest/accounts_test.go | 9 +++++++++ management/client/rest/client.go | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/management/client/rest/accounts_test.go b/management/client/rest/accounts_test.go index f6d48d874..d2ace4ec9 100644 --- a/management/client/rest/accounts_test.go +++ b/management/client/rest/accounts_test.go @@ -66,6 +66,15 @@ func TestAccounts_List_Err(t *testing.T) { }) } +func TestAccounts_List_ConnErr(t *testing.T) { + withMockClient(func(c *rest.Client, mux *http.ServeMux) { + ret, err := c.Accounts.List(context.Background()) + assert.Error(t, err) + assert.Contains(t, err.Error(), "404") + assert.Empty(t, ret) + }) +} + func TestAccounts_Update_200(t *testing.T) { withMockClient(func(c *rest.Client, mux *http.ServeMux) { mux.HandleFunc("/api/accounts/Test", func(w http.ResponseWriter, r *http.Request) { diff --git a/management/client/rest/client.go b/management/client/rest/client.go index 886a59f2c..25e8ad0da 100644 --- a/management/client/rest/client.go +++ b/management/client/rest/client.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" @@ -134,7 +135,8 @@ func (c *Client) NewRequest(ctx context.Context, method, path string, body io.Re if resp.StatusCode > 299 { parsedErr, pErr := parseResponse[util.ErrorResponse](resp) if pErr != nil { - return nil, err + + return nil, pErr } return nil, errors.New(parsedErr.Message) } @@ -145,13 +147,16 @@ func (c *Client) NewRequest(ctx context.Context, method, path string, body io.Re func parseResponse[T any](resp *http.Response) (T, error) { var ret T if resp.Body == nil { - return ret, errors.New("No body") + return ret, fmt.Errorf("Body missing, HTTP Error code %d", resp.StatusCode) } bs, err := io.ReadAll(resp.Body) if err != nil { return ret, err } err = json.Unmarshal(bs, &ret) + if err != nil { + return ret, fmt.Errorf("Error code %d, error unmarshalling body: %w", resp.StatusCode, err) + } - return ret, err + return ret, nil } From 5523040acd7cbe97510b1b3f7bd47c885feb974f Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Tue, 27 May 2025 13:47:53 +0300 Subject: [PATCH 04/37] [management] Add correlated network traffic event schema (#3680) --- management/server/http/api/openapi.yml | 132 ++++++++++++++---------- management/server/http/api/types.gen.go | 82 ++++++++------- management/server/store/store.go | 29 +++--- management/server/testutil/store.go | 56 +++++++--- management/server/testutil/store_ios.go | 8 +- 5 files changed, 183 insertions(+), 124 deletions(-) diff --git a/management/server/http/api/openapi.yml b/management/server/http/api/openapi.yml index 64feab975..f6ea39a8e 100644 --- a/management/server/http/api/openapi.yml +++ b/management/server/http/api/openapi.yml @@ -1925,13 +1925,71 @@ components: - os - address - dns_label - NetworkTrafficEvent: + NetworkTrafficUser: type: object properties: id: type: string - description: "ID of the event. Unique." - example: "18e204d6-f7c6-405d-8025-70becb216add" + description: "UserID is the ID of the user that initiated the event (can be empty as not every event is user-initiated)." + example: "google-oauth2|123456789012345678901" + email: + type: string + description: "Email of the user who initiated the event (if any)." + example: "alice@netbird.io" + name: + type: string + description: "Name of the user who initiated the event (if any)." + example: "Alice Smith" + required: + - id + - email + - name + NetworkTrafficPolicy: + type: object + properties: + id: + type: string + description: "ID of the policy that allowed this event." + example: "ch8i4ug6lnn4g9hqv7m0" + name: + type: string + description: "Name of the policy that allowed this event." + example: "All to All" + required: + - id + - name + NetworkTrafficICMP: + type: object + properties: + type: + type: integer + description: "ICMP type (if applicable)." + example: 8 + code: + type: integer + description: "ICMP code (if applicable)." + example: 0 + required: + - type + - code + NetworkTrafficSubEvent: + type: object + properties: + type: + type: string + description: Type of the event (e.g., TYPE_UNKNOWN, TYPE_START, TYPE_END, TYPE_DROP). + example: TYPE_START + timestamp: + type: string + format: date-time + description: Timestamp of the event as sent by the peer. + example: 2025-03-20T16:23:58.125397Z + required: + - type + - timestamp + NetworkTrafficEvent: + type: object + properties: flow_id: type: string description: "FlowID is the ID of the connection flow. Not unique because it can be the same for multiple events (e.g., start and end of the connection)." @@ -1940,43 +1998,20 @@ components: type: string description: "ID of the reporter of the event (e.g., the peer that reported the event)." example: "ch8i4ug6lnn4g9hqv7m0" - timestamp: - type: string - format: date-time - description: "Timestamp of the event. Send by the peer." - example: "2025-03-20T16:23:58.125397Z" - receive_timestamp: - type: string - format: date-time - description: "Timestamp when the event was received by our API." - example: "2025-03-20T16:23:58.125397Z" source: $ref: '#/components/schemas/NetworkTrafficEndpoint' - user_id: - type: string - nullable: true - description: "UserID is the ID of the user that initiated the event (can be empty as not every event is user-initiated)." - example: "google-oauth2|123456789012345678901" - user_email: - type: string - nullable: true - description: "Email of the user who initiated the event (if any)." - example: "alice@netbird.io" - user_name: - type: string - nullable: true - description: "Name of the user who initiated the event (if any)." - example: "Alice Smith" destination: $ref: '#/components/schemas/NetworkTrafficEndpoint' + user: + $ref: '#/components/schemas/NetworkTrafficUser' + policy: + $ref: '#/components/schemas/NetworkTrafficPolicy' + icmp: + $ref: '#/components/schemas/NetworkTrafficICMP' protocol: type: integer description: "Protocol is the protocol of the traffic (e.g. 1 = ICMP, 6 = TCP, 17 = UDP, etc.)." example: 6 - type: - type: string - description: "Type of the event (e.g. TYPE_UNKNOWN, TYPE_START, TYPE_END, TYPE_DROP)." - example: "TYPE_START" direction: type: string description: "Direction of the traffic (e.g. DIRECTION_UNKNOWN, INGRESS, EGRESS)." @@ -1997,43 +2032,28 @@ components: type: integer description: "Number of packets transmitted." example: 5 - policy_id: - type: string - description: "ID of the policy that allowed this event." - example: "ch8i4ug6lnn4g9hqv7m0" - policy_name: - type: string - description: "Name of the policy that allowed this event." - example: "All to All" - icmp_type: - type: integer - description: "ICMP type (if applicable)." - example: 8 - icmp_code: - type: integer - description: "ICMP code (if applicable)." - example: 0 + events: + type: array + description: "List of events that are correlated to this flow (e.g., start, end)." + items: + $ref: '#/components/schemas/NetworkTrafficSubEvent' required: - id - flow_id - reporter_id - - timestamp - receive_timestamp - source - - user_id - - user_email - destination + - user + - policy + - icmp - protocol - - type - direction - rx_bytes - rx_packets - tx_bytes - tx_packets - - policy_id - - policy_name - - icmp_type - - icmp_code + - events NetworkTrafficEventsResponse: type: object properties: diff --git a/management/server/http/api/types.gen.go b/management/server/http/api/types.gen.go index 647b17e32..0a09d7ca2 100644 --- a/management/server/http/api/types.gen.go +++ b/management/server/http/api/types.gen.go @@ -883,30 +883,17 @@ type NetworkTrafficEvent struct { // Direction Direction of the traffic (e.g. DIRECTION_UNKNOWN, INGRESS, EGRESS). Direction string `json:"direction"` + // Events List of events that are correlated to this flow (e.g., start, end). + Events []NetworkTrafficSubEvent `json:"events"` + // FlowId FlowID is the ID of the connection flow. Not unique because it can be the same for multiple events (e.g., start and end of the connection). - FlowId string `json:"flow_id"` - - // IcmpCode ICMP code (if applicable). - IcmpCode int `json:"icmp_code"` - - // IcmpType ICMP type (if applicable). - IcmpType int `json:"icmp_type"` - - // Id ID of the event. Unique. - Id string `json:"id"` - - // PolicyId ID of the policy that allowed this event. - PolicyId string `json:"policy_id"` - - // PolicyName Name of the policy that allowed this event. - PolicyName string `json:"policy_name"` + FlowId string `json:"flow_id"` + Icmp NetworkTrafficICMP `json:"icmp"` + Policy NetworkTrafficPolicy `json:"policy"` // Protocol Protocol is the protocol of the traffic (e.g. 1 = ICMP, 6 = TCP, 17 = UDP, etc.). Protocol int `json:"protocol"` - // ReceiveTimestamp Timestamp when the event was received by our API. - ReceiveTimestamp time.Time `json:"receive_timestamp"` - // ReporterId ID of the reporter of the event (e.g., the peer that reported the event). ReporterId string `json:"reporter_id"` @@ -917,26 +904,12 @@ type NetworkTrafficEvent struct { RxPackets int `json:"rx_packets"` Source NetworkTrafficEndpoint `json:"source"` - // Timestamp Timestamp of the event. Send by the peer. - Timestamp time.Time `json:"timestamp"` - // TxBytes Number of bytes transmitted. TxBytes int `json:"tx_bytes"` // TxPackets Number of packets transmitted. - TxPackets int `json:"tx_packets"` - - // Type Type of the event (e.g. TYPE_UNKNOWN, TYPE_START, TYPE_END, TYPE_DROP). - Type string `json:"type"` - - // UserEmail Email of the user who initiated the event (if any). - UserEmail *string `json:"user_email"` - - // UserId UserID is the ID of the user that initiated the event (can be empty as not every event is user-initiated). - UserId *string `json:"user_id"` - - // UserName Name of the user who initiated the event (if any). - UserName *string `json:"user_name"` + TxPackets int `json:"tx_packets"` + User NetworkTrafficUser `json:"user"` } // NetworkTrafficEventsResponse defines model for NetworkTrafficEventsResponse. @@ -957,6 +930,15 @@ type NetworkTrafficEventsResponse struct { TotalRecords int `json:"total_records"` } +// NetworkTrafficICMP defines model for NetworkTrafficICMP. +type NetworkTrafficICMP struct { + // Code ICMP code (if applicable). + Code int `json:"code"` + + // Type ICMP type (if applicable). + Type int `json:"type"` +} + // NetworkTrafficLocation defines model for NetworkTrafficLocation. type NetworkTrafficLocation struct { // CityName Name of the city (if known). @@ -966,6 +948,36 @@ type NetworkTrafficLocation struct { CountryCode string `json:"country_code"` } +// NetworkTrafficPolicy defines model for NetworkTrafficPolicy. +type NetworkTrafficPolicy struct { + // Id ID of the policy that allowed this event. + Id string `json:"id"` + + // Name Name of the policy that allowed this event. + Name string `json:"name"` +} + +// NetworkTrafficSubEvent defines model for NetworkTrafficSubEvent. +type NetworkTrafficSubEvent struct { + // Timestamp Timestamp of the event as sent by the peer. + Timestamp time.Time `json:"timestamp"` + + // Type Type of the event (e.g., TYPE_UNKNOWN, TYPE_START, TYPE_END, TYPE_DROP). + Type string `json:"type"` +} + +// NetworkTrafficUser defines model for NetworkTrafficUser. +type NetworkTrafficUser struct { + // Email Email of the user who initiated the event (if any). + Email string `json:"email"` + + // Id UserID is the ID of the user that initiated the event (can be empty as not every event is user-initiated). + Id string `json:"id"` + + // Name Name of the user who initiated the event (if any). + Name string `json:"name"` +} + // OSVersionCheck Posture check for the version of operating system type OSVersionCheck struct { // Android Posture check for the version of operating system diff --git a/management/server/store/store.go b/management/server/store/store.go index b3c2fceff..fff809247 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -365,11 +365,14 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) ( return nil, nil, fmt.Errorf("failed to add all group to account: %v", err) } + var sqlStore Store + var cleanup func() + maxRetries := 2 for i := 0; i < maxRetries; i++ { - sqlStore, cleanUp, err := getSqlStoreEngine(ctx, store, kind) + sqlStore, cleanup, err = getSqlStoreEngine(ctx, store, kind) if err == nil { - return sqlStore, cleanUp, nil + return sqlStore, cleanup, nil } if i < maxRetries-1 { time.Sleep(100 * time.Millisecond) @@ -427,16 +430,16 @@ func getSqlStoreEngine(ctx context.Context, store *SqlStore, kind types.Engine) } func newReusedPostgresStore(ctx context.Context, store *SqlStore, kind types.Engine) (*SqlStore, func(), error) { - if envDsn, ok := os.LookupEnv(postgresDsnEnv); !ok || envDsn == "" { + dsn, ok := os.LookupEnv(postgresDsnEnv) + if !ok || dsn == "" { var err error - _, err = testutil.CreatePostgresTestContainer() + _, dsn, err = testutil.CreatePostgresTestContainer() if err != nil { return nil, nil, err } } - dsn, ok := os.LookupEnv(postgresDsnEnv) - if !ok { + if dsn == "" { return nil, nil, fmt.Errorf("%s is not set", postgresDsnEnv) } @@ -447,28 +450,28 @@ func newReusedPostgresStore(ctx context.Context, store *SqlStore, kind types.Eng dsn, cleanup, err := createRandomDB(dsn, db, kind) if err != nil { - return nil, cleanup, err + return nil, nil, err } store, err = NewPostgresqlStoreFromSqlStore(ctx, store, dsn, nil) if err != nil { - return nil, cleanup, err + return nil, nil, err } return store, cleanup, nil } func newReusedMysqlStore(ctx context.Context, store *SqlStore, kind types.Engine) (*SqlStore, func(), error) { - if envDsn, ok := os.LookupEnv(mysqlDsnEnv); !ok || envDsn == "" { + dsn, ok := os.LookupEnv(mysqlDsnEnv) + if !ok || dsn == "" { var err error - _, err = testutil.CreateMysqlTestContainer() + _, dsn, err = testutil.CreateMysqlTestContainer() if err != nil { return nil, nil, err } } - dsn, ok := os.LookupEnv(mysqlDsnEnv) - if !ok { + if dsn == "" { return nil, nil, fmt.Errorf("%s is not set", mysqlDsnEnv) } @@ -479,7 +482,7 @@ func newReusedMysqlStore(ctx context.Context, store *SqlStore, kind types.Engine dsn, cleanup, err := createRandomDB(dsn, db, kind) if err != nil { - return nil, cleanup, err + return nil, nil, err } store, err = NewMysqlStoreFromSqlStore(ctx, store, dsn, nil) diff --git a/management/server/testutil/store.go b/management/server/testutil/store.go index ca022bfef..7f6a824a4 100644 --- a/management/server/testutil/store.go +++ b/management/server/testutil/store.go @@ -5,7 +5,6 @@ package testutil import ( "context" - "os" "time" log "github.com/sirupsen/logrus" @@ -16,11 +15,25 @@ import ( "github.com/testcontainers/testcontainers-go/wait" ) +var ( + pgContainer *postgres.PostgresContainer + mysqlContainer *mysql.MySQLContainer +) + // CreateMysqlTestContainer creates a new MySQL container for testing. -func CreateMysqlTestContainer() (func(), error) { +func CreateMysqlTestContainer() (func(), string, error) { ctx := context.Background() - myContainer, err := mysql.RunContainer(ctx, + if mysqlContainer != nil { + connStr, err := mysqlContainer.ConnectionString(ctx) + if err != nil { + return nil, "", err + } + return noOpCleanup, connStr, nil + } + + var err error + mysqlContainer, err = mysql.RunContainer(ctx, testcontainers.WithImage("mlsmaycon/warmed-mysql:8"), mysql.WithDatabase("testing"), mysql.WithUsername("root"), @@ -31,31 +44,39 @@ func CreateMysqlTestContainer() (func(), error) { ), ) if err != nil { - return nil, err + return nil, "", err } cleanup := func() { - os.Unsetenv("NETBIRD_STORE_ENGINE_MYSQL_DSN") timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) defer cancelFunc() - if err = myContainer.Terminate(timeoutCtx); err != nil { - log.WithContext(ctx).Warnf("failed to stop mysql container %s: %s", myContainer.GetContainerID(), err) + if err = mysqlContainer.Terminate(timeoutCtx); err != nil { + log.WithContext(ctx).Warnf("failed to stop mysql container %s: %s", mysqlContainer.GetContainerID(), err) } } - talksConn, err := myContainer.ConnectionString(ctx) + talksConn, err := mysqlContainer.ConnectionString(ctx) if err != nil { - return nil, err + return nil, "", err } - return cleanup, os.Setenv("NETBIRD_STORE_ENGINE_MYSQL_DSN", talksConn) + return cleanup, talksConn, nil } // CreatePostgresTestContainer creates a new PostgreSQL container for testing. -func CreatePostgresTestContainer() (func(), error) { +func CreatePostgresTestContainer() (func(), string, error) { ctx := context.Background() - pgContainer, err := postgres.RunContainer(ctx, + if pgContainer != nil { + connStr, err := pgContainer.ConnectionString(ctx) + if err != nil { + return nil, "", err + } + return noOpCleanup, connStr, nil + } + + var err error + pgContainer, err = postgres.RunContainer(ctx, testcontainers.WithImage("postgres:16-alpine"), postgres.WithDatabase("netbird"), postgres.WithUsername("root"), @@ -66,11 +87,10 @@ func CreatePostgresTestContainer() (func(), error) { ), ) if err != nil { - return nil, err + return nil, "", err } cleanup := func() { - os.Unsetenv("NETBIRD_STORE_ENGINE_POSTGRES_DSN") timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) defer cancelFunc() if err = pgContainer.Terminate(timeoutCtx); err != nil { @@ -80,10 +100,14 @@ func CreatePostgresTestContainer() (func(), error) { talksConn, err := pgContainer.ConnectionString(ctx) if err != nil { - return nil, err + return nil, "", err } - return cleanup, os.Setenv("NETBIRD_STORE_ENGINE_POSTGRES_DSN", talksConn) + return cleanup, talksConn, nil +} + +func noOpCleanup() { + // no-op } // CreateRedisTestContainer creates a new Redis container for testing. diff --git a/management/server/testutil/store_ios.go b/management/server/testutil/store_ios.go index a614258d2..c3dd839d3 100644 --- a/management/server/testutil/store_ios.go +++ b/management/server/testutil/store_ios.go @@ -3,16 +3,16 @@ package testutil -func CreatePostgresTestContainer() (func(), error) { +func CreatePostgresTestContainer() (func(), string, error) { return func() { // Empty function for Postgres - }, nil + }, "", nil } -func CreateMysqlTestContainer() (func(), error) { +func CreateMysqlTestContainer() (func(), string, error) { return func() { // Empty function for MySQL - }, nil + }, "", nil } func CreateRedisTestContainer() (func(), string, error) { From cdd27a9fe56932bb1b780e39fbaa112e475dd155 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 27 May 2025 13:32:54 +0200 Subject: [PATCH 05/37] [client, android] Fix/android enable server route (#3806) Enable the server route; otherwise, the manager throws an error and the engine will restart. --- .../{server_nonandroid.go => server.go} | 2 -- .../internal/routemanager/server_android.go | 27 ------------------- 2 files changed, 29 deletions(-) rename client/internal/routemanager/{server_nonandroid.go => server.go} (99%) delete mode 100644 client/internal/routemanager/server_android.go diff --git a/client/internal/routemanager/server_nonandroid.go b/client/internal/routemanager/server.go similarity index 99% rename from client/internal/routemanager/server_nonandroid.go rename to client/internal/routemanager/server.go index 131d4c170..5bacb856c 100644 --- a/client/internal/routemanager/server_nonandroid.go +++ b/client/internal/routemanager/server.go @@ -1,5 +1,3 @@ -//go:build !android - package routemanager import ( diff --git a/client/internal/routemanager/server_android.go b/client/internal/routemanager/server_android.go deleted file mode 100644 index 953210e9e..000000000 --- a/client/internal/routemanager/server_android.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build android - -package routemanager - -import ( - "context" - "fmt" - - firewall "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/internal/peer" - "github.com/netbirdio/netbird/client/internal/routemanager/iface" - "github.com/netbirdio/netbird/route" -) - -type serverRouter struct { -} - -func (r serverRouter) cleanUp() { -} - -func (r serverRouter) updateRoutes(map[route.ID]*route.Route, bool) error { - return nil -} - -func newServerRouter(context.Context, iface.WGIface, firewall.Manager, *peer.Status) (*serverRouter, error) { - return nil, fmt.Errorf("server route not supported on this os") -} From a0d28f9851bc607ea8458024c629303b343174c7 Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Tue, 27 May 2025 14:42:00 +0300 Subject: [PATCH 06/37] [management] Reset test containers after cleanup (#3885) --- management/server/testutil/store.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/management/server/testutil/store.go b/management/server/testutil/store.go index 7f6a824a4..db418c45b 100644 --- a/management/server/testutil/store.go +++ b/management/server/testutil/store.go @@ -48,10 +48,13 @@ func CreateMysqlTestContainer() (func(), string, error) { } cleanup := func() { - timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) - defer cancelFunc() - if err = mysqlContainer.Terminate(timeoutCtx); err != nil { - log.WithContext(ctx).Warnf("failed to stop mysql container %s: %s", mysqlContainer.GetContainerID(), err) + if mysqlContainer != nil { + timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) + defer cancelFunc() + if err = mysqlContainer.Terminate(timeoutCtx); err != nil { + log.WithContext(ctx).Warnf("failed to stop mysql container %s: %s", mysqlContainer.GetContainerID(), err) + } + mysqlContainer = nil // reset the container to allow recreation } } @@ -91,11 +94,15 @@ func CreatePostgresTestContainer() (func(), string, error) { } cleanup := func() { - timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) - defer cancelFunc() - if err = pgContainer.Terminate(timeoutCtx); err != nil { - log.WithContext(ctx).Warnf("failed to stop postgres container %s: %s", pgContainer.GetContainerID(), err) + if pgContainer != nil { + timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) + defer cancelFunc() + if err = pgContainer.Terminate(timeoutCtx); err != nil { + log.WithContext(ctx).Warnf("failed to stop postgres container %s: %s", pgContainer.GetContainerID(), err) + } + pgContainer = nil // reset the container to allow recreation } + } talksConn, err := pgContainer.ConnectionString(ctx) From 6f436e57b50f0ef7efa71267173c1f27cf0988e7 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 27 May 2025 16:42:06 +0200 Subject: [PATCH 07/37] [server-test] Install libs for i386 tests (#3887) Install libs for i386 tests --- .github/workflows/golang-test-linux.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index d585ba209..cbce3e6e4 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -223,6 +223,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Install dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: sudo apt update && sudo apt install -y gcc-multilib g++-multilib libc6-dev-i386 + - name: Get Go environment run: | echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV @@ -269,6 +273,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Install dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: sudo apt update && sudo apt install -y gcc-multilib g++-multilib libc6-dev-i386 + - name: Get Go environment run: | echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV From 0492c1724ad28d152f34a2ead28970558ae57053 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 27 May 2025 17:12:04 +0200 Subject: [PATCH 08/37] [client, android] Fix/notifier threading (#3807) - Fix potential deadlocks - When adding a listener, immediately notify with the last known IP and fqdn. --- client/internal/peer/notifier.go | 102 ++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/client/internal/peer/notifier.go b/client/internal/peer/notifier.go index f1175c2c4..8d1954fe5 100644 --- a/client/internal/peer/notifier.go +++ b/client/internal/peer/notifier.go @@ -18,6 +18,8 @@ type notifier struct { currentClientState bool lastNotification int lastNumberOfPeers int + lastFqdnAddress string + lastIPAddress string } func newNotifier() *notifier { @@ -25,15 +27,22 @@ func newNotifier() *notifier { } func (n *notifier) setListener(listener Listener) { + n.serverStateLock.Lock() + lastNotification := n.lastNotification + numOfPeers := n.lastNumberOfPeers + fqdnAddress := n.lastFqdnAddress + address := n.lastIPAddress + n.serverStateLock.Unlock() + n.listenersLock.Lock() defer n.listenersLock.Unlock() - n.serverStateLock.Lock() - n.notifyListener(listener, n.lastNotification) - listener.OnPeersListChanged(n.lastNumberOfPeers) - n.serverStateLock.Unlock() - n.listener = listener + + listener.OnAddressChanged(fqdnAddress, address) + notifyListener(listener, lastNotification) + // run on go routine to avoid on Java layer to call go functions on same thread + go listener.OnPeersListChanged(numOfPeers) } func (n *notifier) removeListener() { @@ -44,41 +53,44 @@ func (n *notifier) removeListener() { func (n *notifier) updateServerStates(mgmState bool, signalState bool) { n.serverStateLock.Lock() - defer n.serverStateLock.Unlock() - calculatedState := n.calculateState(mgmState, signalState) if !n.isServerStateChanged(calculatedState) { + n.serverStateLock.Unlock() return } n.lastNotification = calculatedState + n.serverStateLock.Unlock() - n.notify(n.lastNotification) + n.notify(calculatedState) } func (n *notifier) clientStart() { n.serverStateLock.Lock() - defer n.serverStateLock.Unlock() n.currentClientState = true n.lastNotification = stateConnecting - n.notify(n.lastNotification) + n.serverStateLock.Unlock() + + n.notify(stateConnecting) } func (n *notifier) clientStop() { n.serverStateLock.Lock() - defer n.serverStateLock.Unlock() n.currentClientState = false n.lastNotification = stateDisconnected - n.notify(n.lastNotification) + n.serverStateLock.Unlock() + + n.notify(stateDisconnected) } func (n *notifier) clientTearDown() { n.serverStateLock.Lock() - defer n.serverStateLock.Unlock() n.currentClientState = false n.lastNotification = stateDisconnecting - n.notify(n.lastNotification) + n.serverStateLock.Unlock() + + n.notify(stateDisconnecting) } func (n *notifier) isServerStateChanged(newState int) bool { @@ -87,26 +99,14 @@ func (n *notifier) isServerStateChanged(newState int) bool { func (n *notifier) notify(state int) { n.listenersLock.Lock() - defer n.listenersLock.Unlock() - if n.listener == nil { + listener := n.listener + n.listenersLock.Unlock() + + if listener == nil { return } - n.notifyListener(n.listener, state) -} -func (n *notifier) notifyListener(l Listener, state int) { - go func() { - switch state { - case stateDisconnected: - l.OnDisconnected() - case stateConnected: - l.OnConnected() - case stateConnecting: - l.OnConnecting() - case stateDisconnecting: - l.OnDisconnecting() - } - }() + notifyListener(listener, state) } func (n *notifier) calculateState(managementConn, signalConn bool) int { @@ -126,20 +126,48 @@ func (n *notifier) calculateState(managementConn, signalConn bool) int { } func (n *notifier) peerListChanged(numOfPeers int) { + n.serverStateLock.Lock() n.lastNumberOfPeers = numOfPeers + n.serverStateLock.Unlock() + n.listenersLock.Lock() - defer n.listenersLock.Unlock() - if n.listener == nil { + listener := n.listener + n.listenersLock.Unlock() + + if listener == nil { return } - n.listener.OnPeersListChanged(numOfPeers) + + // run on go routine to avoid on Java layer to call go functions on same thread + go listener.OnPeersListChanged(numOfPeers) } func (n *notifier) localAddressChanged(fqdn, address string) { + n.serverStateLock.Lock() + n.lastFqdnAddress = fqdn + n.lastIPAddress = address + n.serverStateLock.Unlock() + n.listenersLock.Lock() - defer n.listenersLock.Unlock() - if n.listener == nil { + listener := n.listener + n.listenersLock.Unlock() + + if listener == nil { return } - n.listener.OnAddressChanged(fqdn, address) + + listener.OnAddressChanged(fqdn, address) +} + +func notifyListener(l Listener, state int) { + switch state { + case stateDisconnected: + l.OnDisconnected() + case stateConnected: + l.OnConnected() + case stateConnecting: + l.OnConnecting() + case stateDisconnecting: + l.OnDisconnecting() + } } From 684501fd35c8b2222e899129243ff50a2fb736bf Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Thu, 29 May 2025 18:50:00 +0300 Subject: [PATCH 09/37] [management] Prevent deletion of peers linked to network routers (#3881) - Prevent deletion of peers linked to network routers - Add API endpoint to list all network routers --- management/server/http/api/openapi.yml | 25 +++++++++++++++++ .../http/handlers/networks/routers_handler.go | 28 ++++++++++++++++++- management/server/peer.go | 27 ++++++++++++++++-- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/management/server/http/api/openapi.yml b/management/server/http/api/openapi.yml index f6ea39a8e..58134d375 100644 --- a/management/server/http/api/openapi.yml +++ b/management/server/http/api/openapi.yml @@ -4068,6 +4068,31 @@ paths: "$ref": "#/components/responses/forbidden" '500': "$ref": "#/components/responses/internal_error" + /api/networks/routers: + get: + summary: List all Network Routers + description: Returns a list of all routers in a network + tags: [ Networks ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + responses: + '200': + description: A JSON Array of Routers + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/NetworkRouter' + '400': + "$ref": "#/components/responses/bad_request" + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '500': + "$ref": "#/components/responses/internal_error" /api/dns/nameservers: get: summary: List all Nameserver Groups diff --git a/management/server/http/handlers/networks/routers_handler.go b/management/server/http/handlers/networks/routers_handler.go index f1a3fba0b..6b00534fc 100644 --- a/management/server/http/handlers/networks/routers_handler.go +++ b/management/server/http/handlers/networks/routers_handler.go @@ -19,7 +19,8 @@ type routersHandler struct { func addRouterEndpoints(routersManager routers.Manager, router *mux.Router) { routersHandler := newRoutersHandler(routersManager) - router.HandleFunc("/networks/{networkId}/routers", routersHandler.getAllRouters).Methods("GET", "OPTIONS") + router.HandleFunc("/networks/routers", routersHandler.getAllRouters).Methods("GET", "OPTIONS") + router.HandleFunc("/networks/{networkId}/routers", routersHandler.getNetworkRouters).Methods("GET", "OPTIONS") router.HandleFunc("/networks/{networkId}/routers", routersHandler.createRouter).Methods("POST", "OPTIONS") router.HandleFunc("/networks/{networkId}/routers/{routerId}", routersHandler.getRouter).Methods("GET", "OPTIONS") router.HandleFunc("/networks/{networkId}/routers/{routerId}", routersHandler.updateRouter).Methods("PUT", "OPTIONS") @@ -41,6 +42,31 @@ func (h *routersHandler) getAllRouters(w http.ResponseWriter, r *http.Request) { accountID, userID := userAuth.AccountId, userAuth.UserId + routersMap, err := h.routersManager.GetAllRoutersInAccount(r.Context(), accountID, userID) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + routersResponse := make([]*api.NetworkRouter, 0) + for _, routers := range routersMap { + for _, router := range routers { + routersResponse = append(routersResponse, router.ToAPIResponse()) + } + } + + util.WriteJSONObject(r.Context(), w, routersResponse) +} + +func (h *routersHandler) getNetworkRouters(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + accountID, userID := userAuth.AccountId, userAuth.UserId + networkID := mux.Vars(r)["networkId"] routers, err := h.routersManager.GetAllRoutersInNetwork(r.Context(), accountID, userID, networkID) if err != nil { diff --git a/management/server/peer.go b/management/server/peer.go index f91db928d..4a468a6cd 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -17,6 +17,7 @@ import ( "github.com/netbirdio/netbird/management/domain" "github.com/netbirdio/netbird/management/server/geolocation" + routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types" "github.com/netbirdio/netbird/management/server/permissions/modules" "github.com/netbirdio/netbird/management/server/permissions/operations" @@ -352,7 +353,7 @@ func (am *DefaultAccountManager) DeletePeer(ctx context.Context, accountID, peer return err } - if err = am.validatePeerDelete(ctx, accountID, peerID); err != nil { + if err = am.validatePeerDelete(ctx, transaction, accountID, peerID); err != nil { return err } @@ -1543,7 +1544,7 @@ func ConvertSliceToMap(existingLabels []string) map[string]struct{} { } // validatePeerDelete checks if the peer can be deleted. -func (am *DefaultAccountManager) validatePeerDelete(ctx context.Context, accountId, peerId string) error { +func (am *DefaultAccountManager) validatePeerDelete(ctx context.Context, transaction store.Store, accountId, peerId string) error { linkedInIngressPorts, err := am.proxyController.IsPeerInIngressPorts(ctx, accountId, peerId) if err != nil { return err @@ -1553,5 +1554,27 @@ func (am *DefaultAccountManager) validatePeerDelete(ctx context.Context, account return status.Errorf(status.PreconditionFailed, "peer is linked to ingress ports: %s", peerId) } + linked, router := isPeerLinkedToNetworkRouter(ctx, transaction, accountId, peerId) + if linked { + return status.Errorf(status.PreconditionFailed, "peer is linked to a network router: %s", router.ID) + } + return nil } + +// isPeerLinkedToNetworkRouter checks if a peer is linked to any network router in the account. +func isPeerLinkedToNetworkRouter(ctx context.Context, transaction store.Store, accountID string, peerID string) (bool, *routerTypes.NetworkRouter) { + routers, err := transaction.GetNetworkRoutersByAccountID(ctx, store.LockingStrengthShare, accountID) + if err != nil { + log.WithContext(ctx).Errorf("error retrieving network routers while checking peer linkage: %v", err) + return false, nil + } + + for _, router := range routers { + if router.Peer == peerID { + return true, router + } + } + + return false, nil +} From cfb2d8235205f2d5db9e81d52677702a11176218 Mon Sep 17 00:00:00 2001 From: hakansa <43675540+hakansa@users.noreply.github.com> Date: Fri, 30 May 2025 16:54:49 +0300 Subject: [PATCH 10/37] [client] Refactor exclude list handling to use a map for permanent connections (#3901) [client] Refactor exclude list handling to use a map for permanent connections (#3901) --- client/internal/conn_mgr.go | 4 ++-- client/internal/engine.go | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/client/internal/conn_mgr.go b/client/internal/conn_mgr.go index 119ddc1bd..f7b1f6a05 100644 --- a/client/internal/conn_mgr.go +++ b/client/internal/conn_mgr.go @@ -98,14 +98,14 @@ func (e *ConnMgr) UpdatedRemoteFeatureFlag(ctx context.Context, enabled bool) er } // SetExcludeList sets the list of peer IDs that should always have permanent connections. -func (e *ConnMgr) SetExcludeList(peerIDs []string) { +func (e *ConnMgr) SetExcludeList(peerIDs map[string]bool) { if e.lazyConnMgr == nil { return } excludedPeers := make([]lazyconn.PeerConfig, 0, len(peerIDs)) - for _, peerID := range peerIDs { + for peerID := range peerIDs { var peerConn *peer.Conn var exists bool if peerConn, exists = e.peerStore.PeerConn(peerID); !exists { diff --git a/client/internal/engine.go b/client/internal/engine.go index d6bcc66f6..e04ccc9b8 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -1927,14 +1927,16 @@ func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewal return forwardingRules, nberrors.FormatErrorOrNil(merr) } -func (e *Engine) toExcludedLazyPeers(routes []*route.Route, rules []firewallManager.ForwardRule, peers []*mgmProto.RemotePeerConfig) []string { - excludedPeers := make([]string, 0) +func (e *Engine) toExcludedLazyPeers(routes []*route.Route, rules []firewallManager.ForwardRule, peers []*mgmProto.RemotePeerConfig) map[string]bool { + excludedPeers := make(map[string]bool) for _, r := range routes { if r.Peer == "" { continue } - log.Infof("exclude router peer from lazy connection: %s", r.Peer) - excludedPeers = append(excludedPeers, r.Peer) + if !excludedPeers[r.Peer] { + log.Infof("exclude router peer from lazy connection: %s", r.Peer) + excludedPeers[r.Peer] = true + } } for _, r := range rules { @@ -1945,7 +1947,7 @@ func (e *Engine) toExcludedLazyPeers(routes []*route.Route, rules []firewallMana continue } log.Infof("exclude forwarder peer from lazy connection: %s", p.GetWgPubKey()) - excludedPeers = append(excludedPeers, p.GetWgPubKey()) + excludedPeers[p.GetWgPubKey()] = true } } } From 2bef214cc07ff7f641a102f326da0bdf6d9d4cb8 Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Fri, 30 May 2025 18:12:30 +0300 Subject: [PATCH 11/37] [management] Fix user groups propagation (#3902) --- management/server/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/server/user.go b/management/server/user.go index 2c762a8eb..5c162c50b 100644 --- a/management/server/user.go +++ b/management/server/user.go @@ -676,7 +676,7 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact return false, nil, nil, nil, fmt.Errorf("error modifying user peers in groups: %w", err) } - if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, update.AccountID, updatedGroups); err != nil { + if err = transaction.SaveGroups(ctx, store.LockingStrengthUpdate, accountID, updatedGroups); err != nil { return false, nil, nil, nil, fmt.Errorf("error saving groups: %w", err) } } From aa07b3b87b262cbf3eb262d939d8c9e2d70ee6de Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 30 May 2025 23:38:02 +0200 Subject: [PATCH 12/37] Fix deadlock (#3904) --- client/internal/peer/conn.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 5037a0bd0..b33023873 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -691,8 +691,7 @@ func (conn *Conn) evalStatus() ConnStatus { } func (conn *Conn) isConnectedOnAllWay() (connected bool) { - conn.mu.Lock() - defer conn.mu.Unlock() + // would be better to protect this with a mutex, but it could cause deadlock with Close function defer func() { if !connected { From f16f0c78318add10cef54f9b70d58ee2f2d97376 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 1 Jun 2025 16:08:27 +0200 Subject: [PATCH 13/37] [client] Fix HA router switch (#3889) * Fix HA router switch. - Simplify the notification filter logic. Always send notification if a state has been changed - Remove IP changes check because we never modify * Notify only the proper listeners * Fix test * Fix TestGetPeerStateChangeNotifierLogic test * Before lazy connection, when the peer disconnected, the status switched to disconnected. After implementing lazy connection, the peer state is connecting, so we did not decrease the reference counters on the routes. * When switch to idle notify the route mgr --- client/internal/peer/status.go | 85 +++++++++++++------------- client/internal/peer/status_test.go | 27 ++++---- client/internal/routemanager/client.go | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 69e333bf1..40956e68e 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -289,11 +289,7 @@ func (d *Status) UpdatePeerState(receivedState State) error { return errors.New("peer doesn't exist") } - if receivedState.IP != "" { - peerState.IP = receivedState.IP - } - - skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + oldState := peerState.ConnStatus if receivedState.ConnStatus != peerState.ConnStatus { peerState.ConnStatus = receivedState.ConnStatus @@ -309,11 +305,15 @@ func (d *Status) UpdatePeerState(receivedState State) error { d.peers[receivedState.PubKey] = peerState - if skipNotification { - return nil + if hasConnStatusChanged(oldState, receivedState.ConnStatus) { + d.notifyPeerListChanged() } - d.notifyPeerListChanged() + // when we close the connection we will not notify the router manager + if receivedState.ConnStatus == StatusIdle { + d.notifyPeerStateChangeListeners(receivedState.PubKey) + + } return nil } @@ -380,11 +380,8 @@ func (d *Status) UpdatePeerICEState(receivedState State) error { return errors.New("peer doesn't exist") } - if receivedState.IP != "" { - peerState.IP = receivedState.IP - } - - skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + oldState := peerState.ConnStatus + oldIsRelayed := peerState.Relayed peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate @@ -397,12 +394,13 @@ func (d *Status) UpdatePeerICEState(receivedState State) error { d.peers[receivedState.PubKey] = peerState - if skipNotification { - return nil + if hasConnStatusChanged(oldState, receivedState.ConnStatus) { + d.notifyPeerListChanged() } - d.notifyPeerStateChangeListeners(receivedState.PubKey) - d.notifyPeerListChanged() + if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) { + d.notifyPeerStateChangeListeners(receivedState.PubKey) + } return nil } @@ -415,7 +413,8 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error { return errors.New("peer doesn't exist") } - skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + oldState := peerState.ConnStatus + oldIsRelayed := peerState.Relayed peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate @@ -425,12 +424,13 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error { d.peers[receivedState.PubKey] = peerState - if skipNotification { - return nil + if hasConnStatusChanged(oldState, receivedState.ConnStatus) { + d.notifyPeerListChanged() } - d.notifyPeerStateChangeListeners(receivedState.PubKey) - d.notifyPeerListChanged() + if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) { + d.notifyPeerStateChangeListeners(receivedState.PubKey) + } return nil } @@ -443,7 +443,8 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error return errors.New("peer doesn't exist") } - skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + oldState := peerState.ConnStatus + oldIsRelayed := peerState.Relayed peerState.ConnStatus = receivedState.ConnStatus peerState.Relayed = receivedState.Relayed @@ -452,12 +453,13 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error d.peers[receivedState.PubKey] = peerState - if skipNotification { - return nil + if hasConnStatusChanged(oldState, receivedState.ConnStatus) { + d.notifyPeerListChanged() } - d.notifyPeerStateChangeListeners(receivedState.PubKey) - d.notifyPeerListChanged() + if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) { + d.notifyPeerStateChangeListeners(receivedState.PubKey) + } return nil } @@ -470,7 +472,8 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error { return errors.New("peer doesn't exist") } - skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + oldState := peerState.ConnStatus + oldIsRelayed := peerState.Relayed peerState.ConnStatus = receivedState.ConnStatus peerState.Relayed = receivedState.Relayed @@ -482,12 +485,13 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error { d.peers[receivedState.PubKey] = peerState - if skipNotification { - return nil + if hasConnStatusChanged(oldState, receivedState.ConnStatus) { + d.notifyPeerListChanged() } - d.notifyPeerStateChangeListeners(receivedState.PubKey) - d.notifyPeerListChanged() + if hasStatusOrRelayedChange(oldState, receivedState.ConnStatus, oldIsRelayed, receivedState.Relayed) { + d.notifyPeerStateChangeListeners(receivedState.PubKey) + } return nil } @@ -510,17 +514,12 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats configurer.WGSt return nil } -func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool { - switch { - case receivedConnStatus == StatusConnecting: - return true - case receivedConnStatus == StatusIdle && curr.ConnStatus == StatusConnecting: - return true - case receivedConnStatus == StatusIdle && curr.ConnStatus == StatusIdle: - return curr.IP != "" - default: - return false - } +func hasStatusOrRelayedChange(oldConnStatus, newConnStatus ConnStatus, oldRelayed, newRelayed bool) bool { + return oldRelayed != newRelayed || hasConnStatusChanged(newConnStatus, oldConnStatus) +} + +func hasConnStatusChanged(oldStatus, newStatus ConnStatus) bool { + return newStatus != oldStatus } // UpdatePeerFQDN update peer's state fqdn only diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index bdf8f087a..8f28a9862 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -4,6 +4,7 @@ import ( "errors" "sync" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -42,16 +43,16 @@ func TestGetPeer(t *testing.T) { func TestUpdatePeerState(t *testing.T) { key := "abc" ip := "10.10.10.10" + fqdn := "peer-a.netbird.local" status := NewRecorder("https://mgm") + _ = status.AddPeer(key, fqdn, ip) + peerState := State{ - PubKey: key, - Mux: new(sync.RWMutex), + PubKey: key, + ConnStatusUpdate: time.Now(), + ConnStatus: StatusConnecting, } - status.peers[key] = peerState - - peerState.IP = ip - err := status.UpdatePeerState(peerState) assert.NoError(t, err, "shouldn't return error") @@ -83,17 +84,17 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) { key := "abc" ip := "10.10.10.10" status := NewRecorder("https://mgm") - peerState := State{ - PubKey: key, - Mux: new(sync.RWMutex), - } - - status.peers[key] = peerState + _ = status.AddPeer(key, "abc.netbird", ip) ch := status.GetPeerStateChangeNotifier(key) assert.NotNil(t, ch, "channel shouldn't be nil") - peerState.IP = ip + peerState := State{ + PubKey: key, + ConnStatus: StatusConnecting, + Relayed: false, + ConnStatusUpdate: time.Now(), + } err := status.UpdatePeerRelayedStateToDisconnected(peerState) assert.NoError(t, err, "shouldn't return error") diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index 847949a53..137e00d31 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -232,7 +232,7 @@ func (c *clientNetwork) watchPeerStatusChanges(ctx context.Context, peerKey stri return case <-c.statusRecorder.GetPeerStateChangeNotifier(peerKey): state, err := c.statusRecorder.GetPeer(peerKey) - if err != nil || state.ConnStatus == peer.StatusConnecting { + if err != nil { continue } peerStateUpdate <- struct{}{} From 41cd4952f17d4e117bddf078d6ab56704f73b809 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 2 Jun 2025 12:11:54 +0200 Subject: [PATCH 14/37] [client] Apply return traffic rules only if firewall is stateless (#3895) --- client/firewall/iptables/manager_linux.go | 4 + client/firewall/manager/firewall.go | 2 + client/firewall/nftables/manager_linux.go | 4 + client/firewall/uspfilter/uspfilter.go | 9 +- client/internal/acl/manager.go | 6 +- client/internal/acl/manager_test.go | 187 +++++++++++-------- client/internal/routemanager/static/route.go | 1 - 7 files changed, 130 insertions(+), 83 deletions(-) diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index b229688fc..0897f831f 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -147,6 +147,10 @@ func (m *Manager) IsServerRouteSupported() bool { return true } +func (m *Manager) IsStateful() bool { + return true +} + func (m *Manager) AddNatRule(pair firewall.RouterPair) error { m.mutex.Lock() defer m.mutex.Unlock() diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 084d19423..3b3164823 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -116,6 +116,8 @@ type Manager interface { // IsServerRouteSupported returns true if the firewall supports server side routing operations IsServerRouteSupported() bool + IsStateful() bool + AddRouteFiltering( id []byte, sources []netip.Prefix, diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index e6b3a031b..2f8ee81a4 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -170,6 +170,10 @@ func (m *Manager) IsServerRouteSupported() bool { return true } +func (m *Manager) IsStateful() bool { + return true +} + func (m *Manager) AddNatRule(pair firewall.RouterPair) error { m.mutex.Lock() defer m.mutex.Unlock() diff --git a/client/firewall/uspfilter/uspfilter.go b/client/firewall/uspfilter/uspfilter.go index 11730dbb3..287e52773 100644 --- a/client/firewall/uspfilter/uspfilter.go +++ b/client/firewall/uspfilter/uspfilter.go @@ -326,6 +326,10 @@ func (m *Manager) IsServerRouteSupported() bool { return true } +func (m *Manager) IsStateful() bool { + return m.stateful +} + func (m *Manager) AddNatRule(pair firewall.RouterPair) error { if m.nativeRouter.Load() && m.nativeFirewall != nil { return m.nativeFirewall.AddNatRule(pair) @@ -606,9 +610,8 @@ func (m *Manager) processOutgoingHooks(packetData []byte, size int) bool { return true } - if m.stateful { - m.trackOutbound(d, srcIP, dstIP, size) - } + // for netflow we keep track even if the firewall is stateless + m.trackOutbound(d, srcIP, dstIP, size) return false } diff --git a/client/internal/acl/manager.go b/client/internal/acl/manager.go index a6316d7a2..5caf2b770 100644 --- a/client/internal/acl/manager.go +++ b/client/internal/acl/manager.go @@ -285,8 +285,10 @@ func (d *DefaultManager) protoRuleToFirewallRule( case mgmProto.RuleDirection_IN: rules, err = d.addInRules(r.PolicyID, ip, protocol, port, action, ipsetName) case mgmProto.RuleDirection_OUT: - // TODO: Remove this soon. Outbound rules are obsolete. - // We only maintain this for return traffic (inbound dir) which is now handled by the stateful firewall already + if d.firewall.IsStateful() { + return "", nil, nil + } + // return traffic for outbound connections if firewall is stateless rules, err = d.addOutRules(r.PolicyID, ip, protocol, port, action, ipsetName) default: return "", nil, fmt.Errorf("invalid direction, skipping firewall rule") diff --git a/client/internal/acl/manager_test.go b/client/internal/acl/manager_test.go index 3595ca600..532d70a24 100644 --- a/client/internal/acl/manager_test.go +++ b/client/internal/acl/manager_test.go @@ -5,9 +5,10 @@ import ( "testing" "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/netbirdio/netbird/client/firewall" - "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/internal/acl/mocks" "github.com/netbirdio/netbird/client/internal/netflow" @@ -43,9 +44,7 @@ func TestDefaultManager(t *testing.T) { ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() ifaceMock.EXPECT().SetFilter(gomock.Any()) ip, network, err := net.ParseCIDR("172.0.0.1/32") - if err != nil { - t.Fatalf("failed to parse IP address: %v", err) - } + require.NoError(t, err) ifaceMock.EXPECT().Name().Return("lo").AnyTimes() ifaceMock.EXPECT().Address().Return(wgaddr.Address{ @@ -54,23 +53,22 @@ func TestDefaultManager(t *testing.T) { }).AnyTimes() ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() - // we receive one rule from the management so for testing purposes ignore it fw, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false) - if err != nil { - t.Errorf("create firewall: %v", err) - return - } - defer func(fw manager.Manager) { - _ = fw.Close(nil) - }(fw) + require.NoError(t, err) + defer func() { + err = fw.Close(nil) + require.NoError(t, err) + }() + acl := NewDefaultManager(fw) t.Run("apply firewall rules", func(t *testing.T) { acl.ApplyFiltering(networkMap, false) - if len(acl.peerRulesPairs) != 2 { - t.Errorf("firewall rules not applied: %v", acl.peerRulesPairs) - return + if fw.IsStateful() { + assert.Equal(t, 0, len(acl.peerRulesPairs)) + } else { + assert.Equal(t, 2, len(acl.peerRulesPairs)) } }) @@ -94,12 +92,13 @@ func TestDefaultManager(t *testing.T) { acl.ApplyFiltering(networkMap, false) - // we should have one old and one new rule in the existed rules - if len(acl.peerRulesPairs) != 2 { - t.Errorf("firewall rules not applied") - return + expectedRules := 2 + if fw.IsStateful() { + expectedRules = 1 // only the inbound rule } + assert.Equal(t, expectedRules, len(acl.peerRulesPairs)) + // check that old rule was removed previousCount := 0 for id := range acl.peerRulesPairs { @@ -107,26 +106,87 @@ func TestDefaultManager(t *testing.T) { previousCount++ } } - if previousCount != 1 { - t.Errorf("old rule was not removed") + + expectedPreviousCount := 0 + if !fw.IsStateful() { + expectedPreviousCount = 1 } + assert.Equal(t, expectedPreviousCount, previousCount) }) t.Run("handle default rules", func(t *testing.T) { networkMap.FirewallRules = networkMap.FirewallRules[:0] networkMap.FirewallRulesIsEmpty = true - if acl.ApplyFiltering(networkMap, false); len(acl.peerRulesPairs) != 0 { - t.Errorf("rules should be empty if FirewallRulesIsEmpty is set, got: %v", len(acl.peerRulesPairs)) - return - } + acl.ApplyFiltering(networkMap, false) + assert.Equal(t, 0, len(acl.peerRulesPairs)) networkMap.FirewallRulesIsEmpty = false acl.ApplyFiltering(networkMap, false) - if len(acl.peerRulesPairs) != 1 { - t.Errorf("rules should contain 1 rules if FirewallRulesIsEmpty is not set, got: %v", len(acl.peerRulesPairs)) - return + + expectedRules := 1 + if fw.IsStateful() { + expectedRules = 1 // only inbound allow-all rule } + assert.Equal(t, expectedRules, len(acl.peerRulesPairs)) + }) +} + +func TestDefaultManagerStateless(t *testing.T) { + // stateless currently only in userspace, so we have to disable kernel + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv("NB_DISABLE_CONNTRACK", "true") + + networkMap := &mgmProto.NetworkMap{ + FirewallRules: []*mgmProto.FirewallRule{ + { + PeerIP: "10.93.0.1", + Direction: mgmProto.RuleDirection_OUT, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "80", + }, + { + PeerIP: "10.93.0.2", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_UDP, + Port: "53", + }, + }, + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + ip, network, err := net.ParseCIDR("172.0.0.1/32") + require.NoError(t, err) + + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{ + IP: ip, + Network: network, + }).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + fw, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false) + require.NoError(t, err) + defer func() { + err = fw.Close(nil) + require.NoError(t, err) + }() + + acl := NewDefaultManager(fw) + + t.Run("stateless firewall creates outbound rules", func(t *testing.T) { + acl.ApplyFiltering(networkMap, false) + + // In stateless mode, we should have both inbound and outbound rules + assert.False(t, fw.IsStateful()) + assert.Equal(t, 2, len(acl.peerRulesPairs)) }) } @@ -192,42 +252,19 @@ func TestDefaultManagerSquashRules(t *testing.T) { manager := &DefaultManager{} rules, _ := manager.squashAcceptRules(networkMap) - if len(rules) != 2 { - t.Errorf("rules should contain 2, got: %v", rules) - return - } + assert.Equal(t, 2, len(rules)) r := rules[0] - switch { - case r.PeerIP != "0.0.0.0": - t.Errorf("IP should be 0.0.0.0, got: %v", r.PeerIP) - return - case r.Direction != mgmProto.RuleDirection_IN: - t.Errorf("direction should be IN, got: %v", r.Direction) - return - case r.Protocol != mgmProto.RuleProtocol_ALL: - t.Errorf("protocol should be ALL, got: %v", r.Protocol) - return - case r.Action != mgmProto.RuleAction_ACCEPT: - t.Errorf("action should be ACCEPT, got: %v", r.Action) - return - } + assert.Equal(t, "0.0.0.0", r.PeerIP) + assert.Equal(t, mgmProto.RuleDirection_IN, r.Direction) + assert.Equal(t, mgmProto.RuleProtocol_ALL, r.Protocol) + assert.Equal(t, mgmProto.RuleAction_ACCEPT, r.Action) r = rules[1] - switch { - case r.PeerIP != "0.0.0.0": - t.Errorf("IP should be 0.0.0.0, got: %v", r.PeerIP) - return - case r.Direction != mgmProto.RuleDirection_OUT: - t.Errorf("direction should be OUT, got: %v", r.Direction) - return - case r.Protocol != mgmProto.RuleProtocol_ALL: - t.Errorf("protocol should be ALL, got: %v", r.Protocol) - return - case r.Action != mgmProto.RuleAction_ACCEPT: - t.Errorf("action should be ACCEPT, got: %v", r.Action) - return - } + assert.Equal(t, "0.0.0.0", r.PeerIP) + assert.Equal(t, mgmProto.RuleDirection_OUT, r.Direction) + assert.Equal(t, mgmProto.RuleProtocol_ALL, r.Protocol) + assert.Equal(t, mgmProto.RuleAction_ACCEPT, r.Action) } func TestDefaultManagerSquashRulesNoAffect(t *testing.T) { @@ -291,9 +328,8 @@ func TestDefaultManagerSquashRulesNoAffect(t *testing.T) { } manager := &DefaultManager{} - if rules, _ := manager.squashAcceptRules(networkMap); len(rules) != len(networkMap.FirewallRules) { - t.Errorf("we should get the same amount of rules as output, got %v", len(rules)) - } + rules, _ := manager.squashAcceptRules(networkMap) + assert.Equal(t, len(networkMap.FirewallRules), len(rules)) } func TestDefaultManagerEnableSSHRules(t *testing.T) { @@ -337,9 +373,7 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) { ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() ifaceMock.EXPECT().SetFilter(gomock.Any()) ip, network, err := net.ParseCIDR("172.0.0.1/32") - if err != nil { - t.Fatalf("failed to parse IP address: %v", err) - } + require.NoError(t, err) ifaceMock.EXPECT().Name().Return("lo").AnyTimes() ifaceMock.EXPECT().Address().Return(wgaddr.Address{ @@ -348,21 +382,20 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) { }).AnyTimes() ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() - // we receive one rule from the management so for testing purposes ignore it fw, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false) - if err != nil { - t.Errorf("create firewall: %v", err) - return - } - defer func(fw manager.Manager) { - _ = fw.Close(nil) - }(fw) + require.NoError(t, err) + defer func() { + err = fw.Close(nil) + require.NoError(t, err) + }() + acl := NewDefaultManager(fw) acl.ApplyFiltering(networkMap, false) - if len(acl.peerRulesPairs) != 3 { - t.Errorf("expect 3 rules (last must be SSH), got: %d", len(acl.peerRulesPairs)) - return + expectedRules := 3 + if fw.IsStateful() { + expectedRules = 3 // 2 inbound rules + SSH rule } + assert.Equal(t, expectedRules, len(acl.peerRulesPairs)) } diff --git a/client/internal/routemanager/static/route.go b/client/internal/routemanager/static/route.go index 98c34dbee..681c192fb 100644 --- a/client/internal/routemanager/static/route.go +++ b/client/internal/routemanager/static/route.go @@ -24,7 +24,6 @@ func NewRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allo } } -// Route route methods func (r *Route) String() string { return r.route.Network.String() } From 07b220d91ba77248dbf320ea943e83ea6da18e69 Mon Sep 17 00:00:00 2001 From: Pedro Maia Costa <550684+pnmcosta@users.noreply.github.com> Date: Mon, 2 Jun 2025 21:11:28 +0100 Subject: [PATCH 15/37] [management] REST client impersonation (#3879) --- management/client/rest/client.go | 2 + management/client/rest/impersonation.go | 48 ++++++++++++ management/client/rest/impersonation_test.go | 77 ++++++++++++++++++++ management/client/rest/options.go | 9 +++ management/server/types/settings.go | 2 +- 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 management/client/rest/impersonation.go create mode 100644 management/client/rest/impersonation_test.go diff --git a/management/client/rest/client.go b/management/client/rest/client.go index 25e8ad0da..8bf11caae 100644 --- a/management/client/rest/client.go +++ b/management/client/rest/client.go @@ -86,6 +86,7 @@ func NewWithBearerToken(managementURL, token string) *Client { ) } +// NewWithOptions initialize new Client instance with options func NewWithOptions(opts ...option) *Client { client := &Client{ httpClient: http.DefaultClient, @@ -115,6 +116,7 @@ func (c *Client) initialize() { c.Events = &EventsAPI{c} } +// NewRequest creates and executes new management API request func (c *Client) NewRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, method, c.managementURL+path, body) if err != nil { diff --git a/management/client/rest/impersonation.go b/management/client/rest/impersonation.go new file mode 100644 index 000000000..4d47c9373 --- /dev/null +++ b/management/client/rest/impersonation.go @@ -0,0 +1,48 @@ +package rest + +import ( + "net/http" + "net/url" +) + +// Impersonate returns a Client impersonated for a specific account +func (c *Client) Impersonate(account string) *Client { + client := NewWithOptions( + WithManagementURL(c.managementURL), + WithAuthHeader(c.authHeader), + WithHttpClient(newImpersonatedHttpClient(c, account)), + ) + return client +} + +type impersonatedHttpClient struct { + baseClient HttpClient + account string +} + +func newImpersonatedHttpClient(c *Client, account string) *impersonatedHttpClient { + if hc, ok := c.httpClient.(*impersonatedHttpClient); ok { + hc.account = account + return hc + } + + return &impersonatedHttpClient{ + baseClient: c.httpClient, + account: account, + } +} + +func (c *impersonatedHttpClient) Do(req *http.Request) (*http.Response, error) { + parsedURL, err := url.Parse(req.URL.String()) + if err != nil { + return nil, err + } + + query := parsedURL.Query() + query.Set("account", c.account) + parsedURL.RawQuery = query.Encode() + + req.URL = parsedURL + + return c.baseClient.Do(req) +} diff --git a/management/client/rest/impersonation_test.go b/management/client/rest/impersonation_test.go new file mode 100644 index 000000000..69c0f9728 --- /dev/null +++ b/management/client/rest/impersonation_test.go @@ -0,0 +1,77 @@ +//go:build integration +// +build integration + +package rest_test + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/management/client/rest" + "github.com/netbirdio/netbird/management/server/http/api" +) + +var ( + testImpersonatedAccount = api.Account{ + Id: "ImpersonatedTest", + Settings: api.AccountSettings{ + Extra: &api.AccountExtraSettings{ + PeerApprovalEnabled: false, + }, + GroupsPropagationEnabled: ptr(true), + JwtGroupsEnabled: ptr(false), + PeerInactivityExpiration: 7, + PeerInactivityExpirationEnabled: true, + PeerLoginExpiration: 24, + PeerLoginExpirationEnabled: true, + RegularUsersViewBlocked: false, + RoutingPeerDnsResolutionEnabled: ptr(false), + }, + } +) + +func TestImpersonation_Peers_List_200(t *testing.T) { + withMockClient(func(c *rest.Client, mux *http.ServeMux) { + impersonatedClient := c.Impersonate(testImpersonatedAccount.Id) + mux.HandleFunc("/api/peers", func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, r.URL.Query().Get("account"), testImpersonatedAccount.Id) + retBytes, _ := json.Marshal([]api.Peer{testPeer}) + _, err := w.Write(retBytes) + require.NoError(t, err) + }) + ret, err := impersonatedClient.Peers.List(context.Background()) + require.NoError(t, err) + assert.Len(t, ret, 1) + assert.Equal(t, testPeer, ret[0]) + }) +} + +func TestImpersonation_Change_Account(t *testing.T) { + withMockClient(func(c *rest.Client, mux *http.ServeMux) { + impersonatedClient := c.Impersonate(testImpersonatedAccount.Id) + mux.HandleFunc("/api/peers", func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, r.URL.Query().Get("account"), testImpersonatedAccount.Id) + retBytes, _ := json.Marshal([]api.Peer{testPeer}) + _, err := w.Write(retBytes) + require.NoError(t, err) + }) + _, err := impersonatedClient.Peers.List(context.Background()) + require.NoError(t, err) + + impersonatedClient = impersonatedClient.Impersonate("another-test-account") + mux.HandleFunc("/api/peers/Test", func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, r.URL.Query().Get("account"), "another-test-account") + retBytes, _ := json.Marshal(testPeer) + _, err := w.Write(retBytes) + require.NoError(t, err) + }) + + _, err = impersonatedClient.Peers.Get(context.Background(), "Test") + require.NoError(t, err) + }) +} diff --git a/management/client/rest/options.go b/management/client/rest/options.go index 5aad7dd7e..21f2394e9 100644 --- a/management/client/rest/options.go +++ b/management/client/rest/options.go @@ -2,32 +2,41 @@ package rest import "net/http" +// option modifier for creation of Client type option func(*Client) +// HTTPClient interface for HTTP client type HttpClient interface { Do(req *http.Request) (*http.Response, error) } +// WithHTTPClient overrides HTTPClient used func WithHttpClient(client HttpClient) option { return func(c *Client) { c.httpClient = client } } +// WithBearerToken uses provided bearer token acquired from SSO for authentication func WithBearerToken(token string) option { return WithAuthHeader("Bearer " + token) } +// WithPAT uses provided Personal Access Token +// (created from NetBird Management Dashboard) for authentication func WithPAT(token string) option { return WithAuthHeader("Token " + token) } +// WithManagementURL overrides target NetBird Management server func WithManagementURL(url string) option { return func(c *Client) { c.managementURL = url } } +// WithAuthHeader overrides auth header completely, this should generally not be used +// and WithBearerToken or WithPAT should be used instead func WithAuthHeader(value string) option { return func(c *Client) { c.authHeader = value diff --git a/management/server/types/settings.go b/management/server/types/settings.go index bd361f3ff..a22a36b03 100644 --- a/management/server/types/settings.go +++ b/management/server/types/settings.go @@ -45,7 +45,7 @@ type Settings struct { // Extra is a dictionary of Account settings Extra *ExtraSettings `gorm:"embedded;embeddedPrefix:extra_"` - // LazyConnectionEnabled indicates wether the experimental feature is enabled or disabled + // LazyConnectionEnabled indicates if the experimental feature is enabled or disabled LazyConnectionEnabled bool `gorm:"default:false"` } From 35287f8241f4bf3582ed9d1049c5da5c2e6bde2e Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Mon, 2 Jun 2025 23:37:51 +0100 Subject: [PATCH 16/37] [misc] Fail linter workflows on codespell failures (#3913) * Fail linter workflows on codespell failures * testing workflow * remove test --- .github/workflows/golangci-lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index bdd508e9b..7e6583cc6 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -21,7 +21,6 @@ jobs: with: ignore_words_list: erro,clienta,hastable,iif,groupd,testin,groupe skip: go.mod,go.sum - only_warn: 1 golangci: strategy: fail-fast: false From af27aaf9af28124198fcf0b1a90ef3443f6aeb5d Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 3 Jun 2025 09:20:33 +0200 Subject: [PATCH 17/37] [client] Refactor peer state change subscription mechanism (#3910) * Refactor peer state change subscription mechanism Because the code generated new channel for every single event, was easy to miss notification. Use single channel. * Fix lint * Avoid potential deadlock * Fix test * Add context * Fix test --- client/internal/peer/status.go | 81 +++++++++++++++++++++----- client/internal/peer/status_test.go | 13 +++-- client/internal/routemanager/client.go | 11 ++-- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 40956e68e..0c6aac372 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -1,6 +1,7 @@ package peer import ( + "context" "errors" "net/netip" "slices" @@ -146,11 +147,31 @@ type FullStatus struct { LazyConnectionEnabled bool } +type StatusChangeSubscription struct { + peerID string + id string + eventsChan chan struct{} + ctx context.Context +} + +func newStatusChangeSubscription(ctx context.Context, peerID string) *StatusChangeSubscription { + return &StatusChangeSubscription{ + ctx: ctx, + peerID: peerID, + id: uuid.New().String(), + eventsChan: make(chan struct{}, 1), + } +} + +func (s *StatusChangeSubscription) Events() chan struct{} { + return s.eventsChan +} + // Status holds a state of peers, signal, management connections and relays type Status struct { mux sync.Mutex peers map[string]State - changeNotify map[string]chan struct{} + changeNotify map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription signalState bool signalError error managementState bool @@ -187,7 +208,7 @@ type Status struct { func NewRecorder(mgmAddress string) *Status { return &Status{ peers: make(map[string]State), - changeNotify: make(map[string]chan struct{}), + changeNotify: make(map[string]map[string]*StatusChangeSubscription), eventStreams: make(map[string]chan *proto.SystemEvent), eventQueue: NewEventQueue(eventQueueSize), offlinePeers: make([]State, 0), @@ -312,7 +333,6 @@ func (d *Status) UpdatePeerState(receivedState State) error { // when we close the connection we will not notify the router manager if receivedState.ConnStatus == StatusIdle { d.notifyPeerStateChangeListeners(receivedState.PubKey) - } return nil } @@ -552,19 +572,41 @@ func (d *Status) FinishPeerListModifications() { d.notifyPeerListChanged() } -// GetPeerStateChangeNotifier returns a change notifier channel for a peer -func (d *Status) GetPeerStateChangeNotifier(peer string) <-chan struct{} { +func (d *Status) SubscribeToPeerStateChanges(ctx context.Context, peerID string) *StatusChangeSubscription { d.mux.Lock() defer d.mux.Unlock() - ch, found := d.changeNotify[peer] - if found { - return ch + sub := newStatusChangeSubscription(ctx, peerID) + if _, ok := d.changeNotify[peerID]; !ok { + d.changeNotify[peerID] = make(map[string]*StatusChangeSubscription) + } + d.changeNotify[peerID][sub.id] = sub + + return sub +} + +func (d *Status) UnsubscribePeerStateChanges(subscription *StatusChangeSubscription) { + d.mux.Lock() + defer d.mux.Unlock() + + if subscription == nil { + return } - ch = make(chan struct{}) - d.changeNotify[peer] = ch - return ch + channels, ok := d.changeNotify[subscription.peerID] + if !ok { + return + } + + sub, exists := channels[subscription.id] + if !exists { + return + } + + delete(channels, subscription.id) + if len(channels) == 0 { + delete(d.changeNotify, sub.peerID) + } } // GetLocalPeerState returns the local peer state @@ -939,13 +981,20 @@ func (d *Status) onConnectionChanged() { // notifyPeerStateChangeListeners notifies route manager about the change in peer state func (d *Status) notifyPeerStateChangeListeners(peerID string) { - ch, found := d.changeNotify[peerID] - if !found { + subs, ok := d.changeNotify[peerID] + if !ok { return } - - close(ch) - delete(d.changeNotify, peerID) + for _, sub := range subs { + // block the write because we do not want to miss notification + // must have to be sure we will run the GetPeerState() on separated thread + go func() { + select { + case sub.eventsChan <- struct{}{}: + case <-sub.ctx.Done(): + } + }() + } } func (d *Status) notifyPeerListChanged() { diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index 8f28a9862..272638750 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -1,6 +1,7 @@ package peer import ( + "context" "errors" "sync" "testing" @@ -86,8 +87,8 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) { status := NewRecorder("https://mgm") _ = status.AddPeer(key, "abc.netbird", ip) - ch := status.GetPeerStateChangeNotifier(key) - assert.NotNil(t, ch, "channel shouldn't be nil") + sub := status.SubscribeToPeerStateChanges(context.Background(), key) + assert.NotNil(t, sub, "channel shouldn't be nil") peerState := State{ PubKey: key, @@ -99,10 +100,12 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) { err := status.UpdatePeerRelayedStateToDisconnected(peerState) assert.NoError(t, err, "shouldn't return error") + timeoutCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() select { - case <-ch: - default: - t.Errorf("channel wasn't closed after update") + case <-sub.eventsChan: + case <-timeoutCtx.Done(): + t.Errorf("timed out waiting for event") } } diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index 137e00d31..bff954c27 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -224,19 +224,18 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID] } func (c *clientNetwork) watchPeerStatusChanges(ctx context.Context, peerKey string, peerStateUpdate chan struct{}, closer chan struct{}) { + subscription := c.statusRecorder.SubscribeToPeerStateChanges(ctx, peerKey) + defer c.statusRecorder.UnsubscribePeerStateChanges(subscription) + for { select { case <-ctx.Done(): return case <-closer: return - case <-c.statusRecorder.GetPeerStateChangeNotifier(peerKey): - state, err := c.statusRecorder.GetPeer(peerKey) - if err != nil { - continue - } + case <-subscription.Events(): peerStateUpdate <- struct{}{} - log.Debugf("triggered route state update for Peer %s, state: %s", peerKey, state.ConnStatus) + log.Debugf("triggered route state update for Peer: %s", peerKey) } } } From 616b19c0644be0d1b0b5ccf4632653f6402e4faf Mon Sep 17 00:00:00 2001 From: hakansa <43675540+hakansa@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:49:13 +0300 Subject: [PATCH 18/37] [client] Add "Deselect All" Menu Item to Exit Node Menu (#3877) * [client] Enhance exit node menu functionality with deselect all option * Hide exit nodes before removal in recreateExitNodeMenu * recreateExitNodeMenu adding mutex locks * Refetch exit nodes after deselecting all in exit node menu --- client/ui/client_ui.go | 18 +++++---- client/ui/network.go | 90 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/client/ui/client_ui.go b/client/ui/client_ui.go index c23b78582..c0c8692c6 100644 --- a/client/ui/client_ui.go +++ b/client/ui/client_ui.go @@ -235,9 +235,11 @@ type serviceClient struct { eventManager *event.Manager - exitNodeMu sync.Mutex - mExitNodeItems []menuHandler - logFile string + exitNodeMu sync.Mutex + mExitNodeItems []menuHandler + exitNodeStates []exitNodeState + mExitNodeDeselectAll *systray.MenuItem + logFile string } type menuHandler struct { @@ -1035,11 +1037,11 @@ func (s *serviceClient) updateConfig() error { lazyConnectionEnabled := s.mLazyConnEnabled.Checked() loginRequest := proto.LoginRequest{ - IsUnixDesktopClient: runtime.GOOS == "linux" || runtime.GOOS == "freebsd", - ServerSSHAllowed: &sshAllowed, - RosenpassEnabled: &rosenpassEnabled, - DisableAutoConnect: &disableAutoStart, - DisableNotifications: ¬ificationsDisabled, + IsUnixDesktopClient: runtime.GOOS == "linux" || runtime.GOOS == "freebsd", + ServerSSHAllowed: &sshAllowed, + RosenpassEnabled: &rosenpassEnabled, + DisableAutoConnect: &disableAutoStart, + DisableNotifications: ¬ificationsDisabled, LazyConnectionEnabled: &lazyConnectionEnabled, } diff --git a/client/ui/network.go b/client/ui/network.go index 435917f30..b3748a89d 100644 --- a/client/ui/network.go +++ b/client/ui/network.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "runtime" + "slices" "sort" "strings" "time" @@ -33,6 +34,11 @@ const ( type filter string +type exitNodeState struct { + id string + selected bool +} + func (s *serviceClient) showNetworksUI() { s.wNetworks = s.app.NewWindow("Networks") s.wNetworks.SetOnClosed(s.cancel) @@ -357,18 +363,45 @@ func (s *serviceClient) updateExitNodes() { } func (s *serviceClient) recreateExitNodeMenu(exitNodes []*proto.Network) { + var exitNodeIDs []exitNodeState + for _, node := range exitNodes { + exitNodeIDs = append(exitNodeIDs, exitNodeState{ + id: node.ID, + selected: node.Selected, + }) + } + + sort.Slice(exitNodeIDs, func(i, j int) bool { + return exitNodeIDs[i].id < exitNodeIDs[j].id + }) + if slices.Equal(s.exitNodeStates, exitNodeIDs) { + log.Debug("Exit node menu already up to date") + return + } + for _, node := range s.mExitNodeItems { node.cancel() + node.Hide() node.Remove() } s.mExitNodeItems = nil + if s.mExitNodeDeselectAll != nil { + s.mExitNodeDeselectAll.Remove() + s.mExitNodeDeselectAll = nil + } if runtime.GOOS == "linux" || runtime.GOOS == "freebsd" { s.mExitNode.Remove() s.mExitNode = systray.AddMenuItem("Exit Node", exitNodeMenuDescr) } + var showDeselectAll bool + for _, node := range exitNodes { + if node.Selected { + showDeselectAll = true + } + menuItem := s.mExitNode.AddSubMenuItemCheckbox( node.ID, fmt.Sprintf("Use exit node %s", node.ID), @@ -383,6 +416,32 @@ func (s *serviceClient) recreateExitNodeMenu(exitNodes []*proto.Network) { go s.handleChecked(ctx, node.ID, menuItem) } + s.exitNodeStates = exitNodeIDs + + if showDeselectAll { + s.mExitNode.AddSeparator() + deselectAllItem := s.mExitNode.AddSubMenuItem("Deselect All", "Deselect All") + s.mExitNodeDeselectAll = deselectAllItem + go func() { + for { + _, ok := <-deselectAllItem.ClickedCh + if !ok { + // channel closed: exit the goroutine + return + } + exitNodes, err := s.handleExitNodeMenuDeselectAll() + if err != nil { + log.Warnf("failed to handle deselect all exit nodes: %v", err) + } else { + s.exitNodeMu.Lock() + s.recreateExitNodeMenu(exitNodes) + s.exitNodeMu.Unlock() + } + } + + }() + } + } func (s *serviceClient) getExitNodes(conn proto.DaemonServiceClient) ([]*proto.Network, error) { @@ -420,6 +479,37 @@ func (s *serviceClient) handleChecked(ctx context.Context, id string, item *syst } } +func (s *serviceClient) handleExitNodeMenuDeselectAll() ([]*proto.Network, error) { + conn, err := s.getSrvClient(defaultFailTimeout) + if err != nil { + return nil, fmt.Errorf("get client: %v", err) + } + + exitNodes, err := s.getExitNodes(conn) + if err != nil { + return nil, fmt.Errorf("get exit nodes: %v", err) + } + + var ids []string + for _, e := range exitNodes { + if e.Selected { + ids = append(ids, e.ID) + } + } + + // deselect selected exit nodes + if err := s.deselectOtherExitNodes(conn, ids); err != nil { + return nil, err + } + + updatedExitNodes, err := s.getExitNodes(conn) + if err != nil { + return nil, fmt.Errorf("re-fetch exit nodes: %v", err) + } + + return updatedExitNodes, nil +} + // Add function to toggle exit node selection func (s *serviceClient) toggleExitNode(nodeID string, item *systray.MenuItem) error { conn, err := s.getSrvClient(defaultFailTimeout) From f367925496136074a4f5911e965de4fd12a5ef87 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:52:10 +0200 Subject: [PATCH 19/37] [client] Log duplicate client ui pid (#3915) --- client/ui/client_ui.go | 4 ++-- client/ui/process/process.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/ui/client_ui.go b/client/ui/client_ui.go index c0c8692c6..92289a8a3 100644 --- a/client/ui/client_ui.go +++ b/client/ui/client_ui.go @@ -89,13 +89,13 @@ func main() { } // Check for another running process. - running, err := process.IsAnotherProcessRunning() + pid, running, err := process.IsAnotherProcessRunning() if err != nil { log.Errorf("error while checking process: %v", err) return } if running { - log.Warn("another process is running") + log.Warnf("another process is running with pid %d, exiting", pid) return } diff --git a/client/ui/process/process.go b/client/ui/process/process.go index f9a8a4fe9..d0ef54896 100644 --- a/client/ui/process/process.go +++ b/client/ui/process/process.go @@ -8,10 +8,10 @@ import ( "github.com/shirou/gopsutil/v3/process" ) -func IsAnotherProcessRunning() (bool, error) { +func IsAnotherProcessRunning() (int32, bool, error) { processes, err := process.Processes() if err != nil { - return false, err + return 0, false, err } pid := os.Getpid() @@ -29,9 +29,9 @@ func IsAnotherProcessRunning() (bool, error) { } if strings.Contains(strings.ToLower(runningProcessPath), processName) && isProcessOwnedByCurrentUser(p) { - return true, nil + return p.Pid, true, nil } } - return false, nil + return 0, false, nil } From 1ce4ee0cef6a5cae84b3daf1ea8fc534bfbc5ffb Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:53:27 +0200 Subject: [PATCH 20/37] [client] Add block inbound flag to disallow inbound connections of any kind (#3897) --- client/cmd/root.go | 2 - client/cmd/system.go | 11 + client/cmd/up.go | 266 +++--- client/firewall/iptables/manager_linux.go | 8 +- client/firewall/iptables/router_linux.go | 8 - client/firewall/nftables/manager_linux.go | 6 + client/firewall/nftables/router_linux.go | 8 - client/firewall/uspfilter/uspfilter.go | 2 +- client/internal/acl/manager.go | 11 +- client/internal/config.go | 18 +- client/internal/connect.go | 5 +- client/internal/debug/debug.go | 26 +- client/internal/engine.go | 118 +-- client/proto/daemon.pb.go | 1064 +++++++++++---------- client/proto/daemon.proto | 7 +- client/server/server.go | 31 +- client/ui/client_ui.go | 29 +- client/ui/const.go | 3 +- 18 files changed, 878 insertions(+), 745 deletions(-) diff --git a/client/cmd/root.go b/client/cmd/root.go index 9bcf65df9..16e445f4d 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -39,7 +39,6 @@ const ( extraIFaceBlackListFlag = "extra-iface-blacklist" dnsRouteIntervalFlag = "dns-router-interval" systemInfoFlag = "system-info" - blockLANAccessFlag = "block-lan-access" enableLazyConnectionFlag = "enable-lazy-connection" uploadBundle = "upload-bundle" uploadBundleURL = "upload-bundle-url" @@ -78,7 +77,6 @@ var ( anonymizeFlag bool debugSystemInfoFlag bool dnsRouteInterval time.Duration - blockLANAccess bool debugUploadBundle bool debugUploadBundleURL string lazyConnEnabled bool diff --git a/client/cmd/system.go b/client/cmd/system.go index f628867a7..83ce8d215 100644 --- a/client/cmd/system.go +++ b/client/cmd/system.go @@ -6,6 +6,8 @@ const ( disableServerRoutesFlag = "disable-server-routes" disableDNSFlag = "disable-dns" disableFirewallFlag = "disable-firewall" + blockLANAccessFlag = "block-lan-access" + blockInboundFlag = "block-inbound" ) var ( @@ -13,6 +15,8 @@ var ( disableServerRoutes bool disableDNS bool disableFirewall bool + blockLANAccess bool + blockInbound bool ) func init() { @@ -28,4 +32,11 @@ func init() { upCmd.PersistentFlags().BoolVar(&disableFirewall, disableFirewallFlag, false, "Disable firewall configuration. If enabled, the client won't modify firewall rules.") + + upCmd.PersistentFlags().BoolVar(&blockLANAccess, blockLANAccessFlag, false, + "Block access to local networks (LAN) when using this peer as a router or exit node") + + upCmd.PersistentFlags().BoolVar(&blockInbound, blockInboundFlag, false, + "Block inbound connections. If enabled, the client will not allow any inbound connections to the local machine nor routed networks.\n"+ + "This overrides any policies received from the management service.") } diff --git a/client/cmd/up.go b/client/cmd/up.go index 2dcf2282b..b9781c0df 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -55,12 +55,11 @@ func init() { upCmd.PersistentFlags().StringVar(&interfaceName, interfaceNameFlag, iface.WgInterfaceDefault, "Wireguard interface name") upCmd.PersistentFlags().Uint16Var(&wireguardPort, wireguardPortFlag, iface.DefaultWgPort, "Wireguard interface listening port") upCmd.PersistentFlags().BoolVarP(&networkMonitor, networkMonitorFlag, "N", networkMonitor, - `Manage network monitoring. Defaults to true on Windows and macOS, false on Linux. `+ + `Manage network monitoring. Defaults to true on Windows and macOS, false on Linux and FreeBSD. `+ `E.g. --network-monitor=false to disable or --network-monitor=true to enable.`, ) upCmd.PersistentFlags().StringSliceVar(&extraIFaceBlackList, extraIFaceBlackListFlag, nil, "Extra list of default interfaces to ignore for listening") upCmd.PersistentFlags().DurationVar(&dnsRouteInterval, dnsRouteIntervalFlag, time.Minute, "DNS route update interval") - upCmd.PersistentFlags().BoolVar(&blockLANAccess, blockLANAccessFlag, false, "Block access to local networks (LAN) when using this peer as a router or exit node") upCmd.PersistentFlags().StringSliceVar(&dnsLabels, dnsLabelsFlag, nil, `Sets DNS labels`+ @@ -119,83 +118,9 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error { return err } - ic := internal.ConfigInput{ - ManagementURL: managementURL, - AdminURL: adminURL, - ConfigPath: configPath, - NATExternalIPs: natExternalIPs, - CustomDNSAddress: customDNSAddressConverted, - ExtraIFaceBlackList: extraIFaceBlackList, - DNSLabels: dnsLabelsValidated, - } - - if cmd.Flag(enableRosenpassFlag).Changed { - ic.RosenpassEnabled = &rosenpassEnabled - } - - if cmd.Flag(rosenpassPermissiveFlag).Changed { - ic.RosenpassPermissive = &rosenpassPermissive - } - - if cmd.Flag(serverSSHAllowedFlag).Changed { - ic.ServerSSHAllowed = &serverSSHAllowed - } - - if cmd.Flag(interfaceNameFlag).Changed { - if err := parseInterfaceName(interfaceName); err != nil { - return err - } - ic.InterfaceName = &interfaceName - } - - if cmd.Flag(wireguardPortFlag).Changed { - p := int(wireguardPort) - ic.WireguardPort = &p - } - - if cmd.Flag(networkMonitorFlag).Changed { - ic.NetworkMonitor = &networkMonitor - } - - if rootCmd.PersistentFlags().Changed(preSharedKeyFlag) { - ic.PreSharedKey = &preSharedKey - } - - if cmd.Flag(disableAutoConnectFlag).Changed { - ic.DisableAutoConnect = &autoConnectDisabled - - if autoConnectDisabled { - cmd.Println("Autoconnect has been disabled. The client won't connect automatically when the service starts.") - } - - if !autoConnectDisabled { - cmd.Println("Autoconnect has been enabled. The client will connect automatically when the service starts.") - } - } - - if cmd.Flag(dnsRouteIntervalFlag).Changed { - ic.DNSRouteInterval = &dnsRouteInterval - } - - if cmd.Flag(disableClientRoutesFlag).Changed { - ic.DisableClientRoutes = &disableClientRoutes - } - if cmd.Flag(disableServerRoutesFlag).Changed { - ic.DisableServerRoutes = &disableServerRoutes - } - if cmd.Flag(disableDNSFlag).Changed { - ic.DisableDNS = &disableDNS - } - if cmd.Flag(disableFirewallFlag).Changed { - ic.DisableFirewall = &disableFirewall - } - - if cmd.Flag(blockLANAccessFlag).Changed { - ic.BlockLANAccess = &blockLANAccess - } - - if cmd.Flag(enableLazyConnectionFlag).Changed { - ic.LazyConnectionEnabled = &lazyConnEnabled + ic, err := setupConfig(customDNSAddressConverted, cmd) + if err != nil { + return fmt.Errorf("setup config: %v", err) } providedSetupKey, err := getSetupKey() @@ -203,7 +128,7 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error { return err } - config, err := internal.UpdateOrCreateConfig(ic) + config, err := internal.UpdateOrCreateConfig(*ic) if err != nil { return fmt.Errorf("get config file: %v", err) } @@ -262,9 +187,141 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error { providedSetupKey, err := getSetupKey() if err != nil { - return err + return fmt.Errorf("get setup key: %v", err) } + loginRequest, err := setupLoginRequest(providedSetupKey, customDNSAddressConverted, cmd) + if err != nil { + return fmt.Errorf("setup login request: %v", err) + } + + var loginErr error + var loginResp *proto.LoginResponse + + err = WithBackOff(func() error { + var backOffErr error + loginResp, backOffErr = client.Login(ctx, loginRequest) + if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || + s.Code() == codes.PermissionDenied || + s.Code() == codes.NotFound || + s.Code() == codes.Unimplemented) { + loginErr = backOffErr + return nil + } + return backOffErr + }) + if err != nil { + return fmt.Errorf("login backoff cycle failed: %v", err) + } + + if loginErr != nil { + return fmt.Errorf("login failed: %v", loginErr) + } + + if loginResp.NeedsSSOLogin { + + openURL(cmd, loginResp.VerificationURIComplete, loginResp.UserCode, noBrowser) + + _, err = client.WaitSSOLogin(ctx, &proto.WaitSSOLoginRequest{UserCode: loginResp.UserCode, Hostname: hostName}) + if err != nil { + return fmt.Errorf("waiting sso login failed with: %v", err) + } + } + + if _, err := client.Up(ctx, &proto.UpRequest{}); err != nil { + return fmt.Errorf("call service up method: %v", err) + } + cmd.Println("Connected") + return nil +} + +func setupConfig(customDNSAddressConverted []byte, cmd *cobra.Command) (*internal.ConfigInput, error) { + ic := internal.ConfigInput{ + ManagementURL: managementURL, + AdminURL: adminURL, + ConfigPath: configPath, + NATExternalIPs: natExternalIPs, + CustomDNSAddress: customDNSAddressConverted, + ExtraIFaceBlackList: extraIFaceBlackList, + DNSLabels: dnsLabelsValidated, + } + + if cmd.Flag(enableRosenpassFlag).Changed { + ic.RosenpassEnabled = &rosenpassEnabled + } + + if cmd.Flag(rosenpassPermissiveFlag).Changed { + ic.RosenpassPermissive = &rosenpassPermissive + } + + if cmd.Flag(serverSSHAllowedFlag).Changed { + ic.ServerSSHAllowed = &serverSSHAllowed + } + + if cmd.Flag(interfaceNameFlag).Changed { + if err := parseInterfaceName(interfaceName); err != nil { + return nil, err + } + ic.InterfaceName = &interfaceName + } + + if cmd.Flag(wireguardPortFlag).Changed { + p := int(wireguardPort) + ic.WireguardPort = &p + } + + if cmd.Flag(networkMonitorFlag).Changed { + ic.NetworkMonitor = &networkMonitor + } + + if rootCmd.PersistentFlags().Changed(preSharedKeyFlag) { + ic.PreSharedKey = &preSharedKey + } + + if cmd.Flag(disableAutoConnectFlag).Changed { + ic.DisableAutoConnect = &autoConnectDisabled + + if autoConnectDisabled { + cmd.Println("Autoconnect has been disabled. The client won't connect automatically when the service starts.") + } + + if !autoConnectDisabled { + cmd.Println("Autoconnect has been enabled. The client will connect automatically when the service starts.") + } + } + + if cmd.Flag(dnsRouteIntervalFlag).Changed { + ic.DNSRouteInterval = &dnsRouteInterval + } + + if cmd.Flag(disableClientRoutesFlag).Changed { + ic.DisableClientRoutes = &disableClientRoutes + } + if cmd.Flag(disableServerRoutesFlag).Changed { + ic.DisableServerRoutes = &disableServerRoutes + } + if cmd.Flag(disableDNSFlag).Changed { + ic.DisableDNS = &disableDNS + } + if cmd.Flag(disableFirewallFlag).Changed { + ic.DisableFirewall = &disableFirewall + } + + if cmd.Flag(blockLANAccessFlag).Changed { + ic.BlockLANAccess = &blockLANAccess + } + + if cmd.Flag(blockInboundFlag).Changed { + ic.BlockInbound = &blockInbound + } + + if cmd.Flag(enableLazyConnectionFlag).Changed { + ic.LazyConnectionEnabled = &lazyConnEnabled + } + return &ic, nil +} + +func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte, cmd *cobra.Command) (*proto.LoginRequest, error) { loginRequest := proto.LoginRequest{ SetupKey: providedSetupKey, ManagementUrl: managementURL, @@ -301,7 +358,7 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error { if cmd.Flag(interfaceNameFlag).Changed { if err := parseInterfaceName(interfaceName); err != nil { - return err + return nil, err } loginRequest.InterfaceName = &interfaceName } @@ -336,49 +393,14 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error { loginRequest.BlockLanAccess = &blockLANAccess } + if cmd.Flag(blockInboundFlag).Changed { + loginRequest.BlockInbound = &blockInbound + } + if cmd.Flag(enableLazyConnectionFlag).Changed { loginRequest.LazyConnectionEnabled = &lazyConnEnabled } - - var loginErr error - - var loginResp *proto.LoginResponse - - err = WithBackOff(func() error { - var backOffErr error - loginResp, backOffErr = client.Login(ctx, &loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { - loginErr = backOffErr - return nil - } - return backOffErr - }) - if err != nil { - return fmt.Errorf("login backoff cycle failed: %v", err) - } - - if loginErr != nil { - return fmt.Errorf("login failed: %v", loginErr) - } - - if loginResp.NeedsSSOLogin { - - openURL(cmd, loginResp.VerificationURIComplete, loginResp.UserCode, noBrowser) - - _, err = client.WaitSSOLogin(ctx, &proto.WaitSSOLoginRequest{UserCode: loginResp.UserCode, Hostname: hostName}) - if err != nil { - return fmt.Errorf("waiting sso login failed with: %v", err) - } - } - - if _, err := client.Up(ctx, &proto.UpRequest{}); err != nil { - return fmt.Errorf("call service up method: %v", err) - } - cmd.Println("Connected") - return nil + return &loginRequest, nil } func validateNATExternalIPs(list []string) error { diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 0897f831f..81f7a9125 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -202,7 +202,7 @@ func (m *Manager) AllowNetbird() error { _, err := m.AddPeerFiltering( nil, net.IP{0, 0, 0, 0}, - "all", + firewall.ProtocolALL, nil, nil, firewall.ActionAccept, @@ -223,10 +223,16 @@ func (m *Manager) SetLogLevel(log.Level) { } func (m *Manager) EnableRouting() error { + if err := m.router.ipFwdState.RequestForwarding(); err != nil { + return fmt.Errorf("enable IP forwarding: %w", err) + } return nil } func (m *Manager) DisableRouting() error { + if err := m.router.ipFwdState.ReleaseForwarding(); err != nil { + return fmt.Errorf("disable IP forwarding: %w", err) + } return nil } diff --git a/client/firewall/iptables/router_linux.go b/client/firewall/iptables/router_linux.go index bb799b99b..1e44c7a4d 100644 --- a/client/firewall/iptables/router_linux.go +++ b/client/firewall/iptables/router_linux.go @@ -248,10 +248,6 @@ func (r *router) deleteIpSet(setName string) error { // AddNatRule inserts an iptables rule pair into the nat chain func (r *router) AddNatRule(pair firewall.RouterPair) error { - if err := r.ipFwdState.RequestForwarding(); err != nil { - return err - } - if r.legacyManagement { log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) if err := r.addLegacyRouteRule(pair); err != nil { @@ -278,10 +274,6 @@ func (r *router) AddNatRule(pair firewall.RouterPair) error { // RemoveNatRule removes an iptables rule pair from forwarding and nat chains func (r *router) RemoveNatRule(pair firewall.RouterPair) error { - if err := r.ipFwdState.ReleaseForwarding(); err != nil { - log.Errorf("%v", err) - } - if pair.Masquerade { if err := r.removeNatRule(pair); err != nil { return fmt.Errorf("remove nat rule: %w", err) diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index 2f8ee81a4..560f224f5 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -328,10 +328,16 @@ func (m *Manager) SetLogLevel(log.Level) { } func (m *Manager) EnableRouting() error { + if err := m.router.ipFwdState.RequestForwarding(); err != nil { + return fmt.Errorf("enable IP forwarding: %w", err) + } return nil } func (m *Manager) DisableRouting() error { + if err := m.router.ipFwdState.ReleaseForwarding(); err != nil { + return fmt.Errorf("disable IP forwarding: %w", err) + } return nil } diff --git a/client/firewall/nftables/router_linux.go b/client/firewall/nftables/router_linux.go index 0f6c5bdf6..f8fed4d80 100644 --- a/client/firewall/nftables/router_linux.go +++ b/client/firewall/nftables/router_linux.go @@ -573,10 +573,6 @@ func (r *router) deleteNftRule(rule *nftables.Rule, ruleKey string) error { // AddNatRule appends a nftables rule pair to the nat chain func (r *router) AddNatRule(pair firewall.RouterPair) error { - if err := r.ipFwdState.RequestForwarding(); err != nil { - return err - } - if err := r.refreshRulesMap(); err != nil { return fmt.Errorf(refreshRulesMapError, err) } @@ -1006,10 +1002,6 @@ func (r *router) removeAcceptForwardRulesIptables(ipt *iptables.IPTables) error // RemoveNatRule removes the prerouting mark rule func (r *router) RemoveNatRule(pair firewall.RouterPair) error { - if err := r.ipFwdState.ReleaseForwarding(); err != nil { - log.Errorf("%v", err) - } - if err := r.refreshRulesMap(); err != nil { return fmt.Errorf(refreshRulesMapError, err) } diff --git a/client/firewall/uspfilter/uspfilter.go b/client/firewall/uspfilter/uspfilter.go index 287e52773..8e0a955ca 100644 --- a/client/firewall/uspfilter/uspfilter.go +++ b/client/firewall/uspfilter/uspfilter.go @@ -269,7 +269,7 @@ func (m *Manager) determineRouting() error { log.Info("userspace routing is forced") - case !m.netstack && m.nativeFirewall != nil && m.nativeFirewall.IsServerRouteSupported(): + case !m.netstack && m.nativeFirewall != nil: // if the OS supports routing natively, then we don't need to filter/route ourselves // netstack mode won't support native routing as there is no interface diff --git a/client/internal/acl/manager.go b/client/internal/acl/manager.go index 5caf2b770..c8bc9123b 100644 --- a/client/internal/acl/manager.go +++ b/client/internal/acl/manager.go @@ -58,6 +58,11 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout d.mutex.Lock() defer d.mutex.Unlock() + if d.firewall == nil { + log.Debug("firewall manager is not supported, skipping firewall rules") + return + } + start := time.Now() defer func() { total := 0 @@ -69,14 +74,8 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout time.Since(start), total) }() - if d.firewall == nil { - log.Debug("firewall manager is not supported, skipping firewall rules") - return - } - d.applyPeerACLs(networkMap) - if err := d.applyRouteACLs(networkMap.RoutesFirewallRules, dnsRouteFeatureFlag); err != nil { log.Errorf("Failed to apply route ACLs: %v", err) } diff --git a/client/internal/config.go b/client/internal/config.go index 86dd7ebb1..45a7620e1 100644 --- a/client/internal/config.go +++ b/client/internal/config.go @@ -68,8 +68,8 @@ type ConfigInput struct { DisableServerRoutes *bool DisableDNS *bool DisableFirewall *bool - - BlockLANAccess *bool + BlockLANAccess *bool + BlockInbound *bool DisableNotifications *bool @@ -98,8 +98,8 @@ type Config struct { DisableServerRoutes bool DisableDNS bool DisableFirewall bool - - BlockLANAccess bool + BlockLANAccess bool + BlockInbound bool DisableNotifications *bool @@ -483,6 +483,16 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) { updated = true } + if input.BlockInbound != nil && *input.BlockInbound != config.BlockInbound { + if *input.BlockInbound { + log.Infof("blocking inbound connections") + } else { + log.Infof("allowing inbound connections") + } + config.BlockInbound = *input.BlockInbound + updated = true + } + if input.DisableNotifications != nil && input.DisableNotifications != config.DisableNotifications { if *input.DisableNotifications { log.Infof("disabling notifications") diff --git a/client/internal/connect.go b/client/internal/connect.go index 1428d2656..1cfef77f2 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -436,11 +436,12 @@ func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.Pe DNSRouteInterval: config.DNSRouteInterval, DisableClientRoutes: config.DisableClientRoutes, - DisableServerRoutes: config.DisableServerRoutes, + DisableServerRoutes: config.DisableServerRoutes || config.BlockInbound, DisableDNS: config.DisableDNS, DisableFirewall: config.DisableFirewall, + BlockLANAccess: config.BlockLANAccess, + BlockInbound: config.BlockInbound, - BlockLANAccess: config.BlockLANAccess, LazyConnectionEnabled: config.LazyConnectionEnabled, } diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 2192872df..a753ece0c 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -366,17 +366,33 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString(fmt.Sprintf("RosenpassEnabled: %v\n", g.internalConfig.RosenpassEnabled)) configContent.WriteString(fmt.Sprintf("RosenpassPermissive: %v\n", g.internalConfig.RosenpassPermissive)) if g.internalConfig.ServerSSHAllowed != nil { - configContent.WriteString(fmt.Sprintf("BundleGeneratorSSHAllowed: %v\n", *g.internalConfig.ServerSSHAllowed)) + configContent.WriteString(fmt.Sprintf("ServerSSHAllowed: %v\n", *g.internalConfig.ServerSSHAllowed)) } - configContent.WriteString(fmt.Sprintf("DisableAutoConnect: %v\n", g.internalConfig.DisableAutoConnect)) - configContent.WriteString(fmt.Sprintf("DNSRouteInterval: %s\n", g.internalConfig.DNSRouteInterval)) configContent.WriteString(fmt.Sprintf("DisableClientRoutes: %v\n", g.internalConfig.DisableClientRoutes)) - configContent.WriteString(fmt.Sprintf("DisableBundleGeneratorRoutes: %v\n", g.internalConfig.DisableServerRoutes)) + configContent.WriteString(fmt.Sprintf("DisableServerRoutes: %v\n", g.internalConfig.DisableServerRoutes)) configContent.WriteString(fmt.Sprintf("DisableDNS: %v\n", g.internalConfig.DisableDNS)) configContent.WriteString(fmt.Sprintf("DisableFirewall: %v\n", g.internalConfig.DisableFirewall)) - configContent.WriteString(fmt.Sprintf("BlockLANAccess: %v\n", g.internalConfig.BlockLANAccess)) + configContent.WriteString(fmt.Sprintf("BlockInbound: %v\n", g.internalConfig.BlockInbound)) + + if g.internalConfig.DisableNotifications != nil { + configContent.WriteString(fmt.Sprintf("DisableNotifications: %v\n", *g.internalConfig.DisableNotifications)) + } + + configContent.WriteString(fmt.Sprintf("DNSLabels: %v\n", g.internalConfig.DNSLabels)) + + configContent.WriteString(fmt.Sprintf("DisableAutoConnect: %v\n", g.internalConfig.DisableAutoConnect)) + + configContent.WriteString(fmt.Sprintf("DNSRouteInterval: %s\n", g.internalConfig.DNSRouteInterval)) + + if g.internalConfig.ClientCertPath != "" { + configContent.WriteString(fmt.Sprintf("ClientCertPath: %s\n", g.internalConfig.ClientCertPath)) + } + if g.internalConfig.ClientCertKeyPath != "" { + configContent.WriteString(fmt.Sprintf("ClientCertKeyPath: %s\n", g.internalConfig.ClientCertKeyPath)) + } + configContent.WriteString(fmt.Sprintf("LazyConnectionEnabled: %v\n", g.internalConfig.LazyConnectionEnabled)) } diff --git a/client/internal/engine.go b/client/internal/engine.go index e04ccc9b8..5efc0b92b 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -121,8 +121,8 @@ type EngineConfig struct { DisableServerRoutes bool DisableDNS bool DisableFirewall bool - - BlockLANAccess bool + BlockLANAccess bool + BlockInbound bool LazyConnectionEnabled bool } @@ -431,7 +431,8 @@ func (e *Engine) Start() error { return fmt.Errorf("up wg interface: %w", err) } - if e.firewall != nil { + // if inbound conns are blocked there is no need to create the ACL manager + if e.firewall != nil && !e.config.BlockInbound { e.acl = acl.NewDefaultManager(e.firewall) } @@ -487,11 +488,9 @@ func (e *Engine) createFirewall() error { } func (e *Engine) initFirewall() error { - if e.firewall.IsServerRouteSupported() { - if err := e.routeManager.EnableServerRouter(e.firewall); err != nil { - e.close() - return fmt.Errorf("enable server router: %w", err) - } + if err := e.routeManager.EnableServerRouter(e.firewall); err != nil { + e.close() + return fmt.Errorf("enable server router: %w", err) } if e.config.BlockLANAccess { @@ -525,6 +524,11 @@ func (e *Engine) initFirewall() error { } func (e *Engine) blockLanAccess() { + if e.config.BlockInbound { + // no need to set up extra deny rules if inbound is already blocked in general + return + } + var merr *multierror.Error // TODO: keep this updated @@ -796,56 +800,58 @@ func isNil(server nbssh.Server) bool { } func (e *Engine) updateSSH(sshConf *mgmProto.SSHConfig) error { + if e.config.BlockInbound { + log.Infof("SSH server is disabled because inbound connections are blocked") + return nil + } if !e.config.ServerSSHAllowed { - log.Warnf("running SSH server is not permitted") + log.Info("SSH server is not enabled") return nil - } else { - - if sshConf.GetSshEnabled() { - if runtime.GOOS == "windows" { - log.Warnf("running SSH server on %s is not supported", runtime.GOOS) - return nil - } - // start SSH server if it wasn't running - if isNil(e.sshServer) { - listenAddr := fmt.Sprintf("%s:%d", e.wgInterface.Address().IP.String(), nbssh.DefaultSSHPort) - if nbnetstack.IsEnabled() { - listenAddr = fmt.Sprintf("127.0.0.1:%d", nbssh.DefaultSSHPort) - } - // nil sshServer means it has not yet been started - var err error - e.sshServer, err = e.sshServerFunc(e.config.SSHKey, listenAddr) - - if err != nil { - return fmt.Errorf("create ssh server: %w", err) - } - go func() { - // blocking - err = e.sshServer.Start() - if err != nil { - // will throw error when we stop it even if it is a graceful stop - log.Debugf("stopped SSH server with error %v", err) - } - e.syncMsgMux.Lock() - defer e.syncMsgMux.Unlock() - e.sshServer = nil - log.Infof("stopped SSH server") - }() - } else { - log.Debugf("SSH server is already running") - } - } else if !isNil(e.sshServer) { - // Disable SSH server request, so stop it if it was running - err := e.sshServer.Stop() - if err != nil { - log.Warnf("failed to stop SSH server %v", err) - } - e.sshServer = nil - } - return nil - } + + if sshConf.GetSshEnabled() { + if runtime.GOOS == "windows" { + log.Warnf("running SSH server on %s is not supported", runtime.GOOS) + return nil + } + // start SSH server if it wasn't running + if isNil(e.sshServer) { + listenAddr := fmt.Sprintf("%s:%d", e.wgInterface.Address().IP.String(), nbssh.DefaultSSHPort) + if nbnetstack.IsEnabled() { + listenAddr = fmt.Sprintf("127.0.0.1:%d", nbssh.DefaultSSHPort) + } + // nil sshServer means it has not yet been started + var err error + e.sshServer, err = e.sshServerFunc(e.config.SSHKey, listenAddr) + + if err != nil { + return fmt.Errorf("create ssh server: %w", err) + } + go func() { + // blocking + err = e.sshServer.Start() + if err != nil { + // will throw error when we stop it even if it is a graceful stop + log.Debugf("stopped SSH server with error %v", err) + } + e.syncMsgMux.Lock() + defer e.syncMsgMux.Unlock() + e.sshServer = nil + log.Infof("stopped SSH server") + }() + } else { + log.Debugf("SSH server is already running") + } + } else if !isNil(e.sshServer) { + // Disable SSH server request, so stop it if it was running + err := e.sshServer.Stop() + if err != nil { + log.Warnf("failed to stop SSH server %v", err) + } + e.sshServer = nil + } + return nil } func (e *Engine) updateConfig(conf *mgmProto.PeerConfig) error { @@ -1796,6 +1802,10 @@ func (e *Engine) updateDNSForwarder( enabled bool, fwdEntries []*dnsfwd.ForwarderEntry, ) { + if e.config.DisableServerRoutes { + return + } + if !enabled { if e.dnsForwardMgr == nil { return diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index 402dd2f9a..b88d0aa31 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -279,6 +279,7 @@ type LoginRequest struct { // omits initialized empty slices due to omitempty tags CleanDNSLabels bool `protobuf:"varint,27,opt,name=cleanDNSLabels,proto3" json:"cleanDNSLabels,omitempty"` LazyConnectionEnabled *bool `protobuf:"varint,28,opt,name=lazyConnectionEnabled,proto3,oneof" json:"lazyConnectionEnabled,omitempty"` + BlockInbound *bool `protobuf:"varint,29,opt,name=block_inbound,json=blockInbound,proto3,oneof" json:"block_inbound,omitempty"` } func (x *LoginRequest) Reset() { @@ -510,6 +511,13 @@ func (x *LoginRequest) GetLazyConnectionEnabled() bool { return false } +func (x *LoginRequest) GetBlockInbound() bool { + if x != nil && x.BlockInbound != nil { + return *x.BlockInbound + } + return false +} + type LoginResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -990,14 +998,16 @@ type GetConfigResponse struct { // preSharedKey settings value. PreSharedKey string `protobuf:"bytes,4,opt,name=preSharedKey,proto3" json:"preSharedKey,omitempty"` // adminURL settings value. - AdminURL string `protobuf:"bytes,5,opt,name=adminURL,proto3" json:"adminURL,omitempty"` - InterfaceName string `protobuf:"bytes,6,opt,name=interfaceName,proto3" json:"interfaceName,omitempty"` - WireguardPort int64 `protobuf:"varint,7,opt,name=wireguardPort,proto3" json:"wireguardPort,omitempty"` - DisableAutoConnect bool `protobuf:"varint,9,opt,name=disableAutoConnect,proto3" json:"disableAutoConnect,omitempty"` - ServerSSHAllowed bool `protobuf:"varint,10,opt,name=serverSSHAllowed,proto3" json:"serverSSHAllowed,omitempty"` - RosenpassEnabled bool `protobuf:"varint,11,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` - RosenpassPermissive bool `protobuf:"varint,12,opt,name=rosenpassPermissive,proto3" json:"rosenpassPermissive,omitempty"` - DisableNotifications bool `protobuf:"varint,13,opt,name=disable_notifications,json=disableNotifications,proto3" json:"disable_notifications,omitempty"` + AdminURL string `protobuf:"bytes,5,opt,name=adminURL,proto3" json:"adminURL,omitempty"` + InterfaceName string `protobuf:"bytes,6,opt,name=interfaceName,proto3" json:"interfaceName,omitempty"` + WireguardPort int64 `protobuf:"varint,7,opt,name=wireguardPort,proto3" json:"wireguardPort,omitempty"` + DisableAutoConnect bool `protobuf:"varint,9,opt,name=disableAutoConnect,proto3" json:"disableAutoConnect,omitempty"` + ServerSSHAllowed bool `protobuf:"varint,10,opt,name=serverSSHAllowed,proto3" json:"serverSSHAllowed,omitempty"` + RosenpassEnabled bool `protobuf:"varint,11,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` + RosenpassPermissive bool `protobuf:"varint,12,opt,name=rosenpassPermissive,proto3" json:"rosenpassPermissive,omitempty"` + DisableNotifications bool `protobuf:"varint,13,opt,name=disable_notifications,json=disableNotifications,proto3" json:"disable_notifications,omitempty"` + LazyConnectionEnabled bool `protobuf:"varint,14,opt,name=lazyConnectionEnabled,proto3" json:"lazyConnectionEnabled,omitempty"` + BlockInbound bool `protobuf:"varint,15,opt,name=blockInbound,proto3" json:"blockInbound,omitempty"` } func (x *GetConfigResponse) Reset() { @@ -1116,6 +1126,20 @@ func (x *GetConfigResponse) GetDisableNotifications() bool { return false } +func (x *GetConfigResponse) GetLazyConnectionEnabled() bool { + if x != nil { + return x.LazyConnectionEnabled + } + return false +} + +func (x *GetConfigResponse) GetBlockInbound() bool { + if x != nil { + return x.BlockInbound + } + return false +} + // PeerState contains the latest state of a peer type PeerState struct { state protoimpl.MessageState @@ -3625,7 +3649,7 @@ var file_daemon_proto_rawDesc = []byte{ 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x0e, 0x0a, 0x0c, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x83, 0x0d, 0x0a, 0x0c, 0x4c, 0x6f, + 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x0d, 0x0a, 0x0c, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x53, 0x68, 0x61, @@ -3707,516 +3731,526 @@ var file_daemon_proto_rawDesc = []byte{ 0x62, 0x65, 0x6c, 0x73, 0x12, 0x39, 0x0a, 0x15, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x08, 0x48, 0x0f, 0x52, 0x15, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x88, 0x01, 0x01, 0x42, - 0x13, 0x0a, 0x11, 0x5f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, - 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x77, 0x69, 0x72, 0x65, 0x67, - 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x4b, 0x65, - 0x79, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x75, 0x74, - 0x6f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x53, 0x53, 0x48, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x42, 0x16, 0x0a, - 0x14, 0x5f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, - 0x73, 0x73, 0x69, 0x76, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x6e, 0x73, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x42, 0x18, 0x0a, - 0x16, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x64, 0x69, 0x73, 0x61, - 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, - 0x73, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, 0x6e, - 0x73, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, - 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x5f, 0x6c, 0x61, 0x6e, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x42, 0x18, 0x0a, 0x16, 0x5f, - 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, - 0xb5, 0x01, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x53, - 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x43, - 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x43, - 0x6f, 0x64, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x76, 0x65, - 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, 0x12, 0x38, 0x0a, - 0x17, 0x76, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, - 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, - 0x76, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x22, 0x4d, 0x0a, 0x13, 0x57, 0x61, 0x69, 0x74, 0x53, - 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, - 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, - 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, - 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, - 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0b, - 0x0a, 0x09, 0x55, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0c, 0x0a, 0x0a, 0x55, - 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3d, 0x0a, 0x0d, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x11, 0x67, 0x65, - 0x74, 0x46, 0x75, 0x6c, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x67, 0x65, 0x74, 0x46, 0x75, 0x6c, 0x6c, 0x50, 0x65, - 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x66, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x66, 0x75, 0x6c, - 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x0d, 0x0a, - 0x0b, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, - 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x12, 0x0a, 0x10, - 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0xee, 0x03, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x55, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x55, 0x72, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x6c, 0x6f, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, - 0x6f, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x53, 0x68, 0x61, - 0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x72, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x64, - 0x6d, 0x69, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x64, - 0x6d, 0x69, 0x6e, 0x55, 0x52, 0x4c, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, - 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0d, - 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0d, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, - 0x72, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x75, 0x74, - 0x6f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, + 0x28, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x18, 0x1d, 0x20, 0x01, 0x28, 0x08, 0x48, 0x10, 0x52, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x49, + 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x72, 0x6f, + 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x42, 0x10, + 0x0a, 0x0e, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, + 0x72, 0x74, 0x42, 0x17, 0x0a, 0x15, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x50, + 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x53, 0x48, 0x41, - 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x53, 0x53, 0x48, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x12, 0x2a, - 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, - 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, - 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, - 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, - 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, - 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x33, 0x0a, 0x15, - 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, 0x73, - 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x22, 0xde, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, - 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x46, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, - 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, - 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, - 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, - 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, - 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, - 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, - 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, - 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1a, 0x0a, - 0x08, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, - 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x22, - 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x12, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x22, 0xf0, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, + 0x63, 0x74, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x53, 0x48, + 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x72, 0x6f, 0x73, 0x65, + 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x42, + 0x11, 0x0a, 0x0f, 0x5f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x6e, 0x73, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x64, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, + 0x73, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, 0x6e, 0x73, 0x42, 0x13, 0x0a, 0x11, 0x5f, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, + 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6c, 0x61, 0x6e, 0x5f, 0x61, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, + 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, + 0x18, 0x0a, 0x16, 0x5f, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0xb5, 0x01, 0x0a, 0x0d, + 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, + 0x0d, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x53, 0x53, 0x4f, 0x4c, 0x6f, + 0x67, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x28, 0x0a, 0x0f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, + 0x52, 0x49, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, 0x12, 0x38, 0x0a, 0x17, 0x76, 0x65, 0x72, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, 0x43, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, 0x76, 0x65, 0x72, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x49, 0x43, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x22, 0x4d, 0x0a, 0x13, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, + 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, + 0x65, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, + 0x65, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, + 0x6d, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, + 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0b, 0x0a, 0x09, 0x55, 0x70, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0c, 0x0a, 0x0a, 0x55, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3d, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x11, 0x67, 0x65, 0x74, 0x46, 0x75, 0x6c, + 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x11, 0x67, 0x65, 0x74, 0x46, 0x75, 0x6c, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x32, 0x0a, 0x0a, 0x66, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x46, 0x75, 0x6c, + 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x66, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x0d, 0x0a, 0x0b, 0x44, 0x6f, 0x77, + 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x44, 0x6f, 0x77, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xc8, 0x04, 0x0a, + 0x11, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x55, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x55, 0x72, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6c, 0x6f, 0x67, 0x46, + 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x46, 0x69, + 0x6c, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x4b, + 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x72, 0x65, 0x53, 0x68, 0x61, + 0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x55, + 0x52, 0x4c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x55, + 0x52, 0x4c, 0x12, 0x24, 0x0a, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x66, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x77, 0x69, 0x72, 0x65, + 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0d, 0x77, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2e, + 0x0a, 0x12, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x64, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x2a, + 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x53, 0x48, 0x41, 0x6c, 0x6c, 0x6f, 0x77, + 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x53, 0x53, 0x48, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, + 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, + 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, + 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x33, 0x0a, 0x15, 0x64, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x34, 0x0a, + 0x15, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x6c, 0x61, + 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0xde, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, - 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, - 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, - 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, - 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xef, 0x03, 0x0a, 0x0a, - 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, - 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, - 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x12, 0x38, 0x0a, 0x17, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x4f, 0x66, 0x46, 0x6f, 0x72, 0x77, - 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x17, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x4f, 0x66, 0x46, 0x6f, 0x72, 0x77, 0x61, - 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x06, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x61, 0x7a, 0x79, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x15, 0x0a, - 0x13, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x3f, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x06, - 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x06, 0x72, - 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x61, 0x0a, 0x15, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, - 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x44, 0x73, 0x12, 0x16, - 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x18, 0x0a, 0x16, 0x53, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, - 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, - 0x01, 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x61, - 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, - 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, - 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x42, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, - 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x52, 0x65, 0x73, - 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, - 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x92, 0x01, 0x0a, 0x08, 0x50, - 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2e, 0x0a, - 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x52, - 0x61, 0x6e, 0x67, 0x65, 0x48, 0x00, 0x52, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x1a, 0x2f, 0x0a, - 0x05, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, - 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x42, 0x0f, - 0x0a, 0x0d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, - 0x80, 0x02, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, - 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x3a, - 0x0a, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x72, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, - 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, - 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, - 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x50, 0x6f, - 0x72, 0x74, 0x22, 0x47, 0x0a, 0x17, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, - 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, - 0x05, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, - 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x22, 0x88, 0x01, 0x0a, 0x12, - 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, - 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x70, 0x6c, 0x6f, - 0x61, 0x64, 0x55, 0x52, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x70, 0x6c, - 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x22, 0x7d, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x4b, 0x65, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, - 0x4b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x13, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x13, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, - 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, - 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13, 0x47, - 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, - 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, - 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x1b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x13, 0x0a, 0x11, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x65, 0x73, 0x22, 0x44, - 0x0a, 0x11, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x3b, 0x0a, 0x12, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, - 0x65, 0x61, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x73, 0x22, 0x45, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x3c, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x22, 0x3b, 0x0a, 0x1f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x22, 0x22, 0x0a, 0x20, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x76, 0x0a, 0x08, 0x54, 0x43, 0x50, 0x46, 0x6c, - 0x61, 0x67, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x03, 0x73, 0x79, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x03, 0x61, 0x63, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x66, 0x69, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x66, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x73, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x72, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x70, - 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x70, 0x73, 0x68, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x72, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x75, 0x72, 0x67, 0x22, - 0x80, 0x03, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, 0x65, 0x73, - 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x65, 0x73, 0x74, 0x69, - 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, - 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x32, 0x0a, 0x09, 0x74, 0x63, 0x70, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x54, 0x43, 0x50, - 0x46, 0x6c, 0x61, 0x67, 0x73, 0x48, 0x00, 0x52, 0x08, 0x74, 0x63, 0x70, 0x46, 0x6c, 0x61, 0x67, - 0x73, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x69, 0x63, 0x6d, 0x70, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x01, 0x52, 0x08, 0x69, 0x63, 0x6d, 0x70, 0x54, - 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x69, 0x63, 0x6d, 0x70, 0x5f, 0x63, - 0x6f, 0x64, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x08, 0x69, 0x63, 0x6d, - 0x70, 0x43, 0x6f, 0x64, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x74, 0x63, 0x70, - 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x69, 0x63, 0x6d, 0x70, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x69, 0x63, 0x6d, 0x70, 0x5f, 0x63, 0x6f, - 0x64, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x0a, 0x54, 0x72, 0x61, 0x63, 0x65, 0x53, 0x74, 0x61, 0x67, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x12, 0x66, 0x6f, 0x72, - 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x11, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, - 0x69, 0x6e, 0x67, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x88, 0x01, 0x01, 0x42, 0x15, 0x0a, - 0x13, 0x5f, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x65, 0x74, - 0x61, 0x69, 0x6c, 0x73, 0x22, 0x6e, 0x0a, 0x13, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, - 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x73, - 0x74, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x67, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x66, 0x69, 0x6e, 0x61, 0x6c, - 0x5f, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x10, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x12, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x93, 0x04, 0x0a, 0x0b, 0x53, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x38, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, - 0x72, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, - 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, - 0x74, 0x79, 0x12, 0x38, 0x0a, 0x08, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, - 0x72, 0x79, 0x52, 0x08, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, - 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x12, 0x3d, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x07, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3a, - 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, - 0x46, 0x4f, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, - 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, - 0x43, 0x52, 0x49, 0x54, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x03, 0x22, 0x52, 0x0a, 0x08, 0x43, 0x61, - 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, - 0x4b, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x4e, 0x53, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, - 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, - 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x56, 0x49, 0x54, 0x59, - 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x04, 0x22, 0x12, - 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x40, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x73, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, - 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, - 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, - 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, - 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, - 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb3, 0x0b, 0x0a, 0x0d, 0x44, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, - 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, - 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, - 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, - 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, - 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x53, 0x0a, 0x10, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, - 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4a, 0x0a, 0x0f, 0x46, 0x6f, 0x72, 0x77, 0x61, - 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1f, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, - 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, - 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, - 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, - 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, - 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, - 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x43, 0x6c, 0x65, 0x61, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1a, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x18, 0x53, 0x65, 0x74, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, + 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x46, 0x0a, + 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x12, + 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, + 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, + 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, + 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, + 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, + 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, + 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, + 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, + 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, + 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, + 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, + 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, + 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, + 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, + 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x18, + 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x12, + 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, + 0x65, 0x6e, 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, + 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xf0, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, + 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, + 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, + 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, + 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, + 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, + 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, + 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, + 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, + 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, + 0x1a, 0x0a, 0x08, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x08, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, + 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, + 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, + 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, + 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, + 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, + 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x22, 0xef, 0x03, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, + 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, + 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, + 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, + 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x38, 0x0a, 0x17, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x4f, 0x66, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, + 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x17, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x4f, + 0x66, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, + 0x12, 0x2b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x34, 0x0a, + 0x15, 0x6c, 0x61, 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x6c, 0x61, + 0x7a, 0x79, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3f, 0x0a, 0x14, 0x4c, 0x69, + 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x27, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x61, 0x0a, 0x15, 0x53, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, + 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x18, + 0x0a, 0x16, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, + 0x12, 0x14, 0x0a, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x42, 0x0a, 0x0b, + 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x20, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, + 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, + 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x92, 0x01, 0x0a, 0x08, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, + 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x04, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x2e, 0x0a, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x6f, 0x72, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x48, 0x00, 0x52, 0x05, 0x72, 0x61, + 0x6e, 0x67, 0x65, 0x1a, 0x2f, 0x0a, 0x05, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x03, 0x65, 0x6e, 0x64, 0x42, 0x0f, 0x0a, 0x0d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x80, 0x02, 0x0a, 0x0e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, + 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x3a, 0x0a, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x72, 0x74, + 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2e, + 0x0a, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x38, + 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, + 0x61, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x47, 0x0a, 0x17, 0x46, 0x6f, 0x72, 0x77, + 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x05, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x46, 0x6f, 0x72, 0x77, + 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x72, 0x75, 0x6c, 0x65, + 0x73, 0x22, 0x88, 0x01, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, + 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, + 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, + 0x0a, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, + 0x0a, 0x09, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x22, 0x7d, 0x0a, 0x13, + 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, + 0x64, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x70, + 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x13, 0x75, 0x70, 0x6c, + 0x6f, 0x61, 0x64, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x61, + 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x14, 0x0a, 0x12, 0x47, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, + 0x22, 0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, + 0x0a, 0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x73, 0x22, 0x44, 0x0a, 0x11, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x3b, 0x0a, 0x12, 0x43, 0x6c, + 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x65, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x65, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x22, 0x45, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x61, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x3c, + 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x64, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x22, 0x3b, 0x0a, 0x1f, + 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, + 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x22, 0x0a, 0x20, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x27, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, - 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x54, 0x72, - 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x54, - 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x0f, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, - 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, - 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, - 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x76, 0x0a, + 0x08, 0x54, 0x43, 0x50, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x73, 0x79, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x61, + 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x63, 0x6b, 0x12, 0x10, 0x0a, + 0x03, 0x66, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x66, 0x69, 0x6e, 0x12, + 0x10, 0x0a, 0x03, 0x72, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x72, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, + 0x70, 0x73, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x03, 0x75, 0x72, 0x67, 0x22, 0x80, 0x03, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, + 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x70, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x29, 0x0a, + 0x10, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x09, 0x74, 0x63, 0x70, 0x5f, 0x66, 0x6c, + 0x61, 0x67, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x54, 0x43, 0x50, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x48, 0x00, 0x52, 0x08, 0x74, + 0x63, 0x70, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x69, 0x63, + 0x6d, 0x70, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x01, 0x52, + 0x08, 0x69, 0x63, 0x6d, 0x70, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, + 0x69, 0x63, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x48, + 0x02, 0x52, 0x08, 0x69, 0x63, 0x6d, 0x70, 0x43, 0x6f, 0x64, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, + 0x0a, 0x0a, 0x5f, 0x74, 0x63, 0x70, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x5f, 0x69, 0x63, 0x6d, 0x70, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x69, + 0x63, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x0a, 0x54, 0x72, 0x61, + 0x63, 0x65, 0x53, 0x74, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x12, + 0x32, 0x0a, 0x12, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x65, + 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x11, 0x66, + 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, + 0x88, 0x01, 0x01, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, + 0x6e, 0x67, 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x6e, 0x0a, 0x13, 0x54, 0x72, + 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, + 0x53, 0x74, 0x61, 0x67, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x67, 0x65, 0x73, 0x12, 0x2b, 0x0a, + 0x11, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x44, + 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x12, 0x0a, 0x10, 0x53, 0x75, + 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x93, + 0x04, 0x0a, 0x0b, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x38, + 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x52, 0x08, + 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x38, 0x0a, 0x08, 0x63, 0x61, 0x74, 0x65, + 0x67, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, + 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x52, 0x08, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, + 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x20, 0x0a, 0x0b, + 0x75, 0x73, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x75, 0x73, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x38, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3d, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3a, 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, + 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x41, + 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, + 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x52, 0x49, 0x54, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x03, + 0x22, 0x52, 0x0a, 0x08, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x0b, 0x0a, 0x07, + 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x4e, 0x53, + 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x49, 0x56, 0x49, 0x54, 0x59, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x59, 0x53, 0x54, + 0x45, 0x4d, 0x10, 0x04, 0x22, 0x12, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x40, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, + 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, + 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, + 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, + 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, + 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, + 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, + 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb3, + 0x0b, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, + 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, + 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x1d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x53, 0x0a, 0x10, 0x44, 0x65, 0x73, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x1d, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4a, 0x0a, + 0x0f, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, + 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, + 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, + 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, + 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, + 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, + 0x0a, 0x0a, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x6f, 0x0a, 0x18, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, + 0x50, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x27, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, + 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, + 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x50, 0x65, 0x72, 0x73, 0x69, + 0x73, 0x74, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x48, 0x0a, 0x0b, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, + 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, + 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x0f, 0x53, 0x75, + 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x18, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, + 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x18, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 6fa391c8e..a46ba554a 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -122,7 +122,6 @@ message LoginRequest { optional bool disable_server_routes = 21; optional bool disable_dns = 22; optional bool disable_firewall = 23; - optional bool block_lan_access = 24; optional bool disable_notifications = 25; @@ -135,6 +134,8 @@ message LoginRequest { bool cleanDNSLabels = 27; optional bool lazyConnectionEnabled = 28; + + optional bool block_inbound = 29; } message LoginResponse { @@ -202,6 +203,10 @@ message GetConfigResponse { bool rosenpassPermissive = 12; bool disable_notifications = 13; + + bool lazyConnectionEnabled = 14; + + bool blockInbound = 15; } // PeerState contains the latest state of a peer diff --git a/client/server/server.go b/client/server/server.go index 43b3eb3b7..2025a89ec 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -398,11 +398,14 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro inputConfig.DisableFirewall = msg.DisableFirewall s.latestConfigInput.DisableFirewall = msg.DisableFirewall } - if msg.BlockLanAccess != nil { inputConfig.BlockLANAccess = msg.BlockLanAccess s.latestConfigInput.BlockLANAccess = msg.BlockLanAccess } + if msg.BlockInbound != nil { + inputConfig.BlockInbound = msg.BlockInbound + s.latestConfigInput.BlockInbound = msg.BlockInbound + } if msg.CleanDNSLabels { inputConfig.DNSLabels = domain.List{} @@ -756,18 +759,20 @@ func (s *Server) GetConfig(_ context.Context, _ *proto.GetConfigRequest) (*proto } return &proto.GetConfigResponse{ - ManagementUrl: managementURL, - ConfigFile: s.latestConfigInput.ConfigPath, - LogFile: s.logFile, - PreSharedKey: preSharedKey, - AdminURL: adminURL, - InterfaceName: s.config.WgIface, - WireguardPort: int64(s.config.WgPort), - DisableAutoConnect: s.config.DisableAutoConnect, - ServerSSHAllowed: *s.config.ServerSSHAllowed, - RosenpassEnabled: s.config.RosenpassEnabled, - RosenpassPermissive: s.config.RosenpassPermissive, - DisableNotifications: disableNotifications, + ManagementUrl: managementURL, + ConfigFile: s.latestConfigInput.ConfigPath, + LogFile: s.logFile, + PreSharedKey: preSharedKey, + AdminURL: adminURL, + InterfaceName: s.config.WgIface, + WireguardPort: int64(s.config.WgPort), + DisableAutoConnect: s.config.DisableAutoConnect, + ServerSSHAllowed: *s.config.ServerSSHAllowed, + RosenpassEnabled: s.config.RosenpassEnabled, + RosenpassPermissive: s.config.RosenpassPermissive, + LazyConnectionEnabled: s.config.LazyConnectionEnabled, + BlockInbound: s.config.BlockInbound, + DisableNotifications: disableNotifications, }, nil } diff --git a/client/ui/client_ui.go b/client/ui/client_ui.go index 92289a8a3..f0202b8e7 100644 --- a/client/ui/client_ui.go +++ b/client/ui/client_ui.go @@ -194,6 +194,7 @@ type serviceClient struct { mAutoConnect *systray.MenuItem mEnableRosenpass *systray.MenuItem mLazyConnEnabled *systray.MenuItem + mBlockInbound *systray.MenuItem mNotifications *systray.MenuItem mAdvancedSettings *systray.MenuItem mCreateDebugBundle *systray.MenuItem @@ -635,7 +636,8 @@ func (s *serviceClient) onTrayReady() { s.mAllowSSH = s.mSettings.AddSubMenuItemCheckbox("Allow SSH", allowSSHMenuDescr, false) s.mAutoConnect = s.mSettings.AddSubMenuItemCheckbox("Connect on Startup", autoConnectMenuDescr, false) s.mEnableRosenpass = s.mSettings.AddSubMenuItemCheckbox("Enable Quantum-Resistance", quantumResistanceMenuDescr, false) - s.mLazyConnEnabled = s.mSettings.AddSubMenuItemCheckbox("Enable lazy connection", lazyConnMenuDescr, false) + s.mLazyConnEnabled = s.mSettings.AddSubMenuItemCheckbox("Enable Lazy Connections", lazyConnMenuDescr, false) + s.mBlockInbound = s.mSettings.AddSubMenuItemCheckbox("Block Inbound Connections", blockInboundMenuDescr, false) s.mNotifications = s.mSettings.AddSubMenuItemCheckbox("Notifications", notificationsMenuDescr, false) s.mAdvancedSettings = s.mSettings.AddSubMenuItem("Advanced Settings", advancedSettingsMenuDescr) s.mCreateDebugBundle = s.mSettings.AddSubMenuItem("Create Debug Bundle", debugBundleMenuDescr) @@ -757,6 +759,15 @@ func (s *serviceClient) listenEvents() { if err := s.updateConfig(); err != nil { log.Errorf("failed to update config: %v", err) } + case <-s.mBlockInbound.ClickedCh: + if s.mBlockInbound.Checked() { + s.mBlockInbound.Uncheck() + } else { + s.mBlockInbound.Check() + } + if err := s.updateConfig(); err != nil { + log.Errorf("failed to update config: %v", err) + } case <-s.mAdvancedSettings.ClickedCh: s.mAdvancedSettings.Disable() go func() { @@ -1017,6 +1028,18 @@ func (s *serviceClient) loadSettings() { s.mEnableRosenpass.Uncheck() } + if cfg.LazyConnectionEnabled { + s.mLazyConnEnabled.Check() + } else { + s.mLazyConnEnabled.Uncheck() + } + + if cfg.BlockInbound { + s.mBlockInbound.Check() + } else { + s.mBlockInbound.Uncheck() + } + if cfg.DisableNotifications { s.mNotifications.Uncheck() } else { @@ -1033,8 +1056,9 @@ func (s *serviceClient) updateConfig() error { disableAutoStart := !s.mAutoConnect.Checked() sshAllowed := s.mAllowSSH.Checked() rosenpassEnabled := s.mEnableRosenpass.Checked() - notificationsDisabled := !s.mNotifications.Checked() lazyConnectionEnabled := s.mLazyConnEnabled.Checked() + blockInbound := s.mBlockInbound.Checked() + notificationsDisabled := !s.mNotifications.Checked() loginRequest := proto.LoginRequest{ IsUnixDesktopClient: runtime.GOOS == "linux" || runtime.GOOS == "freebsd", @@ -1043,6 +1067,7 @@ func (s *serviceClient) updateConfig() error { DisableAutoConnect: &disableAutoStart, DisableNotifications: ¬ificationsDisabled, LazyConnectionEnabled: &lazyConnectionEnabled, + BlockInbound: &blockInbound, } if err := s.restartClient(&loginRequest); err != nil { diff --git a/client/ui/const.go b/client/ui/const.go index cd4e7db8e..5a4b27f32 100644 --- a/client/ui/const.go +++ b/client/ui/const.go @@ -5,7 +5,8 @@ const ( allowSSHMenuDescr = "Allow SSH connections" autoConnectMenuDescr = "Connect automatically when the service starts" quantumResistanceMenuDescr = "Enable post-quantum security via Rosenpass" - lazyConnMenuDescr = "[Experimental] Enable lazy connect" + lazyConnMenuDescr = "[Experimental] Enable lazy connections" + blockInboundMenuDescr = "Block inbound connections to the local machine and routed networks" notificationsMenuDescr = "Enable notifications" advancedSettingsMenuDescr = "Advanced settings of the application" debugBundleMenuDescr = "Create and open debug information bundle" From 06980e7fa0a30b7d6d14129c980a4f3edf37cc25 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:53:39 +0200 Subject: [PATCH 21/37] [client] Apply routes right away instead of on peer connection (#3907) --- client/internal/dns/server.go | 2 +- client/internal/engine.go | 18 +- .../routemanager/{ => client}/client.go | 343 +++++++++--------- .../routemanager/{ => client}/client_test.go | 4 +- client/internal/routemanager/manager.go | 146 ++++++-- client/internal/routemanager/manager_test.go | 5 +- .../routemanager/{ => server}/server.go | 78 ++-- client/internal/routemanager/static/route.go | 18 +- 8 files changed, 349 insertions(+), 265 deletions(-) rename client/internal/routemanager/{ => client}/client.go (52%) rename client/internal/routemanager/{ => client}/client_test.go (99%) rename client/internal/routemanager/{ => server}/server.go (63%) diff --git a/client/internal/dns/server.go b/client/internal/dns/server.go index 3f49c23fd..7b845235c 100644 --- a/client/internal/dns/server.go +++ b/client/internal/dns/server.go @@ -489,7 +489,7 @@ func (s *DefaultServer) applyHostConfig() { } } - log.Debugf("extra match domains: %v", s.extraDomains) + log.Debugf("extra match domains: %v", maps.Keys(s.extraDomains)) if err := s.hostManager.applyDNSConfig(config, s.stateManager); err != nil { log.Errorf("failed to apply DNS host manager update: %v", err) diff --git a/client/internal/engine.go b/client/internal/engine.go index 5efc0b92b..d015c1d6c 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -994,6 +994,15 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error { } } + protoDNSConfig := networkMap.GetDNSConfig() + if protoDNSConfig == nil { + protoDNSConfig = &mgmProto.DNSConfig{} + } + + if err := e.dnsServer.UpdateDNSServer(serial, toDNSConfig(protoDNSConfig, e.wgInterface.Address().Network)); err != nil { + log.Errorf("failed to update dns server, err: %v", err) + } + dnsRouteFeatureFlag := toDNSFeatureFlag(networkMap) // apply routes first, route related actions might depend on routing being enabled @@ -1061,15 +1070,6 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error { excludedLazyPeers := e.toExcludedLazyPeers(routes, forwardingRules, networkMap.GetRemotePeers()) e.connMgr.SetExcludeList(excludedLazyPeers) - protoDNSConfig := networkMap.GetDNSConfig() - if protoDNSConfig == nil { - protoDNSConfig = &mgmProto.DNSConfig{} - } - - if err := e.dnsServer.UpdateDNSServer(serial, toDNSConfig(protoDNSConfig, e.wgInterface.Address().Network)); err != nil { - log.Errorf("failed to update dns server, err: %v", err) - } - e.networkSerial = serial // Test received (upstream) servers for availability right away instead of upon usage. diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client/client.go similarity index 52% rename from client/internal/routemanager/client.go rename to client/internal/routemanager/client/client.go index bff954c27..5582591a9 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client/client.go @@ -1,4 +1,4 @@ -package routemanager +package client import ( "context" @@ -7,10 +7,8 @@ import ( "runtime" "time" - "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - nberrors "github.com/netbirdio/netbird/client/errors" nbdns "github.com/netbirdio/netbird/client/internal/dns" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/peerstore" @@ -36,6 +34,7 @@ const ( reasonRouteUpdate reasonPeerUpdate reasonShutdown + reasonHA ) type routerPeerStatus struct { @@ -44,9 +43,9 @@ type routerPeerStatus struct { latency time.Duration } -type routesUpdate struct { - updateSerial uint64 - routes []*route.Route +type RoutesUpdate struct { + UpdateSerial uint64 + Routes []*route.Route } // RouteHandler defines the interface for handling routes @@ -58,64 +57,54 @@ type RouteHandler interface { RemoveAllowedIPs() error } -type clientNetwork struct { +type WatcherConfig struct { + Context context.Context + DNSRouteInterval time.Duration + WGInterface iface.WGIface + StatusRecorder *peer.Status + Route *route.Route + Handler RouteHandler +} + +// Watcher watches route and peer changes and updates allowed IPs accordingly. +// Once stopped, it cannot be reused. +type Watcher struct { ctx context.Context cancel context.CancelFunc statusRecorder *peer.Status wgInterface iface.WGIface routes map[route.ID]*route.Route - routeUpdate chan routesUpdate + routeUpdate chan RoutesUpdate peerStateUpdate chan struct{} - routePeersNotifiers map[string]chan struct{} + routePeersNotifiers map[string]chan struct{} // map of peer key to channel for peer state changes currentChosen *route.Route handler RouteHandler updateSerial uint64 } -func newClientNetworkWatcher( - ctx context.Context, - dnsRouteInterval time.Duration, - wgInterface iface.WGIface, - statusRecorder *peer.Status, - rt *route.Route, - routeRefCounter *refcounter.RouteRefCounter, - allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, - dnsServer nbdns.Server, - peerStore *peerstore.Store, - useNewDNSRoute bool, -) *clientNetwork { - ctx, cancel := context.WithCancel(ctx) +func NewWatcher(config WatcherConfig) *Watcher { + ctx, cancel := context.WithCancel(config.Context) - client := &clientNetwork{ + client := &Watcher{ ctx: ctx, cancel: cancel, - statusRecorder: statusRecorder, - wgInterface: wgInterface, + statusRecorder: config.StatusRecorder, + wgInterface: config.WGInterface, routes: make(map[route.ID]*route.Route), routePeersNotifiers: make(map[string]chan struct{}), - routeUpdate: make(chan routesUpdate), + routeUpdate: make(chan RoutesUpdate), peerStateUpdate: make(chan struct{}), - handler: handlerFromRoute( - rt, - routeRefCounter, - allowedIPsRefCounter, - dnsRouteInterval, - statusRecorder, - wgInterface, - dnsServer, - peerStore, - useNewDNSRoute, - ), + handler: config.Handler, } return client } -func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { +func (w *Watcher) getRouterPeerStatuses() map[route.ID]routerPeerStatus { routePeerStatuses := make(map[route.ID]routerPeerStatus) - for _, r := range c.routes { - peerStatus, err := c.statusRecorder.GetPeer(r.Peer) + for _, r := range w.routes { + peerStatus, err := w.statusRecorder.GetPeer(r.Peer) if err != nil { - log.Debugf("couldn't fetch peer state: %v", err) + log.Debugf("couldn't fetch peer state %v: %v", r.Peer, err) continue } routePeerStatuses[r.ID] = routerPeerStatus{ @@ -128,7 +117,7 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { } // getBestRouteFromStatuses determines the most optimal route from the available routes -// within a clientNetwork, taking into account peer connection status, route metrics, and +// within a Watcher, taking into account peer connection status, route metrics, and // preference for non-relayed and direct connections. // // It follows these prioritization rules: @@ -140,17 +129,17 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { // * Stability: In case of equal scores, the currently active route (if any) is maintained. // // It returns the ID of the selected optimal route. -func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID]routerPeerStatus) route.ID { - chosen := route.ID("") +func (w *Watcher) getBestRouteFromStatuses(routePeerStatuses map[route.ID]routerPeerStatus) route.ID { + var chosen route.ID chosenScore := float64(0) currScore := float64(0) - currID := route.ID("") - if c.currentChosen != nil { - currID = c.currentChosen.ID + var currID route.ID + if w.currentChosen != nil { + currID = w.currentChosen.ID } - for _, r := range c.routes { + for _, r := range w.routes { tempScore := float64(0) peerStatus, found := routePeerStatuses[r.ID] if !found || !peerStatus.connected { @@ -167,7 +156,7 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID] if peerStatus.latency != 0 { latency = peerStatus.latency } else { - log.Tracef("peer %s has 0 latency, range %s", r.Peer, c.handler) + log.Tracef("peer %s has 0 latency, range %s", r.Peer, w.handler) } // avoid negative tempScore on the higher latency calculation @@ -197,35 +186,45 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID] } } - log.Debugf("chosen route: %s, chosen score: %f, current route: %s, current score: %f", chosen, chosenScore, currID, currScore) + chosenID := chosen + if chosen == "" { + chosenID = "" + } + currentID := currID + if currID == "" { + currentID = "" + } + + log.Debugf("chosen route: %s, chosen score: %f, current route: %s, current score: %f", chosenID, chosenScore, currentID, currScore) switch { case chosen == "": var peers []string - for _, r := range c.routes { + for _, r := range w.routes { peers = append(peers, r.Peer) } - log.Warnf("The network [%v] has not been assigned a routing peer as no peers from the list %s are currently connected", c.handler, peers) + log.Infof("network [%v] has not been assigned a routing peer as no peers from the list %s are currently connected", w.handler, peers) case chosen != currID: // we compare the current score + 10ms to the chosen score to avoid flapping between routes if currScore != 0 && currScore+0.01 > chosenScore { - log.Debugf("Keeping current routing peer because the score difference with latency is less than 0.01(10ms), current: %f, new: %f", currScore, chosenScore) + log.Debugf("keeping current routing peer %s for [%v]: the score difference with latency is less than 0.01(10ms): current: %f, new: %f", + w.currentChosen.Peer, w.handler, currScore, chosenScore) return currID } var p string - if rt := c.routes[chosen]; rt != nil { + if rt := w.routes[chosen]; rt != nil { p = rt.Peer } - log.Infof("New chosen route is %s with peer %s with score %f for network [%v]", chosen, p, chosenScore, c.handler) + log.Infof("New chosen route is %s with peer %s with score %f for network [%v]", chosen, p, chosenScore, w.handler) } return chosen } -func (c *clientNetwork) watchPeerStatusChanges(ctx context.Context, peerKey string, peerStateUpdate chan struct{}, closer chan struct{}) { - subscription := c.statusRecorder.SubscribeToPeerStateChanges(ctx, peerKey) - defer c.statusRecorder.UnsubscribePeerStateChanges(subscription) +func (w *Watcher) watchPeerStatusChanges(ctx context.Context, peerKey string, peerStateUpdate chan struct{}, closer chan struct{}) { + subscription := w.statusRecorder.SubscribeToPeerStateChanges(ctx, peerKey) + defer w.statusRecorder.UnsubscribePeerStateChanges(subscription) for { select { @@ -240,105 +239,92 @@ func (c *clientNetwork) watchPeerStatusChanges(ctx context.Context, peerKey stri } } -func (c *clientNetwork) startPeersStatusChangeWatcher() { - for _, r := range c.routes { - _, found := c.routePeersNotifiers[r.Peer] - if found { +func (w *Watcher) startNewPeerStatusWatchers() { + for _, r := range w.routes { + if _, found := w.routePeersNotifiers[r.Peer]; found { continue } closerChan := make(chan struct{}) - c.routePeersNotifiers[r.Peer] = closerChan - go c.watchPeerStatusChanges(c.ctx, r.Peer, c.peerStateUpdate, closerChan) + w.routePeersNotifiers[r.Peer] = closerChan + go w.watchPeerStatusChanges(w.ctx, r.Peer, w.peerStateUpdate, closerChan) } } -func (c *clientNetwork) removeRouteFromWireGuardPeer() error { - if err := c.statusRecorder.RemovePeerStateRoute(c.currentChosen.Peer, c.handler.String()); err != nil { +// addAllowedIPs adds the allowed IPs for the current chosen route to the handler. +func (w *Watcher) addAllowedIPs(route *route.Route) error { + if err := w.handler.AddAllowedIPs(route.Peer); err != nil { + return fmt.Errorf("add allowed IPs for peer %s: %w", route.Peer, err) + } + + if err := w.statusRecorder.AddPeerStateRoute(route.Peer, w.handler.String(), route.GetResourceID()); err != nil { log.Warnf("Failed to update peer state: %v", err) } - if err := c.handler.RemoveAllowedIPs(); err != nil { - return fmt.Errorf("remove allowed IPs: %w", err) - } + w.connectEvent(route) return nil } -func (c *clientNetwork) removeRouteFromPeerAndSystem(rsn reason) error { - if c.currentChosen == nil { - return nil +func (w *Watcher) removeAllowedIPs(route *route.Route, rsn reason) error { + if err := w.statusRecorder.RemovePeerStateRoute(route.Peer, w.handler.String()); err != nil { + log.Warnf("Failed to update peer state: %v", err) } - var merr *multierror.Error - - if err := c.removeRouteFromWireGuardPeer(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove allowed IPs for peer %s: %w", c.currentChosen.Peer, err)) - } - if err := c.handler.RemoveRoute(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove route: %w", err)) + if err := w.handler.RemoveAllowedIPs(); err != nil { + return fmt.Errorf("remove allowed IPs: %w", err) } - c.disconnectEvent(rsn) + w.disconnectEvent(route, rsn) - return nberrors.FormatErrorOrNil(merr) + return nil } -func (c *clientNetwork) recalculateRouteAndUpdatePeerAndSystem(rsn reason) error { - routerPeerStatuses := c.getRouterPeerStatuses() +func (w *Watcher) recalculateRoutes(rsn reason) error { + routerPeerStatuses := w.getRouterPeerStatuses() - newChosenID := c.getBestRouteFromStatuses(routerPeerStatuses) + newChosenID := w.getBestRouteFromStatuses(routerPeerStatuses) - // If no route is chosen, remove the route from the peer and system + // If no route is chosen, remove the route from the peer if newChosenID == "" { - if err := c.removeRouteFromPeerAndSystem(rsn); err != nil { - return fmt.Errorf("remove route for peer %s: %w", c.currentChosen.Peer, err) + if w.currentChosen == nil { + return nil } - c.currentChosen = nil + if err := w.removeAllowedIPs(w.currentChosen, rsn); err != nil { + return fmt.Errorf("remove obsolete: %w", err) + } + + w.currentChosen = nil return nil } // If the chosen route is the same as the current route, do nothing - if c.currentChosen != nil && c.currentChosen.ID == newChosenID && - c.currentChosen.Equal(c.routes[newChosenID]) { + if w.currentChosen != nil && w.currentChosen.ID == newChosenID && + w.currentChosen.Equal(w.routes[newChosenID]) { return nil } - var isNew bool - if c.currentChosen == nil { - // If they were not previously assigned to another peer, add routes to the system first - if err := c.handler.AddRoute(c.ctx); err != nil { - return fmt.Errorf("add route: %w", err) - } - isNew = true - } else { - // Otherwise, remove the allowed IPs from the previous peer first - if err := c.removeRouteFromWireGuardPeer(); err != nil { - return fmt.Errorf("remove allowed IPs for peer %s: %w", c.currentChosen.Peer, err) + // If the chosen route was assigned to a different peer, remove the allowed IPs first + if isNew := w.currentChosen == nil; !isNew { + if err := w.removeAllowedIPs(w.currentChosen, reasonHA); err != nil { + return fmt.Errorf("remove old: %w", err) } } - c.currentChosen = c.routes[newChosenID] - - if err := c.handler.AddAllowedIPs(c.currentChosen.Peer); err != nil { - return fmt.Errorf("add allowed IPs for peer %s: %w", c.currentChosen.Peer, err) + newChosenRoute := w.routes[newChosenID] + if err := w.addAllowedIPs(newChosenRoute); err != nil { + return fmt.Errorf("add new: %w", err) } - if isNew { - c.connectEvent() - } + w.currentChosen = newChosenRoute - err := c.statusRecorder.AddPeerStateRoute(c.currentChosen.Peer, c.handler.String(), c.currentChosen.GetResourceID()) - if err != nil { - return fmt.Errorf("add peer state route: %w", err) - } return nil } -func (c *clientNetwork) connectEvent() { +func (w *Watcher) connectEvent(route *route.Route) { var defaultRoute bool - for _, r := range c.routes { + for _, r := range w.routes { if r.Network.Bits() == 0 { defaultRoute = true break @@ -350,13 +336,13 @@ func (c *clientNetwork) connectEvent() { } meta := map[string]string{ - "network": c.handler.String(), + "network": w.handler.String(), } - if c.currentChosen != nil { - meta["id"] = string(c.currentChosen.NetID) - meta["peer"] = c.currentChosen.Peer + if route != nil { + meta["id"] = string(route.NetID) + meta["peer"] = route.Peer } - c.statusRecorder.PublishEvent( + w.statusRecorder.PublishEvent( proto.SystemEvent_INFO, proto.SystemEvent_NETWORK, "Default route added", @@ -365,9 +351,9 @@ func (c *clientNetwork) connectEvent() { ) } -func (c *clientNetwork) disconnectEvent(rsn reason) { +func (w *Watcher) disconnectEvent(route *route.Route, rsn reason) { var defaultRoute bool - for _, r := range c.routes { + for _, r := range w.routes { if r.Network.Bits() == 0 { defaultRoute = true break @@ -383,11 +369,11 @@ func (c *clientNetwork) disconnectEvent(rsn reason) { var userMessage string meta := make(map[string]string) - if c.currentChosen != nil { - meta["id"] = string(c.currentChosen.NetID) - meta["peer"] = c.currentChosen.Peer + if route != nil { + meta["id"] = string(route.NetID) + meta["peer"] = route.Peer } - meta["network"] = c.handler.String() + meta["network"] = w.handler.String() switch rsn { case reasonShutdown: severity = proto.SystemEvent_INFO @@ -400,13 +386,17 @@ func (c *clientNetwork) disconnectEvent(rsn reason) { severity = proto.SystemEvent_WARNING message = "Default route disconnected due to peer unreachability" userMessage = "Exit node connection lost. Your internet access might be affected." + case reasonHA: + severity = proto.SystemEvent_INFO + message = "Default route disconnected due to high availability change" + userMessage = "Exit node disconnected due to high availability change." default: severity = proto.SystemEvent_ERROR message = "Default route disconnected for unknown reasons" userMessage = "Exit node disconnected for unknown reasons." } - c.statusRecorder.PublishEvent( + w.statusRecorder.PublishEvent( severity, proto.SystemEvent_NETWORK, message, @@ -415,86 +405,101 @@ func (c *clientNetwork) disconnectEvent(rsn reason) { ) } -func (c *clientNetwork) sendUpdateToClientNetworkWatcher(update routesUpdate) { +func (w *Watcher) SendUpdate(update RoutesUpdate) { go func() { - c.routeUpdate <- update + select { + case w.routeUpdate <- update: + case <-w.ctx.Done(): + } }() } -func (c *clientNetwork) handleUpdate(update routesUpdate) bool { +func (w *Watcher) classifyUpdate(update RoutesUpdate) bool { isUpdateMapDifferent := false updateMap := make(map[route.ID]*route.Route) - for _, r := range update.routes { + for _, r := range update.Routes { updateMap[r.ID] = r } - if len(c.routes) != len(updateMap) { + if len(w.routes) != len(updateMap) { isUpdateMapDifferent = true } - for id, r := range c.routes { + for id, r := range w.routes { _, found := updateMap[id] if !found { - close(c.routePeersNotifiers[r.Peer]) - delete(c.routePeersNotifiers, r.Peer) + close(w.routePeersNotifiers[r.Peer]) + delete(w.routePeersNotifiers, r.Peer) isUpdateMapDifferent = true continue } - if !reflect.DeepEqual(c.routes[id], updateMap[id]) { + if !reflect.DeepEqual(w.routes[id], updateMap[id]) { isUpdateMapDifferent = true } } - c.routes = updateMap + w.routes = updateMap return isUpdateMapDifferent } -// peersStateAndUpdateWatcher is the main point of reacting on client network routing events. +// Start is the main point of reacting on client network routing events. // All the processing related to the client network should be done here. Thread-safe. -func (c *clientNetwork) peersStateAndUpdateWatcher() { +func (w *Watcher) Start() { for { select { - case <-c.ctx.Done(): - log.Debugf("Stopping watcher for network [%v]", c.handler) - if err := c.removeRouteFromPeerAndSystem(reasonShutdown); err != nil { - log.Errorf("Failed to remove routes for [%v]: %v", c.handler, err) - } + case <-w.ctx.Done(): return - case <-c.peerStateUpdate: - err := c.recalculateRouteAndUpdatePeerAndSystem(reasonPeerUpdate) - if err != nil { - log.Errorf("Failed to recalculate routes for network [%v]: %v", c.handler, err) + case <-w.peerStateUpdate: + if err := w.recalculateRoutes(reasonPeerUpdate); err != nil { + log.Errorf("Failed to recalculate routes for network [%v]: %v", w.handler, err) } - case update := <-c.routeUpdate: - if update.updateSerial < c.updateSerial { - log.Warnf("Received a routes update with smaller serial number (%d -> %d), ignoring it", c.updateSerial, update.updateSerial) + case update := <-w.routeUpdate: + if update.UpdateSerial < w.updateSerial { + log.Warnf("Received a routes update with smaller serial number (%d -> %d), ignoring it", w.updateSerial, update.UpdateSerial) continue } - log.Debugf("Received a new client network route update for [%v]", c.handler) - - // hash update somehow - isTrueRouteUpdate := c.handleUpdate(update) - - c.updateSerial = update.updateSerial - - if isTrueRouteUpdate { - log.Debug("Client network update contains different routes, recalculating routes") - err := c.recalculateRouteAndUpdatePeerAndSystem(reasonRouteUpdate) - if err != nil { - log.Errorf("Failed to recalculate routes for network [%v]: %v", c.handler, err) - } - } else { - log.Debug("Route update is not different, skipping route recalculation") - } - - c.startPeersStatusChangeWatcher() + w.handleRouteUpdate(update) } } } -func handlerFromRoute( +func (w *Watcher) handleRouteUpdate(update RoutesUpdate) { + log.Debugf("Received a new client network route update for [%v]", w.handler) + + // hash update somehow + isTrueRouteUpdate := w.classifyUpdate(update) + + w.updateSerial = update.UpdateSerial + + if isTrueRouteUpdate { + log.Debugf("client network update %v for [%v] contains different routes, recalculating routes", update.UpdateSerial, w.handler) + if err := w.recalculateRoutes(reasonRouteUpdate); err != nil { + log.Errorf("failed to recalculate routes for network [%v]: %v", w.handler, err) + } + } else { + log.Debugf("route update %v for [%v] is not different, skipping route recalculation", update.UpdateSerial, w.handler) + } + + w.startNewPeerStatusWatchers() +} + +// Stop stops the watcher and cleans up resources. +func (w *Watcher) Stop() { + log.Debugf("Stopping watcher for network [%v]", w.handler) + + w.cancel() + + if w.currentChosen == nil { + return + } + if err := w.removeAllowedIPs(w.currentChosen, reasonShutdown); err != nil { + log.Errorf("Failed to remove routes for [%v]: %v", w.handler, err) + } +} + +func HandlerFromRoute( rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, diff --git a/client/internal/routemanager/client_test.go b/client/internal/routemanager/client/client_test.go similarity index 99% rename from client/internal/routemanager/client_test.go rename to client/internal/routemanager/client/client_test.go index 56fcf1613..48a9495bf 100644 --- a/client/internal/routemanager/client_test.go +++ b/client/internal/routemanager/client/client_test.go @@ -1,4 +1,4 @@ -package routemanager +package client import ( "fmt" @@ -395,7 +395,7 @@ func TestGetBestrouteFromStatuses(t *testing.T) { } // create new clientNetwork - client := &clientNetwork{ + client := &Watcher{ handler: static.NewRoute(&route.Route{Network: netip.MustParsePrefix("192.168.0.0/24")}, nil, nil), routes: tc.existingRoutes, currentChosen: currentRoute, diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 078206ab9..afb74c23e 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -11,9 +11,11 @@ import ( "sync" "time" + "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" "golang.org/x/exp/maps" + nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface/configurer" "github.com/netbirdio/netbird/client/iface/netstack" @@ -21,9 +23,11 @@ import ( "github.com/netbirdio/netbird/client/internal/listener" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/peerstore" + "github.com/netbirdio/netbird/client/internal/routemanager/client" "github.com/netbirdio/netbird/client/internal/routemanager/iface" "github.com/netbirdio/netbird/client/internal/routemanager/notifier" "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" + "github.com/netbirdio/netbird/client/internal/routemanager/server" "github.com/netbirdio/netbird/client/internal/routemanager/systemops" "github.com/netbirdio/netbird/client/internal/routemanager/vars" "github.com/netbirdio/netbird/client/internal/routeselector" @@ -68,9 +72,9 @@ type DefaultManager struct { ctx context.Context stop context.CancelFunc mux sync.Mutex - clientNetworks map[route.HAUniqueID]*clientNetwork + clientNetworks map[route.HAUniqueID]*client.Watcher routeSelector *routeselector.RouteSelector - serverRouter *serverRouter + serverRouter *server.Router sysOps *systemops.SysOps statusRecorder *peer.Status relayMgr *relayClient.Manager @@ -88,6 +92,7 @@ type DefaultManager struct { useNewDNSRoute bool disableClientRoutes bool disableServerRoutes bool + activeRoutes map[route.HAUniqueID]client.RouteHandler } func NewManager(config ManagerConfig) *DefaultManager { @@ -99,7 +104,7 @@ func NewManager(config ManagerConfig) *DefaultManager { ctx: mCTX, stop: cancel, dnsRouteInterval: config.DNSRouteInterval, - clientNetworks: make(map[route.HAUniqueID]*clientNetwork), + clientNetworks: make(map[route.HAUniqueID]*client.Watcher), relayMgr: config.RelayManager, sysOps: sysOps, statusRecorder: config.StatusRecorder, @@ -111,6 +116,7 @@ func NewManager(config ManagerConfig) *DefaultManager { peerStore: config.PeerStore, disableClientRoutes: config.DisableClientRoutes, disableServerRoutes: config.DisableServerRoutes, + activeRoutes: make(map[route.HAUniqueID]client.RouteHandler), } useNoop := netstack.IsEnabled() || config.DisableClientRoutes @@ -226,7 +232,7 @@ func (m *DefaultManager) EnableServerRouter(firewall firewall.Manager) error { } var err error - m.serverRouter, err = newServerRouter(m.ctx, m.wgInterface, firewall, m.statusRecorder) + m.serverRouter, err = server.NewRouter(m.ctx, m.wgInterface, firewall, m.statusRecorder) if err != nil { return err } @@ -237,7 +243,7 @@ func (m *DefaultManager) EnableServerRouter(firewall firewall.Manager) error { func (m *DefaultManager) Stop(stateManager *statemanager.Manager) { m.stop() if m.serverRouter != nil { - m.serverRouter.cleanUp() + m.serverRouter.CleanUp() } if m.routeRefCounter != nil { @@ -265,6 +271,54 @@ func (m *DefaultManager) Stop(stateManager *statemanager.Manager) { } // UpdateRoutes compares received routes with existing routes and removes, updates or adds them to the client and server maps +func (m *DefaultManager) updateSystemRoutes(newRoutes route.HAMap) error { + toAdd := make(map[route.HAUniqueID]*route.Route) + toRemove := make(map[route.HAUniqueID]client.RouteHandler) + + for id, routes := range newRoutes { + if len(routes) > 0 { + toAdd[id] = routes[0] + } + } + + for id, activeHandler := range m.activeRoutes { + if _, exists := toAdd[id]; exists { + delete(toAdd, id) + } else { + toRemove[id] = activeHandler + } + } + + var merr *multierror.Error + for id, handler := range toRemove { + if err := handler.RemoveRoute(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove route %s: %w", handler.String(), err)) + } + delete(m.activeRoutes, id) + } + + for id, route := range toAdd { + handler := client.HandlerFromRoute( + route, + m.routeRefCounter, + m.allowedIPsRefCounter, + m.dnsRouteInterval, + m.statusRecorder, + m.wgInterface, + m.dnsServer, + m.peerStore, + m.useNewDNSRoute, + ) + if err := handler.AddRoute(m.ctx); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add route %s: %w", handler.String(), err)) + continue + } + m.activeRoutes[id] = handler + } + + return nberrors.FormatErrorOrNil(merr) +} + func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Route, useNewDNSRoute bool) error { select { case <-m.ctx.Done(): @@ -281,6 +335,11 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro if !m.disableClientRoutes { filteredClientRoutes := m.routeSelector.FilterSelected(newClientRoutesIDMap) + + if err := m.updateSystemRoutes(filteredClientRoutes); err != nil { + log.Errorf("Failed to update system routes: %v", err) + } + m.updateClientNetworks(updateSerial, filteredClientRoutes) m.notifier.OnNewRoutes(filteredClientRoutes) } @@ -290,7 +349,7 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro return nil } - if err := m.serverRouter.updateRoutes(newServerRoutesMap, useNewDNSRoute); err != nil { + if err := m.serverRouter.UpdateRoutes(newServerRoutesMap, useNewDNSRoute); err != nil { return fmt.Errorf("update routes: %w", err) } @@ -341,6 +400,10 @@ func (m *DefaultManager) TriggerSelection(networks route.HAMap) { m.notifier.OnNewRoutes(networks) + if err := m.updateSystemRoutes(networks); err != nil { + log.Errorf("failed to update system routes during selection: %v", err) + } + m.stopObsoleteClients(networks) for id, routes := range networks { @@ -349,21 +412,24 @@ func (m *DefaultManager) TriggerSelection(networks route.HAMap) { continue } - clientNetworkWatcher := newClientNetworkWatcher( - m.ctx, - m.dnsRouteInterval, - m.wgInterface, - m.statusRecorder, - routes[0], - m.routeRefCounter, - m.allowedIPsRefCounter, - m.dnsServer, - m.peerStore, - m.useNewDNSRoute, - ) + handler := m.activeRoutes[id] + if handler == nil { + log.Warnf("no active handler found for route %s", id) + continue + } + + config := client.WatcherConfig{ + Context: m.ctx, + DNSRouteInterval: m.dnsRouteInterval, + WGInterface: m.wgInterface, + StatusRecorder: m.statusRecorder, + Route: routes[0], + Handler: handler, + } + clientNetworkWatcher := client.NewWatcher(config) m.clientNetworks[id] = clientNetworkWatcher - go clientNetworkWatcher.peersStateAndUpdateWatcher() - clientNetworkWatcher.sendUpdateToClientNetworkWatcher(routesUpdate{routes: routes}) + go clientNetworkWatcher.Start() + clientNetworkWatcher.SendUpdate(client.RoutesUpdate{Routes: routes}) } if err := m.stateManager.UpdateState((*SelectorState)(m.routeSelector)); err != nil { @@ -375,8 +441,7 @@ func (m *DefaultManager) TriggerSelection(networks route.HAMap) { func (m *DefaultManager) stopObsoleteClients(networks route.HAMap) { for id, client := range m.clientNetworks { if _, ok := networks[id]; !ok { - log.Debugf("Stopping client network watcher, %s", id) - client.cancel() + client.Stop() delete(m.clientNetworks, id) } } @@ -389,26 +454,29 @@ func (m *DefaultManager) updateClientNetworks(updateSerial uint64, networks rout for id, routes := range networks { clientNetworkWatcher, found := m.clientNetworks[id] if !found { - clientNetworkWatcher = newClientNetworkWatcher( - m.ctx, - m.dnsRouteInterval, - m.wgInterface, - m.statusRecorder, - routes[0], - m.routeRefCounter, - m.allowedIPsRefCounter, - m.dnsServer, - m.peerStore, - m.useNewDNSRoute, - ) + handler := m.activeRoutes[id] + if handler == nil { + log.Errorf("No active handler found for route %s", id) + continue + } + + config := client.WatcherConfig{ + Context: m.ctx, + DNSRouteInterval: m.dnsRouteInterval, + WGInterface: m.wgInterface, + StatusRecorder: m.statusRecorder, + Route: routes[0], + Handler: handler, + } + clientNetworkWatcher = client.NewWatcher(config) m.clientNetworks[id] = clientNetworkWatcher - go clientNetworkWatcher.peersStateAndUpdateWatcher() + go clientNetworkWatcher.Start() } - update := routesUpdate{ - updateSerial: updateSerial, - routes: routes, + update := client.RoutesUpdate{ + UpdateSerial: updateSerial, + Routes: routes, } - clientNetworkWatcher.sendUpdateToClientNetworkWatcher(update) + clientNetworkWatcher.SendUpdate(update) } } diff --git a/client/internal/routemanager/manager_test.go b/client/internal/routemanager/manager_test.go index 318ef5ae5..680bd813f 100644 --- a/client/internal/routemanager/manager_test.go +++ b/client/internal/routemanager/manager_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/netip" - "runtime" "testing" "github.com/pion/transport/v3/stdnet" @@ -454,8 +453,8 @@ func TestManagerUpdateRoutes(t *testing.T) { } require.Len(t, routeManager.clientNetworks, expectedWatchers, "client networks size should match") - if runtime.GOOS == "linux" && routeManager.serverRouter != nil { - require.Len(t, routeManager.serverRouter.routes, testCase.serverRoutesExpected, "server networks size should match") + if routeManager.serverRouter != nil { + require.Equal(t, testCase.serverRoutesExpected, routeManager.serverRouter.RoutesCount(), "server networks size should match") } }) } diff --git a/client/internal/routemanager/server.go b/client/internal/routemanager/server/server.go similarity index 63% rename from client/internal/routemanager/server.go rename to client/internal/routemanager/server/server.go index 5bacb856c..e674c80cd 100644 --- a/client/internal/routemanager/server.go +++ b/client/internal/routemanager/server/server.go @@ -1,4 +1,4 @@ -package routemanager +package server import ( "context" @@ -14,7 +14,7 @@ import ( "github.com/netbirdio/netbird/route" ) -type serverRouter struct { +type Router struct { mux sync.Mutex ctx context.Context routes map[route.ID]*route.Route @@ -23,8 +23,8 @@ type serverRouter struct { statusRecorder *peer.Status } -func newServerRouter(ctx context.Context, wgInterface iface.WGIface, firewall firewall.Manager, statusRecorder *peer.Status) (*serverRouter, error) { - return &serverRouter{ +func NewRouter(ctx context.Context, wgInterface iface.WGIface, firewall firewall.Manager, statusRecorder *peer.Status) (*Router, error) { + return &Router{ ctx: ctx, routes: make(map[route.ID]*route.Route), firewall: firewall, @@ -33,104 +33,110 @@ func newServerRouter(ctx context.Context, wgInterface iface.WGIface, firewall fi }, nil } -func (m *serverRouter) updateRoutes(routesMap map[route.ID]*route.Route, useNewDNSRoute bool) error { - m.mux.Lock() - defer m.mux.Unlock() +func (r *Router) UpdateRoutes(routesMap map[route.ID]*route.Route, useNewDNSRoute bool) error { + r.mux.Lock() + defer r.mux.Unlock() serverRoutesToRemove := make([]route.ID, 0) - for routeID := range m.routes { + for routeID := range r.routes { update, found := routesMap[routeID] - if !found || !update.Equal(m.routes[routeID]) { + if !found || !update.Equal(r.routes[routeID]) { serverRoutesToRemove = append(serverRoutesToRemove, routeID) } } for _, routeID := range serverRoutesToRemove { - oldRoute := m.routes[routeID] - err := m.removeFromServerNetwork(oldRoute) + oldRoute := r.routes[routeID] + err := r.removeFromServerNetwork(oldRoute) if err != nil { log.Errorf("Unable to remove route id: %s, network %s, from server, got: %v", oldRoute.ID, oldRoute.Network, err) } - delete(m.routes, routeID) + delete(r.routes, routeID) } // If routing is to be disabled, do it after routes have been removed // If routing is to be enabled, do it before adding new routes; addToServerNetwork needs routing to be enabled if len(routesMap) > 0 { - if err := m.firewall.EnableRouting(); err != nil { + if err := r.firewall.EnableRouting(); err != nil { return fmt.Errorf("enable routing: %w", err) } } else { - if err := m.firewall.DisableRouting(); err != nil { + if err := r.firewall.DisableRouting(); err != nil { return fmt.Errorf("disable routing: %w", err) } } for id, newRoute := range routesMap { - _, found := m.routes[id] + _, found := r.routes[id] if found { continue } - err := m.addToServerNetwork(newRoute, useNewDNSRoute) + err := r.addToServerNetwork(newRoute, useNewDNSRoute) if err != nil { log.Errorf("Unable to add route %s from server, got: %v", newRoute.ID, err) continue } - m.routes[id] = newRoute + r.routes[id] = newRoute } return nil } -func (m *serverRouter) removeFromServerNetwork(route *route.Route) error { - if m.ctx.Err() != nil { +func (r *Router) removeFromServerNetwork(route *route.Route) error { + if r.ctx.Err() != nil { log.Infof("Not removing from server network because context is done") - return m.ctx.Err() + return r.ctx.Err() } routerPair := routeToRouterPair(route, false) - if err := m.firewall.RemoveNatRule(routerPair); err != nil { + if err := r.firewall.RemoveNatRule(routerPair); err != nil { return fmt.Errorf("remove routing rules: %w", err) } - delete(m.routes, route.ID) - m.statusRecorder.RemoveLocalPeerStateRoute(route.NetString()) + delete(r.routes, route.ID) + r.statusRecorder.RemoveLocalPeerStateRoute(route.NetString()) return nil } -func (m *serverRouter) addToServerNetwork(route *route.Route, useNewDNSRoute bool) error { - if m.ctx.Err() != nil { +func (r *Router) addToServerNetwork(route *route.Route, useNewDNSRoute bool) error { + if r.ctx.Err() != nil { log.Infof("Not adding to server network because context is done") - return m.ctx.Err() + return r.ctx.Err() } routerPair := routeToRouterPair(route, useNewDNSRoute) - if err := m.firewall.AddNatRule(routerPair); err != nil { + if err := r.firewall.AddNatRule(routerPair); err != nil { return fmt.Errorf("insert routing rules: %w", err) } - m.routes[route.ID] = route - m.statusRecorder.AddLocalPeerStateRoute(route.NetString(), route.GetResourceID()) + r.routes[route.ID] = route + r.statusRecorder.AddLocalPeerStateRoute(route.NetString(), route.GetResourceID()) return nil } -func (m *serverRouter) cleanUp() { - m.mux.Lock() - defer m.mux.Unlock() +func (r *Router) CleanUp() { + r.mux.Lock() + defer r.mux.Unlock() - for _, r := range m.routes { - routerPair := routeToRouterPair(r, false) - if err := m.firewall.RemoveNatRule(routerPair); err != nil { + for _, route := range r.routes { + routerPair := routeToRouterPair(route, false) + if err := r.firewall.RemoveNatRule(routerPair); err != nil { log.Errorf("Failed to remove cleanup route: %v", err) } } - m.statusRecorder.CleanLocalPeerStateRoutes() + r.statusRecorder.CleanLocalPeerStateRoutes() +} + +func (r *Router) RoutesCount() int { + r.mux.Lock() + defer r.mux.Unlock() + return len(r.routes) } func routeToRouterPair(route *route.Route, useNewDNSRoute bool) firewall.RouterPair { diff --git a/client/internal/routemanager/static/route.go b/client/internal/routemanager/static/route.go index 681c192fb..c8b9338e0 100644 --- a/client/internal/routemanager/static/route.go +++ b/client/internal/routemanager/static/route.go @@ -29,13 +29,17 @@ func (r *Route) String() string { } func (r *Route) AddRoute(context.Context) error { - _, err := r.routeRefCounter.Increment(r.route.Network, struct{}{}) - return err + if _, err := r.routeRefCounter.Increment(r.route.Network, struct{}{}); err != nil { + return err + } + return nil } func (r *Route) RemoveRoute() error { - _, err := r.routeRefCounter.Decrement(r.route.Network) - return err + if _, err := r.routeRefCounter.Decrement(r.route.Network); err != nil { + return err + } + return nil } func (r *Route) AddAllowedIPs(peerKey string) error { @@ -51,6 +55,8 @@ func (r *Route) AddAllowedIPs(peerKey string) error { } func (r *Route) RemoveAllowedIPs() error { - _, err := r.allowedIPsRefcounter.Decrement(r.route.Network) - return err + if _, err := r.allowedIPsRefcounter.Decrement(r.route.Network); err != nil { + return err + } + return nil } From 0cd36baf67daf068c6e9194c96a4835a56f68a08 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 3 Jun 2025 13:09:39 +0200 Subject: [PATCH 22/37] [client] Allow the netbird service to log to console (#3916) --- client/cmd/service.go | 2 +- client/cmd/service_installer.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/cmd/service.go b/client/cmd/service.go index 3560088a7..005479306 100644 --- a/client/cmd/service.go +++ b/client/cmd/service.go @@ -30,7 +30,7 @@ func newSVCConfig() *service.Config { return &service.Config{ Name: serviceName, DisplayName: "Netbird", - Description: "A WireGuard-based mesh network that connects your devices into a single private network.", + Description: "Netbird mesh network client", Option: make(service.KeyValue), } } diff --git a/client/cmd/service_installer.go b/client/cmd/service_installer.go index 99a4821b0..c1d6308c6 100644 --- a/client/cmd/service_installer.go +++ b/client/cmd/service_installer.go @@ -39,7 +39,7 @@ var installCmd = &cobra.Command{ svcConfig.Arguments = append(svcConfig.Arguments, "--management-url", managementURL) } - if logFile != "console" { + if logFile != "" { svcConfig.Arguments = append(svcConfig.Arguments, "--log-file", logFile) } From 87148c503f8c236c39492d05bba7681473811d43 Mon Sep 17 00:00:00 2001 From: Pedro Maia Costa <550684+pnmcosta@users.noreply.github.com> Date: Wed, 4 Jun 2025 11:21:31 +0100 Subject: [PATCH 23/37] [management] support account retrieval and creation by private domain (#3825) * [management] sys initiator save user (#3911) * [management] activity events with multiple external account users (#3914) --- management/server/account.go | 60 ++++++++++++------- management/server/account/manager.go | 2 +- management/server/account_test.go | 40 ++++++++++--- management/server/event.go | 43 ++++++------- management/server/mock_server/account_mock.go | 11 ++-- management/server/user.go | 38 ++++++++---- 6 files changed, 121 insertions(+), 73 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index 033ec5fa1..63879802a 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -1730,23 +1730,26 @@ func (am *DefaultAccountManager) GetStore() store.Store { return am.Store } -// Creates account by private domain. -// Expects domain value to be a valid and a private dns domain. -func (am *DefaultAccountManager) CreateAccountByPrivateDomain(ctx context.Context, initiatorId, domain string) (*types.Account, error) { +func (am *DefaultAccountManager) GetOrCreateAccountByPrivateDomain(ctx context.Context, initiatorId, domain string) (*types.Account, bool, error) { cancel := am.Store.AcquireGlobalLock(ctx) defer cancel() - domain = strings.ToLower(domain) - - count, err := am.Store.CountAccountsByPrivateDomain(ctx, domain) - if err != nil { - return nil, err + existingPrimaryAccountID, err := am.Store.GetAccountIDByPrivateDomain(ctx, store.LockingStrengthShare, domain) + if handleNotFound(err) != nil { + return nil, false, err } - if count > 0 { - return nil, status.Errorf(status.InvalidArgument, "account with private domain already exists") + // a primary account already exists for this private domain + if err == nil { + existingAccount, err := am.Store.GetAccount(ctx, existingPrimaryAccountID) + if err != nil { + return nil, false, err + } + + return existingAccount, false, nil } + // create a new account for this private domain // retry twice for new ID clashes for range 2 { accountId := xid.New().String() @@ -1776,7 +1779,7 @@ func (am *DefaultAccountManager) CreateAccountByPrivateDomain(ctx context.Contex Users: users, // @todo check if using the MSP owner id here is ok CreatedBy: initiatorId, - Domain: domain, + Domain: strings.ToLower(domain), DomainCategory: types.PrivateCategory, IsDomainPrimaryAccount: false, Routes: routes, @@ -1795,19 +1798,22 @@ func (am *DefaultAccountManager) CreateAccountByPrivateDomain(ctx context.Contex } if err := newAccount.AddAllGroup(); err != nil { - return nil, status.Errorf(status.Internal, "failed to add all group to new account by private domain") + return nil, false, status.Errorf(status.Internal, "failed to add all group to new account by private domain") } if err := am.Store.SaveAccount(ctx, newAccount); err != nil { - log.WithContext(ctx).Errorf("failed to save new account %s by private domain: %v", newAccount.Id, err) - return nil, err + log.WithContext(ctx).WithFields(log.Fields{ + "accountId": newAccount.Id, + "domain": domain, + }).Errorf("failed to create new account: %v", err) + return nil, false, err } am.StoreEvent(ctx, initiatorId, newAccount.Id, accountId, activity.AccountCreated, nil) - return newAccount, nil + return newAccount, true, nil } - return nil, status.Errorf(status.Internal, "failed to create new account by private domain") + return nil, false, status.Errorf(status.Internal, "failed to get or create new account by private domain") } func (am *DefaultAccountManager) UpdateToPrimaryAccount(ctx context.Context, accountId string) (*types.Account, error) { @@ -1820,21 +1826,29 @@ func (am *DefaultAccountManager) UpdateToPrimaryAccount(ctx context.Context, acc return account, nil } - // additional check to ensure there is only one account for this domain at the time of update - count, err := am.Store.CountAccountsByPrivateDomain(ctx, account.Domain) - if err != nil { + existingPrimaryAccountID, err := am.Store.GetAccountIDByPrivateDomain(ctx, store.LockingStrengthShare, account.Domain) + + // error is not a not found error + if handleNotFound(err) != nil { return nil, err } - if count > 1 { - return nil, status.Errorf(status.Internal, "more than one account exists with the same private domain") + // a primary account already exists for this private domain + if err == nil { + log.WithContext(ctx).WithFields(log.Fields{ + "accountId": accountId, + "existingAccountId": existingPrimaryAccountID, + }).Errorf("cannot update account to primary, another account already exists as primary for the same domain") + return nil, status.Errorf(status.Internal, "cannot update account to primary") } account.IsDomainPrimaryAccount = true if err := am.Store.SaveAccount(ctx, account); err != nil { - log.WithContext(ctx).Errorf("failed to update primary account %s by private domain: %v", account.Id, err) - return nil, status.Errorf(status.Internal, "failed to update primary account %s by private domain", account.Id) + log.WithContext(ctx).WithFields(log.Fields{ + "accountId": accountId, + }).Errorf("failed to update account to primary: %v", err) + return nil, status.Errorf(status.Internal, "failed to update account to primary") } return account, nil diff --git a/management/server/account/manager.go b/management/server/account/manager.go index 9bc4f9605..030bd94ef 100644 --- a/management/server/account/manager.go +++ b/management/server/account/manager.go @@ -113,7 +113,7 @@ type Manager interface { BuildUserInfosForAccount(ctx context.Context, accountID, initiatorUserID string, accountUsers []*types.User) (map[string]*types.UserInfo, error) SyncUserJWTGroups(ctx context.Context, userAuth nbcontext.UserAuth) error GetStore() store.Store - CreateAccountByPrivateDomain(ctx context.Context, initiatorId, domain string) (*types.Account, error) + GetOrCreateAccountByPrivateDomain(ctx context.Context, initiatorId, domain string) (*types.Account, bool, error) UpdateToPrimaryAccount(ctx context.Context, accountId string) (*types.Account, error) GetOwnerInfo(ctx context.Context, accountId string) (*types.UserInfo, error) GetCurrentUserInfo(ctx context.Context, userAuth nbcontext.UserAuth) (*users.UserInfoWithPermissions, error) diff --git a/management/server/account_test.go b/management/server/account_test.go index c5583d226..5ada28ca3 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -14,7 +14,6 @@ import ( "time" "github.com/golang/mock/gomock" - "github.com/netbirdio/netbird/management/server/idp" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,6 +24,7 @@ import ( "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/cache" nbcontext "github.com/netbirdio/netbird/management/server/context" + "github.com/netbirdio/netbird/management/server/idp" "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types" routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types" @@ -3198,7 +3198,7 @@ func BenchmarkLoginPeer_NewPeer(b *testing.B) { } } -func Test_CreateAccountByPrivateDomain(t *testing.T) { +func Test_GetCreateAccountByPrivateDomain(t *testing.T) { manager, err := createManager(t) if err != nil { t.Fatal(err) @@ -3209,9 +3209,10 @@ func Test_CreateAccountByPrivateDomain(t *testing.T) { initiatorId := "test-user" domain := "example.com" - account, err := manager.CreateAccountByPrivateDomain(ctx, initiatorId, domain) + account, created, err := manager.GetOrCreateAccountByPrivateDomain(ctx, initiatorId, domain) assert.NoError(t, err) + assert.True(t, created) assert.False(t, account.IsDomainPrimaryAccount) assert.Equal(t, domain, account.Domain) assert.Equal(t, types.PrivateCategory, account.DomainCategory) @@ -3220,9 +3221,25 @@ func Test_CreateAccountByPrivateDomain(t *testing.T) { assert.Equal(t, 0, len(account.Users)) assert.Equal(t, 0, len(account.SetupKeys)) - // retry should fail - _, err = manager.CreateAccountByPrivateDomain(ctx, initiatorId, domain) - assert.Error(t, err) + // should return a new account because the previous one is not primary + account2, created2, err := manager.GetOrCreateAccountByPrivateDomain(ctx, initiatorId, domain) + assert.NoError(t, err) + + assert.True(t, created2) + assert.False(t, account2.IsDomainPrimaryAccount) + assert.Equal(t, domain, account2.Domain) + assert.Equal(t, types.PrivateCategory, account2.DomainCategory) + assert.Equal(t, initiatorId, account2.CreatedBy) + assert.Equal(t, 1, len(account2.Groups)) + assert.Equal(t, 0, len(account2.Users)) + assert.Equal(t, 0, len(account2.SetupKeys)) + + account, err = manager.UpdateToPrimaryAccount(ctx, account.Id) + assert.NoError(t, err) + assert.True(t, account.IsDomainPrimaryAccount) + + _, err = manager.UpdateToPrimaryAccount(ctx, account2.Id) + assert.Error(t, err, "should not be able to update a second account to primary") } func Test_UpdateToPrimaryAccount(t *testing.T) { @@ -3236,14 +3253,21 @@ func Test_UpdateToPrimaryAccount(t *testing.T) { initiatorId := "test-user" domain := "example.com" - account, err := manager.CreateAccountByPrivateDomain(ctx, initiatorId, domain) + account, created, err := manager.GetOrCreateAccountByPrivateDomain(ctx, initiatorId, domain) assert.NoError(t, err) + assert.True(t, created) assert.False(t, account.IsDomainPrimaryAccount) + assert.Equal(t, domain, account.Domain) - // retry should fail account, err = manager.UpdateToPrimaryAccount(ctx, account.Id) assert.NoError(t, err) assert.True(t, account.IsDomainPrimaryAccount) + + account2, created2, err := manager.GetOrCreateAccountByPrivateDomain(ctx, initiatorId, domain) + assert.NoError(t, err) + assert.False(t, created2) + assert.True(t, account.IsDomainPrimaryAccount) + assert.Equal(t, account.Id, account2.Id) } func TestDefaultAccountManager_IsCacheCold(t *testing.T) { diff --git a/management/server/event.go b/management/server/event.go index 2952edc8c..d94714e2c 100644 --- a/management/server/event.go +++ b/management/server/event.go @@ -143,11 +143,10 @@ func (am *DefaultAccountManager) getEventsUserInfo(ctx context.Context, events [ return eventUserInfos, nil } - return am.getEventsExternalUserInfo(ctx, externalUserIds, eventUserInfos, userId) + return am.getEventsExternalUserInfo(ctx, externalUserIds, eventUserInfos) } -func (am *DefaultAccountManager) getEventsExternalUserInfo(ctx context.Context, externalUserIds []string, eventUserInfos map[string]eventUserInfo, userId string) (map[string]eventUserInfo, error) { - externalAccountId := "" +func (am *DefaultAccountManager) getEventsExternalUserInfo(ctx context.Context, externalUserIds []string, eventUserInfos map[string]eventUserInfo) (map[string]eventUserInfo, error) { fetched := make(map[string]struct{}) externalUsers := []*types.User{} for _, id := range externalUserIds { @@ -161,34 +160,30 @@ func (am *DefaultAccountManager) getEventsExternalUserInfo(ctx context.Context, continue } - if externalAccountId != "" && externalAccountId != externalUser.AccountID { - return nil, fmt.Errorf("multiple external user accounts in events") - } - - if externalAccountId == "" { - externalAccountId = externalUser.AccountID - } - fetched[id] = struct{}{} externalUsers = append(externalUsers, externalUser) } - // if we couldn't determine an account, return what we have - if externalAccountId == "" { - log.WithContext(ctx).Warnf("failed to determine external user account from users: %v", externalUserIds) - return eventUserInfos, nil + usersByExternalAccount := map[string][]*types.User{} + for _, u := range externalUsers { + if _, ok := usersByExternalAccount[u.AccountID]; !ok { + usersByExternalAccount[u.AccountID] = make([]*types.User, 0) + } + usersByExternalAccount[u.AccountID] = append(usersByExternalAccount[u.AccountID], u) } - externalUserInfos, err := am.BuildUserInfosForAccount(ctx, externalAccountId, userId, externalUsers) - if err != nil { - return nil, err - } + for externalAccountId, externalUsers := range usersByExternalAccount { + externalUserInfos, err := am.BuildUserInfosForAccount(ctx, externalAccountId, "", externalUsers) + if err != nil { + return nil, err + } - for i, k := range externalUserInfos { - eventUserInfos[i] = eventUserInfo{ - email: k.Email, - name: k.Name, - accountId: externalAccountId, + for i, k := range externalUserInfos { + eventUserInfos[i] = eventUserInfo{ + email: k.Email, + name: k.Name, + accountId: externalAccountId, + } } } diff --git a/management/server/mock_server/account_mock.go b/management/server/mock_server/account_mock.go index 0dd3f927e..ed47d3914 100644 --- a/management/server/mock_server/account_mock.go +++ b/management/server/mock_server/account_mock.go @@ -113,11 +113,12 @@ type MockAccountManager struct { DeleteSetupKeyFunc func(ctx context.Context, accountID, userID, keyID string) error BuildUserInfosForAccountFunc func(ctx context.Context, accountID, initiatorUserID string, accountUsers []*types.User) (map[string]*types.UserInfo, error) GetStoreFunc func() store.Store - CreateAccountByPrivateDomainFunc func(ctx context.Context, initiatorId, domain string) (*types.Account, error) UpdateToPrimaryAccountFunc func(ctx context.Context, accountId string) (*types.Account, error) GetOwnerInfoFunc func(ctx context.Context, accountID string) (*types.UserInfo, error) GetCurrentUserInfoFunc func(ctx context.Context, userAuth nbcontext.UserAuth) (*users.UserInfoWithPermissions, error) GetAccountMetaFunc func(ctx context.Context, accountID, userID string) (*types.AccountMeta, error) + + GetOrCreateAccountByPrivateDomainFunc func(ctx context.Context, initiatorId, domain string) (*types.Account, bool, error) } func (am *MockAccountManager) UpdateAccountPeers(ctx context.Context, accountID string) { @@ -862,11 +863,11 @@ func (am *MockAccountManager) GetStore() store.Store { return nil } -func (am *MockAccountManager) CreateAccountByPrivateDomain(ctx context.Context, initiatorId, domain string) (*types.Account, error) { - if am.CreateAccountByPrivateDomainFunc != nil { - return am.CreateAccountByPrivateDomainFunc(ctx, initiatorId, domain) +func (am *MockAccountManager) GetOrCreateAccountByPrivateDomain(ctx context.Context, initiatorId, domain string) (*types.Account, bool, error) { + if am.GetOrCreateAccountByPrivateDomainFunc != nil { + return am.GetOrCreateAccountByPrivateDomainFunc(ctx, initiatorId, domain) } - return nil, status.Errorf(codes.Unimplemented, "method CreateAccountByPrivateDomain is not implemented") + return nil, false, status.Errorf(codes.Unimplemented, "method GetOrCreateAccountByPrivateDomainFunc is not implemented") } func (am *MockAccountManager) UpdateToPrimaryAccount(ctx context.Context, accountId string) (*types.Account, error) { diff --git a/management/server/user.go b/management/server/user.go index 5c162c50b..6d780cda3 100644 --- a/management/server/user.go +++ b/management/server/user.go @@ -531,9 +531,13 @@ func (am *DefaultAccountManager) SaveOrAddUsers(ctx context.Context, accountID, groupsMap[group.ID] = group } - initiatorUser, err := am.Store.GetUserByUserID(ctx, store.LockingStrengthShare, initiatorUserID) - if err != nil { - return nil, err + var initiatorUser *types.User + if initiatorUserID != activity.SystemInitiator { + result, err := am.Store.GetUserByUserID(ctx, store.LockingStrengthShare, initiatorUserID) + if err != nil { + return nil, err + } + initiatorUser = result } err = am.Store.ExecuteInTransaction(ctx, func(transaction store.Store) error { @@ -543,7 +547,7 @@ func (am *DefaultAccountManager) SaveOrAddUsers(ctx context.Context, accountID, } userHadPeers, updatedUser, userPeersToExpire, userEvents, err := am.processUserUpdate( - ctx, transaction, groupsMap, accountID, initiatorUser, update, addIfNotExists, settings, + ctx, transaction, groupsMap, accountID, initiatorUserID, initiatorUser, update, addIfNotExists, settings, ) if err != nil { return fmt.Errorf("failed to process user update: %w", err) @@ -629,7 +633,7 @@ func (am *DefaultAccountManager) prepareUserUpdateEvents(ctx context.Context, ac } func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transaction store.Store, groupsMap map[string]*types.Group, - accountID string, initiatorUser, update *types.User, addIfNotExists bool, settings *types.Settings) (bool, *types.User, []*nbpeer.Peer, []func(), error) { + accountID, initiatorUserId string, initiatorUser, update *types.User, addIfNotExists bool, settings *types.Settings) (bool, *types.User, []*nbpeer.Peer, []func(), error) { if update == nil { return false, nil, nil, nil, status.Errorf(status.InvalidArgument, "provided user update is nil") @@ -653,10 +657,12 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact updatedUser.Issued = update.Issued updatedUser.IntegrationReference = update.IntegrationReference - transferredOwnerRole, err := handleOwnerRoleTransfer(ctx, transaction, initiatorUser, update) + var transferredOwnerRole bool + result, err := handleOwnerRoleTransfer(ctx, transaction, initiatorUser, update) if err != nil { return false, nil, nil, nil, err } + transferredOwnerRole = result userPeers, err := transaction.GetUserPeers(ctx, store.LockingStrengthUpdate, updatedUser.AccountID, update.Id) if err != nil { @@ -682,7 +688,7 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact } updateAccountPeers := len(userPeers) > 0 - userEventsToAdd := am.prepareUserUpdateEvents(ctx, updatedUser.AccountID, initiatorUser.Id, oldUser, updatedUser, transferredOwnerRole) + userEventsToAdd := am.prepareUserUpdateEvents(ctx, updatedUser.AccountID, initiatorUserId, oldUser, updatedUser, transferredOwnerRole) return updateAccountPeers, updatedUser, peersToExpire, userEventsToAdd, nil } @@ -709,7 +715,7 @@ func getUserOrCreateIfNotExists(ctx context.Context, transaction store.Store, ac } func handleOwnerRoleTransfer(ctx context.Context, transaction store.Store, initiatorUser, update *types.User) (bool, error) { - if initiatorUser.Role == types.UserRoleOwner && initiatorUser.Id != update.Id && update.Role == types.UserRoleOwner { + if initiatorUser != nil && initiatorUser.Role == types.UserRoleOwner && initiatorUser.Id != update.Id && update.Role == types.UserRoleOwner { newInitiatorUser := initiatorUser.Copy() newInitiatorUser.Role = types.UserRoleAdmin @@ -737,6 +743,10 @@ func (am *DefaultAccountManager) getUserInfo(ctx context.Context, user *types.Us // validateUserUpdate validates the update operation for a user. func validateUserUpdate(groupsMap map[string]*types.Group, initiatorUser, oldUser, update *types.User) error { + if initiatorUser == nil { + return nil + } + // @todo double check these if initiatorUser.HasAdminPower() && initiatorUser.Id == update.Id && oldUser.Blocked != update.Blocked { return status.Errorf(status.PermissionDenied, "admins can't block or unblock themselves") @@ -818,9 +828,13 @@ func (am *DefaultAccountManager) GetUsersFromAccount(ctx context.Context, accoun return nil, status.NewPermissionValidationError(err) } - user, err := am.Store.GetUserByUserID(ctx, store.LockingStrengthShare, initiatorUserID) - if err != nil { - return nil, fmt.Errorf("failed to get user: %w", err) + var user *types.User + if initiatorUserID != activity.SystemInitiator { + result, err := am.Store.GetUserByUserID(ctx, store.LockingStrengthShare, initiatorUserID) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + user = result } accountUsers := []*types.User{} @@ -830,7 +844,7 @@ func (am *DefaultAccountManager) GetUsersFromAccount(ctx context.Context, accoun if err != nil { return nil, err } - case user.AccountID == accountID: + case user != nil && user.AccountID == accountID: accountUsers = append(accountUsers, user) default: return map[string]*types.UserInfo{}, nil From ea4d13e96d79665bc0e4b489af5a0d492e104515 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:28:58 +0200 Subject: [PATCH 24/37] [client] Use platform-native routing APIs for freeBSD, macOS and Windows --- client/cmd/trace.go | 2 +- .../firewall/iptables/manager_linux_test.go | 39 +- .../firewall/nftables/manager_linux_test.go | 31 +- .../firewall/uspfilter/forwarder/forwarder.go | 12 +- client/firewall/uspfilter/localip.go | 48 +- client/firewall/uspfilter/localip_test.go | 49 +- client/firewall/uspfilter/tracer_test.go | 7 +- client/firewall/uspfilter/uspfilter.go | 6 - .../uspfilter/uspfilter_bench_test.go | 74 +- .../uspfilter/uspfilter_filter_test.go | 22 +- client/firewall/uspfilter/uspfilter_test.go | 20 +- client/iface/bind/udp_mux_universal.go | 2 +- client/iface/device/device_filter.go | 4 - client/iface/device/device_netstack.go | 6 +- client/iface/device/wg_link_freebsd.go | 10 +- client/iface/iface.go | 1 - client/iface/mocks/filter.go | 13 - client/iface/netstack/tun.go | 22 +- client/iface/wgaddr/address.go | 15 +- client/internal/acl/manager_test.go | 17 +- client/internal/dns.go | 34 +- client/internal/dns/server_test.go | 12 +- client/internal/dns/service_memory.go | 8 +- client/internal/dns/service_memory_test.go | 33 - client/internal/dns/upstream_android.go | 5 +- client/internal/dns/upstream_general.go | 6 +- client/internal/dns/upstream_ios.go | 20 +- client/internal/dns/upstream_test.go | 4 +- client/internal/engine.go | 17 +- client/internal/engine_test.go | 7 +- .../internal/netflow/conntrack/conntrack.go | 12 +- client/internal/netflow/logger/logger.go | 13 +- client/internal/netflow/logger/logger_test.go | 4 +- client/internal/netflow/manager.go | 8 +- client/internal/netflow/manager_test.go | 12 +- .../routemanager/dnsinterceptor/handler.go | 2 +- client/internal/routemanager/manager.go | 9 +- client/internal/routemanager/manager_test.go | 30 +- .../routemanager/sysctl/sysctl_linux.go | 9 +- .../routemanager/systemops/systemops.go | 30 +- .../systemops/systemops_bsd_test.go | 78 ++- .../systemops/systemops_generic.go | 121 +--- .../systemops/systemops_generic_test.go | 635 ++++++++++-------- .../routemanager/systemops/systemops_linux.go | 14 +- .../systemops/systemops_linux_test.go | 7 - .../systemops/systemops_nonlinux.go | 6 + .../routemanager/systemops/systemops_test.go | 268 ++++++++ .../routemanager/systemops/systemops_unix.go | 204 ++++-- .../systemops/systemops_windows.go | 244 +++++-- .../systemops/systemops_windows_test.go | 64 +- client/server/trace.go | 169 +++-- util/net/net.go | 19 +- util/net/net_test.go | 94 +++ 53 files changed, 1552 insertions(+), 1046 deletions(-) delete mode 100644 client/internal/dns/service_memory_test.go create mode 100644 client/internal/routemanager/systemops/systemops_test.go create mode 100644 util/net/net_test.go diff --git a/client/cmd/trace.go b/client/cmd/trace.go index b2ff1f1b5..abb73b646 100644 --- a/client/cmd/trace.go +++ b/client/cmd/trace.go @@ -17,7 +17,7 @@ var traceCmd = &cobra.Command{ Example: ` netbird debug trace in 192.168.1.10 10.10.0.2 -p tcp --sport 12345 --dport 443 --syn --ack netbird debug trace out 10.10.0.1 8.8.8.8 -p udp --dport 53 - netbird debug trace in 10.10.0.2 10.10.0.1 -p icmp --type 8 --code 0 + netbird debug trace in 10.10.0.2 10.10.0.1 -p icmp --icmp-type 8 --icmp-code 0 netbird debug trace in 100.64.1.1 self -p tcp --dport 80`, Args: cobra.ExactArgs(3), RunE: tracePacket, diff --git a/client/firewall/iptables/manager_linux_test.go b/client/firewall/iptables/manager_linux_test.go index af9f5dd23..30f391a6d 100644 --- a/client/firewall/iptables/manager_linux_test.go +++ b/client/firewall/iptables/manager_linux_test.go @@ -2,7 +2,7 @@ package iptables import ( "fmt" - "net" + "net/netip" "testing" "time" @@ -19,11 +19,8 @@ var ifaceMock = &iFaceMock{ }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("10.20.0.1"), - Network: &net.IPNet{ - IP: net.ParseIP("10.20.0.0"), - Mask: net.IPv4Mask(255, 255, 255, 0), - }, + IP: netip.MustParseAddr("10.20.0.1"), + Network: netip.MustParsePrefix("10.20.0.0/24"), } }, } @@ -70,12 +67,12 @@ func TestIptablesManager(t *testing.T) { var rule2 []fw.Rule t.Run("add second rule", func(t *testing.T) { - ip := net.ParseIP("10.20.0.3") + ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{ IsRange: true, Values: []uint16{8043, 8046}, } - rule2, err = manager.AddPeerFiltering(nil, ip, "tcp", port, nil, fw.ActionAccept, "") + rule2, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", port, nil, fw.ActionAccept, "") require.NoError(t, err, "failed to add rule") for _, r := range rule2 { @@ -95,9 +92,9 @@ func TestIptablesManager(t *testing.T) { t.Run("reset check", func(t *testing.T) { // add second rule - ip := net.ParseIP("10.20.0.3") + ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{Values: []uint16{5353}} - _, err = manager.AddPeerFiltering(nil, ip, "udp", nil, port, fw.ActionAccept, "") + _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "udp", nil, port, fw.ActionAccept, "") require.NoError(t, err, "failed to add rule") err = manager.Close(nil) @@ -119,11 +116,8 @@ func TestIptablesManagerIPSet(t *testing.T) { }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("10.20.0.1"), - Network: &net.IPNet{ - IP: net.ParseIP("10.20.0.0"), - Mask: net.IPv4Mask(255, 255, 255, 0), - }, + IP: netip.MustParseAddr("10.20.0.1"), + Network: netip.MustParsePrefix("10.20.0.0/24"), } }, } @@ -144,11 +138,11 @@ func TestIptablesManagerIPSet(t *testing.T) { var rule2 []fw.Rule t.Run("add second rule", func(t *testing.T) { - ip := net.ParseIP("10.20.0.3") + ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{ Values: []uint16{443}, } - rule2, err = manager.AddPeerFiltering(nil, ip, "tcp", port, nil, fw.ActionAccept, "default") + rule2, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", port, nil, fw.ActionAccept, "default") for _, r := range rule2 { require.NoError(t, err, "failed to add rule") require.Equal(t, r.(*Rule).ipsetName, "default-sport", "ipset name must be set") @@ -186,11 +180,8 @@ func TestIptablesCreatePerformance(t *testing.T) { }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("10.20.0.1"), - Network: &net.IPNet{ - IP: net.ParseIP("10.20.0.0"), - Mask: net.IPv4Mask(255, 255, 255, 0), - }, + IP: netip.MustParseAddr("10.20.0.1"), + Network: netip.MustParsePrefix("10.20.0.0/24"), } }, } @@ -212,11 +203,11 @@ func TestIptablesCreatePerformance(t *testing.T) { require.NoError(t, err) - ip := net.ParseIP("10.20.0.100") + ip := netip.MustParseAddr("10.20.0.100") start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip, "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "") require.NoError(t, err, "failed to add rule") } diff --git a/client/firewall/nftables/manager_linux_test.go b/client/firewall/nftables/manager_linux_test.go index 602a6b8dc..1dd3e9183 100644 --- a/client/firewall/nftables/manager_linux_test.go +++ b/client/firewall/nftables/manager_linux_test.go @@ -3,7 +3,6 @@ package nftables import ( "bytes" "fmt" - "net" "net/netip" "os/exec" "testing" @@ -25,11 +24,8 @@ var ifaceMock = &iFaceMock{ }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("100.96.0.1"), - Network: &net.IPNet{ - IP: net.ParseIP("100.96.0.0"), - Mask: net.IPv4Mask(255, 255, 255, 0), - }, + IP: netip.MustParseAddr("100.96.0.1"), + Network: netip.MustParsePrefix("100.96.0.0/16"), } }, } @@ -70,11 +66,11 @@ func TestNftablesManager(t *testing.T) { time.Sleep(time.Second) }() - ip := net.ParseIP("100.96.0.1") + ip := netip.MustParseAddr("100.96.0.1").Unmap() testClient := &nftables.Conn{} - rule, err := manager.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop, "") + rule, err := manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop, "") require.NoError(t, err, "failed to add rule") err = manager.Flush() @@ -109,8 +105,6 @@ func TestNftablesManager(t *testing.T) { } compareExprsIgnoringCounters(t, rules[0].Exprs, expectedExprs1) - ipToAdd, _ := netip.AddrFromSlice(ip) - add := ipToAdd.Unmap() expectedExprs2 := []expr.Any{ &expr.Payload{ DestRegister: 1, @@ -132,7 +126,7 @@ func TestNftablesManager(t *testing.T) { &expr.Cmp{ Op: expr.CmpOpEq, Register: 1, - Data: add.AsSlice(), + Data: ip.AsSlice(), }, &expr.Payload{ DestRegister: 1, @@ -173,11 +167,8 @@ func TestNFtablesCreatePerformance(t *testing.T) { }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("100.96.0.1"), - Network: &net.IPNet{ - IP: net.ParseIP("100.96.0.0"), - Mask: net.IPv4Mask(255, 255, 255, 0), - }, + IP: netip.MustParseAddr("100.96.0.1"), + Network: netip.MustParsePrefix("100.96.0.0/16"), } }, } @@ -197,11 +188,11 @@ func TestNFtablesCreatePerformance(t *testing.T) { time.Sleep(time.Second) }() - ip := net.ParseIP("10.20.0.100") + ip := netip.MustParseAddr("10.20.0.100") start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip, "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "") require.NoError(t, err, "failed to add rule") if i%100 == 0 { @@ -282,8 +273,8 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) { verifyIptablesOutput(t, stdout, stderr) }) - ip := net.ParseIP("100.96.0.1") - _, err = manager.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + ip := netip.MustParseAddr("100.96.0.1") + _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") require.NoError(t, err, "failed to add peer filtering rule") _, err = manager.AddRouteFiltering( diff --git a/client/firewall/uspfilter/forwarder/forwarder.go b/client/firewall/uspfilter/forwarder/forwarder.go index 2ae983f6e..42a3e0800 100644 --- a/client/firewall/uspfilter/forwarder/forwarder.go +++ b/client/firewall/uspfilter/forwarder/forwarder.go @@ -41,7 +41,7 @@ type Forwarder struct { udpForwarder *udpForwarder ctx context.Context cancel context.CancelFunc - ip net.IP + ip tcpip.Address netstack bool } @@ -71,12 +71,11 @@ func New(iface common.IFaceMapper, logger *nblog.Logger, flowLogger nftypes.Flow return nil, fmt.Errorf("failed to create NIC: %v", err) } - ones, _ := iface.Address().Network.Mask.Size() protoAddr := tcpip.ProtocolAddress{ Protocol: ipv4.ProtocolNumber, AddressWithPrefix: tcpip.AddressWithPrefix{ - Address: tcpip.AddrFromSlice(iface.Address().IP.To4()), - PrefixLen: ones, + Address: tcpip.AddrFromSlice(iface.Address().IP.AsSlice()), + PrefixLen: iface.Address().Network.Bits(), }, } @@ -116,7 +115,7 @@ func New(iface common.IFaceMapper, logger *nblog.Logger, flowLogger nftypes.Flow ctx: ctx, cancel: cancel, netstack: netstack, - ip: iface.Address().IP, + ip: tcpip.AddrFromSlice(iface.Address().IP.AsSlice()), } receiveWindow := defaultReceiveWindow @@ -167,7 +166,7 @@ func (f *Forwarder) Stop() { } func (f *Forwarder) determineDialAddr(addr tcpip.Address) net.IP { - if f.netstack && f.ip.Equal(addr.AsSlice()) { + if f.netstack && f.ip.Equal(addr) { return net.IPv4(127, 0, 0, 1) } return addr.AsSlice() @@ -179,7 +178,6 @@ func (f *Forwarder) RegisterRuleID(srcIP, dstIP netip.Addr, srcPort, dstPort uin } func (f *Forwarder) getRuleID(srcIP, dstIP netip.Addr, srcPort, dstPort uint16) ([]byte, bool) { - if value, ok := f.ruleIdMap.Load(buildKey(srcIP, dstIP, srcPort, dstPort)); ok { return value.([]byte), true } else if value, ok := f.ruleIdMap.Load(buildKey(dstIP, srcIP, dstPort, srcPort)); ok { diff --git a/client/firewall/uspfilter/localip.go b/client/firewall/uspfilter/localip.go index f093f3429..7f6b52c71 100644 --- a/client/firewall/uspfilter/localip.go +++ b/client/firewall/uspfilter/localip.go @@ -45,24 +45,26 @@ func (m *localIPManager) setBitmapBit(ip net.IP) { m.ipv4Bitmap[high].bitmap[index] |= 1 << bit } -func (m *localIPManager) setBitInBitmap(ip net.IP, bitmap *[256]*ipv4LowBitmap, ipv4Set map[string]struct{}, ipv4Addresses *[]string) { - if ipv4 := ip.To4(); ipv4 != nil { - high := uint16(ipv4[0]) - low := (uint16(ipv4[1]) << 8) | (uint16(ipv4[2]) << 4) | uint16(ipv4[3]) +func (m *localIPManager) setBitInBitmap(ip netip.Addr, bitmap *[256]*ipv4LowBitmap, ipv4Set map[netip.Addr]struct{}, ipv4Addresses *[]netip.Addr) { + if !ip.Is4() { + return + } + ipv4 := ip.AsSlice() - if bitmap[high] == nil { - bitmap[high] = &ipv4LowBitmap{} - } + high := uint16(ipv4[0]) + low := (uint16(ipv4[1]) << 8) | (uint16(ipv4[2]) << 4) | uint16(ipv4[3]) - index := low / 32 - bit := low % 32 - bitmap[high].bitmap[index] |= 1 << bit + if bitmap[high] == nil { + bitmap[high] = &ipv4LowBitmap{} + } - ipStr := ipv4.String() - if _, exists := ipv4Set[ipStr]; !exists { - ipv4Set[ipStr] = struct{}{} - *ipv4Addresses = append(*ipv4Addresses, ipStr) - } + index := low / 32 + bit := low % 32 + bitmap[high].bitmap[index] |= 1 << bit + + if _, exists := ipv4Set[ip]; !exists { + ipv4Set[ip] = struct{}{} + *ipv4Addresses = append(*ipv4Addresses, ip) } } @@ -79,12 +81,12 @@ func (m *localIPManager) checkBitmapBit(ip []byte) bool { return (m.ipv4Bitmap[high].bitmap[index] & (1 << bit)) != 0 } -func (m *localIPManager) processIP(ip net.IP, bitmap *[256]*ipv4LowBitmap, ipv4Set map[string]struct{}, ipv4Addresses *[]string) error { +func (m *localIPManager) processIP(ip netip.Addr, bitmap *[256]*ipv4LowBitmap, ipv4Set map[netip.Addr]struct{}, ipv4Addresses *[]netip.Addr) error { m.setBitInBitmap(ip, bitmap, ipv4Set, ipv4Addresses) return nil } -func (m *localIPManager) processInterface(iface net.Interface, bitmap *[256]*ipv4LowBitmap, ipv4Set map[string]struct{}, ipv4Addresses *[]string) { +func (m *localIPManager) processInterface(iface net.Interface, bitmap *[256]*ipv4LowBitmap, ipv4Set map[netip.Addr]struct{}, ipv4Addresses *[]netip.Addr) { addrs, err := iface.Addrs() if err != nil { log.Debugf("get addresses for interface %s failed: %v", iface.Name, err) @@ -102,7 +104,13 @@ func (m *localIPManager) processInterface(iface net.Interface, bitmap *[256]*ipv continue } - if err := m.processIP(ip, bitmap, ipv4Set, ipv4Addresses); err != nil { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + log.Warnf("invalid IP address %s in interface %s", ip.String(), iface.Name) + continue + } + + if err := m.processIP(addr.Unmap(), bitmap, ipv4Set, ipv4Addresses); err != nil { log.Debugf("process IP failed: %v", err) } } @@ -116,8 +124,8 @@ func (m *localIPManager) UpdateLocalIPs(iface common.IFaceMapper) (err error) { }() var newIPv4Bitmap [256]*ipv4LowBitmap - ipv4Set := make(map[string]struct{}) - var ipv4Addresses []string + ipv4Set := make(map[netip.Addr]struct{}) + var ipv4Addresses []netip.Addr // 127.0.0.0/8 newIPv4Bitmap[127] = &ipv4LowBitmap{} diff --git a/client/firewall/uspfilter/localip_test.go b/client/firewall/uspfilter/localip_test.go index 0104c9603..45ac912cd 100644 --- a/client/firewall/uspfilter/localip_test.go +++ b/client/firewall/uspfilter/localip_test.go @@ -20,11 +20,8 @@ func TestLocalIPManager(t *testing.T) { { name: "Localhost range", setupAddr: wgaddr.Address{ - IP: net.ParseIP("192.168.1.1"), - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.0"), - Mask: net.CIDRMask(24, 32), - }, + IP: netip.MustParseAddr("192.168.1.1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("127.0.0.2"), expected: true, @@ -32,11 +29,8 @@ func TestLocalIPManager(t *testing.T) { { name: "Localhost standard address", setupAddr: wgaddr.Address{ - IP: net.ParseIP("192.168.1.1"), - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.0"), - Mask: net.CIDRMask(24, 32), - }, + IP: netip.MustParseAddr("192.168.1.1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("127.0.0.1"), expected: true, @@ -44,11 +38,8 @@ func TestLocalIPManager(t *testing.T) { { name: "Localhost range edge", setupAddr: wgaddr.Address{ - IP: net.ParseIP("192.168.1.1"), - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.0"), - Mask: net.CIDRMask(24, 32), - }, + IP: netip.MustParseAddr("192.168.1.1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("127.255.255.255"), expected: true, @@ -56,11 +47,8 @@ func TestLocalIPManager(t *testing.T) { { name: "Local IP matches", setupAddr: wgaddr.Address{ - IP: net.ParseIP("192.168.1.1"), - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.0"), - Mask: net.CIDRMask(24, 32), - }, + IP: netip.MustParseAddr("192.168.1.1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("192.168.1.1"), expected: true, @@ -68,11 +56,8 @@ func TestLocalIPManager(t *testing.T) { { name: "Local IP doesn't match", setupAddr: wgaddr.Address{ - IP: net.ParseIP("192.168.1.1"), - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.0"), - Mask: net.CIDRMask(24, 32), - }, + IP: netip.MustParseAddr("192.168.1.1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("192.168.1.2"), expected: false, @@ -80,11 +65,8 @@ func TestLocalIPManager(t *testing.T) { { name: "Local IP doesn't match - addresses 32 apart", setupAddr: wgaddr.Address{ - IP: net.ParseIP("192.168.1.1"), - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.0"), - Mask: net.CIDRMask(24, 32), - }, + IP: netip.MustParseAddr("192.168.1.1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("192.168.1.33"), expected: false, @@ -92,11 +74,8 @@ func TestLocalIPManager(t *testing.T) { { name: "IPv6 address", setupAddr: wgaddr.Address{ - IP: net.ParseIP("fe80::1"), - Network: &net.IPNet{ - IP: net.ParseIP("fe80::"), - Mask: net.CIDRMask(64, 128), - }, + IP: netip.MustParseAddr("fe80::1"), + Network: netip.MustParsePrefix("192.168.1.0/24"), }, testIP: netip.MustParseAddr("fe80::1"), expected: false, diff --git a/client/firewall/uspfilter/tracer_test.go b/client/firewall/uspfilter/tracer_test.go index bd87879a5..46c115787 100644 --- a/client/firewall/uspfilter/tracer_test.go +++ b/client/firewall/uspfilter/tracer_test.go @@ -38,11 +38,8 @@ func TestTracePacket(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("100.10.0.100"), - Network: &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - }, + IP: netip.MustParseAddr("100.10.0.100"), + Network: netip.MustParsePrefix("100.10.0.0/16"), } }, } diff --git a/client/firewall/uspfilter/uspfilter.go b/client/firewall/uspfilter/uspfilter.go index 8e0a955ca..eede1ab13 100644 --- a/client/firewall/uspfilter/uspfilter.go +++ b/client/firewall/uspfilter/uspfilter.go @@ -71,7 +71,6 @@ type Manager struct { // incomingRules is used for filtering and hooks incomingRules map[netip.Addr]RuleSet routeRules RouteRules - wgNetwork *net.IPNet decoders sync.Pool wgIface common.IFaceMapper nativeFirewall firewall.Manager @@ -1091,11 +1090,6 @@ func (m *Manager) ruleMatches(rule *RouteRule, srcAddr, dstAddr netip.Addr, prot return true } -// SetNetwork of the wireguard interface to which filtering applied -func (m *Manager) SetNetwork(network *net.IPNet) { - m.wgNetwork = network -} - // AddUDPPacketHook calls hook when UDP packet from given direction matched // // Hook function returns flag which indicates should be the matched package dropped or not diff --git a/client/firewall/uspfilter/uspfilter_bench_test.go b/client/firewall/uspfilter/uspfilter_bench_test.go index beb5b9336..c03e60640 100644 --- a/client/firewall/uspfilter/uspfilter_bench_test.go +++ b/client/firewall/uspfilter/uspfilter_bench_test.go @@ -174,11 +174,6 @@ func BenchmarkCoreFiltering(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } - // Apply scenario-specific setup sc.setupFunc(manager) @@ -219,11 +214,6 @@ func BenchmarkStateScaling(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } - // Pre-populate connection table srcIPs := generateRandomIPs(count) dstIPs := generateRandomIPs(count) @@ -267,11 +257,6 @@ func BenchmarkEstablishmentOverhead(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } - srcIP := generateRandomIPs(1)[0] dstIP := generateRandomIPs(1)[0] outbound := generatePacket(b, srcIP, dstIP, 1024, 80, layers.IPProtocolTCP) @@ -304,10 +289,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolTCP, state: "new", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } b.Setenv("NB_DISABLE_CONNTRACK", "1") }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -321,10 +302,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolTCP, state: "established", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } b.Setenv("NB_DISABLE_CONNTRACK", "1") }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -339,10 +316,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolUDP, state: "new", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } b.Setenv("NB_DISABLE_CONNTRACK", "1") }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -356,10 +329,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolUDP, state: "established", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - } b.Setenv("NB_DISABLE_CONNTRACK", "1") }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -373,10 +342,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolTCP, state: "new", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - } require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -390,10 +355,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolTCP, state: "established", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - } require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -408,10 +369,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolTCP, state: "post_handshake", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - } require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -426,10 +383,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolUDP, state: "new", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - } require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -443,10 +396,6 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { proto: layers.IPProtocolUDP, state: "established", setupFunc: func(m *Manager) { - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("0.0.0.0"), - Mask: net.CIDRMask(0, 32), - } require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) }, genPackets: func(srcIP, dstIP net.IP) ([]byte, []byte) { @@ -593,11 +542,6 @@ func BenchmarkLongLivedConnections(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.SetNetwork(&net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - }) - // Setup initial state based on scenario if sc.rules { // Single rule to allow all return traffic from port 80 @@ -681,11 +625,6 @@ func BenchmarkShortLivedConnections(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.SetNetwork(&net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - }) - // Setup initial state based on scenario if sc.rules { // Single rule to allow all return traffic from port 80 @@ -797,11 +736,6 @@ func BenchmarkParallelLongLivedConnections(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.SetNetwork(&net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - }) - // Setup initial state based on scenario if sc.rules { _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") @@ -882,11 +816,6 @@ func BenchmarkParallelShortLivedConnections(b *testing.B) { require.NoError(b, manager.Close(nil)) }) - manager.SetNetwork(&net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.CIDRMask(10, 32), - }) - if sc.rules { _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") require.NoError(b, err) @@ -1032,7 +961,8 @@ func BenchmarkRouteACLs(b *testing.B) { } for _, r := range rules { - _, err := manager.AddRouteFiltering(nil, r.sources, r.dest, r.proto, nil, r.port, fw.ActionAccept) + dst := fw.Network{Prefix: r.dest} + _, err := manager.AddRouteFiltering(nil, r.sources, dst, r.proto, nil, r.port, fw.ActionAccept) if err != nil { b.Fatal(err) } diff --git a/client/firewall/uspfilter/uspfilter_filter_test.go b/client/firewall/uspfilter/uspfilter_filter_test.go index 04a398d1f..318f86a87 100644 --- a/client/firewall/uspfilter/uspfilter_filter_test.go +++ b/client/firewall/uspfilter/uspfilter_filter_test.go @@ -19,12 +19,8 @@ import ( ) func TestPeerACLFiltering(t *testing.T) { - localIP := net.ParseIP("100.10.0.100") - wgNet := &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - } - + localIP := netip.MustParseAddr("100.10.0.100") + wgNet := netip.MustParsePrefix("100.10.0.0/16") ifaceMock := &IFaceMock{ SetFilterFunc: func(device.PacketFilter) error { return nil }, AddressFunc: func() wgaddr.Address { @@ -43,8 +39,6 @@ func TestPeerACLFiltering(t *testing.T) { require.NoError(t, manager.Close(nil)) }) - manager.wgNetwork = wgNet - err = manager.UpdateLocalIPs() require.NoError(t, err) @@ -581,14 +575,13 @@ func setupRoutedManager(tb testing.TB, network string) *Manager { dev := mocks.NewMockDevice(ctrl) dev.EXPECT().MTU().Return(1500, nil).AnyTimes() - localIP, wgNet, err := net.ParseCIDR(network) - require.NoError(tb, err) + wgNet := netip.MustParsePrefix(network) ifaceMock := &IFaceMock{ SetFilterFunc: func(device.PacketFilter) error { return nil }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: localIP, + IP: wgNet.Addr(), Network: wgNet, } }, @@ -1440,11 +1433,8 @@ func TestRouteACLSet(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("100.10.0.100"), - Network: &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - }, + IP: netip.MustParseAddr("100.10.0.100"), + Network: netip.MustParsePrefix("100.10.0.0/16"), } }, } diff --git a/client/firewall/uspfilter/uspfilter_test.go b/client/firewall/uspfilter/uspfilter_test.go index 24a6a2c40..88de1ddcd 100644 --- a/client/firewall/uspfilter/uspfilter_test.go +++ b/client/firewall/uspfilter/uspfilter_test.go @@ -271,11 +271,8 @@ func TestNotMatchByIP(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("100.10.0.100"), - Network: &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - }, + IP: netip.MustParseAddr("100.10.0.100"), + Network: netip.MustParsePrefix("100.10.0.0/16"), } }, } @@ -285,10 +282,6 @@ func TestNotMatchByIP(t *testing.T) { t.Errorf("failed to create Manager: %v", err) return } - m.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - } ip := net.ParseIP("0.0.0.0") proto := fw.ProtocolUDP @@ -396,10 +389,6 @@ func TestProcessOutgoingHooks(t *testing.T) { }, false, flowLogger) require.NoError(t, err) - manager.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - } manager.udpTracker.Close() manager.udpTracker = conntrack.NewUDPTracker(100*time.Millisecond, logger, flowLogger) defer func() { @@ -509,11 +498,6 @@ func TestStatefulFirewall_UDPTracking(t *testing.T) { }, false, flowLogger) require.NoError(t, err) - manager.wgNetwork = &net.IPNet{ - IP: net.ParseIP("100.10.0.0"), - Mask: net.CIDRMask(16, 32), - } - manager.udpTracker.Close() // Close the existing tracker manager.udpTracker = conntrack.NewUDPTracker(200*time.Millisecond, logger, flowLogger) manager.decoders = sync.Pool{ diff --git a/client/iface/bind/udp_mux_universal.go b/client/iface/bind/udp_mux_universal.go index 9fed02bb7..5cc634955 100644 --- a/client/iface/bind/udp_mux_universal.go +++ b/client/iface/bind/udp_mux_universal.go @@ -164,7 +164,7 @@ func (u *udpConn) performFilterCheck(addr net.Addr) error { return nil } - if u.address.Network.Contains(a.AsSlice()) { + if u.address.Network.Contains(a) { log.Warnf("Address %s is part of the NetBird network %s, refusing to write", addr, u.address) return fmt.Errorf("address %s is part of the NetBird network %s, refusing to write", addr, u.address) } diff --git a/client/iface/device/device_filter.go b/client/iface/device/device_filter.go index c9b7e2448..5a1a0e96a 100644 --- a/client/iface/device/device_filter.go +++ b/client/iface/device/device_filter.go @@ -1,7 +1,6 @@ package device import ( - "net" "net/netip" "sync" @@ -24,9 +23,6 @@ type PacketFilter interface { // RemovePacketHook removes hook by ID RemovePacketHook(hookID string) error - - // SetNetwork of the wireguard interface to which filtering applied - SetNetwork(*net.IPNet) } // FilteredDevice to override Read or Write of packets diff --git a/client/iface/device/device_netstack.go b/client/iface/device/device_netstack.go index d3c92235e..d2f2c87a1 100644 --- a/client/iface/device/device_netstack.go +++ b/client/iface/device/device_netstack.go @@ -51,7 +51,11 @@ func (t *TunNetstackDevice) Create() (WGConfigurer, error) { log.Info("create nbnetstack tun interface") // TODO: get from service listener runtime IP - dnsAddr := nbnet.GetLastIPFromNetwork(t.address.Network, 1) + dnsAddr, err := nbnet.GetLastIPFromNetwork(t.address.Network, 1) + if err != nil { + return nil, fmt.Errorf("last ip: %w", err) + } + log.Debugf("netstack using address: %s", t.address.IP) t.nsTun = nbnetstack.NewNetStackTun(t.listenAddress, t.address.IP, dnsAddr, t.mtu) log.Debugf("netstack using dns address: %s", dnsAddr) diff --git a/client/iface/device/wg_link_freebsd.go b/client/iface/device/wg_link_freebsd.go index 9067790e4..1b06e0e15 100644 --- a/client/iface/device/wg_link_freebsd.go +++ b/client/iface/device/wg_link_freebsd.go @@ -64,7 +64,15 @@ func (l *wgLink) assignAddr(address wgaddr.Address) error { } ip := address.IP.String() - mask := "0x" + address.Network.Mask.String() + + // Convert prefix length to hex netmask + prefixLen := address.Network.Bits() + if !address.IP.Is4() { + return fmt.Errorf("IPv6 not supported for interface assignment") + } + + maskBits := uint32(0xffffffff) << (32 - prefixLen) + mask := fmt.Sprintf("0x%08x", maskBits) log.Infof("assign addr %s mask %s to %s interface", ip, mask, l.name) diff --git a/client/iface/iface.go b/client/iface/iface.go index c78a252da..1f659af29 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -185,7 +185,6 @@ func (w *WGIface) SetFilter(filter device.PacketFilter) error { } w.filter = filter - w.filter.SetNetwork(w.tun.WgAddress().Network) w.tun.FilteredDevice().SetFilter(filter) return nil diff --git a/client/iface/mocks/filter.go b/client/iface/mocks/filter.go index faac55d68..8cd2a1231 100644 --- a/client/iface/mocks/filter.go +++ b/client/iface/mocks/filter.go @@ -5,7 +5,6 @@ package mocks import ( - net "net" "net/netip" reflect "reflect" @@ -90,15 +89,3 @@ func (mr *MockPacketFilterMockRecorder) RemovePacketHook(arg0 interface{}) *gomo mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePacketHook", reflect.TypeOf((*MockPacketFilter)(nil).RemovePacketHook), arg0) } - -// SetNetwork mocks base method. -func (m *MockPacketFilter) SetNetwork(arg0 *net.IPNet) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "SetNetwork", arg0) -} - -// SetNetwork indicates an expected call of SetNetwork. -func (mr *MockPacketFilterMockRecorder) SetNetwork(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNetwork", reflect.TypeOf((*MockPacketFilter)(nil).SetNetwork), arg0) -} diff --git a/client/iface/netstack/tun.go b/client/iface/netstack/tun.go index a271a1954..aec9d4faa 100644 --- a/client/iface/netstack/tun.go +++ b/client/iface/netstack/tun.go @@ -1,8 +1,6 @@ package netstack import ( - "fmt" - "net" "net/netip" "os" "strconv" @@ -15,8 +13,8 @@ import ( const EnvSkipProxy = "NB_NETSTACK_SKIP_PROXY" type NetStackTun struct { //nolint:revive - address net.IP - dnsAddress net.IP + address netip.Addr + dnsAddress netip.Addr mtu int listenAddress string @@ -24,7 +22,7 @@ type NetStackTun struct { //nolint:revive tundev tun.Device } -func NewNetStackTun(listenAddress string, address net.IP, dnsAddress net.IP, mtu int) *NetStackTun { +func NewNetStackTun(listenAddress string, address netip.Addr, dnsAddress netip.Addr, mtu int) *NetStackTun { return &NetStackTun{ address: address, dnsAddress: dnsAddress, @@ -34,19 +32,9 @@ func NewNetStackTun(listenAddress string, address net.IP, dnsAddress net.IP, mtu } func (t *NetStackTun) Create() (tun.Device, *netstack.Net, error) { - addr, ok := netip.AddrFromSlice(t.address) - if !ok { - return nil, nil, fmt.Errorf("convert address to netip.Addr: %v", t.address) - } - - dnsAddr, ok := netip.AddrFromSlice(t.dnsAddress) - if !ok { - return nil, nil, fmt.Errorf("convert dns address to netip.Addr: %v", t.dnsAddress) - } - nsTunDev, tunNet, err := netstack.CreateNetTUN( - []netip.Addr{addr.Unmap()}, - []netip.Addr{dnsAddr.Unmap()}, + []netip.Addr{t.address}, + []netip.Addr{t.dnsAddress}, t.mtu) if err != nil { return nil, nil, err diff --git a/client/iface/wgaddr/address.go b/client/iface/wgaddr/address.go index e5079258c..078f8be95 100644 --- a/client/iface/wgaddr/address.go +++ b/client/iface/wgaddr/address.go @@ -2,28 +2,27 @@ package wgaddr import ( "fmt" - "net" + "net/netip" ) // Address WireGuard parsed address type Address struct { - IP net.IP - Network *net.IPNet + IP netip.Addr + Network netip.Prefix } // ParseWGAddress parse a string ("1.2.3.4/24") address to WG Address func ParseWGAddress(address string) (Address, error) { - ip, network, err := net.ParseCIDR(address) + prefix, err := netip.ParsePrefix(address) if err != nil { return Address{}, err } return Address{ - IP: ip, - Network: network, + IP: prefix.Addr().Unmap(), + Network: prefix.Masked(), }, nil } func (addr Address) String() string { - maskSize, _ := addr.Network.Mask.Size() - return fmt.Sprintf("%s/%d", addr.IP.String(), maskSize) + return fmt.Sprintf("%s/%d", addr.IP.String(), addr.Network.Bits()) } diff --git a/client/internal/acl/manager_test.go b/client/internal/acl/manager_test.go index 532d70a24..16620033e 100644 --- a/client/internal/acl/manager_test.go +++ b/client/internal/acl/manager_test.go @@ -1,7 +1,7 @@ package acl import ( - "net" + "net/netip" "testing" "github.com/golang/mock/gomock" @@ -43,12 +43,11 @@ func TestDefaultManager(t *testing.T) { ifaceMock := mocks.NewMockIFaceMapper(ctrl) ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() ifaceMock.EXPECT().SetFilter(gomock.Any()) - ip, network, err := net.ParseCIDR("172.0.0.1/32") - require.NoError(t, err) + network := netip.MustParsePrefix("172.0.0.1/32") ifaceMock.EXPECT().Name().Return("lo").AnyTimes() ifaceMock.EXPECT().Address().Return(wgaddr.Address{ - IP: ip, + IP: network.Addr(), Network: network, }).AnyTimes() ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() @@ -162,12 +161,11 @@ func TestDefaultManagerStateless(t *testing.T) { ifaceMock := mocks.NewMockIFaceMapper(ctrl) ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() ifaceMock.EXPECT().SetFilter(gomock.Any()) - ip, network, err := net.ParseCIDR("172.0.0.1/32") - require.NoError(t, err) + network := netip.MustParsePrefix("172.0.0.1/32") ifaceMock.EXPECT().Name().Return("lo").AnyTimes() ifaceMock.EXPECT().Address().Return(wgaddr.Address{ - IP: ip, + IP: network.Addr(), Network: network, }).AnyTimes() ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() @@ -372,12 +370,11 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) { ifaceMock := mocks.NewMockIFaceMapper(ctrl) ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() ifaceMock.EXPECT().SetFilter(gomock.Any()) - ip, network, err := net.ParseCIDR("172.0.0.1/32") - require.NoError(t, err) + network := netip.MustParsePrefix("172.0.0.1/32") ifaceMock.EXPECT().Name().Return("lo").AnyTimes() ifaceMock.EXPECT().Address().Return(wgaddr.Address{ - IP: ip, + IP: network.Addr(), Network: network, }).AnyTimes() ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() diff --git a/client/internal/dns.go b/client/internal/dns.go index 8a73f50f2..5e604bec5 100644 --- a/client/internal/dns.go +++ b/client/internal/dns.go @@ -2,7 +2,7 @@ package internal import ( "fmt" - "net" + "net/netip" "slices" "strings" @@ -12,13 +12,14 @@ import ( nbdns "github.com/netbirdio/netbird/dns" ) -func createPTRRecord(aRecord nbdns.SimpleRecord, ipNet *net.IPNet) (nbdns.SimpleRecord, bool) { - ip := net.ParseIP(aRecord.RData) - if ip == nil || ip.To4() == nil { +func createPTRRecord(aRecord nbdns.SimpleRecord, prefix netip.Prefix) (nbdns.SimpleRecord, bool) { + ip, err := netip.ParseAddr(aRecord.RData) + if err != nil { + log.Warnf("failed to parse IP address %s: %v", aRecord.RData, err) return nbdns.SimpleRecord{}, false } - if !ipNet.Contains(ip) { + if !prefix.Contains(ip) { return nbdns.SimpleRecord{}, false } @@ -36,16 +37,19 @@ func createPTRRecord(aRecord nbdns.SimpleRecord, ipNet *net.IPNet) (nbdns.Simple } // generateReverseZoneName creates the reverse DNS zone name for a given network -func generateReverseZoneName(ipNet *net.IPNet) (string, error) { - networkIP := ipNet.IP.Mask(ipNet.Mask) - maskOnes, _ := ipNet.Mask.Size() +func generateReverseZoneName(network netip.Prefix) (string, error) { + networkIP := network.Masked().Addr() + + if !networkIP.Is4() { + return "", fmt.Errorf("reverse DNS is only supported for IPv4 networks, got: %s", networkIP) + } // round up to nearest byte - octetsToUse := (maskOnes + 7) / 8 + octetsToUse := (network.Bits() + 7) / 8 octets := strings.Split(networkIP.String(), ".") if octetsToUse > len(octets) { - return "", fmt.Errorf("invalid network mask size for reverse DNS: %d", maskOnes) + return "", fmt.Errorf("invalid network mask size for reverse DNS: %d", network.Bits()) } reverseOctets := make([]string, octetsToUse) @@ -68,7 +72,7 @@ func zoneExists(config *nbdns.Config, zoneName string) bool { } // collectPTRRecords gathers all PTR records for the given network from A records -func collectPTRRecords(config *nbdns.Config, ipNet *net.IPNet) []nbdns.SimpleRecord { +func collectPTRRecords(config *nbdns.Config, prefix netip.Prefix) []nbdns.SimpleRecord { var records []nbdns.SimpleRecord for _, zone := range config.CustomZones { @@ -77,7 +81,7 @@ func collectPTRRecords(config *nbdns.Config, ipNet *net.IPNet) []nbdns.SimpleRec continue } - if ptrRecord, ok := createPTRRecord(record, ipNet); ok { + if ptrRecord, ok := createPTRRecord(record, prefix); ok { records = append(records, ptrRecord) } } @@ -87,8 +91,8 @@ func collectPTRRecords(config *nbdns.Config, ipNet *net.IPNet) []nbdns.SimpleRec } // addReverseZone adds a reverse DNS zone to the configuration for the given network -func addReverseZone(config *nbdns.Config, ipNet *net.IPNet) { - zoneName, err := generateReverseZoneName(ipNet) +func addReverseZone(config *nbdns.Config, network netip.Prefix) { + zoneName, err := generateReverseZoneName(network) if err != nil { log.Warn(err) return @@ -99,7 +103,7 @@ func addReverseZone(config *nbdns.Config, ipNet *net.IPNet) { return } - records := collectPTRRecords(config, ipNet) + records := collectPTRRecords(config, network) reverseZone := nbdns.CustomZone{ Domain: zoneName, diff --git a/client/internal/dns/server_test.go b/client/internal/dns/server_test.go index 1c7c9b117..e55b27910 100644 --- a/client/internal/dns/server_test.go +++ b/client/internal/dns/server_test.go @@ -46,10 +46,9 @@ func (w *mocWGIface) Name() string { } func (w *mocWGIface) Address() wgaddr.Address { - ip, network, _ := net.ParseCIDR("100.66.100.0/24") return wgaddr.Address{ - IP: ip, - Network: network, + IP: netip.MustParseAddr("100.66.100.1"), + Network: netip.MustParsePrefix("100.66.100.0/24"), } } @@ -464,17 +463,10 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - _, ipNet, err := net.ParseCIDR("100.66.100.1/32") - if err != nil { - t.Errorf("parse CIDR: %v", err) - return - } - packetfilter := pfmock.NewMockPacketFilter(ctrl) packetfilter.EXPECT().DropOutgoing(gomock.Any(), gomock.Any()).AnyTimes() packetfilter.EXPECT().AddUDPPacketHook(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) packetfilter.EXPECT().RemovePacketHook(gomock.Any()) - packetfilter.EXPECT().SetNetwork(ipNet) if err := wgIface.SetFilter(packetfilter); err != nil { t.Errorf("set packet filter: %v", err) diff --git a/client/internal/dns/service_memory.go b/client/internal/dns/service_memory.go index 34c563757..226202cf7 100644 --- a/client/internal/dns/service_memory.go +++ b/client/internal/dns/service_memory.go @@ -24,11 +24,15 @@ type ServiceViaMemory struct { } func NewServiceViaMemory(wgIface WGIface) *ServiceViaMemory { + lastIP, err := nbnet.GetLastIPFromNetwork(wgIface.Address().Network, 1) + if err != nil { + log.Errorf("get last ip from network: %v", err) + } s := &ServiceViaMemory{ wgInterface: wgIface, dnsMux: dns.NewServeMux(), - runtimeIP: nbnet.GetLastIPFromNetwork(wgIface.Address().Network, 1).String(), + runtimeIP: lastIP.String(), runtimePort: defaultPort, } return s @@ -91,7 +95,7 @@ func (s *ServiceViaMemory) filterDNSTraffic() (string, error) { } firstLayerDecoder := layers.LayerTypeIPv4 - if s.wgInterface.Address().Network.IP.To4() == nil { + if s.wgInterface.Address().IP.Is6() { firstLayerDecoder = layers.LayerTypeIPv6 } diff --git a/client/internal/dns/service_memory_test.go b/client/internal/dns/service_memory_test.go deleted file mode 100644 index 244adfaef..000000000 --- a/client/internal/dns/service_memory_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package dns - -import ( - "net" - "testing" - - nbnet "github.com/netbirdio/netbird/util/net" -) - -func TestGetLastIPFromNetwork(t *testing.T) { - tests := []struct { - addr string - ip string - }{ - {"2001:db8::/32", "2001:db8:ffff:ffff:ffff:ffff:ffff:fffe"}, - {"192.168.0.0/30", "192.168.0.2"}, - {"192.168.0.0/16", "192.168.255.254"}, - {"192.168.0.0/24", "192.168.0.254"}, - } - - for _, tt := range tests { - _, ipnet, err := net.ParseCIDR(tt.addr) - if err != nil { - t.Errorf("Error parsing CIDR: %v", err) - return - } - - lastIP := nbnet.GetLastIPFromNetwork(ipnet, 1).String() - if lastIP != tt.ip { - t.Errorf("wrong IP address, expected %s: got %s", tt.ip, lastIP) - } - } -} diff --git a/client/internal/dns/upstream_android.go b/client/internal/dns/upstream_android.go index 06ffcba11..52d2ba58b 100644 --- a/client/internal/dns/upstream_android.go +++ b/client/internal/dns/upstream_android.go @@ -3,6 +3,7 @@ package dns import ( "context" "net" + "net/netip" "syscall" "time" @@ -23,8 +24,8 @@ type upstreamResolver struct { func newUpstreamResolver( ctx context.Context, _ string, - _ net.IP, - _ *net.IPNet, + _ netip.Addr, + _ netip.Prefix, statusRecorder *peer.Status, hostsDNSHolder *hostsDNSHolder, domain string, diff --git a/client/internal/dns/upstream_general.go b/client/internal/dns/upstream_general.go index 9bb5feab0..1bc06a7c1 100644 --- a/client/internal/dns/upstream_general.go +++ b/client/internal/dns/upstream_general.go @@ -4,7 +4,7 @@ package dns import ( "context" - "net" + "net/netip" "time" "github.com/miekg/dns" @@ -19,8 +19,8 @@ type upstreamResolver struct { func newUpstreamResolver( ctx context.Context, _ string, - _ net.IP, - _ *net.IPNet, + _ netip.Addr, + _ netip.Prefix, statusRecorder *peer.Status, _ *hostsDNSHolder, domain string, diff --git a/client/internal/dns/upstream_ios.go b/client/internal/dns/upstream_ios.go index ca5b31132..648cab176 100644 --- a/client/internal/dns/upstream_ios.go +++ b/client/internal/dns/upstream_ios.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "net" + "net/netip" "syscall" "time" @@ -18,16 +19,16 @@ import ( type upstreamResolverIOS struct { *upstreamResolverBase - lIP net.IP - lNet *net.IPNet + lIP netip.Addr + lNet netip.Prefix interfaceName string } func newUpstreamResolver( ctx context.Context, interfaceName string, - ip net.IP, - net *net.IPNet, + ip netip.Addr, + net netip.Prefix, statusRecorder *peer.Status, _ *hostsDNSHolder, domain string, @@ -58,8 +59,11 @@ func (u *upstreamResolverIOS) exchange(ctx context.Context, upstream string, r * } client.DialTimeout = timeout - upstreamIP := net.ParseIP(upstreamHost) - if u.lNet.Contains(upstreamIP) || net.IP.IsPrivate(upstreamIP) { + upstreamIP, err := netip.ParseAddr(upstreamHost) + if err != nil { + log.Warnf("failed to parse upstream host %s: %s", upstreamHost, err) + } + if u.lNet.Contains(upstreamIP) || upstreamIP.IsPrivate() { log.Debugf("using private client to query upstream: %s", upstream) client, err = GetClientPrivate(u.lIP, u.interfaceName, timeout) if err != nil { @@ -73,7 +77,7 @@ func (u *upstreamResolverIOS) exchange(ctx context.Context, upstream string, r * // GetClientPrivate returns a new DNS client bound to the local IP address of the Netbird interface // This method is needed for iOS -func GetClientPrivate(ip net.IP, interfaceName string, dialTimeout time.Duration) (*dns.Client, error) { +func GetClientPrivate(ip netip.Addr, interfaceName string, dialTimeout time.Duration) (*dns.Client, error) { index, err := getInterfaceIndex(interfaceName) if err != nil { log.Debugf("unable to get interface index for %s: %s", interfaceName, err) @@ -82,7 +86,7 @@ func GetClientPrivate(ip net.IP, interfaceName string, dialTimeout time.Duration dialer := &net.Dialer{ LocalAddr: &net.UDPAddr{ - IP: ip, + IP: ip.AsSlice(), Port: 0, // Let the OS pick a free port }, Timeout: dialTimeout, diff --git a/client/internal/dns/upstream_test.go b/client/internal/dns/upstream_test.go index 13bc91a37..e440995d9 100644 --- a/client/internal/dns/upstream_test.go +++ b/client/internal/dns/upstream_test.go @@ -2,7 +2,7 @@ package dns import ( "context" - "net" + "net/netip" "strings" "testing" "time" @@ -58,7 +58,7 @@ func TestUpstreamResolver_ServeDNS(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) - resolver, _ := newUpstreamResolver(ctx, "", net.IP{}, &net.IPNet{}, nil, nil, ".") + resolver, _ := newUpstreamResolver(ctx, "", netip.Addr{}, netip.Prefix{}, nil, nil, ".") resolver.upstreamServers = testCase.InputServers resolver.upstreamTimeout = testCase.timeout if testCase.cancelCTX { diff --git a/client/internal/engine.go b/client/internal/engine.go index d015c1d6c..0dec799bf 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -1008,7 +1008,7 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error { // apply routes first, route related actions might depend on routing being enabled routes := toRoutes(networkMap.GetRoutes()) if err := e.routeManager.UpdateRoutes(serial, routes, dnsRouteFeatureFlag); err != nil { - log.Errorf("failed to update clientRoutes, err: %v", err) + log.Errorf("failed to update routes: %v", err) } if e.acl != nil { @@ -1104,7 +1104,7 @@ func toRoutes(protoRoutes []*mgmProto.Route) []*route.Route { convertedRoute := &route.Route{ ID: route.ID(protoRoute.ID), - Network: prefix, + Network: prefix.Masked(), Domains: domain.FromPunycodeList(protoRoute.Domains), NetID: route.NetID(protoRoute.NetID), NetworkType: route.NetworkType(protoRoute.NetworkType), @@ -1138,7 +1138,7 @@ func toRouteDomains(myPubKey string, routes []*route.Route) []*dnsfwd.ForwarderE return entries } -func toDNSConfig(protoDNSConfig *mgmProto.DNSConfig, network *net.IPNet) nbdns.Config { +func toDNSConfig(protoDNSConfig *mgmProto.DNSConfig, network netip.Prefix) nbdns.Config { dnsUpdate := nbdns.Config{ ServiceEnable: protoDNSConfig.GetServiceEnable(), CustomZones: make([]nbdns.CustomZone, 0), @@ -1790,9 +1790,9 @@ func (e *Engine) GetLatestNetworkMap() (*mgmProto.NetworkMap, error) { } // GetWgAddr returns the wireguard address -func (e *Engine) GetWgAddr() net.IP { +func (e *Engine) GetWgAddr() netip.Addr { if e.wgInterface == nil { - return nil + return netip.Addr{} } return e.wgInterface.Address().IP } @@ -1861,12 +1861,7 @@ func (e *Engine) Address() (netip.Addr, error) { return netip.Addr{}, errors.New("wireguard interface not initialized") } - addr := e.wgInterface.Address() - ip, ok := netip.AddrFromSlice(addr.IP) - if !ok { - return netip.Addr{}, errors.New("failed to convert address to netip.Addr") - } - return ip.Unmap(), nil + return e.wgInterface.Address().IP, nil } func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewallManager.ForwardRule, error) { diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 422059bd8..82c1ba0e2 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -371,11 +371,8 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { }, AddressFunc: func() wgaddr.Address { return wgaddr.Address{ - IP: net.ParseIP("10.20.0.1"), - Network: &net.IPNet{ - IP: net.ParseIP("10.20.0.0"), - Mask: net.IPv4Mask(255, 255, 255, 0), - }, + IP: netip.MustParseAddr("10.20.0.1"), + Network: netip.MustParsePrefix("10.20.0.0/24"), } }, UpdatePeerFunc: func(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { diff --git a/client/internal/netflow/conntrack/conntrack.go b/client/internal/netflow/conntrack/conntrack.go index f8440b913..d01adf135 100644 --- a/client/internal/netflow/conntrack/conntrack.go +++ b/client/internal/netflow/conntrack/conntrack.go @@ -232,7 +232,7 @@ func (c *ConnTrack) relevantFlow(mark uint32, srcIP, dstIP netip.Addr) bool { // fallback if mark rules are not in place wgnet := c.iface.Address().Network - return wgnet.Contains(srcIP.AsSlice()) || wgnet.Contains(dstIP.AsSlice()) + return wgnet.Contains(srcIP) || wgnet.Contains(dstIP) } // mapRxPackets maps packet counts to RX based on flow direction @@ -293,17 +293,15 @@ func (c *ConnTrack) inferDirection(mark uint32, srcIP, dstIP netip.Addr) nftypes // fallback if marks are not set wgaddr := c.iface.Address().IP wgnetwork := c.iface.Address().Network - src, dst := srcIP.AsSlice(), dstIP.AsSlice() - switch { - case wgaddr.Equal(src): + case wgaddr == srcIP: return nftypes.Egress - case wgaddr.Equal(dst): + case wgaddr == dstIP: return nftypes.Ingress - case wgnetwork.Contains(src): + case wgnetwork.Contains(srcIP): // netbird network -> resource network return nftypes.Ingress - case wgnetwork.Contains(dst): + case wgnetwork.Contains(dstIP): // resource network -> netbird network return nftypes.Egress } diff --git a/client/internal/netflow/logger/logger.go b/client/internal/netflow/logger/logger.go index a3bd091b6..e28fdf2f4 100644 --- a/client/internal/netflow/logger/logger.go +++ b/client/internal/netflow/logger/logger.go @@ -2,7 +2,7 @@ package logger import ( "context" - "net" + "net/netip" "sync" "sync/atomic" "time" @@ -23,17 +23,16 @@ type Logger struct { rcvChan atomic.Pointer[rcvChan] cancel context.CancelFunc statusRecorder *peer.Status - wgIfaceIPNet net.IPNet + wgIfaceNet netip.Prefix dnsCollection atomic.Bool exitNodeCollection atomic.Bool Store types.Store } -func New(statusRecorder *peer.Status, wgIfaceIPNet net.IPNet) *Logger { - +func New(statusRecorder *peer.Status, wgIfaceIPNet netip.Prefix) *Logger { return &Logger{ statusRecorder: statusRecorder, - wgIfaceIPNet: wgIfaceIPNet, + wgIfaceNet: wgIfaceIPNet, Store: store.NewMemoryStore(), } } @@ -89,11 +88,11 @@ func (l *Logger) startReceiver() { var isSrcExitNode bool var isDestExitNode bool - if !l.wgIfaceIPNet.Contains(net.IP(event.SourceIP.AsSlice())) { + if !l.wgIfaceNet.Contains(event.SourceIP) { event.SourceResourceID, isSrcExitNode = l.statusRecorder.CheckRoutes(event.SourceIP) } - if !l.wgIfaceIPNet.Contains(net.IP(event.DestIP.AsSlice())) { + if !l.wgIfaceNet.Contains(event.DestIP) { event.DestResourceID, isDestExitNode = l.statusRecorder.CheckRoutes(event.DestIP) } diff --git a/client/internal/netflow/logger/logger_test.go b/client/internal/netflow/logger/logger_test.go index 06e10c36c..1144544d8 100644 --- a/client/internal/netflow/logger/logger_test.go +++ b/client/internal/netflow/logger/logger_test.go @@ -1,7 +1,7 @@ package logger_test import ( - "net" + "net/netip" "testing" "time" @@ -12,7 +12,7 @@ import ( ) func TestStore(t *testing.T) { - logger := logger.New(nil, net.IPNet{}) + logger := logger.New(nil, netip.Prefix{}) logger.Enable() event := types.EventFields{ diff --git a/client/internal/netflow/manager.go b/client/internal/netflow/manager.go index bf80e5a9f..e3b188468 100644 --- a/client/internal/netflow/manager.go +++ b/client/internal/netflow/manager.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "net" + "net/netip" "runtime" "sync" "time" @@ -34,11 +34,11 @@ type Manager struct { // NewManager creates a new netflow manager func NewManager(iface nftypes.IFaceMapper, publicKey []byte, statusRecorder *peer.Status) *Manager { - var ipNet net.IPNet + var prefix netip.Prefix if iface != nil { - ipNet = *iface.Address().Network + prefix = iface.Address().Network } - flowLogger := logger.New(statusRecorder, ipNet) + flowLogger := logger.New(statusRecorder, prefix) var ct nftypes.ConnTracker if runtime.GOOS == "linux" && iface != nil && !iface.IsUserspaceBind() { diff --git a/client/internal/netflow/manager_test.go b/client/internal/netflow/manager_test.go index bf7e05f8e..0b5eb3be6 100644 --- a/client/internal/netflow/manager_test.go +++ b/client/internal/netflow/manager_test.go @@ -1,7 +1,7 @@ package netflow import ( - "net" + "net/netip" "testing" "time" @@ -33,10 +33,7 @@ func (m *mockIFaceMapper) IsUserspaceBind() bool { func TestManager_Update(t *testing.T) { mockIFace := &mockIFaceMapper{ address: wgaddr.Address{ - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.1"), - Mask: net.CIDRMask(24, 32), - }, + Network: netip.MustParsePrefix("192.168.1.1/32"), }, isUserspaceBind: true, } @@ -102,10 +99,7 @@ func TestManager_Update(t *testing.T) { func TestManager_Update_TokenPreservation(t *testing.T) { mockIFace := &mockIFaceMapper{ address: wgaddr.Address{ - Network: &net.IPNet{ - IP: net.ParseIP("192.168.1.1"), - Mask: net.CIDRMask(24, 32), - }, + Network: netip.MustParsePrefix("192.168.1.1/32"), }, isUserspaceBind: true, } diff --git a/client/internal/routemanager/dnsinterceptor/handler.go b/client/internal/routemanager/dnsinterceptor/handler.go index 6d51c88c0..78d5e3b30 100644 --- a/client/internal/routemanager/dnsinterceptor/handler.go +++ b/client/internal/routemanager/dnsinterceptor/handler.go @@ -264,7 +264,7 @@ func (d *DnsInterceptor) writeMsg(w dns.ResponseWriter, r *dns.Msg) error { continue } - prefix := netip.PrefixFrom(ip, ip.BitLen()) + prefix := netip.PrefixFrom(ip.Unmap(), ip.BitLen()) newPrefixes = append(newPrefixes, prefix) } diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index afb74c23e..8dbbb5f77 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -333,11 +333,12 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro newServerRoutesMap, newClientRoutesIDMap := m.classifyRoutes(newRoutes) + var merr *multierror.Error if !m.disableClientRoutes { filteredClientRoutes := m.routeSelector.FilterSelected(newClientRoutesIDMap) if err := m.updateSystemRoutes(filteredClientRoutes); err != nil { - log.Errorf("Failed to update system routes: %v", err) + merr = multierror.Append(merr, fmt.Errorf("update system routes: %w", err)) } m.updateClientNetworks(updateSerial, filteredClientRoutes) @@ -346,14 +347,14 @@ func (m *DefaultManager) UpdateRoutes(updateSerial uint64, newRoutes []*route.Ro m.clientRoutes = newClientRoutesIDMap if m.serverRouter == nil { - return nil + return nberrors.FormatErrorOrNil(merr) } if err := m.serverRouter.UpdateRoutes(newServerRoutesMap, useNewDNSRoute); err != nil { - return fmt.Errorf("update routes: %w", err) + merr = multierror.Append(merr, fmt.Errorf("update server routes: %w", err)) } - return nil + return nberrors.FormatErrorOrNil(merr) } // SetRouteChangeListener set RouteListener for route change Notifier diff --git a/client/internal/routemanager/manager_test.go b/client/internal/routemanager/manager_test.go index 680bd813f..a46ae080e 100644 --- a/client/internal/routemanager/manager_test.go +++ b/client/internal/routemanager/manager_test.go @@ -44,7 +44,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -71,7 +71,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: localPeerKey, - Network: netip.MustParsePrefix("100.64.252.250/30"), + Network: netip.MustParsePrefix("100.64.252.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -99,7 +99,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: localPeerKey, - Network: netip.MustParsePrefix("100.64.30.250/30"), + Network: netip.MustParsePrefix("100.64.30.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -127,7 +127,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: localPeerKey, - Network: netip.MustParsePrefix("100.64.30.250/30"), + Network: netip.MustParsePrefix("100.64.30.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -211,7 +211,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -233,7 +233,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -250,7 +250,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -272,7 +272,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -282,7 +282,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "b", NetID: "routeA", Peer: remotePeerKey2, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -299,7 +299,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -327,7 +327,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "a", NetID: "routeA", Peer: localPeerKey, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -356,7 +356,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "l1", NetID: "routeA", Peer: localPeerKey, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -376,7 +376,7 @@ func TestManagerUpdateRoutes(t *testing.T) { ID: "r1", NetID: "routeA", Peer: remotePeerKey1, - Network: netip.MustParsePrefix("100.64.251.250/30"), + Network: netip.MustParsePrefix("100.64.251.248/30"), NetworkType: route.IPv4Network, Metric: 9999, Masquerade: false, @@ -440,11 +440,11 @@ func TestManagerUpdateRoutes(t *testing.T) { } if len(testCase.inputInitRoutes) > 0 { - _ = routeManager.UpdateRoutes(testCase.inputSerial, testCase.inputRoutes, false) + err = routeManager.UpdateRoutes(testCase.inputSerial, testCase.inputRoutes, false) require.NoError(t, err, "should update routes with init routes") } - _ = routeManager.UpdateRoutes(testCase.inputSerial+uint64(len(testCase.inputInitRoutes)), testCase.inputRoutes, false) + err = routeManager.UpdateRoutes(testCase.inputSerial+uint64(len(testCase.inputInitRoutes)), testCase.inputRoutes, false) require.NoError(t, err, "should update routes") expectedWatchers := testCase.clientNetworkWatchersExpected diff --git a/client/internal/routemanager/sysctl/sysctl_linux.go b/client/internal/routemanager/sysctl/sysctl_linux.go index ea63f02fc..f96a57f37 100644 --- a/client/internal/routemanager/sysctl/sysctl_linux.go +++ b/client/internal/routemanager/sysctl/sysctl_linux.go @@ -13,7 +13,7 @@ import ( log "github.com/sirupsen/logrus" nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/internal/routemanager/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" ) const ( @@ -22,8 +22,13 @@ const ( srcValidMarkPath = "net.ipv4.conf.all.src_valid_mark" ) +type iface interface { + Address() wgaddr.Address + Name() string +} + // Setup configures sysctl settings for RP filtering and source validation. -func Setup(wgIface iface.WGIface) (map[string]int, error) { +func Setup(wgIface iface) (map[string]int, error) { keys := map[string]int{} var result *multierror.Error diff --git a/client/internal/routemanager/systemops/systemops.go b/client/internal/routemanager/systemops/systemops.go index fd511fc20..261567dc3 100644 --- a/client/internal/routemanager/systemops/systemops.go +++ b/client/internal/routemanager/systemops/systemops.go @@ -6,9 +6,10 @@ import ( "net/netip" "sync" - "github.com/netbirdio/netbird/client/internal/routemanager/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/internal/routemanager/notifier" "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" + "github.com/netbirdio/netbird/client/internal/routemanager/vars" ) type Nexthop struct { @@ -30,11 +31,16 @@ func (n Nexthop) String() string { return fmt.Sprintf("%s @ %d (%s)", n.IP.String(), n.Intf.Index, n.Intf.Name) } +type wgIface interface { + Address() wgaddr.Address + Name() string +} + type ExclusionCounter = refcounter.Counter[netip.Prefix, struct{}, Nexthop] type SysOps struct { refCounter *ExclusionCounter - wgInterface iface.WGIface + wgInterface wgIface // prefixes is tracking all the current added prefixes im memory // (this is used in iOS as all route updates require a full table update) //nolint @@ -45,9 +51,27 @@ type SysOps struct { notifier *notifier.Notifier } -func NewSysOps(wgInterface iface.WGIface, notifier *notifier.Notifier) *SysOps { +func NewSysOps(wgInterface wgIface, notifier *notifier.Notifier) *SysOps { return &SysOps{ wgInterface: wgInterface, notifier: notifier, } } + +func (r *SysOps) validateRoute(prefix netip.Prefix) error { + addr := prefix.Addr() + + switch { + case + !addr.IsValid(), + addr.IsLoopback(), + addr.IsLinkLocalUnicast(), + addr.IsLinkLocalMulticast(), + addr.IsInterfaceLocalMulticast(), + addr.IsMulticast(), + addr.IsUnspecified() && prefix.Bits() != 0, + r.wgInterface.Address().Network.Contains(addr): + return vars.ErrRouteNotAllowed + } + return nil +} diff --git a/client/internal/routemanager/systemops/systemops_bsd_test.go b/client/internal/routemanager/systemops/systemops_bsd_test.go index a83d7f1de..0d892c162 100644 --- a/client/internal/routemanager/systemops/systemops_bsd_test.go +++ b/client/internal/routemanager/systemops/systemops_bsd_test.go @@ -8,6 +8,8 @@ import ( "net/netip" "os/exec" "regexp" + "runtime" + "strings" "sync" "testing" @@ -33,7 +35,12 @@ func init() { func TestConcurrentRoutes(t *testing.T) { baseIP := netip.MustParseAddr("192.0.2.0") - intf := &net.Interface{Name: "lo0"} + + var intf *net.Interface + var nexthop Nexthop + + _, intf = setupDummyInterface(t) + nexthop = Nexthop{netip.Addr{}, intf} r := NewSysOps(nil, nil) @@ -43,7 +50,7 @@ func TestConcurrentRoutes(t *testing.T) { go func(ip netip.Addr) { defer wg.Done() prefix := netip.PrefixFrom(ip, 32) - if err := r.addToRouteTable(prefix, Nexthop{netip.Addr{}, intf}); err != nil { + if err := r.addToRouteTable(prefix, nexthop); err != nil { t.Errorf("Failed to add route for %s: %v", prefix, err) } }(baseIP) @@ -59,7 +66,7 @@ func TestConcurrentRoutes(t *testing.T) { go func(ip netip.Addr) { defer wg.Done() prefix := netip.PrefixFrom(ip, 32) - if err := r.removeFromRouteTable(prefix, Nexthop{netip.Addr{}, intf}); err != nil { + if err := r.removeFromRouteTable(prefix, nexthop); err != nil { t.Errorf("Failed to remove route for %s: %v", prefix, err) } }(baseIP) @@ -119,18 +126,39 @@ func TestBits(t *testing.T) { func createAndSetupDummyInterface(t *testing.T, intf string, ipAddressCIDR string) string { t.Helper() - err := exec.Command("ifconfig", intf, "alias", ipAddressCIDR).Run() - require.NoError(t, err, "Failed to create loopback alias") + if runtime.GOOS == "darwin" { + err := exec.Command("ifconfig", intf, "alias", ipAddressCIDR).Run() + require.NoError(t, err, "Failed to create loopback alias") + + t.Cleanup(func() { + err := exec.Command("ifconfig", intf, ipAddressCIDR, "-alias").Run() + assert.NoError(t, err, "Failed to remove loopback alias") + }) + + return intf + } + + prefix, err := netip.ParsePrefix(ipAddressCIDR) + require.NoError(t, err, "Failed to parse prefix") + + netIntf, err := net.InterfaceByName(intf) + require.NoError(t, err, "Failed to get interface by name") + + nexthop := Nexthop{netip.Addr{}, netIntf} + + r := NewSysOps(nil, nil) + err = r.addToRouteTable(prefix, nexthop) + require.NoError(t, err, "Failed to add route to table") t.Cleanup(func() { - err := exec.Command("ifconfig", intf, ipAddressCIDR, "-alias").Run() - assert.NoError(t, err, "Failed to remove loopback alias") + err := r.removeFromRouteTable(prefix, nexthop) + assert.NoError(t, err, "Failed to remove route from table") }) - return "lo0" + return intf } -func addDummyRoute(t *testing.T, dstCIDR string, gw net.IP, _ string) { +func addDummyRoute(t *testing.T, dstCIDR string, gw netip.Addr, _ string) { t.Helper() var originalNexthop net.IP @@ -176,12 +204,40 @@ func fetchOriginalGateway() (net.IP, error) { return net.ParseIP(matches[1]), nil } +// setupDummyInterface creates a dummy tun interface for FreeBSD route testing +func setupDummyInterface(t *testing.T) (netip.Addr, *net.Interface) { + t.Helper() + + if runtime.GOOS == "darwin" { + return netip.AddrFrom4([4]byte{192, 168, 1, 2}), &net.Interface{Name: "lo0"} + } + + output, err := exec.Command("ifconfig", "tun", "create").CombinedOutput() + require.NoError(t, err, "Failed to create tun interface: %s", string(output)) + + tunName := strings.TrimSpace(string(output)) + + output, err = exec.Command("ifconfig", tunName, "192.168.1.1", "netmask", "255.255.0.0", "192.168.1.2", "up").CombinedOutput() + require.NoError(t, err, "Failed to configure tun interface: %s", string(output)) + + intf, err := net.InterfaceByName(tunName) + require.NoError(t, err, "Failed to get interface by name") + + t.Cleanup(func() { + if err := exec.Command("ifconfig", tunName, "destroy").Run(); err != nil { + t.Logf("Failed to destroy tun interface %s: %v", tunName, err) + } + }) + + return netip.AddrFrom4([4]byte{192, 168, 1, 2}), intf +} + func setupDummyInterfacesAndRoutes(t *testing.T) { t.Helper() defaultDummy := createAndSetupDummyInterface(t, expectedExternalInt, "192.168.0.1/24") - addDummyRoute(t, "0.0.0.0/0", net.IPv4(192, 168, 0, 1), defaultDummy) + addDummyRoute(t, "0.0.0.0/0", netip.AddrFrom4([4]byte{192, 168, 0, 1}), defaultDummy) otherDummy := createAndSetupDummyInterface(t, expectedInternalInt, "192.168.1.1/24") - addDummyRoute(t, "10.0.0.0/8", net.IPv4(192, 168, 1, 1), otherDummy) + addDummyRoute(t, "10.0.0.0/8", netip.AddrFrom4([4]byte{192, 168, 1, 1}), otherDummy) } diff --git a/client/internal/routemanager/systemops/systemops_generic.go b/client/internal/routemanager/systemops/systemops_generic.go index eaef01815..d223a27b2 100644 --- a/client/internal/routemanager/systemops/systemops_generic.go +++ b/client/internal/routemanager/systemops/systemops_generic.go @@ -17,7 +17,6 @@ import ( nberrors "github.com/netbirdio/netbird/client/errors" "github.com/netbirdio/netbird/client/iface/netstack" - "github.com/netbirdio/netbird/client/internal/routemanager/iface" "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" "github.com/netbirdio/netbird/client/internal/routemanager/util" "github.com/netbirdio/netbird/client/internal/routemanager/vars" @@ -106,59 +105,15 @@ func (r *SysOps) cleanupRefCounter(stateManager *statemanager.Manager) error { return nil } -// TODO: fix: for default our wg address now appears as the default gw -func (r *SysOps) addRouteForCurrentDefaultGateway(prefix netip.Prefix) error { - addr := netip.IPv4Unspecified() - if prefix.Addr().Is6() { - addr = netip.IPv6Unspecified() - } - - nexthop, err := GetNextHop(addr) - if err != nil && !errors.Is(err, vars.ErrRouteNotFound) { - return fmt.Errorf("get existing route gateway: %s", err) - } - - if !prefix.Contains(nexthop.IP) { - log.Debugf("Skipping adding a new route for gateway %s because it is not in the network %s", nexthop.IP, prefix) - return nil - } - - gatewayPrefix := netip.PrefixFrom(nexthop.IP, 32) - if nexthop.IP.Is6() { - gatewayPrefix = netip.PrefixFrom(nexthop.IP, 128) - } - - ok, err := existsInRouteTable(gatewayPrefix) - if err != nil { - return fmt.Errorf("unable to check if there is an existing route for gateway %s. error: %s", gatewayPrefix, err) - } - - if ok { - log.Debugf("Skipping adding a new route for gateway %s because it already exists", gatewayPrefix) - return nil - } - - nexthop, err = GetNextHop(nexthop.IP) - if err != nil && !errors.Is(err, vars.ErrRouteNotFound) { - return fmt.Errorf("unable to get the next hop for the default gateway address. error: %s", err) - } - - log.Debugf("Adding a new route for gateway %s with next hop %s", gatewayPrefix, nexthop.IP) - return r.addToRouteTable(gatewayPrefix, nexthop) -} - // addRouteToNonVPNIntf adds a new route to the routing table for the given prefix and returns the next hop and interface. // If the next hop or interface is pointing to the VPN interface, it will return the initial values. -func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf iface.WGIface, initialNextHop Nexthop) (Nexthop, error) { - addr := prefix.Addr() - switch { - case addr.IsLoopback(), - addr.IsLinkLocalUnicast(), - addr.IsLinkLocalMulticast(), - addr.IsInterfaceLocalMulticast(), - addr.IsUnspecified(), - addr.IsMulticast(): +func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf wgIface, initialNextHop Nexthop) (Nexthop, error) { + if err := r.validateRoute(prefix); err != nil { + return Nexthop{}, err + } + addr := prefix.Addr() + if addr.IsUnspecified() { return Nexthop{}, vars.ErrRouteNotAllowed } @@ -179,10 +134,7 @@ func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf iface.WGIface Intf: nexthop.Intf, } - vpnAddr, ok := netip.AddrFromSlice(vpnIntf.Address().IP) - if !ok { - return Nexthop{}, fmt.Errorf("failed to convert vpn address to netip.Addr") - } + vpnAddr := vpnIntf.Address().IP // if next hop is the VPN address or the interface is the VPN interface, we should use the initial values if exitNextHop.IP == vpnAddr || exitNextHop.Intf != nil && exitNextHop.Intf.Name == vpnIntf.Name() { @@ -271,32 +223,7 @@ func (r *SysOps) genericAddVPNRoute(prefix netip.Prefix, intf *net.Interface) er return nil } - return r.addNonExistingRoute(prefix, intf) -} - -// addNonExistingRoute adds a new route to the vpn interface if it doesn't exist in the current routing table -func (r *SysOps) addNonExistingRoute(prefix netip.Prefix, intf *net.Interface) error { - ok, err := existsInRouteTable(prefix) - if err != nil { - return fmt.Errorf("exists in route table: %w", err) - } - if ok { - log.Warnf("Skipping adding a new route for network %s because it already exists", prefix) - return nil - } - - ok, err = isSubRange(prefix) - if err != nil { - return fmt.Errorf("sub range: %w", err) - } - - if ok { - if err := r.addRouteForCurrentDefaultGateway(prefix); err != nil { - log.Warnf("Unable to add route for current default gateway route. Will proceed without it. error: %s", err) - } - } - - return r.addToRouteTable(prefix, Nexthop{netip.Addr{}, intf}) + return r.addToRouteTable(prefix, nextHop) } // genericRemoveVPNRoute removes the route from the vpn interface. If a default prefix is given, @@ -408,12 +335,8 @@ func GetNextHop(ip netip.Addr) (Nexthop, error) { log.Debugf("Route for %s: interface %v nexthop %v, preferred source %v", ip, intf, gateway, preferredSrc) if gateway == nil { - if runtime.GOOS == "freebsd" { - return Nexthop{Intf: intf}, nil - } - if preferredSrc == nil { - return Nexthop{}, vars.ErrRouteNotFound + return Nexthop{Intf: intf}, nil } log.Debugf("No next hop found for IP %s, using preferred source %s", ip, preferredSrc) @@ -457,32 +380,6 @@ func ipToAddr(ip net.IP, intf *net.Interface) (netip.Addr, error) { return addr.Unmap(), nil } -func existsInRouteTable(prefix netip.Prefix) (bool, error) { - routes, err := GetRoutesFromTable() - if err != nil { - return false, fmt.Errorf("get routes from table: %w", err) - } - for _, tableRoute := range routes { - if tableRoute == prefix { - return true, nil - } - } - return false, nil -} - -func isSubRange(prefix netip.Prefix) (bool, error) { - routes, err := GetRoutesFromTable() - if err != nil { - return false, fmt.Errorf("get routes from table: %w", err) - } - for _, tableRoute := range routes { - if tableRoute.Bits() > vars.MinRangeBits && tableRoute.Contains(prefix.Addr()) && tableRoute.Bits() < prefix.Bits() { - return true, nil - } - } - return false, nil -} - // IsAddrRouted checks if the candidate address would route to the vpn, in which case it returns true and the matched prefix. func IsAddrRouted(addr netip.Addr, vpnRoutes []netip.Prefix) (bool, netip.Prefix) { localRoutes, err := hasSeparateRouting() diff --git a/client/internal/routemanager/systemops/systemops_generic_test.go b/client/internal/routemanager/systemops/systemops_generic_test.go index 5b7b13f97..2a57e6044 100644 --- a/client/internal/routemanager/systemops/systemops_generic_test.go +++ b/client/internal/routemanager/systemops/systemops_generic_test.go @@ -3,23 +3,25 @@ package systemops import ( - "bytes" "context" + "errors" "fmt" "net" "net/netip" - "os" + "os/exec" "runtime" + "strconv" "strings" + "syscall" "testing" "github.com/pion/transport/v3/stdnet" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/internal/routemanager/vars" ) type dialer interface { @@ -27,105 +29,370 @@ type dialer interface { DialContext(ctx context.Context, network, address string) (net.Conn, error) } -func TestAddRemoveRoutes(t *testing.T) { +func TestAddVPNRoute(t *testing.T) { testCases := []struct { - name string - prefix netip.Prefix - shouldRouteToWireguard bool - shouldBeRemoved bool + name string + prefix netip.Prefix + expectError bool }{ { - name: "Should Add And Remove Route 100.66.120.0/24", - prefix: netip.MustParsePrefix("100.66.120.0/24"), - shouldRouteToWireguard: true, - shouldBeRemoved: true, + name: "IPv4 - Private network route", + prefix: netip.MustParsePrefix("10.10.100.0/24"), }, { - name: "Should Not Add Or Remove Route 127.0.0.1/32", - prefix: netip.MustParsePrefix("127.0.0.1/32"), - shouldRouteToWireguard: false, - shouldBeRemoved: false, + name: "IPv4 Single host", + prefix: netip.MustParsePrefix("10.111.111.111/32"), + }, + { + name: "IPv4 RFC3927 test range", + prefix: netip.MustParsePrefix("198.51.100.0/24"), + }, + { + name: "IPv4 Default route", + prefix: netip.MustParsePrefix("0.0.0.0/0"), + }, + + { + name: "IPv6 Subnet", + prefix: netip.MustParsePrefix("fdb1:848a:7e16::/48"), + }, + { + name: "IPv6 Single host", + prefix: netip.MustParsePrefix("fdb1:848a:7e16:a::b/128"), + }, + { + name: "IPv6 Default route", + prefix: netip.MustParsePrefix("::/0"), + }, + + // IPv4 addresses that should be rejected (matches validateRoute logic) + { + name: "IPv4 Loopback", + prefix: netip.MustParsePrefix("127.0.0.1/32"), + expectError: true, + }, + { + name: "IPv4 Link-local unicast", + prefix: netip.MustParsePrefix("169.254.1.1/32"), + expectError: true, + }, + { + name: "IPv4 Link-local multicast", + prefix: netip.MustParsePrefix("224.0.0.251/32"), + expectError: true, + }, + { + name: "IPv4 Multicast", + prefix: netip.MustParsePrefix("239.255.255.250/32"), + expectError: true, + }, + { + name: "IPv4 Unspecified with prefix", + prefix: netip.MustParsePrefix("0.0.0.0/32"), + expectError: true, + }, + + // IPv6 addresses that should be rejected (matches validateRoute logic) + { + name: "IPv6 Loopback", + prefix: netip.MustParsePrefix("::1/128"), + expectError: true, + }, + { + name: "IPv6 Link-local unicast", + prefix: netip.MustParsePrefix("fe80::1/128"), + expectError: true, + }, + { + name: "IPv6 Link-local multicast", + prefix: netip.MustParsePrefix("ff02::1/128"), + expectError: true, + }, + { + name: "IPv6 Interface-local multicast", + prefix: netip.MustParsePrefix("ff01::1/128"), + expectError: true, + }, + { + name: "IPv6 Multicast", + prefix: netip.MustParsePrefix("ff00::1/128"), + expectError: true, + }, + { + name: "IPv6 Unspecified with prefix", + prefix: netip.MustParsePrefix("::/128"), + expectError: true, + }, + + { + name: "IPv4 WireGuard interface network overlap", + prefix: netip.MustParsePrefix("100.65.75.0/24"), + expectError: true, + }, + { + name: "IPv4 WireGuard interface network subnet", + prefix: netip.MustParsePrefix("100.65.75.0/32"), + expectError: true, }, } for n, testCase := range testCases { - // todo resolve test execution on freebsd - if runtime.GOOS == "freebsd" { - t.Skip("skipping ", testCase.name, " on freebsd") - } t.Run(testCase.name, func(t *testing.T) { t.Setenv("NB_DISABLE_ROUTE_CACHE", "true") - peerPrivateKey, _ := wgtypes.GeneratePrivateKey() - newNet, err := stdnet.NewNet() - if err != nil { - t.Fatal(err) - } - opts := iface.WGIFaceOpts{ - IFaceName: fmt.Sprintf("utun53%d", n), - Address: "100.65.75.2/24", - WGPrivKey: peerPrivateKey.String(), - MTU: iface.DefaultMTU, - TransportNet: newNet, - } - wgInterface, err := iface.NewWGIFace(opts) - require.NoError(t, err, "should create testing WGIface interface") - defer wgInterface.Close() - - err = wgInterface.Create() - require.NoError(t, err, "should create testing wireguard interface") + wgInterface := createWGInterface(t, fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100+n) r := NewSysOps(wgInterface, nil) - - _, _, err = r.SetupRouting(nil, nil) + _, _, err := r.SetupRouting(nil, nil) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, r.CleanupRouting(nil)) }) - index, err := net.InterfaceByName(wgInterface.Name()) - require.NoError(t, err, "InterfaceByName should not return err") - intf := &net.Interface{Index: index.Index, Name: wgInterface.Name()} + intf, err := net.InterfaceByName(wgInterface.Name()) + require.NoError(t, err) + // add the route err = r.AddVPNRoute(testCase.prefix, intf) - require.NoError(t, err, "genericAddVPNRoute should not return err") + if testCase.expectError { + assert.ErrorIs(t, err, vars.ErrRouteNotAllowed) + return + } - if testCase.shouldRouteToWireguard { - assertWGOutInterface(t, testCase.prefix, wgInterface, false) + // validate it's pointing to the WireGuard interface + require.NoError(t, err) + + nextHop := getNextHop(t, testCase.prefix.Addr()) + assert.Equal(t, wgInterface.Name(), nextHop.Intf.Name, "next hop interface should be WireGuard interface") + + // remove route again + err = r.RemoveVPNRoute(testCase.prefix, intf) + require.NoError(t, err) + + // validate it's gone + nextHop, err = GetNextHop(testCase.prefix.Addr()) + require.True(t, + errors.Is(err, vars.ErrRouteNotFound) || err == nil && nextHop.Intf != nil && nextHop.Intf.Name != wgInterface.Name(), + "err: %v, next hop: %v", err, nextHop) + }) + } +} + +func getNextHop(t *testing.T, addr netip.Addr) Nexthop { + t.Helper() + + if runtime.GOOS == "windows" || runtime.GOOS == "linux" { + nextHop, err := GetNextHop(addr) + + if runtime.GOOS == "windows" && errors.Is(err, vars.ErrRouteNotFound) && addr.Is6() { + // TODO: Fix this test. It doesn't return the route when running in a windows github runner, but it is + // present in the route table. + t.Skip("Skipping windows test") + } + + require.NoError(t, err) + require.NotNil(t, nextHop.Intf, "next hop interface should not be nil for %s", addr) + + return nextHop + } + // GetNextHop for bsd is buggy and returns the wrong interface for the default route. + + if addr.IsUnspecified() { + // On macOS, querying 0.0.0.0 returns the wrong interface + if addr.Is4() { + addr = netip.MustParseAddr("1.2.3.4") + } else { + addr = netip.MustParseAddr("2001:db8::1") + } + } + + cmd := exec.Command("route", "-n", "get", addr.String()) + if addr.Is6() { + cmd = exec.Command("route", "-n", "get", "-inet6", addr.String()) + } + + output, err := cmd.CombinedOutput() + t.Logf("route output: %s", output) + require.NoError(t, err, "%s failed") + + lines := strings.Split(string(output), "\n") + var intf string + var gateway string + + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "interface:") { + intf = strings.TrimSpace(strings.TrimPrefix(line, "interface:")) + } else if strings.HasPrefix(line, "gateway:") { + gateway = strings.TrimSpace(strings.TrimPrefix(line, "gateway:")) + } + } + + require.NotEmpty(t, intf, "interface should be found in route output") + + iface, err := net.InterfaceByName(intf) + require.NoError(t, err, "interface %s should exist", intf) + + nexthop := Nexthop{Intf: iface} + + if gateway != "" && gateway != "link#"+strconv.Itoa(iface.Index) { + addr, err := netip.ParseAddr(gateway) + if err == nil { + nexthop.IP = addr + } + } + + return nexthop +} + +func TestAddRouteToNonVPNIntf(t *testing.T) { + testCases := []struct { + name string + prefix netip.Prefix + expectError bool + errorType error + }{ + { + name: "IPv4 RFC3927 test range", + prefix: netip.MustParsePrefix("198.51.100.0/24"), + }, + { + name: "IPv4 Single host", + prefix: netip.MustParsePrefix("8.8.8.8/32"), + }, + { + name: "IPv6 External network route", + prefix: netip.MustParsePrefix("2001:db8:1000::/48"), + }, + { + name: "IPv6 Single host", + prefix: netip.MustParsePrefix("2001:db8::1/128"), + }, + { + name: "IPv6 Subnet", + prefix: netip.MustParsePrefix("2a05:d014:1f8d::/48"), + }, + { + name: "IPv6 Single host", + prefix: netip.MustParsePrefix("2a05:d014:1f8d:7302:ebca:ec15:b24d:d07e/128"), + }, + + // Addresses that should be rejected + { + name: "IPv4 Loopback", + prefix: netip.MustParsePrefix("127.0.0.1/32"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv4 Link-local unicast", + prefix: netip.MustParsePrefix("169.254.1.1/32"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv4 Multicast", + prefix: netip.MustParsePrefix("239.255.255.250/32"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv4 Unspecified", + prefix: netip.MustParsePrefix("0.0.0.0/0"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv6 Loopback", + prefix: netip.MustParsePrefix("::1/128"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv6 Link-local unicast", + prefix: netip.MustParsePrefix("fe80::1/128"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv6 Multicast", + prefix: netip.MustParsePrefix("ff00::1/128"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv6 Unspecified", + prefix: netip.MustParsePrefix("::/0"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + { + name: "IPv4 WireGuard interface network overlap", + prefix: netip.MustParsePrefix("100.65.75.0/24"), + expectError: true, + errorType: vars.ErrRouteNotAllowed, + }, + } + + for n, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Setenv("NB_DISABLE_ROUTE_CACHE", "true") + + wgInterface := createWGInterface(t, fmt.Sprintf("utun54%d", n), "100.65.75.2/24", 33200+n) + + r := NewSysOps(wgInterface, nil) + _, _, err := r.SetupRouting(nil, nil) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, r.CleanupRouting(nil)) + }) + + initialNextHopV4, err := GetNextHop(netip.IPv4Unspecified()) + require.NoError(t, err, "Should be able to get IPv4 default route") + t.Logf("Initial IPv4 next hop: %s", initialNextHopV4) + + initialNextHopV6, err := GetNextHop(netip.IPv6Unspecified()) + if testCase.prefix.Addr().Is6() && + (errors.Is(err, vars.ErrRouteNotFound) || initialNextHopV6.Intf != nil && strings.HasPrefix(initialNextHopV6.Intf.Name, "utun")) { + t.Skip("Skipping test as no ipv6 default route is available") + } + if err != nil && !errors.Is(err, vars.ErrRouteNotFound) { + t.Fatalf("Failed to get IPv6 default route: %v", err) + } + + var initialNextHop Nexthop + if testCase.prefix.Addr().Is6() { + initialNextHop = initialNextHopV6 } else { - assertWGOutInterface(t, testCase.prefix, wgInterface, true) + initialNextHop = initialNextHopV4 } - exists, err := existsInRouteTable(testCase.prefix) - require.NoError(t, err, "existsInRouteTable should not return err") - if exists && testCase.shouldRouteToWireguard { - err = r.RemoveVPNRoute(testCase.prefix, intf) - require.NoError(t, err, "genericRemoveVPNRoute should not return err") - prefixNexthop, err := GetNextHop(testCase.prefix.Addr()) - require.NoError(t, err, "GetNextHop should not return err") + nexthop, err := r.addRouteToNonVPNIntf(testCase.prefix, wgInterface, initialNextHop) - internetNexthop, err := GetNextHop(netip.MustParseAddr("0.0.0.0")) - require.NoError(t, err) - - if testCase.shouldBeRemoved { - require.Equal(t, internetNexthop.IP, prefixNexthop.IP, "route should be pointing to default internet gateway") - } else { - require.NotEqual(t, internetNexthop.IP, prefixNexthop.IP, "route should be pointing to a different gateway than the internet gateway") - } + if testCase.expectError { + require.ErrorIs(t, err, vars.ErrRouteNotAllowed) + return } + require.NoError(t, err) + t.Logf("Next hop for %s: %s", testCase.prefix, nexthop) + + // Verify the route was added and points to non-VPN interface + currentNextHop, err := GetNextHop(testCase.prefix.Addr()) + require.NoError(t, err) + assert.NotEqual(t, wgInterface.Name(), currentNextHop.Intf.Name, "Route should not point to VPN interface") + + err = r.removeFromRouteTable(testCase.prefix, nexthop) + assert.NoError(t, err) }) } } func TestGetNextHop(t *testing.T) { - if runtime.GOOS == "freebsd" { - t.Skip("skipping on freebsd") - } - nexthop, err := GetNextHop(netip.MustParseAddr("0.0.0.0")) + defaultNh, err := GetNextHop(netip.MustParseAddr("0.0.0.0")) if err != nil { t.Fatal("shouldn't return error when fetching the gateway: ", err) } - if !nexthop.IP.IsValid() { + if !defaultNh.IP.IsValid() { t.Fatal("should return a gateway") } addresses, err := net.InterfaceAddrs() @@ -133,7 +400,6 @@ func TestGetNextHop(t *testing.T) { t.Fatal("shouldn't return error when fetching interface addresses: ", err) } - var testingIP string var testingPrefix netip.Prefix for _, address := range addresses { if address.Network() != "ip+net" { @@ -141,213 +407,23 @@ func TestGetNextHop(t *testing.T) { } prefix := netip.MustParsePrefix(address.String()) if !prefix.Addr().IsLoopback() && prefix.Addr().Is4() { - testingIP = prefix.Addr().String() testingPrefix = prefix.Masked() break } } - localIP, err := GetNextHop(testingPrefix.Addr()) + nh, err := GetNextHop(testingPrefix.Addr()) if err != nil { t.Fatal("shouldn't return error: ", err) } - if !localIP.IP.IsValid() { + if nh.Intf == nil { t.Fatal("should return a gateway for local network") } - if localIP.IP.String() == nexthop.IP.String() { - t.Fatal("local IP should not match with gateway IP") + if nh.IP.String() == defaultNh.IP.String() { + t.Fatal("next hop IP should not match with default gateway IP") } - if localIP.IP.String() != testingIP { - t.Fatalf("local IP should match with testing IP: want %s got %s", testingIP, localIP.IP.String()) - } -} - -func TestAddExistAndRemoveRoute(t *testing.T) { - defaultNexthop, err := GetNextHop(netip.MustParseAddr("0.0.0.0")) - t.Log("defaultNexthop: ", defaultNexthop) - if err != nil { - t.Fatal("shouldn't return error when fetching the gateway: ", err) - } - testCases := []struct { - name string - prefix netip.Prefix - preExistingPrefix netip.Prefix - shouldAddRoute bool - }{ - { - name: "Should Add And Remove random Route", - prefix: netip.MustParsePrefix("99.99.99.99/32"), - shouldAddRoute: true, - }, - { - name: "Should Not Add Route if overlaps with default gateway", - prefix: netip.MustParsePrefix(defaultNexthop.IP.String() + "/31"), - shouldAddRoute: false, - }, - { - name: "Should Add Route if bigger network exists", - prefix: netip.MustParsePrefix("100.100.100.0/24"), - preExistingPrefix: netip.MustParsePrefix("100.100.0.0/16"), - shouldAddRoute: true, - }, - { - name: "Should Add Route if smaller network exists", - prefix: netip.MustParsePrefix("100.100.0.0/16"), - preExistingPrefix: netip.MustParsePrefix("100.100.100.0/24"), - shouldAddRoute: true, - }, - { - name: "Should Not Add Route if same network exists", - prefix: netip.MustParsePrefix("100.100.0.0/16"), - preExistingPrefix: netip.MustParsePrefix("100.100.0.0/16"), - shouldAddRoute: false, - }, - } - - for n, testCase := range testCases { - - var buf bytes.Buffer - log.SetOutput(&buf) - defer func() { - log.SetOutput(os.Stderr) - }() - t.Run(testCase.name, func(t *testing.T) { - t.Setenv("NB_USE_LEGACY_ROUTING", "true") - t.Setenv("NB_DISABLE_ROUTE_CACHE", "true") - - peerPrivateKey, _ := wgtypes.GeneratePrivateKey() - newNet, err := stdnet.NewNet() - if err != nil { - t.Fatal(err) - } - opts := iface.WGIFaceOpts{ - IFaceName: fmt.Sprintf("utun53%d", n), - Address: "100.65.75.2/24", - WGPort: 33100, - WGPrivKey: peerPrivateKey.String(), - MTU: iface.DefaultMTU, - TransportNet: newNet, - } - wgInterface, err := iface.NewWGIFace(opts) - require.NoError(t, err, "should create testing WGIface interface") - defer wgInterface.Close() - - err = wgInterface.Create() - require.NoError(t, err, "should create testing wireguard interface") - - index, err := net.InterfaceByName(wgInterface.Name()) - require.NoError(t, err, "InterfaceByName should not return err") - intf := &net.Interface{Index: index.Index, Name: wgInterface.Name()} - - r := NewSysOps(wgInterface, nil) - - // Prepare the environment - if testCase.preExistingPrefix.IsValid() { - err := r.AddVPNRoute(testCase.preExistingPrefix, intf) - require.NoError(t, err, "should not return err when adding pre-existing route") - } - - // Add the route - err = r.AddVPNRoute(testCase.prefix, intf) - require.NoError(t, err, "should not return err when adding route") - - if testCase.shouldAddRoute { - // test if route exists after adding - ok, err := existsInRouteTable(testCase.prefix) - require.NoError(t, err, "should not return err") - require.True(t, ok, "route should exist") - - // remove route again if added - err = r.RemoveVPNRoute(testCase.prefix, intf) - require.NoError(t, err, "should not return err") - } - - // route should either not have been added or should have been removed - // In case of already existing route, it should not have been added (but still exist) - ok, err := existsInRouteTable(testCase.prefix) - t.Log("Buffer string: ", buf.String()) - require.NoError(t, err, "should not return err") - - if !strings.Contains(buf.String(), "because it already exists") { - require.False(t, ok, "route should not exist") - } - }) - } -} - -func TestIsSubRange(t *testing.T) { - addresses, err := net.InterfaceAddrs() - if err != nil { - t.Fatal("shouldn't return error when fetching interface addresses: ", err) - } - - var subRangeAddressPrefixes []netip.Prefix - var nonSubRangeAddressPrefixes []netip.Prefix - for _, address := range addresses { - p := netip.MustParsePrefix(address.String()) - if !p.Addr().IsLoopback() && p.Addr().Is4() && p.Bits() < 32 { - p2 := netip.PrefixFrom(p.Masked().Addr(), p.Bits()+1) - subRangeAddressPrefixes = append(subRangeAddressPrefixes, p2) - nonSubRangeAddressPrefixes = append(nonSubRangeAddressPrefixes, p.Masked()) - } - } - - for _, prefix := range subRangeAddressPrefixes { - isSubRangePrefix, err := isSubRange(prefix) - if err != nil { - t.Fatal("shouldn't return error when checking if address is sub-range: ", err) - } - if !isSubRangePrefix { - t.Fatalf("address %s should be sub-range of an existing route in the table", prefix) - } - } - - for _, prefix := range nonSubRangeAddressPrefixes { - isSubRangePrefix, err := isSubRange(prefix) - if err != nil { - t.Fatal("shouldn't return error when checking if address is sub-range: ", err) - } - if isSubRangePrefix { - t.Fatalf("address %s should not be sub-range of an existing route in the table", prefix) - } - } -} - -func TestExistsInRouteTable(t *testing.T) { - addresses, err := net.InterfaceAddrs() - if err != nil { - t.Fatal("shouldn't return error when fetching interface addresses: ", err) - } - - var addressPrefixes []netip.Prefix - for _, address := range addresses { - p := netip.MustParsePrefix(address.String()) - - switch { - case p.Addr().Is6(): - continue - // Windows sometimes has hidden interface link local addrs that don't turn up on any interface - case runtime.GOOS == "windows" && p.Addr().IsLinkLocalUnicast(): - continue - // Linux loopback 127/8 is in the local table, not in the main table and always takes precedence - case runtime.GOOS == "linux" && p.Addr().IsLoopback(): - continue - // FreeBSD loopback 127/8 is not added to the routing table - case runtime.GOOS == "freebsd" && p.Addr().IsLoopback(): - continue - default: - addressPrefixes = append(addressPrefixes, p.Masked()) - } - } - - for _, prefix := range addressPrefixes { - exists, err := existsInRouteTable(prefix) - if err != nil { - t.Fatal("shouldn't return error when checking if address exists in route table: ", err) - } - if !exists { - t.Fatalf("address %s should exist in route table", prefix) - } + if nh.Intf.Name != defaultNh.Intf.Name { + t.Fatalf("next hop interface name should match with default gateway interface name, got: %s, want: %s", nh.Intf.Name, defaultNh.Intf.Name) } } @@ -384,11 +460,16 @@ func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, listen func setupRouteAndCleanup(t *testing.T, r *SysOps, prefix netip.Prefix, intf *net.Interface) { t.Helper() - err := r.AddVPNRoute(prefix, intf) - require.NoError(t, err, "addVPNRoute should not return err") + if err := r.AddVPNRoute(prefix, intf); err != nil { + if !errors.Is(err, syscall.EEXIST) && !errors.Is(err, vars.ErrRouteNotAllowed) { + t.Fatalf("addVPNRoute should not return err: %v", err) + } + t.Logf("addVPNRoute %v returned: %v", prefix, err) + } t.Cleanup(func() { - err = r.RemoveVPNRoute(prefix, intf) - assert.NoError(t, err, "removeVPNRoute should not return err") + if err := r.RemoveVPNRoute(prefix, intf); err != nil && !errors.Is(err, vars.ErrRouteNotAllowed) { + t.Fatalf("removeVPNRoute should not return err: %v", err) + } }) } @@ -422,28 +503,10 @@ func setupTestEnv(t *testing.T) { // 10.10.0.0/24 more specific route exists in vpn table setupRouteAndCleanup(t, r, netip.MustParsePrefix("10.10.0.0/24"), intf) - // 127.0.10.0/24 more specific route exists in vpn table - setupRouteAndCleanup(t, r, netip.MustParsePrefix("127.0.10.0/24"), intf) - // unique route in vpn table setupRouteAndCleanup(t, r, netip.MustParsePrefix("172.16.0.0/12"), intf) } -func assertWGOutInterface(t *testing.T, prefix netip.Prefix, wgIface *iface.WGIface, invert bool) { - t.Helper() - if runtime.GOOS == "linux" && prefix.Addr().IsLoopback() { - return - } - - prefixNexthop, err := GetNextHop(prefix.Addr()) - require.NoError(t, err, "GetNextHop should not return err") - if invert { - assert.NotEqual(t, wgIface.Address().IP.String(), prefixNexthop.IP.String(), "route should not point to wireguard interface IP") - } else { - assert.Equal(t, wgIface.Address().IP.String(), prefixNexthop.IP.String(), "route should point to wireguard interface IP") - } -} - func TestIsVpnRoute(t *testing.T) { tests := []struct { name string diff --git a/client/internal/routemanager/systemops/systemops_linux.go b/client/internal/routemanager/systemops/systemops_linux.go index 59b6346c6..b48cfa242 100644 --- a/client/internal/routemanager/systemops/systemops_linux.go +++ b/client/internal/routemanager/systemops/systemops_linux.go @@ -149,6 +149,10 @@ func (r *SysOps) removeFromRouteTable(prefix netip.Prefix, nexthop Nexthop) erro } func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error { + if err := r.validateRoute(prefix); err != nil { + return err + } + if !nbnet.AdvancedRouting() { return r.genericAddVPNRoute(prefix, intf) } @@ -172,6 +176,10 @@ func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error { } func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error { + if err := r.validateRoute(prefix); err != nil { + return err + } + if !nbnet.AdvancedRouting() { return r.genericRemoveVPNRoute(prefix, intf) } @@ -219,7 +227,7 @@ func getRoutes(tableID, family int) ([]netip.Prefix, error) { ones, _ := route.Dst.Mask.Size() - prefix := netip.PrefixFrom(addr, ones) + prefix := netip.PrefixFrom(addr.Unmap(), ones) if prefix.IsValid() { prefixList = append(prefixList, prefix) } @@ -247,7 +255,7 @@ func addRoute(prefix netip.Prefix, nexthop Nexthop, tableID int) error { return fmt.Errorf("add gateway and device: %w", err) } - if err := netlink.RouteAdd(route); err != nil && !errors.Is(err, syscall.EEXIST) && !isOpErr(err) { + if err := netlink.RouteAdd(route); err != nil && !isOpErr(err) { return fmt.Errorf("netlink add route: %w", err) } @@ -270,7 +278,7 @@ func addUnreachableRoute(prefix netip.Prefix, tableID int) error { Dst: ipNet, } - if err := netlink.RouteAdd(route); err != nil && !errors.Is(err, syscall.EEXIST) && !isOpErr(err) { + if err := netlink.RouteAdd(route); err != nil && !isOpErr(err) { return fmt.Errorf("netlink add unreachable route: %w", err) } diff --git a/client/internal/routemanager/systemops/systemops_linux_test.go b/client/internal/routemanager/systemops/systemops_linux_test.go index f0d7472dc..880296d91 100644 --- a/client/internal/routemanager/systemops/systemops_linux_test.go +++ b/client/internal/routemanager/systemops/systemops_linux_test.go @@ -19,7 +19,6 @@ import ( ) var expectedVPNint = "wgtest0" -var expectedLoopbackInt = "lo" var expectedExternalInt = "dummyext0" var expectedInternalInt = "dummyint0" @@ -31,12 +30,6 @@ func init() { dialer: &net.Dialer{}, expectedPacket: createPacketExpectation("192.168.1.1", 12345, "10.10.0.2", 53), }, - { - name: "To more specific route (local) without custom dialer via physical interface", - expectedInterface: expectedLoopbackInt, - dialer: &net.Dialer{}, - expectedPacket: createPacketExpectation("127.0.0.1", 12345, "127.0.10.1", 53), - }, }...) } diff --git a/client/internal/routemanager/systemops/systemops_nonlinux.go b/client/internal/routemanager/systemops/systemops_nonlinux.go index 3b52fc7af..59581255f 100644 --- a/client/internal/routemanager/systemops/systemops_nonlinux.go +++ b/client/internal/routemanager/systemops/systemops_nonlinux.go @@ -11,10 +11,16 @@ import ( ) func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error { + if err := r.validateRoute(prefix); err != nil { + return err + } return r.genericAddVPNRoute(prefix, intf) } func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error { + if err := r.validateRoute(prefix); err != nil { + return err + } return r.genericRemoveVPNRoute(prefix, intf) } diff --git a/client/internal/routemanager/systemops/systemops_test.go b/client/internal/routemanager/systemops/systemops_test.go new file mode 100644 index 000000000..1d1f78830 --- /dev/null +++ b/client/internal/routemanager/systemops/systemops_test.go @@ -0,0 +1,268 @@ +package systemops + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/routemanager/notifier" + "github.com/netbirdio/netbird/client/internal/routemanager/vars" +) + +type mockWGIface struct { + address wgaddr.Address + name string +} + +func (m *mockWGIface) Address() wgaddr.Address { + return m.address +} + +func (m *mockWGIface) Name() string { + return m.name +} + +func TestSysOps_validateRoute(t *testing.T) { + wgNetwork := netip.MustParsePrefix("10.0.0.0/24") + mockWG := &mockWGIface{ + address: wgaddr.Address{ + IP: wgNetwork.Addr(), + Network: wgNetwork, + }, + name: "wg0", + } + + sysOps := &SysOps{ + wgInterface: mockWG, + notifier: ¬ifier.Notifier{}, + } + + tests := []struct { + name string + prefix string + expectError bool + }{ + // Valid routes + { + name: "valid IPv4 route", + prefix: "192.168.1.0/24", + expectError: false, + }, + { + name: "valid IPv6 route", + prefix: "2001:db8::/32", + expectError: false, + }, + { + name: "valid single IPv4 host", + prefix: "8.8.8.8/32", + expectError: false, + }, + { + name: "valid single IPv6 host", + prefix: "2001:4860:4860::8888/128", + expectError: false, + }, + + // Invalid routes - loopback + { + name: "IPv4 loopback", + prefix: "127.0.0.1/32", + expectError: true, + }, + { + name: "IPv6 loopback", + prefix: "::1/128", + expectError: true, + }, + + // Invalid routes - link-local unicast + { + name: "IPv4 link-local unicast", + prefix: "169.254.1.1/32", + expectError: true, + }, + { + name: "IPv6 link-local unicast", + prefix: "fe80::1/128", + expectError: true, + }, + + // Invalid routes - multicast + { + name: "IPv4 multicast", + prefix: "224.0.0.1/32", + expectError: true, + }, + { + name: "IPv6 multicast", + prefix: "ff02::1/128", + expectError: true, + }, + + // Invalid routes - link-local multicast + { + name: "IPv4 link-local multicast", + prefix: "224.0.0.0/24", + expectError: true, + }, + { + name: "IPv6 link-local multicast", + prefix: "ff02::/16", + expectError: true, + }, + + // Invalid routes - interface-local multicast (IPv6 only) + { + name: "IPv6 interface-local multicast", + prefix: "ff01::1/128", + expectError: true, + }, + + // Invalid routes - overlaps with WG interface network + { + name: "overlaps with WG network - exact match", + prefix: "10.0.0.0/24", + expectError: true, + }, + { + name: "overlaps with WG network - subset", + prefix: "10.0.0.1/32", + expectError: true, + }, + { + name: "overlaps with WG network - host in range", + prefix: "10.0.0.100/32", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix, err := netip.ParsePrefix(tt.prefix) + require.NoError(t, err, "Failed to parse test prefix %s", tt.prefix) + + err = sysOps.validateRoute(prefix) + + if tt.expectError { + require.Error(t, err, "validateRoute() expected error for %s", tt.prefix) + assert.Equal(t, vars.ErrRouteNotAllowed, err, "validateRoute() expected ErrRouteNotAllowed for %s", tt.prefix) + } else { + assert.NoError(t, err, "validateRoute() expected no error for %s", tt.prefix) + } + }) + } +} + +func TestSysOps_validateRoute_SubnetOverlap(t *testing.T) { + wgNetwork := netip.MustParsePrefix("192.168.100.0/24") + mockWG := &mockWGIface{ + address: wgaddr.Address{ + IP: wgNetwork.Addr(), + Network: wgNetwork, + }, + name: "wg0", + } + + sysOps := &SysOps{ + wgInterface: mockWG, + notifier: ¬ifier.Notifier{}, + } + + tests := []struct { + name string + prefix string + expectError bool + description string + }{ + { + name: "identical subnet", + prefix: "192.168.100.0/24", + expectError: true, + description: "exact same network as WG interface", + }, + { + name: "broader subnet containing WG network", + prefix: "192.168.0.0/16", + expectError: false, + description: "broader network that contains WG network should be allowed", + }, + { + name: "host within WG network", + prefix: "192.168.100.50/32", + expectError: true, + description: "specific host within WG network", + }, + { + name: "subnet within WG network", + prefix: "192.168.100.128/25", + expectError: true, + description: "smaller subnet within WG network", + }, + { + name: "adjacent subnet - same /23", + prefix: "192.168.101.0/24", + expectError: false, + description: "adjacent subnet, no overlap", + }, + { + name: "adjacent subnet - different /16", + prefix: "192.167.100.0/24", + expectError: false, + description: "different network, no overlap", + }, + { + name: "WG network broadcast address", + prefix: "192.168.100.255/32", + expectError: true, + description: "broadcast address of WG network", + }, + { + name: "WG network first usable", + prefix: "192.168.100.1/32", + expectError: true, + description: "first usable address in WG network", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix, err := netip.ParsePrefix(tt.prefix) + require.NoError(t, err, "Failed to parse test prefix %s", tt.prefix) + + err = sysOps.validateRoute(prefix) + + if tt.expectError { + require.Error(t, err, "validateRoute() expected error for %s (%s)", tt.prefix, tt.description) + assert.Equal(t, vars.ErrRouteNotAllowed, err, "validateRoute() expected ErrRouteNotAllowed for %s (%s)", tt.prefix, tt.description) + } else { + assert.NoError(t, err, "validateRoute() expected no error for %s (%s)", tt.prefix, tt.description) + } + }) + } +} + +func TestSysOps_validateRoute_InvalidPrefix(t *testing.T) { + wgNetwork := netip.MustParsePrefix("10.0.0.0/24") + mockWG := &mockWGIface{ + address: wgaddr.Address{ + IP: wgNetwork.Addr(), + Network: wgNetwork, + }, + name: "wt0", + } + + sysOps := &SysOps{ + wgInterface: mockWG, + notifier: ¬ifier.Notifier{}, + } + + var invalidPrefix netip.Prefix + err := sysOps.validateRoute(invalidPrefix) + + require.Error(t, err, "validateRoute() expected error for invalid prefix") + assert.Equal(t, vars.ErrRouteNotAllowed, err, "validateRoute() expected ErrRouteNotAllowed for invalid prefix") +} diff --git a/client/internal/routemanager/systemops/systemops_unix.go b/client/internal/routemanager/systemops/systemops_unix.go index 0f8f2a341..f284e131b 100644 --- a/client/internal/routemanager/systemops/systemops_unix.go +++ b/client/internal/routemanager/systemops/systemops_unix.go @@ -3,15 +3,19 @@ package systemops import ( + "errors" "fmt" "net" "net/netip" - "os/exec" - "strings" + "strconv" + "syscall" "time" + "unsafe" "github.com/cenkalti/backoff/v4" log "github.com/sirupsen/logrus" + "golang.org/x/net/route" + "golang.org/x/sys/unix" "github.com/netbirdio/netbird/client/internal/statemanager" nbnet "github.com/netbirdio/netbird/util/net" @@ -26,48 +30,16 @@ func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error { } func (r *SysOps) addToRouteTable(prefix netip.Prefix, nexthop Nexthop) error { - return r.routeCmd("add", prefix, nexthop) + return r.routeSocket(unix.RTM_ADD, prefix, nexthop) } func (r *SysOps) removeFromRouteTable(prefix netip.Prefix, nexthop Nexthop) error { - return r.routeCmd("delete", prefix, nexthop) + return r.routeSocket(unix.RTM_DELETE, prefix, nexthop) } -func (r *SysOps) routeCmd(action string, prefix netip.Prefix, nexthop Nexthop) error { - inet := "-inet" - if prefix.Addr().Is6() { - inet = "-inet6" - } - - network := prefix.String() - if prefix.IsSingleIP() { - network = prefix.Addr().String() - } - - args := []string{"-n", action, inet, network} - if nexthop.IP.IsValid() { - args = append(args, nexthop.IP.Unmap().String()) - } else if nexthop.Intf != nil { - args = append(args, "-interface", nexthop.Intf.Name) - } - - if err := retryRouteCmd(args); err != nil { - return fmt.Errorf("failed to %s route for %s: %w", action, prefix, err) - } - return nil -} - -func retryRouteCmd(args []string) error { - operation := func() error { - out, err := exec.Command("route", args...).CombinedOutput() - log.Tracef("route %s: %s", strings.Join(args, " "), out) - // https://github.com/golang/go/issues/45736 - if err != nil && strings.Contains(string(out), "sysctl: cannot allocate memory") { - return err - } else if err != nil { - return backoff.Permanent(err) - } - return nil +func (r *SysOps) routeSocket(action int, prefix netip.Prefix, nexthop Nexthop) error { + if !prefix.IsValid() { + return fmt.Errorf("invalid prefix: %s", prefix) } expBackOff := backoff.NewExponentialBackOff() @@ -75,9 +47,157 @@ func retryRouteCmd(args []string) error { expBackOff.MaxInterval = 500 * time.Millisecond expBackOff.MaxElapsedTime = 1 * time.Second - err := backoff.Retry(operation, expBackOff) - if err != nil { - return fmt.Errorf("route cmd retry failed: %w", err) + if err := backoff.Retry(r.routeOp(action, prefix, nexthop), expBackOff); err != nil { + a := "add" + if action == unix.RTM_DELETE { + a = "remove" + } + return fmt.Errorf("%s route for %s: %w", a, prefix, err) } return nil } + +func (r *SysOps) routeOp(action int, prefix netip.Prefix, nexthop Nexthop) func() error { + operation := func() error { + fd, err := unix.Socket(syscall.AF_ROUTE, syscall.SOCK_RAW, syscall.AF_UNSPEC) + if err != nil { + return fmt.Errorf("open routing socket: %w", err) + } + defer func() { + if err := unix.Close(fd); err != nil && !errors.Is(err, unix.EBADF) { + log.Warnf("failed to close routing socket: %v", err) + } + }() + + msg, err := r.buildRouteMessage(action, prefix, nexthop) + if err != nil { + return backoff.Permanent(fmt.Errorf("build route message: %w", err)) + } + + msgBytes, err := msg.Marshal() + if err != nil { + return backoff.Permanent(fmt.Errorf("marshal route message: %w", err)) + } + + if _, err = unix.Write(fd, msgBytes); err != nil { + if errors.Is(err, unix.ENOBUFS) || errors.Is(err, unix.EAGAIN) { + return fmt.Errorf("write: %w", err) + } + return backoff.Permanent(fmt.Errorf("write: %w", err)) + } + + respBuf := make([]byte, 2048) + n, err := unix.Read(fd, respBuf) + if err != nil { + return backoff.Permanent(fmt.Errorf("read route response: %w", err)) + } + + if n > 0 { + if err := r.parseRouteResponse(respBuf[:n]); err != nil { + return backoff.Permanent(err) + } + } + + return nil + } + return operation +} + +func (r *SysOps) buildRouteMessage(action int, prefix netip.Prefix, nexthop Nexthop) (msg *route.RouteMessage, err error) { + msg = &route.RouteMessage{ + Type: action, + Flags: unix.RTF_UP, + Version: unix.RTM_VERSION, + Seq: 1, + } + + const numAddrs = unix.RTAX_NETMASK + 1 + addrs := make([]route.Addr, numAddrs) + + addrs[unix.RTAX_DST], err = addrToRouteAddr(prefix.Addr()) + if err != nil { + return nil, fmt.Errorf("build destination address for %s: %w", prefix.Addr(), err) + } + + if prefix.IsSingleIP() { + msg.Flags |= unix.RTF_HOST + } else { + addrs[unix.RTAX_NETMASK], err = prefixToRouteNetmask(prefix) + if err != nil { + return nil, fmt.Errorf("build netmask for %s: %w", prefix, err) + } + } + + if nexthop.IP.IsValid() { + msg.Flags |= unix.RTF_GATEWAY + addrs[unix.RTAX_GATEWAY], err = addrToRouteAddr(nexthop.IP.Unmap()) + if err != nil { + return nil, fmt.Errorf("build gateway IP address for %s: %w", nexthop.IP, err) + } + } else if nexthop.Intf != nil { + msg.Index = nexthop.Intf.Index + addrs[unix.RTAX_GATEWAY] = &route.LinkAddr{ + Index: nexthop.Intf.Index, + Name: nexthop.Intf.Name, + } + } + + msg.Addrs = addrs + return msg, nil +} + +func (r *SysOps) parseRouteResponse(buf []byte) error { + if len(buf) < int(unsafe.Sizeof(unix.RtMsghdr{})) { + return nil + } + + rtMsg := (*unix.RtMsghdr)(unsafe.Pointer(&buf[0])) + if rtMsg.Errno != 0 { + return fmt.Errorf("parse: %d", rtMsg.Errno) + } + + return nil +} + +// addrToRouteAddr converts a netip.Addr to the appropriate route.Addr (*route.Inet4Addr or *route.Inet6Addr). +func addrToRouteAddr(addr netip.Addr) (route.Addr, error) { + if addr.Is4() { + return &route.Inet4Addr{IP: addr.As4()}, nil + } + + if addr.Zone() == "" { + return &route.Inet6Addr{IP: addr.As16()}, nil + } + + var zone int + // zone can be either a numeric zone ID or an interface name. + if z, err := strconv.Atoi(addr.Zone()); err == nil { + zone = z + } else { + iface, err := net.InterfaceByName(addr.Zone()) + if err != nil { + return nil, fmt.Errorf("resolve zone '%s': %w", addr.Zone(), err) + } + zone = iface.Index + } + return &route.Inet6Addr{IP: addr.As16(), ZoneID: zone}, nil +} + +func prefixToRouteNetmask(prefix netip.Prefix) (route.Addr, error) { + bits := prefix.Bits() + if prefix.Addr().Is4() { + m := net.CIDRMask(bits, 32) + var maskBytes [4]byte + copy(maskBytes[:], m) + return &route.Inet4Addr{IP: maskBytes}, nil + } + + if prefix.Addr().Is6() { + m := net.CIDRMask(bits, 128) + var maskBytes [16]byte + copy(maskBytes[:], m) + return &route.Inet6Addr{IP: maskBytes}, nil + } + + return nil, fmt.Errorf("unknown IP version in prefix: %s", prefix.Addr().String()) +} diff --git a/client/internal/routemanager/systemops/systemops_windows.go b/client/internal/routemanager/systemops/systemops_windows.go index f66161595..11eaa435e 100644 --- a/client/internal/routemanager/systemops/systemops_windows.go +++ b/client/internal/routemanager/systemops/systemops_windows.go @@ -1,5 +1,3 @@ -//go:build windows - package systemops import ( @@ -9,9 +7,8 @@ import ( "net" "net/netip" "os" - "os/exec" + "runtime/debug" "strconv" - "strings" "sync" "syscall" "time" @@ -21,11 +18,12 @@ import ( "github.com/yusufpapurcu/wmi" "golang.org/x/sys/windows" - "github.com/netbirdio/netbird/client/firewall/uspfilter" "github.com/netbirdio/netbird/client/internal/statemanager" nbnet "github.com/netbirdio/netbird/util/net" ) +const InfiniteLifetime = 0xffffffff + type RouteUpdateType int // RouteUpdate represents a change in the routing table. @@ -58,9 +56,13 @@ type MSFT_NetRoute struct { AddressFamily uint16 } -// MIB_IPFORWARD_ROW2 is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_ipforward_row2 +// luid represents a locally unique identifier for network interfaces +type luid uint64 + +// MIB_IPFORWARD_ROW2 represents a route entry in the routing table. +// It is defined in https://learn.microsoft.com/en-us/windows/win32/api/netioapi/ns-netioapi-mib_ipforward_row2 type MIB_IPFORWARD_ROW2 struct { - InterfaceLuid uint64 + InterfaceLuid luid InterfaceIndex uint32 DestinationPrefix IP_ADDRESS_PREFIX NextHop SOCKADDR_INET_NEXTHOP @@ -108,9 +110,14 @@ type SOCKADDR_INET_NEXTHOP struct { type MIB_NOTIFICATION_TYPE int32 var ( - modiphlpapi = windows.NewLazyDLL("iphlpapi.dll") - procNotifyRouteChange2 = modiphlpapi.NewProc("NotifyRouteChange2") - procCancelMibChangeNotify2 = modiphlpapi.NewProc("CancelMibChangeNotify2") + modiphlpapi = windows.NewLazyDLL("iphlpapi.dll") + procNotifyRouteChange2 = modiphlpapi.NewProc("NotifyRouteChange2") + procCancelMibChangeNotify2 = modiphlpapi.NewProc("CancelMibChangeNotify2") + procCreateIpForwardEntry2 = modiphlpapi.NewProc("CreateIpForwardEntry2") + procDeleteIpForwardEntry2 = modiphlpapi.NewProc("DeleteIpForwardEntry2") + procGetIpForwardEntry2 = modiphlpapi.NewProc("GetIpForwardEntry2") + procInitializeIpForwardEntry = modiphlpapi.NewProc("InitializeIpForwardEntry") + procConvertInterfaceIndexToLuid = modiphlpapi.NewProc("ConvertInterfaceIndexToLuid") prefixList []netip.Prefix lastUpdate time.Time @@ -139,6 +146,8 @@ func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error { } func (r *SysOps) addToRouteTable(prefix netip.Prefix, nexthop Nexthop) error { + log.Debugf("Adding route to %s via %s", prefix, nexthop) + // if we don't have an interface but a zone, extract the interface index from the zone if nexthop.IP.Zone() != "" && nexthop.Intf == nil { zone, err := strconv.Atoi(nexthop.IP.Zone()) if err != nil { @@ -147,23 +156,187 @@ func (r *SysOps) addToRouteTable(prefix netip.Prefix, nexthop Nexthop) error { nexthop.Intf = &net.Interface{Index: zone} } - return addRouteCmd(prefix, nexthop) + return addRoute(prefix, nexthop) } func (r *SysOps) removeFromRouteTable(prefix netip.Prefix, nexthop Nexthop) error { - args := []string{"delete", prefix.String()} - if nexthop.IP.IsValid() { - ip := nexthop.IP.WithZone("") - args = append(args, ip.Unmap().String()) + log.Debugf("Removing route to %s via %s", prefix, nexthop) + return deleteRoute(prefix, nexthop) +} + +// setupRouteEntry prepares a route entry with common configuration +func setupRouteEntry(prefix netip.Prefix, nexthop Nexthop) (*MIB_IPFORWARD_ROW2, error) { + route := &MIB_IPFORWARD_ROW2{} + + initializeIPForwardEntry(route) + + // Convert interface index to luid if interface is specified + if nexthop.Intf != nil { + var luid luid + if err := convertInterfaceIndexToLUID(uint32(nexthop.Intf.Index), &luid); err != nil { + return nil, fmt.Errorf("convert interface index to luid: %w", err) + } + route.InterfaceLuid = luid + route.InterfaceIndex = uint32(nexthop.Intf.Index) } - routeCmd := uspfilter.GetSystem32Command("route") + if err := setDestinationPrefix(&route.DestinationPrefix, prefix); err != nil { + return nil, fmt.Errorf("set destination prefix: %w", err) + } - out, err := exec.Command(routeCmd, args...).CombinedOutput() - log.Tracef("route %s: %s", strings.Join(args, " "), out) + if nexthop.IP.IsValid() { + if err := setNextHop(&route.NextHop, nexthop.IP); err != nil { + return nil, fmt.Errorf("set next hop: %w", err) + } + } - if err != nil { - return fmt.Errorf("remove route: %w", err) + return route, nil +} + +// addRoute adds a route using Windows iphelper APIs +func addRoute(prefix netip.Prefix, nexthop Nexthop) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic in addRoute: %v, stack trace: %s", r, debug.Stack()) + } + }() + + route, setupErr := setupRouteEntry(prefix, nexthop) + if setupErr != nil { + return fmt.Errorf("setup route entry: %w", setupErr) + } + + route.Metric = 1 + route.ValidLifetime = InfiniteLifetime + route.PreferredLifetime = InfiniteLifetime + + return createIPForwardEntry2(route) +} + +// deleteRoute deletes a route using Windows iphelper APIs +func deleteRoute(prefix netip.Prefix, nexthop Nexthop) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic in deleteRoute: %v, stack trace: %s", r, debug.Stack()) + } + }() + + route, setupErr := setupRouteEntry(prefix, nexthop) + if setupErr != nil { + return fmt.Errorf("setup route entry: %w", setupErr) + } + + if err := getIPForwardEntry2(route); err != nil { + return fmt.Errorf("get route entry: %w", err) + } + + return deleteIPForwardEntry2(route) +} + +// setDestinationPrefix sets the destination prefix in the route structure +func setDestinationPrefix(prefix *IP_ADDRESS_PREFIX, dest netip.Prefix) error { + addr := dest.Addr() + prefix.PrefixLength = uint8(dest.Bits()) + + if addr.Is4() { + prefix.Prefix.sin6_family = windows.AF_INET + ip4 := addr.As4() + binary.BigEndian.PutUint32(prefix.Prefix.data[:4], + uint32(ip4[0])<<24|uint32(ip4[1])<<16|uint32(ip4[2])<<8|uint32(ip4[3])) + return nil + } + + if addr.Is6() { + prefix.Prefix.sin6_family = windows.AF_INET6 + ip6 := addr.As16() + copy(prefix.Prefix.data[4:20], ip6[:]) + + if zone := addr.Zone(); zone != "" { + if scopeID, err := strconv.ParseUint(zone, 10, 32); err == nil { + binary.BigEndian.PutUint32(prefix.Prefix.data[20:24], uint32(scopeID)) + } + } + return nil + } + + return fmt.Errorf("invalid address family") +} + +// setNextHop sets the next hop address in the route structure +func setNextHop(nextHop *SOCKADDR_INET_NEXTHOP, addr netip.Addr) error { + if addr.Is4() { + nextHop.sin6_family = windows.AF_INET + ip4 := addr.As4() + binary.BigEndian.PutUint32(nextHop.data[:4], + uint32(ip4[0])<<24|uint32(ip4[1])<<16|uint32(ip4[2])<<8|uint32(ip4[3])) + return nil + } + + if addr.Is6() { + nextHop.sin6_family = windows.AF_INET6 + ip6 := addr.As16() + copy(nextHop.data[4:20], ip6[:]) + + // Handle zone if present + if zone := addr.Zone(); zone != "" { + if scopeID, err := strconv.ParseUint(zone, 10, 32); err == nil { + binary.BigEndian.PutUint32(nextHop.data[20:24], uint32(scopeID)) + } + } + return nil + } + + return fmt.Errorf("invalid address family") +} + +// Windows API wrappers +func createIPForwardEntry2(route *MIB_IPFORWARD_ROW2) error { + r1, _, e1 := syscall.SyscallN(procCreateIpForwardEntry2.Addr(), uintptr(unsafe.Pointer(route))) + if r1 != 0 { + if e1 != 0 { + return fmt.Errorf("CreateIpForwardEntry2: %w", e1) + } + return fmt.Errorf("CreateIpForwardEntry2: code %d", r1) + } + return nil +} + +func deleteIPForwardEntry2(route *MIB_IPFORWARD_ROW2) error { + r1, _, e1 := syscall.SyscallN(procDeleteIpForwardEntry2.Addr(), uintptr(unsafe.Pointer(route))) + if r1 != 0 { + if e1 != 0 { + return fmt.Errorf("DeleteIpForwardEntry2: %w", e1) + } + return fmt.Errorf("DeleteIpForwardEntry2: code %d", r1) + } + return nil +} + +func getIPForwardEntry2(route *MIB_IPFORWARD_ROW2) error { + r1, _, e1 := syscall.SyscallN(procGetIpForwardEntry2.Addr(), uintptr(unsafe.Pointer(route))) + if r1 != 0 { + if e1 != 0 { + return fmt.Errorf("GetIpForwardEntry2: %w", e1) + } + return fmt.Errorf("GetIpForwardEntry2: code %d", r1) + } + return nil +} + +// https://learn.microsoft.com/en-us/windows/win32/api/netioapi/nf-netioapi-initializeipforwardentry +func initializeIPForwardEntry(route *MIB_IPFORWARD_ROW2) { + // Does not return anything. Trying to handle the error might return an uninitialized value. + _, _, _ = syscall.SyscallN(procInitializeIpForwardEntry.Addr(), uintptr(unsafe.Pointer(route))) +} + +func convertInterfaceIndexToLUID(interfaceIndex uint32, interfaceLUID *luid) error { + r1, _, e1 := syscall.SyscallN(procConvertInterfaceIndexToLuid.Addr(), + uintptr(interfaceIndex), uintptr(unsafe.Pointer(interfaceLUID))) + if r1 != 0 { + if e1 != 0 { + return fmt.Errorf("ConvertInterfaceIndexToLuid: %w", e1) + } + return fmt.Errorf("ConvertInterfaceIndexToLuid: code %d", r1) } return nil } @@ -319,7 +492,7 @@ func cancelMibChangeNotify2(handle windows.Handle) error { } // GetRoutesFromTable returns the current routing table from with prefixes only. -// It ccaches the result for 2 seconds to avoid blocking the caller. +// It caches the result for 2 seconds to avoid blocking the caller. func GetRoutesFromTable() ([]netip.Prefix, error) { mux.Lock() defer mux.Unlock() @@ -388,35 +561,6 @@ func GetRoutes() ([]Route, error) { return routes, nil } -func addRouteCmd(prefix netip.Prefix, nexthop Nexthop) error { - args := []string{"add", prefix.String()} - - if nexthop.IP.IsValid() { - ip := nexthop.IP.WithZone("") - args = append(args, ip.Unmap().String()) - } else { - addr := "0.0.0.0" - if prefix.Addr().Is6() { - addr = "::" - } - args = append(args, addr) - } - - if nexthop.Intf != nil { - args = append(args, "if", strconv.Itoa(nexthop.Intf.Index)) - } - - routeCmd := uspfilter.GetSystem32Command("route") - - out, err := exec.Command(routeCmd, args...).CombinedOutput() - log.Tracef("route %s: %s", strings.Join(args, " "), out) - if err != nil { - return fmt.Errorf("route add: %w", err) - } - - return nil -} - func isCacheDisabled() bool { return os.Getenv("NB_DISABLE_ROUTE_CACHE") == "true" } diff --git a/client/internal/routemanager/systemops/systemops_windows_test.go b/client/internal/routemanager/systemops/systemops_windows_test.go index 19b006017..523bd0b0d 100644 --- a/client/internal/routemanager/systemops/systemops_windows_test.go +++ b/client/internal/routemanager/systemops/systemops_windows_test.go @@ -5,18 +5,23 @@ import ( "encoding/json" "fmt" "net" + "net/netip" "os/exec" "strings" "testing" "time" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" nbnet "github.com/netbirdio/netbird/util/net" ) -var expectedExtInt = "Ethernet1" +var ( + expectedExternalInt = "Ethernet1" + expectedVPNint = "wgtest0" +) type RouteInfo struct { NextHop string `json:"nexthop"` @@ -43,8 +48,6 @@ type testCase struct { dialer dialer } -var expectedVPNint = "wgtest0" - var testCases = []testCase{ { name: "To external host without custom dialer via vpn", @@ -52,14 +55,14 @@ var testCases = []testCase{ expectedSourceIP: "100.64.0.1", expectedDestPrefix: "128.0.0.0/1", expectedNextHop: "0.0.0.0", - expectedInterface: "wgtest0", + expectedInterface: expectedVPNint, dialer: &net.Dialer{}, }, { name: "To external host with custom dialer via physical interface", destination: "192.0.2.1:53", expectedDestPrefix: "192.0.2.1/32", - expectedInterface: expectedExtInt, + expectedInterface: expectedExternalInt, dialer: nbnet.NewDialer(), }, @@ -67,24 +70,15 @@ var testCases = []testCase{ name: "To duplicate internal route with custom dialer via physical interface", destination: "10.0.0.2:53", expectedDestPrefix: "10.0.0.2/32", - expectedInterface: expectedExtInt, + expectedInterface: expectedExternalInt, dialer: nbnet.NewDialer(), }, - { - name: "To duplicate internal route without custom dialer via physical interface", // local route takes precedence - destination: "10.0.0.2:53", - expectedSourceIP: "127.0.0.1", - expectedDestPrefix: "10.0.0.0/8", - expectedNextHop: "0.0.0.0", - expectedInterface: "Loopback Pseudo-Interface 1", - dialer: &net.Dialer{}, - }, { name: "To unique vpn route with custom dialer via physical interface", destination: "172.16.0.2:53", expectedDestPrefix: "172.16.0.2/32", - expectedInterface: expectedExtInt, + expectedInterface: expectedExternalInt, dialer: nbnet.NewDialer(), }, { @@ -93,7 +87,7 @@ var testCases = []testCase{ expectedSourceIP: "100.64.0.1", expectedDestPrefix: "172.16.0.0/12", expectedNextHop: "0.0.0.0", - expectedInterface: "wgtest0", + expectedInterface: expectedVPNint, dialer: &net.Dialer{}, }, @@ -103,22 +97,14 @@ var testCases = []testCase{ expectedSourceIP: "100.64.0.1", expectedDestPrefix: "10.10.0.0/24", expectedNextHop: "0.0.0.0", - expectedInterface: "wgtest0", - dialer: &net.Dialer{}, - }, - - { - name: "To more specific route (local) without custom dialer via physical interface", - destination: "127.0.10.2:53", - expectedSourceIP: "127.0.0.1", - expectedDestPrefix: "127.0.0.0/8", - expectedNextHop: "0.0.0.0", - expectedInterface: "Loopback Pseudo-Interface 1", + expectedInterface: expectedVPNint, dialer: &net.Dialer{}, }, } func TestRouting(t *testing.T) { + log.SetLevel(log.DebugLevel) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { setupTestEnv(t) @@ -129,7 +115,7 @@ func TestRouting(t *testing.T) { require.NoError(t, err, "Failed to fetch interface IP") output := testRoute(t, tc.destination, tc.dialer) - if tc.expectedInterface == expectedExtInt { + if tc.expectedInterface == expectedExternalInt { verifyOutput(t, output, ip, tc.expectedDestPrefix, route.NextHop, route.InterfaceAlias) } else { verifyOutput(t, output, tc.expectedSourceIP, tc.expectedDestPrefix, tc.expectedNextHop, tc.expectedInterface) @@ -242,19 +228,23 @@ func setupDummyInterfacesAndRoutes(t *testing.T) { func addDummyRoute(t *testing.T, dstCIDR string) { t.Helper() - script := fmt.Sprintf(`New-NetRoute -DestinationPrefix "%s" -InterfaceIndex 1 -PolicyStore ActiveStore`, dstCIDR) - - output, err := exec.Command("powershell", "-Command", script).CombinedOutput() + prefix, err := netip.ParsePrefix(dstCIDR) if err != nil { - t.Logf("Failed to add dummy route: %v\nOutput: %s", err, output) - t.FailNow() + t.Fatalf("Failed to parse destination CIDR %s: %v", dstCIDR, err) + } + + nexthop := Nexthop{ + Intf: &net.Interface{Index: 1}, + } + + if err = addRoute(prefix, nexthop); err != nil { + t.Fatalf("Failed to add dummy route: %v", err) } t.Cleanup(func() { - script = fmt.Sprintf(`Remove-NetRoute -DestinationPrefix "%s" -InterfaceIndex 1 -Confirm:$false`, dstCIDR) - output, err := exec.Command("powershell", "-Command", script).CombinedOutput() + err := deleteRoute(prefix, nexthop) if err != nil { - t.Logf("Failed to remove dummy route: %v\nOutput: %s", err, output) + t.Logf("Failed to remove dummy route: %v", err) } }) } diff --git a/client/server/trace.go b/client/server/trace.go index 8b9d375f3..e4ac91487 100644 --- a/client/server/trace.go +++ b/client/server/trace.go @@ -3,11 +3,11 @@ package server import ( "context" "fmt" - "net" "net/netip" fw "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/firewall/uspfilter" + "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/proto" ) @@ -19,81 +19,32 @@ func (s *Server) TracePacket(_ context.Context, req *proto.TracePacketRequest) ( s.mutex.Lock() defer s.mutex.Unlock() - if s.connectClient == nil { - return nil, fmt.Errorf("connect client not initialized") - } - engine := s.connectClient.Engine() - if engine == nil { - return nil, fmt.Errorf("engine not initialized") + tracer, engine, err := s.getPacketTracer() + if err != nil { + return nil, err } - fwManager := engine.GetFirewallManager() - if fwManager == nil { - return nil, fmt.Errorf("firewall manager not initialized") + srcAddr, err := s.parseAddress(req.GetSourceIp(), engine) + if err != nil { + return nil, fmt.Errorf("invalid source IP address: %w", err) } - tracer, ok := fwManager.(packetTracer) - if !ok { - return nil, fmt.Errorf("firewall manager does not support packet tracing") + dstAddr, err := s.parseAddress(req.GetDestinationIp(), engine) + if err != nil { + return nil, fmt.Errorf("invalid destination IP address: %w", err) } - srcIP := net.ParseIP(req.GetSourceIp()) - if req.GetSourceIp() == "self" { - srcIP = engine.GetWgAddr() + protocol, err := s.parseProtocol(req.GetProtocol()) + if err != nil { + return nil, err } - srcAddr, ok := netip.AddrFromSlice(srcIP) - if !ok { - return nil, fmt.Errorf("invalid source IP address") + direction, err := s.parseDirection(req.GetDirection()) + if err != nil { + return nil, err } - dstIP := net.ParseIP(req.GetDestinationIp()) - if req.GetDestinationIp() == "self" { - dstIP = engine.GetWgAddr() - } - - dstAddr, ok := netip.AddrFromSlice(dstIP) - if !ok { - return nil, fmt.Errorf("invalid source IP address") - } - - if srcIP == nil || dstIP == nil { - return nil, fmt.Errorf("invalid IP address") - } - - var tcpState *uspfilter.TCPState - if flags := req.GetTcpFlags(); flags != nil { - tcpState = &uspfilter.TCPState{ - SYN: flags.GetSyn(), - ACK: flags.GetAck(), - FIN: flags.GetFin(), - RST: flags.GetRst(), - PSH: flags.GetPsh(), - URG: flags.GetUrg(), - } - } - - var dir fw.RuleDirection - switch req.GetDirection() { - case "in": - dir = fw.RuleDirectionIN - case "out": - dir = fw.RuleDirectionOUT - default: - return nil, fmt.Errorf("invalid direction") - } - - var protocol fw.Protocol - switch req.GetProtocol() { - case "tcp": - protocol = fw.ProtocolTCP - case "udp": - protocol = fw.ProtocolUDP - case "icmp": - protocol = fw.ProtocolICMP - default: - return nil, fmt.Errorf("invalid protocolcol") - } + tcpState := s.parseTCPFlags(req.GetTcpFlags()) builder := &uspfilter.PacketBuilder{ SrcIP: srcAddr, @@ -101,16 +52,96 @@ func (s *Server) TracePacket(_ context.Context, req *proto.TracePacketRequest) ( Protocol: protocol, SrcPort: uint16(req.GetSourcePort()), DstPort: uint16(req.GetDestinationPort()), - Direction: dir, + Direction: direction, TCPState: tcpState, ICMPType: uint8(req.GetIcmpType()), ICMPCode: uint8(req.GetIcmpCode()), } + trace, err := tracer.TracePacketFromBuilder(builder) if err != nil { return nil, fmt.Errorf("trace packet: %w", err) } + return s.buildTraceResponse(trace), nil +} + +func (s *Server) getPacketTracer() (packetTracer, *internal.Engine, error) { + if s.connectClient == nil { + return nil, nil, fmt.Errorf("connect client not initialized") + } + + engine := s.connectClient.Engine() + if engine == nil { + return nil, nil, fmt.Errorf("engine not initialized") + } + + fwManager := engine.GetFirewallManager() + if fwManager == nil { + return nil, nil, fmt.Errorf("firewall manager not initialized") + } + + tracer, ok := fwManager.(packetTracer) + if !ok { + return nil, nil, fmt.Errorf("firewall manager does not support packet tracing") + } + + return tracer, engine, nil +} + +func (s *Server) parseAddress(addr string, engine *internal.Engine) (netip.Addr, error) { + if addr == "self" { + return engine.GetWgAddr(), nil + } + + a, err := netip.ParseAddr(addr) + if err != nil { + return netip.Addr{}, err + } + + return a.Unmap(), nil +} + +func (s *Server) parseProtocol(protocol string) (fw.Protocol, error) { + switch protocol { + case "tcp": + return fw.ProtocolTCP, nil + case "udp": + return fw.ProtocolUDP, nil + case "icmp": + return fw.ProtocolICMP, nil + default: + return "", fmt.Errorf("invalid protocol") + } +} + +func (s *Server) parseDirection(direction string) (fw.RuleDirection, error) { + switch direction { + case "in": + return fw.RuleDirectionIN, nil + case "out": + return fw.RuleDirectionOUT, nil + default: + return 0, fmt.Errorf("invalid direction") + } +} + +func (s *Server) parseTCPFlags(flags *proto.TCPFlags) *uspfilter.TCPState { + if flags == nil { + return nil + } + + return &uspfilter.TCPState{ + SYN: flags.GetSyn(), + ACK: flags.GetAck(), + FIN: flags.GetFin(), + RST: flags.GetRst(), + PSH: flags.GetPsh(), + URG: flags.GetUrg(), + } +} + +func (s *Server) buildTraceResponse(trace *uspfilter.PacketTrace) *proto.TracePacketResponse { resp := &proto.TracePacketResponse{} for _, result := range trace.Results { @@ -119,10 +150,12 @@ func (s *Server) TracePacket(_ context.Context, req *proto.TracePacketRequest) ( Message: result.Message, Allowed: result.Allowed, } + if result.ForwarderAction != nil { details := fmt.Sprintf("%s to %s", result.ForwarderAction.Action, result.ForwarderAction.RemoteAddr) stage.ForwardingDetails = &details } + resp.Stages = append(resp.Stages, stage) } @@ -130,5 +163,5 @@ func (s *Server) TracePacket(_ context.Context, req *proto.TracePacketRequest) ( resp.FinalDisposition = trace.Results[len(trace.Results)-1].Allowed } - return resp, nil + return resp } diff --git a/util/net/net.go b/util/net/net.go index b573f9aeb..fdcf4ee6a 100644 --- a/util/net/net.go +++ b/util/net/net.go @@ -1,8 +1,10 @@ package net import ( + "fmt" "math/big" "net" + "net/netip" "github.com/google/uuid" ) @@ -54,11 +56,13 @@ func GenerateConnID() ConnectionID { return ConnectionID(uuid.NewString()) } -func GetLastIPFromNetwork(network *net.IPNet, fromEnd int) net.IP { - // Calculate the last IP in the CIDR range +func GetLastIPFromNetwork(network netip.Prefix, fromEnd int) (netip.Addr, error) { var endIP net.IP - for i := 0; i < len(network.IP); i++ { - endIP = append(endIP, network.IP[i]|^network.Mask[i]) + addr := network.Addr().AsSlice() + mask := net.CIDRMask(network.Bits(), len(addr)*8) + + for i := 0; i < len(addr); i++ { + endIP = append(endIP, addr[i]|^mask[i]) } // convert to big.Int @@ -70,5 +74,10 @@ func GetLastIPFromNetwork(network *net.IPNet, fromEnd int) net.IP { resultInt := big.NewInt(0) resultInt.Sub(endInt, fromEndBig) - return resultInt.Bytes() + ip, ok := netip.AddrFromSlice(resultInt.Bytes()) + if !ok { + return netip.Addr{}, fmt.Errorf("invalid IP address from network %s", network) + } + + return ip.Unmap(), nil } diff --git a/util/net/net_test.go b/util/net/net_test.go new file mode 100644 index 000000000..e0633cb6a --- /dev/null +++ b/util/net/net_test.go @@ -0,0 +1,94 @@ +package net + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetLastIPFromNetwork(t *testing.T) { + tests := []struct { + name string + network string + fromEnd int + expected string + expectErr bool + }{ + { + name: "IPv4 /24 network - last IP (fromEnd=0)", + network: "192.168.1.0/24", + fromEnd: 0, + expected: "192.168.1.255", + }, + { + name: "IPv4 /24 network - fromEnd=1", + network: "192.168.1.0/24", + fromEnd: 1, + expected: "192.168.1.254", + }, + { + name: "IPv4 /24 network - fromEnd=5", + network: "192.168.1.0/24", + fromEnd: 5, + expected: "192.168.1.250", + }, + { + name: "IPv4 /16 network - last IP", + network: "10.0.0.0/16", + fromEnd: 0, + expected: "10.0.255.255", + }, + { + name: "IPv4 /16 network - fromEnd=256", + network: "10.0.0.0/16", + fromEnd: 256, + expected: "10.0.254.255", + }, + { + name: "IPv4 /32 network - single host", + network: "192.168.1.100/32", + fromEnd: 0, + expected: "192.168.1.100", + }, + { + name: "IPv6 /64 network - last IP", + network: "2001:db8::/64", + fromEnd: 0, + expected: "2001:db8::ffff:ffff:ffff:ffff", + }, + { + name: "IPv6 /64 network - fromEnd=1", + network: "2001:db8::/64", + fromEnd: 1, + expected: "2001:db8::ffff:ffff:ffff:fffe", + }, + { + name: "IPv6 /128 network - single host", + network: "2001:db8::1/128", + fromEnd: 0, + expected: "2001:db8::1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + network, err := netip.ParsePrefix(tt.network) + require.NoError(t, err, "Failed to parse network prefix") + + result, err := GetLastIPFromNetwork(network, tt.fromEnd) + + if tt.expectErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + + expectedIP, err := netip.ParseAddr(tt.expected) + require.NoError(t, err, "Failed to parse expected IP") + + assert.Equal(t, expectedIP, result, "IP mismatch for network %s with fromEnd=%d", tt.network, tt.fromEnd) + }) + } +} From b604c6614037e3908bf0e137675ef2ffaf27999b Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Wed, 4 Jun 2025 17:38:49 +0300 Subject: [PATCH 25/37] [management] Add postgres support for activity event store (#3890) --- go.mod | 2 +- go.sum | 4 +- .../activity/{sqlite => store}/crypt.go | 2 +- .../activity/{sqlite => store}/crypt_test.go | 2 +- .../activity/{sqlite => store}/migration.go | 22 +++--- .../{sqlite => store}/migration_test.go | 13 ++-- .../{sqlite/sqlite.go => store/sql_store.go} | 77 +++++++++++++++---- .../sql_store_test.go} | 6 +- 8 files changed, 92 insertions(+), 36 deletions(-) rename management/server/activity/{sqlite => store}/crypt.go (99%) rename management/server/activity/{sqlite => store}/crypt_test.go (99%) rename management/server/activity/{sqlite => store}/migration.go (91%) rename management/server/activity/{sqlite => store}/migration_test.go (93%) rename management/server/activity/{sqlite/sqlite.go => store/sql_store.go} (74%) rename management/server/activity/{sqlite/sqlite_test.go => store/sql_store_test.go} (90%) diff --git a/go.mod b/go.mod index c86acdf26..11dc88c43 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/miekg/dns v1.1.59 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/nadoo/ipset v0.5.0 - github.com/netbirdio/management-integrations/integrations v0.0.0-20250330143713-7901e0a82203 + github.com/netbirdio/management-integrations/integrations v0.0.0-20250529122842-6700aa91190c github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250514131221-a464fd5f30cb github.com/okta/okta-sdk-golang/v2 v2.18.0 github.com/oschwald/maxminddb-golang v1.12.0 diff --git a/go.sum b/go.sum index 226ee94c2..f887cee94 100644 --- a/go.sum +++ b/go.sum @@ -503,8 +503,8 @@ github.com/netbirdio/go-netroute v0.0.0-20240611143515-f59b0e1d3944 h1:TDtJKmM6S github.com/netbirdio/go-netroute v0.0.0-20240611143515-f59b0e1d3944/go.mod h1:sHA6TRxjQ6RLbnI+3R4DZo2Eseg/iKiPRfNmcuNySVQ= github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e h1:PURA50S8u4mF6RrkYYCAvvPCixhqqEiEy3Ej6avh04c= github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e/go.mod h1:YMLU7qbKfVjmEv7EoZPIVEI+kNYxWCdPK3VS0BU+U4Q= -github.com/netbirdio/management-integrations/integrations v0.0.0-20250330143713-7901e0a82203 h1:uxxbLPXQgC9VO15epNPtrD6zazyd5rZeqC5hQSmCdZU= -github.com/netbirdio/management-integrations/integrations v0.0.0-20250330143713-7901e0a82203/go.mod h1:2ZE6/tBBCKHQggPfO2UOQjyjXI7k+JDVl2ymorTOVQs= +github.com/netbirdio/management-integrations/integrations v0.0.0-20250529122842-6700aa91190c h1:SdZxYjR9XXHLyRsTbS1EHBr6+RI15oie1K9Q8yvi3FY= +github.com/netbirdio/management-integrations/integrations v0.0.0-20250529122842-6700aa91190c/go.mod h1:Gi9raplYzCCyh07Olw/DVfCJTFgpr1WCXJ/Q+8TSA9Q= github.com/netbirdio/service v0.0.0-20240911161631-f62744f42502 h1:3tHlFmhTdX9axERMVN63dqyFqnvuD+EMJHzM7mNGON8= github.com/netbirdio/service v0.0.0-20240911161631-f62744f42502/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM= github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250514131221-a464fd5f30cb h1:Cr6age+ePALqlSvtp7wc6lYY97XN7rkD1K4XEDmY+TU= diff --git a/management/server/activity/sqlite/crypt.go b/management/server/activity/store/crypt.go similarity index 99% rename from management/server/activity/sqlite/crypt.go rename to management/server/activity/store/crypt.go index 096f49ea3..ce97347d4 100644 --- a/management/server/activity/sqlite/crypt.go +++ b/management/server/activity/store/crypt.go @@ -1,4 +1,4 @@ -package sqlite +package store import ( "bytes" diff --git a/management/server/activity/sqlite/crypt_test.go b/management/server/activity/store/crypt_test.go similarity index 99% rename from management/server/activity/sqlite/crypt_test.go rename to management/server/activity/store/crypt_test.go index aff3a08b1..700bbcd6b 100644 --- a/management/server/activity/sqlite/crypt_test.go +++ b/management/server/activity/store/crypt_test.go @@ -1,4 +1,4 @@ -package sqlite +package store import ( "bytes" diff --git a/management/server/activity/sqlite/migration.go b/management/server/activity/store/migration.go similarity index 91% rename from management/server/activity/sqlite/migration.go rename to management/server/activity/store/migration.go index 6da7893a0..af19a34eb 100644 --- a/management/server/activity/sqlite/migration.go +++ b/management/server/activity/store/migration.go @@ -1,4 +1,4 @@ -package sqlite +package store import ( "context" @@ -6,6 +6,7 @@ import ( log "github.com/sirupsen/logrus" "gorm.io/gorm" + "gorm.io/gorm/clause" "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/migration" @@ -132,11 +133,6 @@ func migrateDuplicateDeletedUsers(ctx context.Context, db *gorm.DB) error { } if err = db.Transaction(func(tx *gorm.DB) error { - groupById := tx.Model(model).Select("MAX(rowid)").Group("id") - if err = tx.Delete(model, "rowid NOT IN (?)", groupById).Error; err != nil { - return err - } - if err = tx.Migrator().RenameTable("deleted_users", "deleted_users_old"); err != nil { return err } @@ -145,12 +141,20 @@ func migrateDuplicateDeletedUsers(ctx context.Context, db *gorm.DB) error { return err } - if err = tx.Exec(` - INSERT INTO deleted_users (id, email, name, enc_algo) SELECT id, email, name, enc_algo - FROM deleted_users_old;`).Error; err != nil { + var deletedUsers []activity.DeletedUser + if err = tx.Table("deleted_users_old").Find(&deletedUsers).Error; err != nil { return err } + for _, deletedUser := range deletedUsers { + if err = tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "id"}}, + DoUpdates: clause.AssignmentColumns([]string{"email", "name", "enc_algo"}), + }).Create(&deletedUser).Error; err != nil { + return err + } + } + return tx.Migrator().DropTable("deleted_users_old") }); err != nil { return err diff --git a/management/server/activity/sqlite/migration_test.go b/management/server/activity/store/migration_test.go similarity index 93% rename from management/server/activity/sqlite/migration_test.go rename to management/server/activity/store/migration_test.go index 498c976d9..e3261d9fa 100644 --- a/management/server/activity/sqlite/migration_test.go +++ b/management/server/activity/store/migration_test.go @@ -1,17 +1,17 @@ -package sqlite +package store import ( "context" - "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gorm.io/driver/sqlite" + "gorm.io/driver/postgres" "gorm.io/gorm" "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/migration" + "github.com/netbirdio/netbird/management/server/testutil" ) const ( @@ -21,8 +21,11 @@ const ( func setupDatabase(t *testing.T) *gorm.DB { t.Helper() - dbFile := filepath.Join(t.TempDir(), eventSinkDB) - db, err := gorm.Open(sqlite.Open(dbFile)) + cleanup, dsn, err := testutil.CreatePostgresTestContainer() + require.NoError(t, err, "Failed to create Postgres test container") + t.Cleanup(cleanup) + + db, err := gorm.Open(postgres.Open(dsn)) require.NoError(t, err) sql, err := db.DB() diff --git a/management/server/activity/sqlite/sqlite.go b/management/server/activity/store/sql_store.go similarity index 74% rename from management/server/activity/sqlite/sqlite.go rename to management/server/activity/store/sql_store.go index 6d198fca9..80b165938 100644 --- a/management/server/activity/sqlite/sqlite.go +++ b/management/server/activity/store/sql_store.go @@ -1,17 +1,22 @@ -package sqlite +package store import ( "context" "fmt" + "os" "path/filepath" + "runtime" + "strconv" log "github.com/sirupsen/logrus" + "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/logger" "github.com/netbirdio/netbird/management/server/activity" + "github.com/netbirdio/netbird/management/server/types" ) const ( @@ -22,6 +27,10 @@ const ( fallbackEmail = "unknown@unknown.com" gcmEncAlgo = "GCM" + + storeEngineEnv = "NB_ACTIVITY_EVENT_STORE_ENGINE" + postgresDsnEnv = "NB_ACTIVITY_EVENT_POSTGRES_DSN" + sqlMaxOpenConnsEnv = "NB_SQL_MAX_OPEN_CONNS" ) type eventWithNames struct { @@ -38,28 +47,19 @@ type Store struct { fieldEncrypt *FieldEncrypt } -// NewSQLiteStore creates a new Store with an event table if not exists. -func NewSQLiteStore(ctx context.Context, dataDir string, encryptionKey string) (*Store, error) { +// NewSqlStore creates a new Store with an event table if not exists. +func NewSqlStore(ctx context.Context, dataDir string, encryptionKey string) (*Store, error) { crypt, err := NewFieldEncrypt(encryptionKey) if err != nil { return nil, err } - dbFile := filepath.Join(dataDir, eventSinkDB) - db, err := gorm.Open(sqlite.Open(dbFile), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }) + db, err := initDatabase(ctx, dataDir) if err != nil { - return nil, err + return nil, fmt.Errorf("initialize database: %w", err) } - sql, err := db.DB() - if err != nil { - return nil, err - } - sql.SetMaxOpenConns(1) - if err = migrate(ctx, crypt, db); err != nil { return nil, fmt.Errorf("events database migration: %w", err) } @@ -236,3 +236,52 @@ func (store *Store) Close(_ context.Context) error { } return nil } + +func initDatabase(ctx context.Context, dataDir string) (*gorm.DB, error) { + var dialector gorm.Dialector + var storeEngine = types.SqliteStoreEngine + + if engine, ok := os.LookupEnv(storeEngineEnv); ok { + storeEngine = types.Engine(engine) + } + + switch storeEngine { + case types.SqliteStoreEngine: + dialector = sqlite.Open(filepath.Join(dataDir, eventSinkDB)) + case types.PostgresStoreEngine: + dsn, ok := os.LookupEnv(postgresDsnEnv) + if !ok { + return nil, fmt.Errorf("%s environment variable not set", postgresDsnEnv) + } + dialector = postgres.Open(dsn) + default: + return nil, fmt.Errorf("unsupported store engine: %s", storeEngine) + } + log.WithContext(ctx).Infof("using %s as activity event store engine", storeEngine) + + db, err := gorm.Open(dialector, &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + return nil, fmt.Errorf("open db connection: %w", err) + } + + return configureConnectionPool(db, storeEngine) +} + +func configureConnectionPool(db *gorm.DB, storeEngine types.Engine) (*gorm.DB, error) { + sqlDB, err := db.DB() + if err != nil { + return nil, err + } + + if storeEngine == types.SqliteStoreEngine { + sqlDB.SetMaxOpenConns(1) + } else { + conns, err := strconv.Atoi(os.Getenv(sqlMaxOpenConnsEnv)) + if err != nil { + conns = runtime.NumCPU() + } + sqlDB.SetMaxOpenConns(conns) + } + + return db, nil +} diff --git a/management/server/activity/sqlite/sqlite_test.go b/management/server/activity/store/sql_store_test.go similarity index 90% rename from management/server/activity/sqlite/sqlite_test.go rename to management/server/activity/store/sql_store_test.go index b10f9b58a..8c0d159df 100644 --- a/management/server/activity/sqlite/sqlite_test.go +++ b/management/server/activity/store/sql_store_test.go @@ -1,4 +1,4 @@ -package sqlite +package store import ( "context" @@ -11,10 +11,10 @@ import ( "github.com/netbirdio/netbird/management/server/activity" ) -func TestNewSQLiteStore(t *testing.T) { +func TestNewSqlStore(t *testing.T) { dataDir := t.TempDir() key, _ := GenerateKey() - store, err := NewSQLiteStore(context.Background(), dataDir, key) + store, err := NewSqlStore(context.Background(), dataDir, key) if err != nil { t.Fatal(err) return From 609654eee7d441846021959bc386338b4696815a Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Wed, 4 Jun 2025 18:12:48 +0200 Subject: [PATCH 26/37] [client] Allow userspace local forwarding to internal interfaces if requested (#3884) --- client/firewall/uspfilter/forwarder/tcp.go | 4 ++-- client/firewall/uspfilter/forwarder/udp.go | 4 ++-- client/firewall/uspfilter/uspfilter.go | 23 +++++++++++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/client/firewall/uspfilter/forwarder/tcp.go b/client/firewall/uspfilter/forwarder/tcp.go index 04b3ae233..64e54e293 100644 --- a/client/firewall/uspfilter/forwarder/tcp.go +++ b/client/firewall/uspfilter/forwarder/tcp.go @@ -111,12 +111,12 @@ func (f *Forwarder) proxyTCP(id stack.TransportEndpointID, inConn *gonet.TCPConn if errInToOut != nil { if !isClosedError(errInToOut) { - f.logger.Error("proxyTCP: copy error (in -> out): %v", errInToOut) + f.logger.Error("proxyTCP: copy error (in -> out) for %s: %v", epID(id), errInToOut) } } if errOutToIn != nil { if !isClosedError(errOutToIn) { - f.logger.Error("proxyTCP: copy error (out -> in): %v", errOutToIn) + f.logger.Error("proxyTCP: copy error (out -> in) for %s: %v", epID(id), errOutToIn) } } diff --git a/client/firewall/uspfilter/forwarder/udp.go b/client/firewall/uspfilter/forwarder/udp.go index cb88aa59a..f237a313d 100644 --- a/client/firewall/uspfilter/forwarder/udp.go +++ b/client/firewall/uspfilter/forwarder/udp.go @@ -250,10 +250,10 @@ func (f *Forwarder) proxyUDP(ctx context.Context, pConn *udpPacketConn, id stack wg.Wait() if outboundErr != nil && !isClosedError(outboundErr) { - f.logger.Error("proxyUDP: copy error (outbound->inbound): %v", outboundErr) + f.logger.Error("proxyUDP: copy error (outbound->inbound) for %s: %v", epID(id), outboundErr) } if inboundErr != nil && !isClosedError(inboundErr) { - f.logger.Error("proxyUDP: copy error (inbound->outbound): %v", inboundErr) + f.logger.Error("proxyUDP: copy error (inbound->outbound) for %s: %v", epID(id), inboundErr) } var rxPackets, txPackets uint64 diff --git a/client/firewall/uspfilter/uspfilter.go b/client/firewall/uspfilter/uspfilter.go index eede1ab13..c216bc302 100644 --- a/client/firewall/uspfilter/uspfilter.go +++ b/client/firewall/uspfilter/uspfilter.go @@ -39,8 +39,12 @@ const ( // EnvForceUserspaceRouter forces userspace routing even if native routing is available. EnvForceUserspaceRouter = "NB_FORCE_USERSPACE_ROUTER" - // EnvEnableNetstackLocalForwarding enables forwarding of local traffic to the native stack when running netstack - // Leaving this on by default introduces a security risk as sockets on listening on localhost only will be accessible + // EnvEnableLocalForwarding enables forwarding of local traffic to the native stack for internal (non-NetBird) interfaces. + // Default off as it might be security risk because sockets listening on localhost only will become accessible. + EnvEnableLocalForwarding = "NB_ENABLE_LOCAL_FORWARDING" + + // EnvEnableNetstackLocalForwarding is an alias for EnvEnableLocalForwarding. + // In netstack mode, it enables forwarding of local traffic to the native stack for all interfaces. EnvEnableNetstackLocalForwarding = "NB_ENABLE_NETSTACK_LOCAL_FORWARDING" ) @@ -147,6 +151,11 @@ func parseCreateEnv() (bool, bool) { if err != nil { log.Warnf("failed to parse %s: %v", EnvEnableNetstackLocalForwarding, err) } + } else if val := os.Getenv(EnvEnableLocalForwarding); val != "" { + enableLocalForwarding, err = strconv.ParseBool(val) + if err != nil { + log.Warnf("failed to parse %s: %v", EnvEnableLocalForwarding, err) + } } return disableConntrack, enableLocalForwarding @@ -779,9 +788,10 @@ func (m *Manager) handleLocalTraffic(d *decoder, srcIP, dstIP netip.Addr, packet return true } - // if running in netstack mode we need to pass this to the forwarder - if m.netstack && m.localForwarding { - return m.handleNetstackLocalTraffic(packetData) + // If requested we pass local traffic to internal interfaces to the forwarder. + // netstack doesn't have an interface to forward packets to the native stack so we always need to use the forwarder. + if m.localForwarding && (m.netstack || dstIP != m.wgIface.Address().IP) { + return m.handleForwardedLocalTraffic(packetData) } // track inbound packets to get the correct direction and session id for flows @@ -791,8 +801,7 @@ func (m *Manager) handleLocalTraffic(d *decoder, srcIP, dstIP netip.Addr, packet return false } -func (m *Manager) handleNetstackLocalTraffic(packetData []byte) bool { - +func (m *Manager) handleForwardedLocalTraffic(packetData []byte) bool { fwd := m.forwarder.Load() if fwd == nil { m.logger.Trace("Dropping local packet (forwarder not initialized)") From 9424b88db2239a5205b429110806d6f5ab1397ec Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 5 Jun 2025 11:51:39 +0200 Subject: [PATCH 27/37] [client] Add output similar to wg show to the debug package (#3922) --- client/anonymize/anonymize.go | 16 +++ client/iface/configurer/kernel_unix.go | 43 +++++++ client/iface/configurer/usp.go | 149 +++++++++++++++++++++++++ client/iface/configurer/wgshow.go | 24 ++++ client/iface/device/interface.go | 1 + client/iface/iface.go | 4 + client/internal/debug/debug.go | 5 + client/internal/debug/wgshow.go | 66 +++++++++++ client/internal/engine.go | 3 +- client/internal/engine_test.go | 4 + client/internal/iface_common.go | 1 + client/internal/peer/status.go | 23 ++++ 12 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 client/iface/configurer/wgshow.go create mode 100644 client/internal/debug/wgshow.go diff --git a/client/anonymize/anonymize.go b/client/anonymize/anonymize.go index 2fc9d49d3..89e653300 100644 --- a/client/anonymize/anonymize.go +++ b/client/anonymize/anonymize.go @@ -69,6 +69,22 @@ func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { return a.ipAnonymizer[ip] } +func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { + // Convert IP to netip.Addr + ip, ok := netip.AddrFromSlice(addr.IP) + if !ok { + return addr + } + + anonIP := a.AnonymizeIP(ip) + + return net.UDPAddr{ + IP: anonIP.AsSlice(), + Port: addr.Port, + Zone: addr.Zone, + } +} + // isInAnonymizedRange checks if an IP is within the range of already assigned anonymized IPs func (a *Anonymizer) isInAnonymizedRange(ip netip.Addr) bool { if ip.Is4() && ip.Compare(a.startAnonIPv4) >= 0 && ip.Compare(a.currentAnonIPv4) <= 0 { diff --git a/client/iface/configurer/kernel_unix.go b/client/iface/configurer/kernel_unix.go index 87076fea8..91991177e 100644 --- a/client/iface/configurer/kernel_unix.go +++ b/client/iface/configurer/kernel_unix.go @@ -12,6 +12,8 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) +var zeroKey wgtypes.Key + type KernelConfigurer struct { deviceName string } @@ -201,6 +203,47 @@ func (c *KernelConfigurer) configure(config wgtypes.Config) error { func (c *KernelConfigurer) Close() { } +func (c *KernelConfigurer) FullStats() (*Stats, error) { + wg, err := wgctrl.New() + if err != nil { + return nil, fmt.Errorf("wgctl: %w", err) + } + defer func() { + err = wg.Close() + if err != nil { + log.Errorf("Got error while closing wgctl: %v", err) + } + }() + + wgDevice, err := wg.Device(c.deviceName) + if err != nil { + return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + } + fullStats := &Stats{ + DeviceName: wgDevice.Name, + PublicKey: wgDevice.PublicKey.String(), + ListenPort: wgDevice.ListenPort, + FWMark: wgDevice.FirewallMark, + Peers: []Peer{}, + } + + for _, p := range wgDevice.Peers { + peer := Peer{ + PublicKey: p.PublicKey.String(), + AllowedIPs: p.AllowedIPs, + TxBytes: p.TransmitBytes, + RxBytes: p.ReceiveBytes, + LastHandshake: p.LastHandshakeTime, + PresharedKey: p.PresharedKey != zeroKey, + } + if p.Endpoint != nil { + peer.Endpoint = *p.Endpoint + } + fullStats.Peers = append(fullStats.Peers, peer) + } + return fullStats, nil +} + func (c *KernelConfigurer) GetStats() (map[string]WGStats, error) { stats := make(map[string]WGStats) wg, err := wgctrl.New() diff --git a/client/iface/configurer/usp.go b/client/iface/configurer/usp.go index d7ab1ec6f..914788821 100644 --- a/client/iface/configurer/usp.go +++ b/client/iface/configurer/usp.go @@ -19,10 +19,17 @@ import ( ) const ( + privateKey = "private_key" ipcKeyLastHandshakeTimeSec = "last_handshake_time_sec" ipcKeyLastHandshakeTimeNsec = "last_handshake_time_nsec" ipcKeyTxBytes = "tx_bytes" ipcKeyRxBytes = "rx_bytes" + allowedIP = "allowed_ip" + endpoint = "endpoint" + fwmark = "fwmark" + listenPort = "listen_port" + publicKey = "public_key" + presharedKey = "preshared_key" ) var ErrAllowedIPNotFound = fmt.Errorf("allowed IP not found") @@ -186,6 +193,15 @@ func (c *WGUSPConfigurer) RemoveAllowedIP(peerKey string, ip string) error { return c.device.IpcSet(toWgUserspaceString(config)) } +func (c *WGUSPConfigurer) FullStats() (*Stats, error) { + ipcStr, err := c.device.IpcGet() + if err != nil { + return nil, fmt.Errorf("IpcGet failed: %w", err) + } + + return parseStatus(c.deviceName, ipcStr) +} + // startUAPI starts the UAPI listener for managing the WireGuard interface via external tool func (t *WGUSPConfigurer) startUAPI() { var err error @@ -365,3 +381,136 @@ func getFwmark() int { } return 0 } + +func hexToWireguardKey(hexKey string) (wgtypes.Key, error) { + // Decode hex string to bytes + keyBytes, err := hex.DecodeString(hexKey) + if err != nil { + return wgtypes.Key{}, fmt.Errorf("failed to decode hex key: %w", err) + } + + // Check if we have the right number of bytes (WireGuard keys are 32 bytes) + if len(keyBytes) != 32 { + return wgtypes.Key{}, fmt.Errorf("invalid key length: expected 32 bytes, got %d", len(keyBytes)) + } + + // Convert to wgtypes.Key + var key wgtypes.Key + copy(key[:], keyBytes) + + return key, nil +} + +func parseStatus(deviceName, ipcStr string) (*Stats, error) { + stats := &Stats{DeviceName: deviceName} + var currentPeer *Peer + for _, line := range strings.Split(strings.TrimSpace(ipcStr), "\n") { + if line == "" { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + continue + } + key := parts[0] + val := parts[1] + + switch key { + case privateKey: + key, err := hexToWireguardKey(val) + if err != nil { + log.Errorf("failed to parse private key: %v", err) + continue + } + stats.PublicKey = key.PublicKey().String() + case publicKey: + // Save previous peer + if currentPeer != nil { + stats.Peers = append(stats.Peers, *currentPeer) + } + key, err := hexToWireguardKey(val) + if err != nil { + log.Errorf("failed to parse public key: %v", err) + continue + } + currentPeer = &Peer{ + PublicKey: key.String(), + } + case listenPort: + if port, err := strconv.Atoi(val); err == nil { + stats.ListenPort = port + } + case fwmark: + if fwmark, err := strconv.Atoi(val); err == nil { + stats.FWMark = fwmark + } + case endpoint: + if currentPeer == nil { + continue + } + + host, portStr, err := net.SplitHostPort(strings.Trim(val, "[]")) + if err != nil { + log.Errorf("failed to parse endpoint: %v", err) + continue + } + port, err := strconv.Atoi(portStr) + if err != nil { + log.Errorf("failed to parse endpoint port: %v", err) + continue + } + currentPeer.Endpoint = net.UDPAddr{ + IP: net.ParseIP(host), + Port: port, + } + case allowedIP: + if currentPeer == nil { + continue + } + _, ipnet, err := net.ParseCIDR(val) + if err == nil { + currentPeer.AllowedIPs = append(currentPeer.AllowedIPs, *ipnet) + } + case ipcKeyTxBytes: + if currentPeer == nil { + continue + } + rxBytes, err := toBytes(val) + if err != nil { + continue + } + currentPeer.TxBytes = rxBytes + case ipcKeyRxBytes: + if currentPeer == nil { + continue + } + rxBytes, err := toBytes(val) + if err != nil { + continue + } + currentPeer.RxBytes = rxBytes + + case ipcKeyLastHandshakeTimeSec: + if currentPeer == nil { + continue + } + + ts, err := toLastHandshake(val) + if err != nil { + continue + } + currentPeer.LastHandshake = ts + case presharedKey: + if currentPeer == nil { + continue + } + if val != "" { + currentPeer.PresharedKey = true + } + } + } + if currentPeer != nil { + stats.Peers = append(stats.Peers, *currentPeer) + } + return stats, nil +} diff --git a/client/iface/configurer/wgshow.go b/client/iface/configurer/wgshow.go new file mode 100644 index 000000000..604264026 --- /dev/null +++ b/client/iface/configurer/wgshow.go @@ -0,0 +1,24 @@ +package configurer + +import ( + "net" + "time" +) + +type Peer struct { + PublicKey string + Endpoint net.UDPAddr + AllowedIPs []net.IPNet + TxBytes int64 + RxBytes int64 + LastHandshake time.Time + PresharedKey bool +} + +type Stats struct { + DeviceName string + PublicKey string + ListenPort int + FWMark int + Peers []Peer +} diff --git a/client/iface/device/interface.go b/client/iface/device/interface.go index a1d44a150..31ebdf4b8 100644 --- a/client/iface/device/interface.go +++ b/client/iface/device/interface.go @@ -17,4 +17,5 @@ type WGConfigurer interface { RemoveAllowedIP(peerKey string, allowedIP string) error Close() GetStats() (map[string]configurer.WGStats, error) + FullStats() (*configurer.Stats, error) } diff --git a/client/iface/iface.go b/client/iface/iface.go index 1f659af29..f4394c476 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -216,6 +216,10 @@ func (w *WGIface) GetStats() (map[string]configurer.WGStats, error) { return w.configurer.GetStats() } +func (w *WGIface) FullStats() (*configurer.Stats, error) { + return w.configurer.FullStats() +} + func (w *WGIface) waitUntilRemoved() error { maxWaitTime := 5 * time.Second timeout := time.NewTimer(maxWaitTime) diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index a753ece0c..a7d873c8f 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -270,11 +270,16 @@ func (g *BundleGenerator) createArchive() error { log.Errorf("Failed to add corrupted state files to debug bundle: %v", err) } + if err := g.addWgShow(); err != nil { + log.Errorf("Failed to add wg show output: %v", err) + } + if g.logFile != "console" { if err := g.addLogfile(); err != nil { return fmt.Errorf("add log file: %w", err) } } + return nil } diff --git a/client/internal/debug/wgshow.go b/client/internal/debug/wgshow.go new file mode 100644 index 000000000..e4b4c2368 --- /dev/null +++ b/client/internal/debug/wgshow.go @@ -0,0 +1,66 @@ +package debug + +import ( + "bytes" + "fmt" + "strings" + "time" + + "github.com/netbirdio/netbird/client/iface/configurer" +) + +type WGIface interface { + FullStats() (*configurer.Stats, error) +} + +func (g *BundleGenerator) addWgShow() error { + result, err := g.statusRecorder.PeersStatus() + if err != nil { + return err + } + + output := g.toWGShowFormat(result) + reader := bytes.NewReader([]byte(output)) + + if err := g.addFileToZip(reader, "wgshow.txt"); err != nil { + return fmt.Errorf("add wg show to zip: %w", err) + } + return nil +} + +func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("interface: %s\n", s.DeviceName)) + sb.WriteString(fmt.Sprintf(" public key: %s\n", s.PublicKey)) + sb.WriteString(fmt.Sprintf(" listen port: %d\n", s.ListenPort)) + if s.FWMark != 0 { + sb.WriteString(fmt.Sprintf(" fwmark: %#x\n", s.FWMark)) + } + + for _, peer := range s.Peers { + sb.WriteString(fmt.Sprintf("\npeer: %s\n", peer.PublicKey)) + if peer.Endpoint.IP != nil { + if g.anonymize { + anonEndpoint := g.anonymizer.AnonymizeUDPAddr(peer.Endpoint) + sb.WriteString(fmt.Sprintf(" endpoint: %s\n", anonEndpoint.String())) + } else { + sb.WriteString(fmt.Sprintf(" endpoint: %s\n", peer.Endpoint.String())) + } + } + if len(peer.AllowedIPs) > 0 { + var ipStrings []string + for _, ipnet := range peer.AllowedIPs { + ipStrings = append(ipStrings, ipnet.String()) + } + sb.WriteString(fmt.Sprintf(" allowed ips: %s\n", strings.Join(ipStrings, ", "))) + } + sb.WriteString(fmt.Sprintf(" latest handshake: %s\n", peer.LastHandshake.Format(time.RFC1123))) + sb.WriteString(fmt.Sprintf(" transfer: %d B received, %d B sent\n", peer.RxBytes, peer.TxBytes)) + if peer.PresharedKey { + sb.WriteString(" preshared key: (hidden)\n") + } + } + + return sb.String() +} diff --git a/client/internal/engine.go b/client/internal/engine.go index 0dec799bf..e47007749 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -359,6 +359,7 @@ func (e *Engine) Start() error { return fmt.Errorf("new wg interface: %w", err) } e.wgInterface = wgIface + e.statusRecorder.SetWgIface(wgIface) // start flow manager right after interface creation publicKey := e.config.WgPrivateKey.PublicKey() @@ -380,7 +381,6 @@ func (e *Engine) Start() error { return fmt.Errorf("run rosenpass manager: %w", err) } } - e.stateManager.Start() initialRoutes, dnsServer, err := e.newDnsServer() @@ -1453,6 +1453,7 @@ func (e *Engine) close() { log.Errorf("failed closing Netbird interface %s %v", e.config.WgIfaceName, err) } e.wgInterface = nil + e.statusRecorder.SetWgIface(nil) } if !isNil(e.sshServer) { diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 82c1ba0e2..6bdd9ae3c 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -99,6 +99,10 @@ type MockWGIface struct { GetNetFunc func() *netstack.Net } +func (m *MockWGIface) FullStats() (*configurer.Stats, error) { + return nil, fmt.Errorf("not implemented") +} + func (m *MockWGIface) GetInterfaceGUIDString() (string, error) { return m.GetInterfaceGUIDStringFunc() } diff --git a/client/internal/iface_common.go b/client/internal/iface_common.go index e1761ff84..95bf146f9 100644 --- a/client/internal/iface_common.go +++ b/client/internal/iface_common.go @@ -37,4 +37,5 @@ type wgIfaceBase interface { GetWGDevice() *wgdevice.Device GetStats() (map[string]configurer.WGStats, error) GetNet() *netstack.Net + FullStats() (*configurer.Stats, error) } diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 0c6aac372..ed2f1fe47 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -3,6 +3,7 @@ package peer import ( "context" "errors" + "fmt" "net/netip" "slices" "sync" @@ -32,6 +33,10 @@ type ResolvedDomainInfo struct { ParentDomain domain.Domain } +type WGIfaceStatus interface { + FullStats() (*configurer.Stats, error) +} + type EventListener interface { OnEvent(event *proto.SystemEvent) } @@ -202,6 +207,7 @@ type Status struct { ingressGwMgr *ingressgw.Manager routeIDLookup routeIDLookup + wgIface WGIfaceStatus } // NewRecorder returns a new Status instance @@ -1078,6 +1084,23 @@ func (d *Status) GetEventHistory() []*proto.SystemEvent { return d.eventQueue.GetAll() } +func (d *Status) SetWgIface(wgInterface WGIfaceStatus) { + d.mux.Lock() + defer d.mux.Unlock() + + d.wgIface = wgInterface +} + +func (d *Status) PeersStatus() (*configurer.Stats, error) { + d.mux.Lock() + defer d.mux.Unlock() + if d.wgIface == nil { + return nil, fmt.Errorf("wgInterface is nil, cannot retrieve peers status") + } + + return d.wgIface.FullStats() +} + type EventQueue struct { maxSize int events []*proto.SystemEvent From df82a45d99a736c026944f89aaccff40b8b3219e Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 5 Jun 2025 12:39:58 +0200 Subject: [PATCH 28/37] [client] Improve dns match trace log (#3928) --- client/internal/dns/handler_chain.go | 81 ++++++++++++++-------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/client/internal/dns/handler_chain.go b/client/internal/dns/handler_chain.go index 6baf9ed95..22caaa761 100644 --- a/client/internal/dns/handler_chain.go +++ b/client/internal/dns/handler_chain.go @@ -1,6 +1,7 @@ package dns import ( + "fmt" "slices" "strings" "sync" @@ -148,61 +149,42 @@ func (c *HandlerChain) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { } qname := strings.ToLower(r.Question[0].Name) - log.Tracef("handling DNS request for domain=%s", qname) c.mu.RLock() handlers := slices.Clone(c.handlers) c.mu.RUnlock() if log.IsLevelEnabled(log.TraceLevel) { - log.Tracef("current handlers (%d):", len(handlers)) + var b strings.Builder + b.WriteString(fmt.Sprintf("DNS request domain=%s, handlers (%d):\n", qname, len(handlers))) for _, h := range handlers { - log.Tracef(" - pattern: domain=%s original: domain=%s wildcard=%v match_subdomain=%v priority=%d", - h.Pattern, h.OrigPattern, h.IsWildcard, h.MatchSubdomains, h.Priority) + b.WriteString(fmt.Sprintf(" - pattern: domain=%s original: domain=%s wildcard=%v match_subdomain=%v priority=%d\n", + h.Pattern, h.OrigPattern, h.IsWildcard, h.MatchSubdomains, h.Priority)) } + log.Trace(strings.TrimSuffix(b.String(), "\n")) } // Try handlers in priority order for _, entry := range handlers { - var matched bool - switch { - case entry.Pattern == ".": - matched = true - case entry.IsWildcard: - parts := strings.Split(strings.TrimSuffix(qname, entry.Pattern), ".") - matched = len(parts) >= 2 && strings.HasSuffix(qname, entry.Pattern) - default: - // For non-wildcard patterns: - // If handler wants subdomain matching, allow suffix match - // Otherwise require exact match - if entry.MatchSubdomains { - matched = strings.EqualFold(qname, entry.Pattern) || strings.HasSuffix(qname, "."+entry.Pattern) - } else { - matched = strings.EqualFold(qname, entry.Pattern) + matched := c.isHandlerMatch(qname, entry) + + if matched { + log.Tracef("handler matched: domain=%s -> pattern=%s wildcard=%v match_subdomain=%v priority=%d", + qname, entry.OrigPattern, entry.IsWildcard, entry.MatchSubdomains, entry.Priority) + + chainWriter := &ResponseWriterChain{ + ResponseWriter: w, + origPattern: entry.OrigPattern, } - } + entry.Handler.ServeDNS(chainWriter, r) - if !matched { - log.Tracef("trying domain match: request: domain=%s pattern: domain=%s wildcard=%v match_subdomain=%v priority=%d matched=false", - qname, entry.OrigPattern, entry.MatchSubdomains, entry.IsWildcard, entry.Priority) - continue + // If handler wants to continue, try next handler + if chainWriter.shouldContinue { + log.Tracef("handler requested continue to next handler for domain=%s", qname) + continue + } + return } - - log.Tracef("handler matched: request: domain=%s pattern: domain=%s wildcard=%v match_subdomain=%v priority=%d", - qname, entry.OrigPattern, entry.IsWildcard, entry.MatchSubdomains, entry.Priority) - - chainWriter := &ResponseWriterChain{ - ResponseWriter: w, - origPattern: entry.OrigPattern, - } - entry.Handler.ServeDNS(chainWriter, r) - - // If handler wants to continue, try next handler - if chainWriter.shouldContinue { - log.Tracef("handler requested continue to next handler") - continue - } - return } // No handler matched or all handlers passed @@ -213,3 +195,22 @@ func (c *HandlerChain) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { log.Errorf("failed to write DNS response: %v", err) } } + +func (c *HandlerChain) isHandlerMatch(qname string, entry HandlerEntry) bool { + switch { + case entry.Pattern == ".": + return true + case entry.IsWildcard: + parts := strings.Split(strings.TrimSuffix(qname, entry.Pattern), ".") + return len(parts) >= 2 && strings.HasSuffix(qname, entry.Pattern) + default: + // For non-wildcard patterns: + // If handler wants subdomain matching, allow suffix match + // Otherwise require exact match + if entry.MatchSubdomains { + return strings.EqualFold(qname, entry.Pattern) || strings.HasSuffix(qname, "."+entry.Pattern) + } else { + return strings.EqualFold(qname, entry.Pattern) + } + } +} From 55957a1960a3bda96f84a9fdc6cbeabba2896153 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 5 Jun 2025 12:40:23 +0200 Subject: [PATCH 29/37] [client] Run registerdns before flushing (#3926) * Run registerdns before flushing * Disable WINS, dynamic updates and registration --- client/internal/dns/host_windows.go | 125 +++++++++++++++++++++++++--- 1 file changed, 113 insertions(+), 12 deletions(-) diff --git a/client/internal/dns/host_windows.go b/client/internal/dns/host_windows.go index cfba29501..f8939328a 100644 --- a/client/internal/dns/host_windows.go +++ b/client/internal/dns/host_windows.go @@ -1,11 +1,14 @@ package dns import ( + "context" "errors" "fmt" "io" + "os/exec" "strings" "syscall" + "time" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" @@ -41,6 +44,20 @@ const ( interfaceConfigNameServerKey = "NameServer" interfaceConfigSearchListKey = "SearchList" + // Network interface DNS registration settings + disableDynamicUpdateKey = "DisableDynamicUpdate" + registrationEnabledKey = "RegistrationEnabled" + maxNumberOfAddressesToRegisterKey = "MaxNumberOfAddressesToRegister" + + // NetBIOS/WINS settings + netbtInterfacePath = `SYSTEM\CurrentControlSet\Services\NetBT\Parameters\Interfaces` + netbiosOptionsKey = "NetbiosOptions" + + // NetBIOS option values: 0 = from DHCP, 1 = enabled, 2 = disabled + netbiosFromDHCP = 0 + netbiosEnabled = 1 + netbiosDisabled = 2 + // RP_FORCE: Reapply all policies even if no policy change was detected rpForce = 0x1 ) @@ -67,16 +84,85 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) { log.Infof("detected GPO DNS policy configuration, using policy store") } - return ®istryConfigurator{ + configurator := ®istryConfigurator{ guid: guid, gpo: useGPO, - }, nil + } + + if err := configurator.configureInterface(); err != nil { + log.Errorf("failed to configure interface settings: %v", err) + } + + return configurator, nil } func (r *registryConfigurator) supportCustomPort() bool { return false } +func (r *registryConfigurator) configureInterface() error { + var merr *multierror.Error + + if err := r.disableDNSRegistrationForInterface(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("disable DNS registration: %w", err)) + } + + if err := r.disableWINSForInterface(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("disable WINS: %w", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *registryConfigurator) disableDNSRegistrationForInterface() error { + regKey, err := r.getInterfaceRegistryKey() + if err != nil { + return fmt.Errorf("get interface registry key: %w", err) + } + defer closer(regKey) + + var merr *multierror.Error + + if err := regKey.SetDWordValue(disableDynamicUpdateKey, 1); err != nil { + merr = multierror.Append(merr, fmt.Errorf("set %s: %w", disableDynamicUpdateKey, err)) + } + + if err := regKey.SetDWordValue(registrationEnabledKey, 0); err != nil { + merr = multierror.Append(merr, fmt.Errorf("set %s: %w", registrationEnabledKey, err)) + } + + if err := regKey.SetDWordValue(maxNumberOfAddressesToRegisterKey, 0); err != nil { + merr = multierror.Append(merr, fmt.Errorf("set %s: %w", maxNumberOfAddressesToRegisterKey, err)) + } + + if merr == nil || len(merr.Errors) == 0 { + log.Infof("disabled DNS registration for interface %s", r.guid) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *registryConfigurator) disableWINSForInterface() error { + netbtKeyPath := fmt.Sprintf(`%s\Tcpip_%s`, netbtInterfacePath, r.guid) + + regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, netbtKeyPath, registry.SET_VALUE) + if err != nil { + regKey, _, err = registry.CreateKey(registry.LOCAL_MACHINE, netbtKeyPath, registry.SET_VALUE) + if err != nil { + return fmt.Errorf("create NetBT interface key %s: %w", netbtKeyPath, err) + } + } + defer closer(regKey) + + // NetbiosOptions: 2 = disabled + if err := regKey.SetDWordValue(netbiosOptionsKey, netbiosDisabled); err != nil { + return fmt.Errorf("set %s: %w", netbiosOptionsKey, err) + } + + log.Infof("disabled WINS/NetBIOS for interface %s", r.guid) + return nil +} + func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error { if config.RouteAll { if err := r.addDNSSetupForAll(config.ServerIP); err != nil { @@ -119,9 +205,7 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager return fmt.Errorf("update search domains: %w", err) } - if err := r.flushDNSCache(); err != nil { - log.Errorf("failed to flush DNS cache: %v", err) - } + go r.flushDNSCache() return nil } @@ -191,7 +275,25 @@ func (r *registryConfigurator) string() string { return "registry" } -func (r *registryConfigurator) flushDNSCache() error { +func (r *registryConfigurator) registerDNS() { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + // nolint:misspell + cmd := exec.CommandContext(ctx, "ipconfig", "/registerdns") + out, err := cmd.CombinedOutput() + + if err != nil { + log.Errorf("failed to register DNS: %v, output: %s", err, out) + return + } + + log.Info("registered DNS names") +} + +func (r *registryConfigurator) flushDNSCache() { + r.registerDNS() + // dnsFlushResolverCacheFn.Call() may panic if the func is not found defer func() { if rec := recover(); rec != nil { @@ -202,13 +304,14 @@ func (r *registryConfigurator) flushDNSCache() error { ret, _, err := dnsFlushResolverCacheFn.Call() if ret == 0 { if err != nil && !errors.Is(err, syscall.Errno(0)) { - return fmt.Errorf("DnsFlushResolverCache failed: %w", err) + log.Errorf("DnsFlushResolverCache failed: %v", err) + return } - return fmt.Errorf("DnsFlushResolverCache failed") + log.Errorf("DnsFlushResolverCache failed") + return } log.Info("flushed DNS cache") - return nil } func (r *registryConfigurator) updateSearchDomains(domains []string) error { @@ -263,9 +366,7 @@ func (r *registryConfigurator) restoreHostDNS() error { return fmt.Errorf("remove interface registry key: %w", err) } - if err := r.flushDNSCache(); err != nil { - log.Errorf("failed to flush DNS cache: %v", err) - } + go r.flushDNSCache() return nil } From 84354951d37fdcff137e0f045cff4190d5ee61a2 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 5 Jun 2025 13:54:15 +0200 Subject: [PATCH 30/37] [client] Add systemd netbird logs to debug bundle (#3917) --- client/cmd/service.go | 10 ++- client/internal/debug/debug.go | 9 ++- client/internal/debug/debug_linux.go | 89 ++++++++++++++++++++++++- client/internal/debug/debug_nonlinux.go | 6 ++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/client/cmd/service.go b/client/cmd/service.go index 005479306..156e67d6d 100644 --- a/client/cmd/service.go +++ b/client/cmd/service.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "runtime" "sync" "github.com/kardianos/service" @@ -27,12 +28,19 @@ func newProgram(ctx context.Context, cancel context.CancelFunc) *program { } func newSVCConfig() *service.Config { - return &service.Config{ + config := &service.Config{ Name: serviceName, DisplayName: "Netbird", Description: "Netbird mesh network client", Option: make(service.KeyValue), + EnvVars: make(map[string]string), } + + if runtime.GOOS == "linux" { + config.EnvVars["SYSTEMD_UNIT"] = serviceName + } + + return config } func newSVC(prg *program, conf *service.Config) (service.Service, error) { diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index a7d873c8f..dfed47f05 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -274,10 +274,15 @@ func (g *BundleGenerator) createArchive() error { log.Errorf("Failed to add wg show output: %v", err) } - if g.logFile != "console" { + if g.logFile != "console" && g.logFile != "" { if err := g.addLogfile(); err != nil { - return fmt.Errorf("add log file: %w", err) + log.Errorf("Failed to add log file to debug bundle: %v", err) + if err := g.trySystemdLogFallback(); err != nil { + log.Errorf("Failed to add systemd logs as fallback: %v", err) + } } + } else if err := g.trySystemdLogFallback(); err != nil { + log.Errorf("Failed to add systemd logs: %v", err) } return nil diff --git a/client/internal/debug/debug_linux.go b/client/internal/debug/debug_linux.go index b4907beca..4626cd9a2 100644 --- a/client/internal/debug/debug_linux.go +++ b/client/internal/debug/debug_linux.go @@ -4,17 +4,104 @@ package debug import ( "bytes" + "context" "encoding/binary" + "errors" "fmt" + "os" "os/exec" "sort" "strings" + "time" "github.com/google/nftables" "github.com/google/nftables/expr" log "github.com/sirupsen/logrus" ) +const ( + maxLogEntries = 100000 + maxLogAge = 7 * 24 * time.Hour // Last 7 days +) + +// trySystemdLogFallback attempts to get logs from systemd journal as fallback +func (g *BundleGenerator) trySystemdLogFallback() error { + log.Debug("Attempting to collect systemd journal logs") + + serviceName := getServiceName() + journalLogs, err := getSystemdLogs(serviceName) + if err != nil { + return fmt.Errorf("get systemd logs for %s: %w", serviceName, err) + } + + if strings.Contains(journalLogs, "No recent log entries found") { + log.Debug("No recent log entries found in systemd journal") + return nil + } + + if g.anonymize { + journalLogs = g.anonymizer.AnonymizeString(journalLogs) + } + + logReader := strings.NewReader(journalLogs) + fileName := fmt.Sprintf("systemd-%s.log", serviceName) + if err := g.addFileToZip(logReader, fileName); err != nil { + return fmt.Errorf("add systemd logs to bundle: %w", err) + } + + log.Infof("Added systemd journal logs for %s to debug bundle", serviceName) + return nil +} + +// getServiceName gets the service name from environment or defaults to netbird +func getServiceName() string { + if unitName := os.Getenv("SYSTEMD_UNIT"); unitName != "" { + log.Debugf("Detected SYSTEMD_UNIT environment variable: %s", unitName) + return unitName + } + + return "netbird" +} + +// getSystemdLogs retrieves logs from systemd journal for a specific service using journalctl +func getSystemdLogs(serviceName string) (string, error) { + args := []string{ + "-u", fmt.Sprintf("%s.service", serviceName), + "--since", fmt.Sprintf("-%s", maxLogAge.String()), + "--lines", fmt.Sprintf("%d", maxLogEntries), + "--no-pager", + "--output", "short-iso", + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "journalctl", args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "", fmt.Errorf("journalctl command timed out after 30 seconds") + } + if strings.Contains(err.Error(), "executable file not found") { + return "", fmt.Errorf("journalctl command not found: %w", err) + } + return "", fmt.Errorf("execute journalctl: %w (stderr: %s)", err, stderr.String()) + } + + logs := stdout.String() + if strings.TrimSpace(logs) == "" { + return "No recent log entries found in systemd journal", nil + } + + header := fmt.Sprintf("=== Systemd Journal Logs for %s.service (last %d entries, max %s) ===\n", + serviceName, maxLogEntries, maxLogAge.String()) + + return header + logs, nil +} + // addFirewallRules collects and adds firewall rules to the archive func (g *BundleGenerator) addFirewallRules() error { log.Info("Collecting firewall rules") @@ -481,7 +568,7 @@ func formatExpr(exp expr.Any) string { case *expr.Fib: return formatFib(e) case *expr.Target: - return fmt.Sprintf("jump %s", e.Name) // Properly format jump targets + return fmt.Sprintf("jump %s", e.Name) case *expr.Immediate: if e.Register == 1 { return formatImmediateData(e.Data) diff --git a/client/internal/debug/debug_nonlinux.go b/client/internal/debug/debug_nonlinux.go index ef93620a0..b0ff55613 100644 --- a/client/internal/debug/debug_nonlinux.go +++ b/client/internal/debug/debug_nonlinux.go @@ -6,3 +6,9 @@ package debug func (g *BundleGenerator) addFirewallRules() error { return nil } + +func (g *BundleGenerator) trySystemdLogFallback() error { + // Systemd is only available on Linux + // TODO: Add BSD support + return nil +} From 6c0cdb6ed17a90992410269eb75362e48ac303db Mon Sep 17 00:00:00 2001 From: Ghazy Abdallah Date: Thu, 5 Jun 2025 15:15:01 +0300 Subject: [PATCH 31/37] [misc] fix: traefik relay accessibility (#3696) --- infrastructure_files/base.setup.env | 2 ++ infrastructure_files/configure.sh | 2 ++ infrastructure_files/docker-compose.yml.tmpl | 2 +- .../docker-compose.yml.tmpl.traefik | 22 +++---------------- infrastructure_files/management.json.tmpl | 2 +- infrastructure_files/setup.env.example | 11 ++++++++++ 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/infrastructure_files/base.setup.env b/infrastructure_files/base.setup.env index ebc38a11f..fdba1f215 100644 --- a/infrastructure_files/base.setup.env +++ b/infrastructure_files/base.setup.env @@ -23,6 +23,7 @@ NETBIRD_SIGNAL_PORT=${NETBIRD_SIGNAL_PORT:-10000} # Relay NETBIRD_RELAY_DOMAIN=${NETBIRD_RELAY_DOMAIN:-$NETBIRD_DOMAIN} NETBIRD_RELAY_PORT=${NETBIRD_RELAY_PORT:-33080} +NETBIRD_RELAY_ENDPOINT=${NETBIRD_RELAY_ENDPOINT:-rel://$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT} # Relay auth secret NETBIRD_RELAY_AUTH_SECRET= @@ -135,5 +136,6 @@ export COTURN_TAG export NETBIRD_TURN_EXTERNAL_IP export NETBIRD_RELAY_DOMAIN export NETBIRD_RELAY_PORT +export NETBIRD_RELAY_ENDPOINT export NETBIRD_RELAY_AUTH_SECRET export NETBIRD_RELAY_TAG diff --git a/infrastructure_files/configure.sh b/infrastructure_files/configure.sh index d02e4f40c..6898555aa 100755 --- a/infrastructure_files/configure.sh +++ b/infrastructure_files/configure.sh @@ -170,6 +170,7 @@ fi if [[ "$NETBIRD_DISABLE_LETSENCRYPT" == "true" ]]; then export NETBIRD_DASHBOARD_ENDPOINT="https://$NETBIRD_DOMAIN:443" export NETBIRD_SIGNAL_ENDPOINT="https://$NETBIRD_DOMAIN:$NETBIRD_SIGNAL_PORT" + export NETBIRD_RELAY_ENDPOINT="rels://$NETBIRD_DOMAIN:$NETBIRD_SIGNAL_PORT/relay" echo "Letsencrypt was disabled, the Https-endpoints cannot be used anymore" echo " and a reverse-proxy with Https needs to be placed in front of netbird!" @@ -178,6 +179,7 @@ if [[ "$NETBIRD_DISABLE_LETSENCRYPT" == "true" ]]; then echo "- $NETBIRD_MGMT_API_ENDPOINT/api -http-> management:$NETBIRD_MGMT_API_PORT" echo "- $NETBIRD_MGMT_API_ENDPOINT/management.ManagementService/ -grpc-> management:$NETBIRD_MGMT_API_PORT" echo "- $NETBIRD_SIGNAL_ENDPOINT/signalexchange.SignalExchange/ -grpc-> signal:80" + echo "- $NETBIRD_RELAY_ENDPOINT/ -http-> relay:33080" echo "You most likely also have to change NETBIRD_MGMT_API_ENDPOINT in base.setup.env and port-mappings in docker-compose.yml.tmpl and rerun this script." echo " The target of the forwards depends on your setup. Beware of the gRPC protocol instead of http for management and signal!" echo "You are also free to remove any occurrences of the Letsencrypt-volume $LETSENCRYPT_VOLUMENAME" diff --git a/infrastructure_files/docker-compose.yml.tmpl b/infrastructure_files/docker-compose.yml.tmpl index dc491ae23..b529f9606 100644 --- a/infrastructure_files/docker-compose.yml.tmpl +++ b/infrastructure_files/docker-compose.yml.tmpl @@ -57,7 +57,7 @@ services: environment: - NB_LOG_LEVEL=info - NB_LISTEN_ADDRESS=:$NETBIRD_RELAY_PORT - - NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT + - NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_ENDPOINT # todo: change to a secure secret - NB_AUTH_SECRET=$NETBIRD_RELAY_AUTH_SECRET ports: diff --git a/infrastructure_files/docker-compose.yml.tmpl.traefik b/infrastructure_files/docker-compose.yml.tmpl.traefik index 8cc3df309..8da3cabb5 100644 --- a/infrastructure_files/docker-compose.yml.tmpl.traefik +++ b/infrastructure_files/docker-compose.yml.tmpl.traefik @@ -3,9 +3,6 @@ services: dashboard: image: netbirdio/dashboard:$NETBIRD_DASHBOARD_TAG restart: unless-stopped - #ports: - # - 80:80 - # - 443:443 environment: # Endpoints - NETBIRD_MGMT_API_ENDPOINT=$NETBIRD_MGMT_API_ENDPOINT @@ -43,11 +40,6 @@ services: restart: unless-stopped volumes: - $SIGNAL_VOLUMENAME:/var/lib/netbird - #ports: - # - $NETBIRD_SIGNAL_PORT:80 - # # port and command for Let's Encrypt validation - # - 443:443 - # command: ["--letsencrypt-domain", "$NETBIRD_LETSENCRYPT_DOMAIN", "--log-file", "console"] labels: - traefik.enable=true - traefik.http.routers.netbird-signal.rule=Host(`$NETBIRD_DOMAIN`) && PathPrefix(`/signalexchange.SignalExchange/`) @@ -65,12 +57,10 @@ services: restart: unless-stopped environment: - NB_LOG_LEVEL=info - - NB_LISTEN_ADDRESS=:$NETBIRD_RELAY_PORT - - NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT + - NB_LISTEN_ADDRESS=:33080 + - NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_ENDPOINT # todo: change to a secure secret - NB_AUTH_SECRET=$NETBIRD_RELAY_AUTH_SECRET - # ports: - # - $NETBIRD_RELAY_PORT:$NETBIRD_RELAY_PORT logging: driver: "json-file" options: @@ -79,7 +69,7 @@ services: labels: - traefik.enable=true - traefik.http.routers.netbird-relay.rule=Host(`$NETBIRD_DOMAIN`) && PathPrefix(`/relay`) - - traefik.http.services.netbird-relay.loadbalancer.server.port=$NETBIRD_RELAY_PORT + - traefik.http.services.netbird-relay.loadbalancer.server.port=33080 # Management management: @@ -91,10 +81,6 @@ services: - $MGMT_VOLUMENAME:/var/lib/netbird - $LETSENCRYPT_VOLUMENAME:/etc/letsencrypt:ro - ./management.json:/etc/netbird/management.json - #ports: - # - $NETBIRD_MGMT_API_PORT:443 #API port - # # command for Let's Encrypt validation without dashboard container - # command: ["--letsencrypt-domain", "$NETBIRD_LETSENCRYPT_DOMAIN", "--log-file", "console"] command: [ "--port", "33073", "--log-file", "console", @@ -129,8 +115,6 @@ services: domainname: $TURN_DOMAIN volumes: - ./turnserver.conf:/etc/turnserver.conf:ro - # - ./privkey.pem:/etc/coturn/private/privkey.pem:ro - # - ./cert.pem:/etc/coturn/certs/cert.pem:ro network_mode: host command: - -c /etc/turnserver.conf diff --git a/infrastructure_files/management.json.tmpl b/infrastructure_files/management.json.tmpl index c0e57b4fd..4d09816ef 100644 --- a/infrastructure_files/management.json.tmpl +++ b/infrastructure_files/management.json.tmpl @@ -21,7 +21,7 @@ "TimeBasedCredentials": false }, "Relay": { - "Addresses": ["rel://$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT"], + "Addresses": ["$NETBIRD_RELAY_ENDPOINT"], "CredentialsTTL": "24h", "Secret": "$NETBIRD_RELAY_AUTH_SECRET" }, diff --git a/infrastructure_files/setup.env.example b/infrastructure_files/setup.env.example index b1b64de78..b5b718a71 100644 --- a/infrastructure_files/setup.env.example +++ b/infrastructure_files/setup.env.example @@ -102,4 +102,15 @@ NETBIRD_RELAY_DOMAIN="" # Relay server connection port. If none is supplied # it will default to 33080 +# should be updated to match TLS-port of reverse proxy when netbird is running behind reverse proxy NETBIRD_RELAY_PORT="" + +# Management API connectin port. If none is supplied +# it will default to 33073 +# should be updated to match TLS-port of reverse proxy when netbird is running behind reverse proxy +NETBIRD_MGMT_API_PORT="" + +# Signal service connectin port. If none is supplied +# it will default to 10000 +# should be updated to match TLS-port of reverse proxy when netbird is running behind reverse proxy +NETBIRD_SIGNAL_PORT="" From c6cceba381b81e7296d4ffec25cadeeda9e6f43c Mon Sep 17 00:00:00 2001 From: Robert Neumann Date: Thu, 5 Jun 2025 14:16:04 +0200 Subject: [PATCH 32/37] Update getting-started-with-zitadel.sh - fix zitadel user console (#3446) --- infrastructure_files/getting-started-with-zitadel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure_files/getting-started-with-zitadel.sh b/infrastructure_files/getting-started-with-zitadel.sh index 9b80058c2..1e67bd177 100644 --- a/infrastructure_files/getting-started-with-zitadel.sh +++ b/infrastructure_files/getting-started-with-zitadel.sh @@ -602,6 +602,7 @@ renderCaddyfile() { reverse_proxy /debug/* h2c://zitadel:8080 reverse_proxy /device/* h2c://zitadel:8080 reverse_proxy /device h2c://zitadel:8080 + reverse_proxy /zitadel.user.v2.UserService/* h2c://zitadel:8080 # Dashboard reverse_proxy /* dashboard:80 } From 122a89c02b4aa3aadd399cb5cb0ec9e7ba12584a Mon Sep 17 00:00:00 2001 From: Abdul Latif <55663276+orchard0@users.noreply.github.com> Date: Thu, 5 Jun 2025 13:28:19 +0100 Subject: [PATCH 33/37] [misc] remove error causing dnf config-manager add (#3925) --- release_files/install.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/release_files/install.sh b/release_files/install.sh index da5c613d5..0f63529ea 100755 --- a/release_files/install.sh +++ b/release_files/install.sh @@ -262,13 +262,6 @@ install_netbird() { ;; dnf) add_rpm_repo - ${SUDO} dnf -y install dnf-plugin-config-manager - if [[ "$(dnf --version | head -n1 | cut -d. -f1)" > "4" ]]; - then - ${SUDO} dnf config-manager addrepo --from-repofile=/etc/yum.repos.d/netbird.repo - else - ${SUDO} dnf config-manager --add-repo /etc/yum.repos.d/netbird.repo - fi ${SUDO} dnf -y install netbird if ! $SKIP_UI_APP; then From 64f111923efbd682f7d627845d8f7831d6b27472 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 5 Jun 2025 15:22:59 +0200 Subject: [PATCH 34/37] [client] Increase stun status probe timeout (#3930) --- client/internal/engine.go | 2 +- client/internal/relay/relay.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index e47007749..0962e9004 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -1678,7 +1678,7 @@ func (e *Engine) RunHealthProbes() bool { func (e *Engine) probeICE(stuns, turns []*stun.URI) []relay.ProbeResult { return append( relay.ProbeAll(e.ctx, relay.ProbeSTUN, stuns), - relay.ProbeAll(e.ctx, relay.ProbeSTUN, turns)..., + relay.ProbeAll(e.ctx, relay.ProbeTURN, turns)..., ) } diff --git a/client/internal/relay/relay.go b/client/internal/relay/relay.go index 7d98a6060..6e1f83a9a 100644 --- a/client/internal/relay/relay.go +++ b/client/internal/relay/relay.go @@ -170,7 +170,7 @@ func ProbeAll( var wg sync.WaitGroup for i, uri := range relays { - ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + ctx, cancel := context.WithTimeout(ctx, 6*time.Second) defer cancel() wg.Add(1) From b56f61bf1b093afaed242a22f861475a40eafe1d Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Thu, 5 Jun 2025 14:44:44 +0100 Subject: [PATCH 35/37] [misc] fix relay exposed address test (#3931) --- .github/workflows/test-infrastructure-files.yml | 4 ++-- infrastructure_files/configure.sh | 2 +- infrastructure_files/setup.env.example | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 8c2d21c8f..e2f9e40c8 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -172,11 +172,11 @@ jobs: grep "NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN" docker-compose.yml grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN" # check relay values - grep "NB_EXPOSED_ADDRESS=$CI_NETBIRD_DOMAIN:33445" docker-compose.yml + grep "NB_EXPOSED_ADDRESS=rels://$CI_NETBIRD_DOMAIN:33445" docker-compose.yml grep "NB_LISTEN_ADDRESS=:33445" docker-compose.yml grep '33445:33445' docker-compose.yml grep -A 10 'relay:' docker-compose.yml | egrep 'NB_AUTH_SECRET=.+$' - grep -A 7 Relay management.json | grep "rel://$CI_NETBIRD_DOMAIN:33445" + grep -A 7 Relay management.json | grep "rels://$CI_NETBIRD_DOMAIN:33445" grep -A 7 Relay management.json | egrep '"Secret": ".+"' grep DisablePromptLogin management.json | grep 'true' grep LoginFlag management.json | grep 0 diff --git a/infrastructure_files/configure.sh b/infrastructure_files/configure.sh index 6898555aa..e3fcbfdde 100755 --- a/infrastructure_files/configure.sh +++ b/infrastructure_files/configure.sh @@ -170,7 +170,7 @@ fi if [[ "$NETBIRD_DISABLE_LETSENCRYPT" == "true" ]]; then export NETBIRD_DASHBOARD_ENDPOINT="https://$NETBIRD_DOMAIN:443" export NETBIRD_SIGNAL_ENDPOINT="https://$NETBIRD_DOMAIN:$NETBIRD_SIGNAL_PORT" - export NETBIRD_RELAY_ENDPOINT="rels://$NETBIRD_DOMAIN:$NETBIRD_SIGNAL_PORT/relay" + export NETBIRD_RELAY_ENDPOINT="rels://$NETBIRD_DOMAIN:$NETBIRD_RELAY_PORT/relay" echo "Letsencrypt was disabled, the Https-endpoints cannot be used anymore" echo " and a reverse-proxy with Https needs to be placed in front of netbird!" diff --git a/infrastructure_files/setup.env.example b/infrastructure_files/setup.env.example index b5b718a71..b6a209953 100644 --- a/infrastructure_files/setup.env.example +++ b/infrastructure_files/setup.env.example @@ -105,12 +105,12 @@ NETBIRD_RELAY_DOMAIN="" # should be updated to match TLS-port of reverse proxy when netbird is running behind reverse proxy NETBIRD_RELAY_PORT="" -# Management API connectin port. If none is supplied +# Management API connecting port. If none is supplied # it will default to 33073 # should be updated to match TLS-port of reverse proxy when netbird is running behind reverse proxy NETBIRD_MGMT_API_PORT="" -# Signal service connectin port. If none is supplied +# Signal service connecting port. If none is supplied # it will default to 10000 # should be updated to match TLS-port of reverse proxy when netbird is running behind reverse proxy NETBIRD_SIGNAL_PORT="" From 0f7c7f1da2f07221d67ea621422ea930fb0da76e Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Sun, 8 Jun 2025 09:53:27 +0100 Subject: [PATCH 36/37] [misc] use generic slack url (#3939) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e0f2df848..1d2a976c2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@
- +
@@ -29,7 +29,7 @@
See Documentation
- Join our Slack channel + Join our Slack channel
From 0f050e5fe1b2f34b484a6f082cc483f0712a3135 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Sun, 8 Jun 2025 11:19:54 +0100 Subject: [PATCH 37/37] [client] Optmize process check time (#3938) This PR optimizes the process check time by updating the implementation of getRunningProcesses and introducing new benchmark tests. Updated getRunningProcesses to use process.Pids() instead of process.Processes() Added benchmark tests for both the new and the legacy implementations Benchmark: https://github.com/netbirdio/netbird/actions/runs/15512741612 todo: evaluate windows optmizations and caching risks --- client/system/process.go | 8 +++-- client/system/process_test.go | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 client/system/process_test.go diff --git a/client/system/process.go b/client/system/process.go index 2e43fcfe0..87e21eb9d 100644 --- a/client/system/process.go +++ b/client/system/process.go @@ -11,16 +11,18 @@ import ( // getRunningProcesses returns a list of running process paths. func getRunningProcesses() ([]string, error) { - processes, err := process.Processes() + processIDs, err := process.Pids() if err != nil { return nil, err } processMap := make(map[string]bool) - for _, p := range processes { + for _, pID := range processIDs { + p := &process.Process{Pid: pID} + path, _ := p.Exe() if path != "" { - processMap[path] = true + processMap[path] = false } } diff --git a/client/system/process_test.go b/client/system/process_test.go new file mode 100644 index 000000000..505808a9e --- /dev/null +++ b/client/system/process_test.go @@ -0,0 +1,58 @@ +package system + +import ( + "testing" + + "github.com/shirou/gopsutil/v3/process" +) + +func Benchmark_getRunningProcesses(b *testing.B) { + b.Run("getRunningProcesses new", func(b *testing.B) { + for i := 0; i < b.N; i++ { + ps, err := getRunningProcesses() + if err != nil { + b.Fatalf("unexpected error: %v", err) + } + if len(ps) == 0 { + b.Fatalf("expected non-empty process list, got empty") + } + } + }) + b.Run("getRunningProcesses old", func(b *testing.B) { + for i := 0; i < b.N; i++ { + ps, err := getRunningProcessesOld() + if err != nil { + b.Fatalf("unexpected error: %v", err) + } + if len(ps) == 0 { + b.Fatalf("expected non-empty process list, got empty") + } + } + }) + s, _ := getRunningProcesses() + b.Logf("getRunningProcesses returned %d processes", len(s)) + s, _ = getRunningProcessesOld() + b.Logf("getRunningProcessesOld returned %d processes", len(s)) +} + +func getRunningProcessesOld() ([]string, error) { + processes, err := process.Processes() + if err != nil { + return nil, err + } + + processMap := make(map[string]bool) + for _, p := range processes { + path, _ := p.Exe() + if path != "" { + processMap[path] = true + } + } + + uniqueProcesses := make([]string, 0, len(processMap)) + for p := range processMap { + uniqueProcesses = append(uniqueProcesses, p) + } + + return uniqueProcesses, nil +}