Merge branch 'main' into feature/add-serial-to-proxy

management/server/account.go

@@ -376,6 +376,7 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco
 	am.handlePeerLoginExpirationSettings(ctx, oldSettings, newSettings, userID, accountID)
 	am.handleGroupsPropagationSettings(ctx, oldSettings, newSettings, userID, accountID)
 	am.handleAutoUpdateVersionSettings(ctx, oldSettings, newSettings, userID, accountID)
+	am.handlePeerExposeSettings(ctx, oldSettings, newSettings, userID, accountID)
 	if err = am.handleInactivityExpirationSettings(ctx, oldSettings, newSettings, userID, accountID); err != nil {
 		return nil, err
 	}
@@ -492,6 +493,21 @@ func (am *DefaultAccountManager) handleAutoUpdateVersionSettings(ctx context.Con
 	}
 }
 
+func (am *DefaultAccountManager) handlePeerExposeSettings(ctx context.Context, oldSettings, newSettings *types.Settings, userID, accountID string) {
+	oldEnabled := oldSettings.PeerExposeEnabled
+	newEnabled := newSettings.PeerExposeEnabled
+
+	if oldEnabled == newEnabled {
+		return
+	}
+
+	event := activity.AccountPeerExposeEnabled
+	if !newEnabled {
+		event = activity.AccountPeerExposeDisabled
+	}
+	am.StoreEvent(ctx, userID, accountID, accountID, event, nil)
+}
+
 func (am *DefaultAccountManager) handleInactivityExpirationSettings(ctx context.Context, oldSettings, newSettings *types.Settings, userID, accountID string) error {
 	if newSettings.PeerInactivityExpirationEnabled {
 		if oldSettings.PeerInactivityExpiration != newSettings.PeerInactivityExpiration {

management/server/account_test.go

@@ -3124,7 +3124,7 @@ func createManager(t testing.TB) (*DefaultAccountManager, *update_channel.PeersU
 	}
 
 	proxyGrpcServer := nbgrpc.NewProxyServiceServer(nil, nil, nbgrpc.ProxyOIDCConfig{}, peersManager, nil)
-	manager.SetServiceManager(reverseproxymanager.NewManager(store, manager, permissionsManager, proxyGrpcServer, nil))
+	manager.SetServiceManager(reverseproxymanager.NewManager(store, manager, permissionsManager, settingsMockManager, proxyGrpcServer, nil))
 
 	return manager, updateManager, nil
 }

management/server/activity/codes.go

@@ -208,6 +208,18 @@ const (
 	ServiceUpdated Activity = 109
 	ServiceDeleted Activity = 110
 
+	// PeerServiceExposed indicates that a peer exposed a service via the reverse proxy
+	PeerServiceExposed Activity = 111
+	// PeerServiceUnexposed indicates that a peer-exposed service was removed
+	PeerServiceUnexposed Activity = 112
+	// PeerServiceExposeExpired indicates that a peer-exposed service was removed due to TTL expiration
+	PeerServiceExposeExpired Activity = 113
+
+	// AccountPeerExposeEnabled indicates that a user enabled peer expose for the account
+	AccountPeerExposeEnabled Activity = 114
+	// AccountPeerExposeDisabled indicates that a user disabled peer expose for the account
+	AccountPeerExposeDisabled Activity = 115
+
 	AccountDeleted Activity = 99999
 )
 
@@ -345,6 +357,13 @@ var activityMap = map[Activity]Code{
 	ServiceCreated: {"Service created", "service.create"},
 	ServiceUpdated: {"Service updated", "service.update"},
 	ServiceDeleted: {"Service deleted", "service.delete"},
+
+	PeerServiceExposed:       {"Peer exposed service", "service.peer.expose"},
+	PeerServiceUnexposed:     {"Peer unexposed service", "service.peer.unexpose"},
+	PeerServiceExposeExpired: {"Peer exposed service expired", "service.peer.expose.expire"},
+
+	AccountPeerExposeEnabled:  {"Account peer expose enabled", "account.setting.peer.expose.enable"},
+	AccountPeerExposeDisabled: {"Account peer expose disabled", "account.setting.peer.expose.disable"},
 }
 
 // StringCode returns a string code of the activity

management/server/http/handlers/accounts/accounts_handler.go

@@ -168,6 +168,10 @@ func (h *handler) getAllAccounts(w http.ResponseWriter, r *http.Request) {
 }
 
 func (h *handler) updateAccountRequestSettings(req api.PutApiAccountsAccountIdJSONRequestBody) (*types.Settings, error) {
+	if req.Settings.PeerExposeEnabled && len(req.Settings.PeerExposeGroups) == 0 {
+		return nil, status.Errorf(status.InvalidArgument, "peer expose requires at least one group")
+	}
+
 	returnSettings := &types.Settings{
 		PeerLoginExpirationEnabled: req.Settings.PeerLoginExpirationEnabled,
 		PeerLoginExpiration:        time.Duration(float64(time.Second.Nanoseconds()) * float64(req.Settings.PeerLoginExpiration)),
@@ -175,6 +179,9 @@ func (h *handler) updateAccountRequestSettings(req api.PutApiAccountsAccountIdJS
 
 		PeerInactivityExpirationEnabled: req.Settings.PeerInactivityExpirationEnabled,
 		PeerInactivityExpiration:        time.Duration(float64(time.Second.Nanoseconds()) * float64(req.Settings.PeerInactivityExpiration)),
+
+		PeerExposeEnabled: req.Settings.PeerExposeEnabled,
+		PeerExposeGroups:  req.Settings.PeerExposeGroups,
 	}
 
 	if req.Settings.Extra != nil {
@@ -336,6 +343,8 @@ func toAccountResponse(accountID string, settings *types.Settings, meta *types.A
 		JwtAllowGroups:                  &jwtAllowGroups,
 		RegularUsersViewBlocked:         settings.RegularUsersViewBlocked,
 		RoutingPeerDnsResolutionEnabled: &settings.RoutingPeerDNSResolutionEnabled,
+		PeerExposeEnabled:               settings.PeerExposeEnabled,
+		PeerExposeGroups:                settings.PeerExposeGroups,
 		LazyConnectionEnabled:           &settings.LazyConnectionEnabled,
 		DnsDomain:                       &settings.DNSDomain,
 		AutoUpdateVersion:               &settings.AutoUpdateVersion,

management/server/http/handlers/proxy/auth_callback_integration_test.go

@@ -414,6 +414,20 @@ func (m *testServiceManager) GetServiceIDByTargetID(_ context.Context, _, _ stri
 	return "", nil
 }
 
+func (m *testServiceManager) CreateServiceFromPeer(_ context.Context, _, _ string, _ *reverseproxy.ExposeServiceRequest) (*reverseproxy.ExposeServiceResponse, error) {
+	return nil, nil
+}
+
+func (m *testServiceManager) RenewServiceFromPeer(_ context.Context, _, _, _ string) error {
+	return nil
+}
+
+func (m *testServiceManager) StopServiceFromPeer(_ context.Context, _, _, _ string) error {
+	return nil
+}
+
+func (m *testServiceManager) StartExposeReaper(_ context.Context) {}
+
 func createTestState(t *testing.T, ps *nbgrpc.ProxyServiceServer, redirectURL string) string {
 	t.Helper()
 

management/server/http/testing/testing_tools/channel/channel.go

@@ -99,7 +99,7 @@ func BuildApiBlackBoxWithDBState(t testing_tools.TB, sqlFile string, expectedPee
 	proxyMgr := proxymanager.NewManager(store)
 	proxyServiceServer := nbgrpc.NewProxyServiceServer(accessLogsManager, proxyTokenStore, nbgrpc.ProxyOIDCConfig{}, peersManager, userManager, proxyMgr)
 	domainManager := manager.NewManager(store, proxyMgr, permissionsManager)
-	serviceManager := reverseproxymanager.NewManager(store, am, permissionsManager, proxyServiceServer, domainManager)
+	serviceManager := reverseproxymanager.NewManager(store, am, permissionsManager, reverseproxymanager.NewGRPCProxyController(proxyServiceServer), domainManager)
 	proxyServiceServer.SetServiceManager(serviceManager)
 	am.SetServiceManager(serviceManager)
 

management/server/metrics/selfhosted.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/hashicorp/go-version"
 	"github.com/netbirdio/netbird/idp/dex"
+	rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/management/server/types"
@@ -51,6 +52,7 @@ type properties map[string]interface{}
 type DataSource interface {
 	GetAllAccounts(ctx context.Context) []*types.Account
 	GetStoreEngine() types.Engine
+	GetCustomDomainsCounts(ctx context.Context) (total int64, validated int64, err error)
 }
 
 // ConnManager peer connection manager that holds state for current active connections
@@ -211,6 +213,16 @@ func (w *Worker) generateProperties(ctx context.Context) properties {
 		localUsers                int
 		idpUsers                  int
 		embeddedIdpTypes          map[string]int
+		services                  int
+		servicesEnabled           int
+		servicesTargets           int
+		servicesStatusActive      int
+		servicesStatusPending     int
+		servicesStatusError       int
+		servicesTargetType        map[string]int
+		servicesAuthPassword      int
+		servicesAuthPin           int
+		servicesAuthOIDC          int
 	)
 	start := time.Now()
 	metricsProperties := make(properties)
@@ -220,10 +232,13 @@ func (w *Worker) generateProperties(ctx context.Context) properties {
 	rulesDirection = make(map[string]int)
 	activeUsersLastDay = make(map[string]struct{})
 	embeddedIdpTypes = make(map[string]int)
+	servicesTargetType = make(map[string]int)
 	uptime = time.Since(w.startupTime).Seconds()
 	connections := w.connManager.GetAllConnectedPeers()
 	version = nbversion.NetbirdVersion()
 
+	customDomains, customDomainsValidated, _ := w.dataSource.GetCustomDomainsCounts(ctx)
+
 	for _, account := range w.dataSource.GetAllAccounts(ctx) {
 		accounts++
 
@@ -335,6 +350,37 @@ func (w *Worker) generateProperties(ctx context.Context) properties {
 				peerActiveVersions = append(peerActiveVersions, peer.Meta.WtVersion)
 			}
 		}
+
+		for _, service := range account.Services {
+			services++
+			if service.Enabled {
+				servicesEnabled++
+			}
+			servicesTargets += len(service.Targets)
+
+			switch rpservice.Status(service.Meta.Status) {
+			case rpservice.StatusActive:
+				servicesStatusActive++
+			case rpservice.StatusPending:
+				servicesStatusPending++
+			case rpservice.StatusError, rpservice.StatusCertificateFailed, rpservice.StatusTunnelNotCreated:
+				servicesStatusError++
+			}
+
+			for _, target := range service.Targets {
+				servicesTargetType[target.TargetType]++
+			}
+
+			if service.Auth.PasswordAuth != nil && service.Auth.PasswordAuth.Enabled {
+				servicesAuthPassword++
+			}
+			if service.Auth.PinAuth != nil && service.Auth.PinAuth.Enabled {
+				servicesAuthPin++
+			}
+			if service.Auth.BearerAuth != nil && service.Auth.BearerAuth.Enabled {
+				servicesAuthOIDC++
+			}
+		}
 	}
 
 	minActivePeerVersion, maxActivePeerVersion := getMinMaxVersion(peerActiveVersions)
@@ -375,6 +421,22 @@ func (w *Worker) generateProperties(ctx context.Context) properties {
 	metricsProperties["idp_users_count"] = idpUsers
 	metricsProperties["embedded_idp_count"] = len(embeddedIdpTypes)
 
+	metricsProperties["services"] = services
+	metricsProperties["services_enabled"] = servicesEnabled
+	metricsProperties["services_targets"] = servicesTargets
+	metricsProperties["services_status_active"] = servicesStatusActive
+	metricsProperties["services_status_pending"] = servicesStatusPending
+	metricsProperties["services_status_error"] = servicesStatusError
+	metricsProperties["services_auth_password"] = servicesAuthPassword
+	metricsProperties["services_auth_pin"] = servicesAuthPin
+	metricsProperties["services_auth_oidc"] = servicesAuthOIDC
+	metricsProperties["custom_domains"] = customDomains
+	metricsProperties["custom_domains_validated"] = customDomainsValidated
+
+	for targetType, count := range servicesTargetType {
+		metricsProperties["services_target_type_"+targetType] = count
+	}
+
 	for idpType, count := range embeddedIdpTypes {
 		metricsProperties["embedded_idp_users_"+idpType] = count
 	}

management/server/metrics/selfhosted_test.go

@@ -6,6 +6,7 @@ import (
 
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/idp/dex"
+	rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
 	resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types"
 	routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types"
 	networkTypes "github.com/netbirdio/netbird/management/server/networks/types"
@@ -115,6 +116,31 @@ func (mockDatasource) GetAllAccounts(_ context.Context) []*types.Account {
 					},
 				},
 			},
+			Services: []*rpservice.Service{
+				{
+					ID:      "svc1",
+					Enabled: true,
+					Targets: []*rpservice.Target{
+						{TargetType: "peer"},
+						{TargetType: "host"},
+					},
+					Auth: rpservice.AuthConfig{
+						PasswordAuth: &rpservice.PasswordAuthConfig{Enabled: true},
+					},
+					Meta: rpservice.ServiceMeta{Status: string(rpservice.StatusActive)},
+				},
+				{
+					ID:      "svc2",
+					Enabled: false,
+					Targets: []*rpservice.Target{
+						{TargetType: "domain"},
+					},
+					Auth: rpservice.AuthConfig{
+						BearerAuth: &rpservice.BearerAuthConfig{Enabled: true},
+					},
+					Meta: rpservice.ServiceMeta{Status: string(rpservice.StatusPending)},
+				},
+			},
 		},
 		{
 			Id:       "2",
@@ -215,6 +241,11 @@ func (mockDatasource) GetStoreEngine() types.Engine {
 	return types.FileStoreEngine
 }
 
+// GetCustomDomainsCounts returns test custom domain counts.
+func (mockDatasource) GetCustomDomainsCounts(_ context.Context) (int64, int64, error) {
+	return 3, 2, nil
+}
+
 // TestGenerateProperties tests and validate the properties generation by using the mockDatasource for the Worker.generateProperties
 func TestGenerateProperties(t *testing.T) {
 	ds := mockDatasource{}
@@ -347,6 +378,49 @@ func TestGenerateProperties(t *testing.T) {
 	if properties["embedded_idp_count"] != 1 {
 		t.Errorf("expected 1 embedded_idp_count, got %v", properties["embedded_idp_count"])
 	}
+
+	if properties["services"] != 2 {
+		t.Errorf("expected 2 services, got %v", properties["services"])
+	}
+	if properties["services_enabled"] != 1 {
+		t.Errorf("expected 1 services_enabled, got %v", properties["services_enabled"])
+	}
+	if properties["services_targets"] != 3 {
+		t.Errorf("expected 3 services_targets, got %v", properties["services_targets"])
+	}
+	if properties["services_status_active"] != 1 {
+		t.Errorf("expected 1 services_status_active, got %v", properties["services_status_active"])
+	}
+	if properties["services_status_pending"] != 1 {
+		t.Errorf("expected 1 services_status_pending, got %v", properties["services_status_pending"])
+	}
+	if properties["services_status_error"] != 0 {
+		t.Errorf("expected 0 services_status_error, got %v", properties["services_status_error"])
+	}
+	if properties["services_target_type_peer"] != 1 {
+		t.Errorf("expected 1 services_target_type_peer, got %v", properties["services_target_type_peer"])
+	}
+	if properties["services_target_type_host"] != 1 {
+		t.Errorf("expected 1 services_target_type_host, got %v", properties["services_target_type_host"])
+	}
+	if properties["services_target_type_domain"] != 1 {
+		t.Errorf("expected 1 services_target_type_domain, got %v", properties["services_target_type_domain"])
+	}
+	if properties["services_auth_password"] != 1 {
+		t.Errorf("expected 1 services_auth_password, got %v", properties["services_auth_password"])
+	}
+	if properties["services_auth_oidc"] != 1 {
+		t.Errorf("expected 1 services_auth_oidc, got %v", properties["services_auth_oidc"])
+	}
+	if properties["services_auth_pin"] != 0 {
+		t.Errorf("expected 0 services_auth_pin, got %v", properties["services_auth_pin"])
+	}
+	if properties["custom_domains"] != int64(3) {
+		t.Errorf("expected 3 custom_domains, got %v", properties["custom_domains"])
+	}
+	if properties["custom_domains_validated"] != int64(2) {
+		t.Errorf("expected 2 custom_domains_validated, got %v", properties["custom_domains_validated"])
+	}
 }
 
 func TestExtractIdpType(t *testing.T) {

management/server/mock_server/account_mock.go

@@ -407,7 +407,7 @@ func (am *MockAccountManager) AddPeer(
 
 // GetGroupByName mock implementation of GetGroupByName from server.AccountManager interface
 func (am *MockAccountManager) GetGroupByName(ctx context.Context, accountID, groupName string) (*types.Group, error) {
-	if am.GetGroupFunc != nil {
+	if am.GetGroupByNameFunc != nil {
 		return am.GetGroupByNameFunc(ctx, accountID, groupName)
 	}
 	return nil, status.Errorf(codes.Unimplemented, "method GetGroupByName is not implemented")

management/server/networks/resources/manager_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 
-	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
+	reverseproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
 	"github.com/netbirdio/netbird/management/server/groups"
 	"github.com/netbirdio/netbird/management/server/mock_server"
 	"github.com/netbirdio/netbird/management/server/networks/resources/types"

management/server/store/file_store.go

@@ -269,3 +269,8 @@ func (s *FileStore) GetStoreEngine() types.Engine {
 func (s *FileStore) SetFieldEncrypt(_ *crypt.FieldEncrypt) {
 	// no-op: FileStore stores data in plaintext JSON; encryption is not supported
 }
+
+// GetCustomDomainsCounts is a no-op for FileStore as it doesn't support custom domains.
+func (s *FileStore) GetCustomDomainsCounts(_ context.Context) (int64, int64, error) {
+	return 0, 0, nil
+}

management/server/store/sql_store.go

@@ -1008,6 +1008,18 @@ func (s *SqlStore) GetAccountsCounter(ctx context.Context) (int64, error) {
 	return count, nil
 }
 
+// GetCustomDomainsCounts returns the total and validated custom domain counts.
+func (s *SqlStore) GetCustomDomainsCounts(ctx context.Context) (int64, int64, error) {
+	var total, validated int64
+	if err := s.db.WithContext(ctx).Model(&domain.Domain{}).Count(&total).Error; err != nil {
+		return 0, 0, err
+	}
+	if err := s.db.WithContext(ctx).Model(&domain.Domain{}).Where("validated = ?", true).Count(&validated).Error; err != nil {
+		return 0, 0, err
+	}
+	return total, validated, nil
+}
+
 func (s *SqlStore) GetAllAccounts(ctx context.Context) (all []*types.Account) {
 	var accounts []types.Account
 	result := s.db.Find(&accounts)
@@ -2115,7 +2127,8 @@ func (s *SqlStore) getServices(ctx context.Context, accountID string) ([]*rpserv
 			s.Meta.CreatedAt = createdAt.Time
 		}
 		if certIssuedAt.Valid {
-			s.Meta.CertificateIssuedAt = certIssuedAt.Time
+			t := certIssuedAt.Time
+			s.Meta.CertificateIssuedAt = &t
 		}
 		if status.Valid {
 			s.Meta.Status = status.String

management/server/store/store.go

@@ -277,6 +277,8 @@ type Store interface {
 	UpdateProxyHeartbeat(ctx context.Context, proxyID string) error
 	GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error)
 	CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error
+
+	GetCustomDomainsCounts(ctx context.Context) (total int64, validated int64, err error)
 }
 
 const (

management/server/store/store_mock.go

@@ -1887,6 +1887,22 @@ func (mr *MockStoreMockRecorder) GetServiceTargetByTargetID(ctx, lockStrength, a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceTargetByTargetID", reflect.TypeOf((*MockStore)(nil).GetServiceTargetByTargetID), ctx, lockStrength, accountID, targetID)
 }
 
+// GetCustomDomainsCounts mocks base method.
+func (m *MockStore) GetCustomDomainsCounts(ctx context.Context) (int64, int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetCustomDomainsCounts", ctx)
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].(int64)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// GetCustomDomainsCounts indicates an expected call of GetCustomDomainsCounts.
+func (mr *MockStoreMockRecorder) GetCustomDomainsCounts(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCustomDomainsCounts", reflect.TypeOf((*MockStore)(nil).GetCustomDomainsCounts), ctx)
+}
+
 // GetServices mocks base method.
 func (m *MockStore) GetServices(ctx context.Context, lockStrength LockingStrength) ([]*service.Service, error) {
 	m.ctrl.T.Helper()

management/server/types/settings.go

@@ -47,6 +47,11 @@ type Settings struct {
 	// NetworkRange is the custom network range for that account
 	NetworkRange netip.Prefix `gorm:"serializer:json"`
 
+	// PeerExposeEnabled enables or disables peer-initiated service expose
+	PeerExposeEnabled bool
+	// PeerExposeGroups list of peer group IDs allowed to expose services
+	PeerExposeGroups []string `gorm:"serializer:json"`
+
 	// Extra is a dictionary of Account settings
 	Extra *ExtraSettings `gorm:"embedded;embeddedPrefix:extra_"`
 
@@ -80,6 +85,8 @@ func (s *Settings) Copy() *Settings {
 		PeerInactivityExpiration:        s.PeerInactivityExpiration,
 
 		RoutingPeerDNSResolutionEnabled: s.RoutingPeerDNSResolutionEnabled,
+		PeerExposeEnabled:               s.PeerExposeEnabled,
+		PeerExposeGroups:                slices.Clone(s.PeerExposeGroups),
 		LazyConnectionEnabled:           s.LazyConnectionEnabled,
 		DNSDomain:                       s.DNSDomain,
 		NetworkRange:                    s.NetworkRange,