Merge branch 'main' into feature/affected-peers

management/server/store/file_store.go

@@ -274,3 +274,9 @@ func (s *FileStore) SetFieldEncrypt(_ *crypt.FieldEncrypt) {
 func (s *FileStore) GetCustomDomainsCounts(_ context.Context) (int64, int64, error) {
 	return 0, 0, nil
 }
+
+// GetProxyMetrics is a no-op for FileStore — proxy/cluster state isn't
+// persisted in the JSON file format.
+func (s *FileStore) GetProxyMetrics(_ context.Context) (ProxyMetrics, error) {
+	return ProxyMetrics{}, nil
+}

management/server/store/sql_store.go

@@ -1091,6 +1091,38 @@ func (s *SqlStore) GetCustomDomainsCounts(ctx context.Context) (int64, int64, er
 	return total, validated, nil
 }
 
+// GetProxyMetrics aggregates per-cluster + per-proxy counts for the
+// self-hosted telemetry payload. Single round-trip via conditional
+// aggregations so a large proxies table doesn't fan out into multiple
+// queries.
+func (s *SqlStore) GetProxyMetrics(ctx context.Context) (ProxyMetrics, error) {
+	var m ProxyMetrics
+	activeCutoff := time.Now().Add(-proxyActiveThreshold)
+
+	// COUNT(DISTINCT ... CASE WHEN ...) is portable across sqlite/postgres
+	// (MySQL too) and keeps the round-trip to one. proxy.StatusConnected
+	// is the same string the cluster-capability queries use; the active
+	// window matches the cluster-capability semantics (only proxies
+	// heartbeating within ~2 * heartbeat interval count as connected).
+	row := s.db.WithContext(ctx).
+		Model(&proxy.Proxy{}).
+		Select(
+			"COUNT(DISTINCT cluster_address) AS clusters, "+
+				"COUNT(DISTINCT CASE WHEN account_id IS NOT NULL THEN cluster_address END) AS clusters_byop, "+
+				"COUNT(DISTINCT CASE WHEN private = ? THEN cluster_address END) AS clusters_private, "+
+				"COUNT(*) AS proxies, "+
+				"COUNT(CASE WHEN status = ? AND last_seen > ? THEN 1 END) AS proxies_connected",
+			true,
+			proxy.StatusConnected,
+			activeCutoff,
+		).
+		Row()
+	if err := row.Scan(&m.Clusters, &m.ClustersBYOP, &m.ClustersPrivate, &m.Proxies, &m.ProxiesConnected); err != nil {
+		return ProxyMetrics{}, fmt.Errorf("scan proxy metrics: %w", err)
+	}
+	return m, nil
+}
+
 func (s *SqlStore) GetAllAccounts(ctx context.Context) (all []*types.Account) {
 	var accounts []types.Account
 	result := s.db.Find(&accounts)
@@ -2179,7 +2211,8 @@ func (s *SqlStore) getServices(ctx context.Context, accountID string) ([]*rpserv
 	const serviceQuery = `SELECT id, account_id, name, domain, enabled, auth,
 		meta_created_at, meta_certificate_issued_at, meta_status, proxy_cluster,
 		pass_host_header, rewrite_redirects, session_private_key, session_public_key,
-		mode, listen_port, port_auto_assigned, source, source_peer, terminated
+		mode, listen_port, port_auto_assigned, source, source_peer, terminated,
+		private, access_groups
 		FROM services WHERE account_id = $1`
 
 	const targetsQuery = `SELECT id, account_id, service_id, path, host, port, protocol,
@@ -2194,10 +2227,11 @@ func (s *SqlStore) getServices(ctx context.Context, accountID string) ([]*rpserv
 	services, err := pgx.CollectRows(serviceRows, func(row pgx.CollectableRow) (*rpservice.Service, error) {
 		var s rpservice.Service
 		var auth []byte
+		var accessGroups []byte
 		var createdAt, certIssuedAt sql.NullTime
 		var status, proxyCluster, sessionPrivateKey, sessionPublicKey sql.NullString
 		var mode, source, sourcePeer sql.NullString
-		var terminated, portAutoAssigned sql.NullBool
+		var terminated, portAutoAssigned, private sql.NullBool
 		var listenPort sql.NullInt64
 		err := row.Scan(
 			&s.ID,
@@ -2220,6 +2254,8 @@ func (s *SqlStore) getServices(ctx context.Context, accountID string) ([]*rpserv
 			&source,
 			&sourcePeer,
 			&terminated,
+			&private,
+			&accessGroups,
 		)
 		if err != nil {
 			return nil, err
@@ -2231,6 +2267,16 @@ func (s *SqlStore) getServices(ctx context.Context, accountID string) ([]*rpserv
 			}
 		}
 
+		if len(accessGroups) > 0 {
+			if err := json.Unmarshal(accessGroups, &s.AccessGroups); err != nil {
+				return nil, fmt.Errorf("unmarshal access_groups: %w", err)
+			}
+		}
+
+		if private.Valid {
+			s.Private = private.Bool
+		}
+
 		s.Meta = rpservice.Meta{}
 		if createdAt.Valid {
 			s.Meta.CreatedAt = createdAt.Time
@@ -5889,6 +5935,7 @@ var validCapabilityColumns = map[string]struct{}{
 	"supports_custom_ports": {},
 	"require_subdomain":     {},
 	"supports_crowdsec":     {},
+	"private":               {},
 }
 
 // GetClusterSupportsCustomPorts returns whether any active proxy in the cluster
@@ -5903,6 +5950,12 @@ func (s *SqlStore) GetClusterRequireSubdomain(ctx context.Context, clusterAddr s
 	return s.getClusterCapability(ctx, clusterAddr, "require_subdomain")
 }
 
+// GetClusterSupportsPrivate reports whether any active proxy in the cluster
+// has the private capability (nil = unreported).
+func (s *SqlStore) GetClusterSupportsPrivate(ctx context.Context, clusterAddr string) *bool {
+	return s.getClusterCapability(ctx, clusterAddr, "private")
+}
+
 // GetClusterSupportsCrowdSec returns whether all active proxies in the cluster
 // have CrowdSec configured. Returns nil when no proxy reported the capability.
 // Unlike other capabilities that use ANY-true (for rolling upgrades), CrowdSec

management/server/store/sql_store_service_test.go

@@ -0,0 +1,46 @@
+package store
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
+)
+
+func TestSqlStore_GetAccount_PrivateServiceRoundtrip(t *testing.T) {
+	if (os.Getenv("CI") == "true" && runtime.GOOS == "darwin") || runtime.GOOS == "windows" {
+		t.Skip("skip CI tests on darwin and windows")
+	}
+
+	runTestForAllEngines(t, "", func(t *testing.T, store Store) {
+		ctx := context.Background()
+		account := newAccountWithId(ctx, "account_private_svc", "testuser", "")
+		require.NoError(t, store.SaveAccount(ctx, account))
+
+		svc := &rpservice.Service{
+			ID:           "svc-private",
+			AccountID:    account.Id,
+			Name:         "private-svc",
+			Domain:       "private.example",
+			ProxyCluster: "cluster.example",
+			Enabled:      true,
+			Mode:         rpservice.ModeHTTP,
+			Private:      true,
+			AccessGroups: []string{"grp-admins", "grp-ops"},
+		}
+		require.NoError(t, store.CreateService(ctx, svc))
+
+		loaded, err := store.GetAccount(ctx, account.Id)
+		require.NoError(t, err)
+		require.Len(t, loaded.Services, 1)
+
+		got := loaded.Services[0]
+		assert.True(t, got.Private)
+		assert.Equal(t, []string{"grp-admins", "grp-ops"}, got.AccessGroups)
+	})
+}

management/server/store/store.go

@@ -315,6 +315,7 @@ type Store interface {
 	GetClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool
 	GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool
 	GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool
+	GetClusterSupportsPrivate(ctx context.Context, clusterAddr string) *bool
 	CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error
 	GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error)
 	CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error)
@@ -323,9 +324,38 @@ type Store interface {
 
 	GetCustomDomainsCounts(ctx context.Context) (total int64, validated int64, err error)
 
+	// GetProxyMetrics returns aggregated proxy / cluster counts for the
+	// self-hosted metrics worker. Self-hosted only — file-based stores
+	// return a zero-valued struct.
+	GetProxyMetrics(ctx context.Context) (ProxyMetrics, error)
+
 	GetRoutingPeerNetworks(ctx context.Context, accountID, peerID string) ([]string, error)
 }
 
+// ProxyMetrics aggregates self-hosted proxy + cluster usage signals
+// surfaced to the telemetry payload. Each field is best-effort: when a
+// store cannot answer (e.g. FileStore) all fields are zero.
+type ProxyMetrics struct {
+	// Clusters counts distinct cluster_address values across the proxies
+	// table — every cluster the management server has heard from, online or not.
+	Clusters int64
+	// ClustersBYOP counts distinct cluster_address values that are owned
+	// by an account (account_id IS NOT NULL). These are bring-your-own-proxy
+	// installations as opposed to NetBird-operated shared clusters.
+	ClustersBYOP int64
+	// ClustersPrivate counts distinct cluster_address values where at
+	// least one proxy reported the private capability (embedded
+	// `netbird proxy` running inside a client).
+	ClustersPrivate int64
+	// Proxies is the total number of proxy rows currently persisted.
+	Proxies int64
+	// ProxiesConnected is the subset of proxies whose status is
+	// "connected" AND last_seen falls within the active heartbeat window
+	// (~2 * heartbeat interval). Proxies the controller hasn't pruned
+	// yet but that are visibly stale don't count.
+	ProxiesConnected int64
+}
+
 const (
 	postgresDsnEnv       = "NB_STORE_ENGINE_POSTGRES_DSN"
 	postgresDsnEnvLegacy = "NETBIRD_STORE_ENGINE_POSTGRES_DSN"

management/server/store/store_mock.go

@@ -1461,6 +1461,20 @@ func (mr *MockStoreMockRecorder) GetClusterSupportsCustomPorts(ctx, clusterAddr
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterSupportsCustomPorts", reflect.TypeOf((*MockStore)(nil).GetClusterSupportsCustomPorts), ctx, clusterAddr)
 }
 
+// GetClusterSupportsPrivate mocks base method.
+func (m *MockStore) GetClusterSupportsPrivate(ctx context.Context, clusterAddr string) *bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetClusterSupportsPrivate", ctx, clusterAddr)
+	ret0, _ := ret[0].(*bool)
+	return ret0
+}
+
+// GetClusterSupportsPrivate indicates an expected call of GetClusterSupportsPrivate.
+func (mr *MockStoreMockRecorder) GetClusterSupportsPrivate(ctx, clusterAddr interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterSupportsPrivate", reflect.TypeOf((*MockStore)(nil).GetClusterSupportsPrivate), ctx, clusterAddr)
+}
+
 // GetCustomDomain mocks base method.
 func (m *MockStore) GetCustomDomain(ctx context.Context, accountID, domainID string) (*domain.Domain, error) {
 	m.ctrl.T.Helper()
@@ -2121,6 +2135,21 @@ func (mr *MockStoreMockRecorder) GetProxyClusters(ctx, accountID interface{}) *g
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyClusters", reflect.TypeOf((*MockStore)(nil).GetProxyClusters), ctx, accountID)
 }
 
+// GetProxyMetrics mocks base method.
+func (m *MockStore) GetProxyMetrics(ctx context.Context) (ProxyMetrics, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetProxyMetrics", ctx)
+	ret0, _ := ret[0].(ProxyMetrics)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetProxyMetrics indicates an expected call of GetProxyMetrics.
+func (mr *MockStoreMockRecorder) GetProxyMetrics(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyMetrics", reflect.TypeOf((*MockStore)(nil).GetProxyMetrics), ctx)
+}
+
 // GetResourceGroups mocks base method.
 func (m *MockStore) GetResourceGroups(ctx context.Context, lockStrength LockingStrength, accountID, resourceID string) ([]*types2.Group, error) {
 	m.ctrl.T.Helper()