[management] Store connected proxies in DB (#5472)

Co-authored-by: mlsmaycon <mlsmaycon@gmail.com>

management/internals/modules/reverseproxy/proxy/manager.go

@@ -0,0 +1,36 @@
+package proxy
+
+//go:generate go run github.com/golang/mock/mockgen -package proxy -destination=manager_mock.go -source=./manager.go -build_flags=-mod=mod
+
+import (
+	"context"
+	"time"
+
+	"github.com/netbirdio/netbird/shared/management/proto"
+)
+
+// Manager defines the interface for proxy operations
+type Manager interface {
+	Connect(ctx context.Context, proxyID, clusterAddress, ipAddress string) error
+	Disconnect(ctx context.Context, proxyID string) error
+	Heartbeat(ctx context.Context, proxyID string) error
+	GetActiveClusterAddresses(ctx context.Context) ([]string, error)
+	CleanupStale(ctx context.Context, inactivityDuration time.Duration) error
+}
+
+// OIDCValidationConfig contains the OIDC configuration needed for token validation.
+type OIDCValidationConfig struct {
+	Issuer             string
+	Audiences          []string
+	KeysLocation       string
+	MaxTokenAgeSeconds int64
+}
+
+// Controller is responsible for managing proxy clusters and routing service updates.
+type Controller interface {
+	SendServiceUpdateToCluster(ctx context.Context, accountID string, update *proto.ProxyMapping, clusterAddr string)
+	GetOIDCValidationConfig() OIDCValidationConfig
+	RegisterProxyToCluster(ctx context.Context, clusterAddr, proxyID string) error
+	UnregisterProxyFromCluster(ctx context.Context, clusterAddr, proxyID string) error
+	GetProxiesForCluster(clusterAddr string) []string
+}

management/internals/modules/reverseproxy/proxy/manager/controller.go

@@ -0,0 +1,88 @@
+package manager
+
+import (
+	"context"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
+	nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
+	"github.com/netbirdio/netbird/shared/management/proto"
+)
+
+// GRPCController is a concrete implementation that manages proxy clusters and sends updates directly via gRPC.
+type GRPCController struct {
+	proxyGRPCServer *nbgrpc.ProxyServiceServer
+	// Map of cluster address -> set of proxy IDs
+	clusterProxies sync.Map
+	metrics        *metrics
+}
+
+// NewGRPCController creates a new GRPCController.
+func NewGRPCController(proxyGRPCServer *nbgrpc.ProxyServiceServer, meter metric.Meter) (*GRPCController, error) {
+	m, err := newMetrics(meter)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GRPCController{
+		proxyGRPCServer: proxyGRPCServer,
+		metrics:         m,
+	}, nil
+}
+
+// SendServiceUpdateToCluster sends a service update to a specific proxy cluster.
+func (c *GRPCController) SendServiceUpdateToCluster(ctx context.Context, accountID string, update *proto.ProxyMapping, clusterAddr string) {
+	c.proxyGRPCServer.SendServiceUpdateToCluster(ctx, update, clusterAddr)
+	c.metrics.IncrementServiceUpdateSendCount(clusterAddr)
+}
+
+// GetOIDCValidationConfig returns the OIDC validation configuration from the gRPC server.
+func (c *GRPCController) GetOIDCValidationConfig() proxy.OIDCValidationConfig {
+	return c.proxyGRPCServer.GetOIDCValidationConfig()
+}
+
+// RegisterProxyToCluster registers a proxy to a specific cluster for routing.
+func (c *GRPCController) RegisterProxyToCluster(ctx context.Context, clusterAddr, proxyID string) error {
+	if clusterAddr == "" {
+		return nil
+	}
+	proxySet, _ := c.clusterProxies.LoadOrStore(clusterAddr, &sync.Map{})
+	proxySet.(*sync.Map).Store(proxyID, struct{}{})
+	log.WithContext(ctx).Debugf("Registered proxy %s to cluster %s", proxyID, clusterAddr)
+
+	c.metrics.IncrementProxyConnectionCount(clusterAddr)
+
+	return nil
+}
+
+// UnregisterProxyFromCluster removes a proxy from a cluster.
+func (c *GRPCController) UnregisterProxyFromCluster(ctx context.Context, clusterAddr, proxyID string) error {
+	if clusterAddr == "" {
+		return nil
+	}
+	if proxySet, ok := c.clusterProxies.Load(clusterAddr); ok {
+		proxySet.(*sync.Map).Delete(proxyID)
+		log.WithContext(ctx).Debugf("Unregistered proxy %s from cluster %s", proxyID, clusterAddr)
+
+		c.metrics.DecrementProxyConnectionCount(clusterAddr)
+	}
+	return nil
+}
+
+// GetProxiesForCluster returns all proxy IDs registered for a specific cluster.
+func (c *GRPCController) GetProxiesForCluster(clusterAddr string) []string {
+	proxySet, ok := c.clusterProxies.Load(clusterAddr)
+	if !ok {
+		return nil
+	}
+
+	var proxies []string
+	proxySet.(*sync.Map).Range(func(key, _ interface{}) bool {
+		proxies = append(proxies, key.(string))
+		return true
+	})
+	return proxies
+}

management/internals/modules/reverseproxy/proxy/manager/manager.go

@@ -0,0 +1,115 @@
+package manager
+
+import (
+	"context"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
+)
+
+// store defines the interface for proxy persistence operations
+type store interface {
+	SaveProxy(ctx context.Context, p *proxy.Proxy) error
+	UpdateProxyHeartbeat(ctx context.Context, proxyID string) error
+	GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error)
+	CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error
+}
+
+// Manager handles all proxy operations
+type Manager struct {
+	store   store
+	metrics *metrics
+}
+
+// NewManager creates a new proxy Manager
+func NewManager(store store, meter metric.Meter) (*Manager, error) {
+	m, err := newMetrics(meter)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Manager{
+		store:   store,
+		metrics: m,
+	}, nil
+}
+
+// Connect registers a new proxy connection in the database
+func (m Manager) Connect(ctx context.Context, proxyID, clusterAddress, ipAddress string) error {
+	now := time.Now()
+	p := &proxy.Proxy{
+		ID:             proxyID,
+		ClusterAddress: clusterAddress,
+		IPAddress:      ipAddress,
+		LastSeen:       now,
+		ConnectedAt:    &now,
+		Status:         "connected",
+	}
+
+	if err := m.store.SaveProxy(ctx, p); err != nil {
+		log.WithContext(ctx).Errorf("failed to register proxy %s: %v", proxyID, err)
+		return err
+	}
+
+	log.WithContext(ctx).WithFields(log.Fields{
+		"proxyID":        proxyID,
+		"clusterAddress": clusterAddress,
+		"ipAddress":      ipAddress,
+	}).Info("proxy connected")
+
+	return nil
+}
+
+// Disconnect marks a proxy as disconnected in the database
+func (m Manager) Disconnect(ctx context.Context, proxyID string) error {
+	now := time.Now()
+	p := &proxy.Proxy{
+		ID:             proxyID,
+		Status:         "disconnected",
+		DisconnectedAt: &now,
+		LastSeen:       now,
+	}
+
+	if err := m.store.SaveProxy(ctx, p); err != nil {
+		log.WithContext(ctx).Errorf("failed to disconnect proxy %s: %v", proxyID, err)
+		return err
+	}
+
+	log.WithContext(ctx).WithFields(log.Fields{
+		"proxyID": proxyID,
+	}).Info("proxy disconnected")
+
+	return nil
+}
+
+// Heartbeat updates the proxy's last seen timestamp
+func (m Manager) Heartbeat(ctx context.Context, proxyID string) error {
+	if err := m.store.UpdateProxyHeartbeat(ctx, proxyID); err != nil {
+		log.WithContext(ctx).Debugf("failed to update proxy %s heartbeat: %v", proxyID, err)
+		return err
+	}
+	m.metrics.IncrementProxyHeartbeatCount()
+	return nil
+}
+
+// GetActiveClusterAddresses returns all unique cluster addresses for active proxies
+func (m Manager) GetActiveClusterAddresses(ctx context.Context) ([]string, error) {
+	addresses, err := m.store.GetActiveProxyClusterAddresses(ctx)
+	if err != nil {
+		log.WithContext(ctx).Errorf("failed to get active proxy cluster addresses: %v", err)
+		return nil, err
+	}
+	return addresses, nil
+}
+
+// CleanupStale removes proxies that haven't sent heartbeat in the specified duration
+func (m Manager) CleanupStale(ctx context.Context, inactivityDuration time.Duration) error {
+	if err := m.store.CleanupStaleProxies(ctx, inactivityDuration); err != nil {
+		log.WithContext(ctx).Errorf("failed to cleanup stale proxies: %v", err)
+		return err
+	}
+	return nil
+}

management/internals/modules/reverseproxy/proxy/manager/metrics.go

@@ -0,0 +1,74 @@
+package manager
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+type metrics struct {
+	proxyConnectionCount   metric.Int64UpDownCounter
+	serviceUpdateSendCount metric.Int64Counter
+	proxyHeartbeatCount    metric.Int64Counter
+}
+
+func newMetrics(meter metric.Meter) (*metrics, error) {
+	proxyConnectionCount, err := meter.Int64UpDownCounter(
+		"management_proxy_connection_count",
+		metric.WithDescription("Number of active proxy connections"),
+		metric.WithUnit("{connection}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	serviceUpdateSendCount, err := meter.Int64Counter(
+		"management_proxy_service_update_send_count",
+		metric.WithDescription("Total number of service updates sent to proxies"),
+		metric.WithUnit("{update}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	proxyHeartbeatCount, err := meter.Int64Counter(
+		"management_proxy_heartbeat_count",
+		metric.WithDescription("Total number of proxy heartbeats received"),
+		metric.WithUnit("{heartbeat}"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &metrics{
+		proxyConnectionCount:   proxyConnectionCount,
+		serviceUpdateSendCount: serviceUpdateSendCount,
+		proxyHeartbeatCount:    proxyHeartbeatCount,
+	}, nil
+}
+
+func (m *metrics) IncrementProxyConnectionCount(clusterAddr string) {
+	m.proxyConnectionCount.Add(context.Background(), 1,
+		metric.WithAttributes(
+			attribute.String("cluster", clusterAddr),
+		))
+}
+
+func (m *metrics) DecrementProxyConnectionCount(clusterAddr string) {
+	m.proxyConnectionCount.Add(context.Background(), -1,
+		metric.WithAttributes(
+			attribute.String("cluster", clusterAddr),
+		))
+}
+
+func (m *metrics) IncrementServiceUpdateSendCount(clusterAddr string) {
+	m.serviceUpdateSendCount.Add(context.Background(), 1,
+		metric.WithAttributes(
+			attribute.String("cluster", clusterAddr),
+		))
+}
+
+func (m *metrics) IncrementProxyHeartbeatCount() {
+	m.proxyHeartbeatCount.Add(context.Background(), 1)
+}

management/internals/modules/reverseproxy/proxy/manager_mock.go

@@ -0,0 +1,199 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: ./manager.go
+
+// Package proxy is a generated GoMock package.
+package proxy
+
+import (
+	context "context"
+	reflect "reflect"
+	time "time"
+
+	gomock "github.com/golang/mock/gomock"
+	proto "github.com/netbirdio/netbird/shared/management/proto"
+)
+
+// MockManager is a mock of Manager interface.
+type MockManager struct {
+	ctrl     *gomock.Controller
+	recorder *MockManagerMockRecorder
+}
+
+// MockManagerMockRecorder is the mock recorder for MockManager.
+type MockManagerMockRecorder struct {
+	mock *MockManager
+}
+
+// NewMockManager creates a new mock instance.
+func NewMockManager(ctrl *gomock.Controller) *MockManager {
+	mock := &MockManager{ctrl: ctrl}
+	mock.recorder = &MockManagerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockManager) EXPECT() *MockManagerMockRecorder {
+	return m.recorder
+}
+
+// CleanupStale mocks base method.
+func (m *MockManager) CleanupStale(ctx context.Context, inactivityDuration time.Duration) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CleanupStale", ctx, inactivityDuration)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CleanupStale indicates an expected call of CleanupStale.
+func (mr *MockManagerMockRecorder) CleanupStale(ctx, inactivityDuration interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupStale", reflect.TypeOf((*MockManager)(nil).CleanupStale), ctx, inactivityDuration)
+}
+
+// Connect mocks base method.
+func (m *MockManager) Connect(ctx context.Context, proxyID, clusterAddress, ipAddress string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Connect", ctx, proxyID, clusterAddress, ipAddress)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Connect indicates an expected call of Connect.
+func (mr *MockManagerMockRecorder) Connect(ctx, proxyID, clusterAddress, ipAddress interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Connect", reflect.TypeOf((*MockManager)(nil).Connect), ctx, proxyID, clusterAddress, ipAddress)
+}
+
+// Disconnect mocks base method.
+func (m *MockManager) Disconnect(ctx context.Context, proxyID string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Disconnect", ctx, proxyID)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Disconnect indicates an expected call of Disconnect.
+func (mr *MockManagerMockRecorder) Disconnect(ctx, proxyID interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Disconnect", reflect.TypeOf((*MockManager)(nil).Disconnect), ctx, proxyID)
+}
+
+// GetActiveClusterAddresses mocks base method.
+func (m *MockManager) GetActiveClusterAddresses(ctx context.Context) ([]string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetActiveClusterAddresses", ctx)
+	ret0, _ := ret[0].([]string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetActiveClusterAddresses indicates an expected call of GetActiveClusterAddresses.
+func (mr *MockManagerMockRecorder) GetActiveClusterAddresses(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveClusterAddresses", reflect.TypeOf((*MockManager)(nil).GetActiveClusterAddresses), ctx)
+}
+
+// Heartbeat mocks base method.
+func (m *MockManager) Heartbeat(ctx context.Context, proxyID string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Heartbeat", ctx, proxyID)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Heartbeat indicates an expected call of Heartbeat.
+func (mr *MockManagerMockRecorder) Heartbeat(ctx, proxyID interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Heartbeat", reflect.TypeOf((*MockManager)(nil).Heartbeat), ctx, proxyID)
+}
+
+// MockController is a mock of Controller interface.
+type MockController struct {
+	ctrl     *gomock.Controller
+	recorder *MockControllerMockRecorder
+}
+
+// MockControllerMockRecorder is the mock recorder for MockController.
+type MockControllerMockRecorder struct {
+	mock *MockController
+}
+
+// NewMockController creates a new mock instance.
+func NewMockController(ctrl *gomock.Controller) *MockController {
+	mock := &MockController{ctrl: ctrl}
+	mock.recorder = &MockControllerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockController) EXPECT() *MockControllerMockRecorder {
+	return m.recorder
+}
+
+// GetOIDCValidationConfig mocks base method.
+func (m *MockController) GetOIDCValidationConfig() OIDCValidationConfig {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetOIDCValidationConfig")
+	ret0, _ := ret[0].(OIDCValidationConfig)
+	return ret0
+}
+
+// GetOIDCValidationConfig indicates an expected call of GetOIDCValidationConfig.
+func (mr *MockControllerMockRecorder) GetOIDCValidationConfig() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOIDCValidationConfig", reflect.TypeOf((*MockController)(nil).GetOIDCValidationConfig))
+}
+
+// GetProxiesForCluster mocks base method.
+func (m *MockController) GetProxiesForCluster(clusterAddr string) []string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetProxiesForCluster", clusterAddr)
+	ret0, _ := ret[0].([]string)
+	return ret0
+}
+
+// GetProxiesForCluster indicates an expected call of GetProxiesForCluster.
+func (mr *MockControllerMockRecorder) GetProxiesForCluster(clusterAddr interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxiesForCluster", reflect.TypeOf((*MockController)(nil).GetProxiesForCluster), clusterAddr)
+}
+
+// RegisterProxyToCluster mocks base method.
+func (m *MockController) RegisterProxyToCluster(ctx context.Context, clusterAddr, proxyID string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RegisterProxyToCluster", ctx, clusterAddr, proxyID)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RegisterProxyToCluster indicates an expected call of RegisterProxyToCluster.
+func (mr *MockControllerMockRecorder) RegisterProxyToCluster(ctx, clusterAddr, proxyID interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterProxyToCluster", reflect.TypeOf((*MockController)(nil).RegisterProxyToCluster), ctx, clusterAddr, proxyID)
+}
+
+// SendServiceUpdateToCluster mocks base method.
+func (m *MockController) SendServiceUpdateToCluster(ctx context.Context, accountID string, update *proto.ProxyMapping, clusterAddr string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SendServiceUpdateToCluster", ctx, accountID, update, clusterAddr)
+}
+
+// SendServiceUpdateToCluster indicates an expected call of SendServiceUpdateToCluster.
+func (mr *MockControllerMockRecorder) SendServiceUpdateToCluster(ctx, accountID, update, clusterAddr interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendServiceUpdateToCluster", reflect.TypeOf((*MockController)(nil).SendServiceUpdateToCluster), ctx, accountID, update, clusterAddr)
+}
+
+// UnregisterProxyFromCluster mocks base method.
+func (m *MockController) UnregisterProxyFromCluster(ctx context.Context, clusterAddr, proxyID string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UnregisterProxyFromCluster", ctx, clusterAddr, proxyID)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UnregisterProxyFromCluster indicates an expected call of UnregisterProxyFromCluster.
+func (mr *MockControllerMockRecorder) UnregisterProxyFromCluster(ctx, clusterAddr, proxyID interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnregisterProxyFromCluster", reflect.TypeOf((*MockController)(nil).UnregisterProxyFromCluster), ctx, clusterAddr, proxyID)
+}

management/internals/modules/reverseproxy/proxy/proxy.go

@@ -0,0 +1,20 @@
+package proxy
+
+import "time"
+
+// Proxy represents a reverse proxy instance
+type Proxy struct {
+	ID             string    `gorm:"primaryKey;type:varchar(255)"`
+	ClusterAddress string    `gorm:"type:varchar(255);not null;index:idx_proxy_cluster_status"`
+	IPAddress      string    `gorm:"type:varchar(45)"`
+	LastSeen       time.Time `gorm:"not null;index:idx_proxy_last_seen"`
+	ConnectedAt    *time.Time
+	DisconnectedAt *time.Time
+	Status         string `gorm:"type:varchar(20);not null;index:idx_proxy_cluster_status"`
+	CreatedAt      time.Time
+	UpdatedAt      time.Time
+}
+
+func (Proxy) TableName() string {
+	return "proxies"
+}