mirror of
https://github.com/netbirdio/netbird.git
synced 2026-07-20 23:41:28 +02:00
[management/client] create job channel between management and client (#4367)
* new bi-directional stream for jobs * create bidirectional job channel to send requests from the server and receive responses from the client * fix tests * fix lint and close bug * fix lint * clean up & fix close of closed channel * add nolint:staticcheck * remove some redundant code from the job channel PR since this one is a cleaner rewrite * cleanup removes a pending job safely * change proto * rename to jobRequest * apply feedback 1 * apply feedback 2 * fix typo * apply feedback 3 * apply last feedback
This commit is contained in:
@@ -20,8 +20,6 @@ import (
|
||||
"github.com/netbirdio/netbird/management/server/activity"
|
||||
"github.com/netbirdio/netbird/management/server/groups"
|
||||
"github.com/netbirdio/netbird/management/server/integrations/port_forwarding"
|
||||
"github.com/netbirdio/netbird/management/server/peers"
|
||||
"github.com/netbirdio/netbird/management/server/peers/ephemeral/manager"
|
||||
"github.com/netbirdio/netbird/management/server/permissions"
|
||||
"github.com/netbirdio/netbird/management/server/settings"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
@@ -85,6 +83,7 @@ func startManagement(t *testing.T, config *config.Config, testFile string) (*grp
|
||||
t.Cleanup(cleanUp)
|
||||
|
||||
peersUpdateManager := mgmt.NewPeersUpdateManager(nil)
|
||||
jobManager := mgmt.NewJobManager(nil, store)
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
@@ -110,13 +109,13 @@ func startManagement(t *testing.T, config *config.Config, testFile string) (*grp
|
||||
Return(&types.Settings{}, nil).
|
||||
AnyTimes()
|
||||
|
||||
accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false)
|
||||
accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, jobManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay, settingsMockManager, groupsManager)
|
||||
mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, secretsManager, nil, &manager.EphemeralManager{}, nil, &mgmt.MockIntegratedValidator{})
|
||||
mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, jobManager, secretsManager, nil, nil, nil, &mgmt.MockIntegratedValidator{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -478,6 +478,7 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL)
|
||||
|
||||
e.receiveSignalEvents()
|
||||
e.receiveManagementEvents()
|
||||
e.receiveJobEvents()
|
||||
|
||||
// starting network monitor at the very last to avoid disruptions
|
||||
e.startNetworkMonitor()
|
||||
@@ -947,6 +948,33 @@ func (e *Engine) updateConfig(conf *mgmProto.PeerConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) receiveJobEvents() {
|
||||
go func() {
|
||||
err := e.mgmClient.Job(e.ctx, func(msg *mgmProto.JobRequest) *mgmProto.JobResponse {
|
||||
// Simple test handler — replace with real logic
|
||||
log.Infof("Received job request: %+v", msg)
|
||||
// TODO: trigger local debug bundle or other job
|
||||
return &mgmProto.JobResponse{
|
||||
ID: msg.ID,
|
||||
WorkloadResults: &mgmProto.JobResponse_Bundle{
|
||||
Bundle: &mgmProto.BundleResult{
|
||||
UploadKey: "upload-key",
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
// happens if management is unavailable for a long time.
|
||||
// We want to cancel the operation of the whole client
|
||||
_ = CtxGetState(e.ctx).Wrap(ErrResetConnection)
|
||||
e.clientCancel()
|
||||
return
|
||||
}
|
||||
log.Debugf("stopped receiving jobs from Management Service")
|
||||
}()
|
||||
log.Debugf("connecting to Management Service jobs stream")
|
||||
}
|
||||
|
||||
// receiveManagementEvents connects to the Management Service event stream to receive updates from the management service
|
||||
// E.g. when a new peer has been registered and we are allowed to connect to it.
|
||||
func (e *Engine) receiveManagementEvents() {
|
||||
|
||||
@@ -1553,6 +1553,7 @@ func startManagement(t *testing.T, dataDir, testFile string) (*grpc.Server, stri
|
||||
t.Cleanup(cleanUp)
|
||||
|
||||
peersUpdateManager := server.NewPeersUpdateManager(nil)
|
||||
jobManager := server.NewJobManager(nil, store)
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
@@ -1580,13 +1581,13 @@ func startManagement(t *testing.T, dataDir, testFile string) (*grpc.Server, stri
|
||||
|
||||
groupsManager := groups.NewManagerMock()
|
||||
|
||||
accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, jobManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay, settingsMockManager, groupsManager)
|
||||
mgmtServer, err := server.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, secretsManager, nil, &manager.EphemeralManager{}, nil, &server.MockIntegratedValidator{})
|
||||
mgmtServer, err := server.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, jobManager, secretsManager, nil, &manager.EphemeralManager{}, nil, &server.MockIntegratedValidator{})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
@@ -291,6 +291,7 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve
|
||||
t.Cleanup(cleanUp)
|
||||
|
||||
peersUpdateManager := server.NewPeersUpdateManager(nil)
|
||||
jobManager := server.NewJobManager(nil, store)
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
@@ -311,13 +312,13 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve
|
||||
settingsMockManager := settings.NewMockManager(ctrl)
|
||||
groupsManager := groups.NewManagerMock()
|
||||
|
||||
accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false)
|
||||
accountManager, err := server.BuildManager(context.Background(), store, peersUpdateManager, jobManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay, settingsMockManager, groupsManager)
|
||||
mgmtServer, err := server.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, secretsManager, nil, &manager.EphemeralManager{}, nil, &server.MockIntegratedValidator{})
|
||||
mgmtServer, err := server.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, jobManager, secretsManager, nil, &manager.EphemeralManager{}, nil, &server.MockIntegratedValidator{})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
@@ -145,7 +145,7 @@ func (s *BaseServer) GRPCServer() *grpc.Server {
|
||||
}
|
||||
|
||||
gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
|
||||
srv, err := server.NewServer(context.Background(), s.config, s.AccountManager(), s.SettingsManager(), s.PeersUpdateManager(), s.SecretsManager(), s.Metrics(), s.EphemeralManager(), s.AuthManager(), s.IntegratedValidator())
|
||||
srv, err := server.NewServer(context.Background(), s.config, s.AccountManager(), s.SettingsManager(), s.PeersUpdateManager(), s.JobManager(), s.SecretsManager(), s.Metrics(), s.EphemeralManager(), s.AuthManager(), s.IntegratedValidator())
|
||||
if err != nil {
|
||||
log.Fatalf("failed to create management server: %v", err)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,12 @@ func (s *BaseServer) PeersUpdateManager() *server.PeersUpdateManager {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BaseServer) JobManager() *server.JobManager {
|
||||
return Create(s, func() *server.JobManager {
|
||||
return server.NewJobManager(s.Metrics(), s.Store())
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BaseServer) IntegratedValidator() integrated_validator.IntegratedValidator {
|
||||
return Create(s, func() integrated_validator.IntegratedValidator {
|
||||
integratedPeerValidator, err := integrations.NewIntegratedValidator(
|
||||
|
||||
@@ -60,7 +60,7 @@ func (s *BaseServer) PeersManager() peers.Manager {
|
||||
|
||||
func (s *BaseServer) AccountManager() account.Manager {
|
||||
return Create(s, func() account.Manager {
|
||||
accountManager, err := server.BuildManager(context.Background(), s.Store(), s.PeersUpdateManager(), s.IdpManager(), s.mgmtSingleAccModeDomain,
|
||||
accountManager, err := server.BuildManager(context.Background(), s.Store(), s.PeersUpdateManager(), s.JobManager(), s.IdpManager(), s.mgmtSingleAccModeDomain,
|
||||
s.dnsDomain, s.EventStore(), s.GeoLocationManager(), s.userDeleteFromIDPEnabled, s.IntegratedValidator(), s.Metrics(), s.ProxyController(), s.SettingsManager(), s.PermissionsManager(), s.config.DisableDefaultPolicy)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to create account manager: %v", err)
|
||||
|
||||
@@ -69,6 +69,7 @@ type DefaultAccountManager struct {
|
||||
// cacheLoading keeps the accountIDs that are currently reloading. The accountID has to be removed once cache has been reloaded
|
||||
cacheLoading map[string]chan struct{}
|
||||
peersUpdateManager *PeersUpdateManager
|
||||
jobManager *JobManager
|
||||
idpManager idp.Manager
|
||||
cacheManager *nbcache.AccountUserDataCache
|
||||
externalCacheManager nbcache.UserDataCache
|
||||
@@ -178,6 +179,7 @@ func BuildManager(
|
||||
ctx context.Context,
|
||||
store store.Store,
|
||||
peersUpdateManager *PeersUpdateManager,
|
||||
jobManager *JobManager,
|
||||
idpManager idp.Manager,
|
||||
singleAccountModeDomain string,
|
||||
dnsDomain string,
|
||||
@@ -200,6 +202,7 @@ func BuildManager(
|
||||
Store: store,
|
||||
geo: geo,
|
||||
peersUpdateManager: peersUpdateManager,
|
||||
jobManager: jobManager,
|
||||
idpManager: idpManager,
|
||||
ctx: context.Background(),
|
||||
cacheMux: sync.Mutex{},
|
||||
|
||||
@@ -2893,7 +2893,7 @@ func createManager(t testing.TB) (*DefaultAccountManager, error) {
|
||||
|
||||
permissionsManager := permissions.NewManager(store)
|
||||
|
||||
manager, err := BuildManager(context.Background(), store, NewPeersUpdateManager(nil), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
manager, err := BuildManager(context.Background(), store, NewPeersUpdateManager(nil), NewJobManager(nil, store), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -218,7 +218,7 @@ func createDNSManager(t *testing.T) (*DefaultAccountManager, error) {
|
||||
// return empty extra settings for expected calls to UpdateAccountPeers
|
||||
settingsMockManager.EXPECT().GetExtraSettings(gomock.Any(), gomock.Any()).Return(&types.ExtraSettings{}, nil).AnyTimes()
|
||||
permissionsManager := permissions.NewManager(store)
|
||||
return BuildManager(context.Background(), store, NewPeersUpdateManager(nil), nil, "", "netbird.test", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
return BuildManager(context.Background(), store, NewPeersUpdateManager(nil), NewJobManager(nil, store), nil, "", "netbird.test", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
}
|
||||
|
||||
func createDNSStore(t *testing.T) (store.Store, error) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
@@ -53,6 +54,7 @@ type GRPCServer struct {
|
||||
wgKey wgtypes.Key
|
||||
proto.UnimplementedManagementServiceServer
|
||||
peersUpdateManager *PeersUpdateManager
|
||||
jobManager *JobManager
|
||||
config *nbconfig.Config
|
||||
secretsManager SecretsManager
|
||||
appMetrics telemetry.AppMetrics
|
||||
@@ -72,6 +74,7 @@ func NewServer(
|
||||
accountManager account.Manager,
|
||||
settingsManager settings.Manager,
|
||||
peersUpdateManager *PeersUpdateManager,
|
||||
jobManager *JobManager,
|
||||
secretsManager SecretsManager,
|
||||
appMetrics telemetry.AppMetrics,
|
||||
ephemeralManager ephemeral.Manager,
|
||||
@@ -85,10 +88,9 @@ func NewServer(
|
||||
|
||||
if appMetrics != nil {
|
||||
// update gauge based on number of connected peers which is equal to open gRPC streams
|
||||
err = appMetrics.GRPCMetrics().RegisterConnectedStreams(func() int64 {
|
||||
if err := appMetrics.GRPCMetrics().RegisterConnectedStreams(func() int64 {
|
||||
return int64(len(peersUpdateManager.peerChannels))
|
||||
})
|
||||
if err != nil {
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -100,6 +102,7 @@ func NewServer(
|
||||
wgKey: key,
|
||||
// peerKey -> event channel
|
||||
peersUpdateManager: peersUpdateManager,
|
||||
jobManager: jobManager,
|
||||
accountManager: accountManager,
|
||||
settingsManager: settingsManager,
|
||||
config: config,
|
||||
@@ -148,6 +151,43 @@ func getRealIP(ctx context.Context) net.IP {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GRPCServer) Job(srv proto.ManagementService_JobServer) error {
|
||||
reqStart := time.Now()
|
||||
ctx := srv.Context()
|
||||
|
||||
peerKey, err := s.handleHandshake(ctx, srv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
accountID, err := s.accountManager.GetAccountIDForPeerKey(ctx, peerKey.String())
|
||||
if err != nil {
|
||||
// nolint:staticcheck
|
||||
ctx = context.WithValue(ctx, nbContext.AccountIDKey, "UNKNOWN")
|
||||
log.WithContext(ctx).Tracef("peer %s is not registered", peerKey.String())
|
||||
if errStatus, ok := internalStatus.FromError(err); ok && errStatus.Type() == internalStatus.NotFound {
|
||||
return status.Errorf(codes.PermissionDenied, "peer is not registered")
|
||||
}
|
||||
return err
|
||||
}
|
||||
// nolint:staticcheck
|
||||
ctx = context.WithValue(ctx, nbContext.AccountIDKey, accountID)
|
||||
peer, err := s.accountManager.GetStore().GetPeerByPeerPubKey(ctx, store.LockingStrengthNone, peerKey.String())
|
||||
if err != nil {
|
||||
return status.Errorf(codes.Unauthenticated, "peer is not registered")
|
||||
}
|
||||
|
||||
// Start background response handler
|
||||
s.startResponseReceiver(ctx, accountID, srv)
|
||||
|
||||
// Prepare per-peer state
|
||||
updates := s.jobManager.CreateJobChannel(peer.ID)
|
||||
log.WithContext(ctx).Debugf("Sync: took %v", time.Since(reqStart))
|
||||
|
||||
// Main loop: forward jobs to client
|
||||
return s.sendJobsLoop(ctx, accountID, peerKey, peer, updates, srv)
|
||||
}
|
||||
|
||||
// Sync validates the existence of a connecting peer, sends an initial state (all available for the connecting peers) and
|
||||
// notifies the connected peer of any updates (e.g. new peers under the same account)
|
||||
func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_SyncServer) error {
|
||||
@@ -222,10 +262,9 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare per-peer state
|
||||
updates := s.peersUpdateManager.CreateChannel(ctx, peer.ID)
|
||||
|
||||
s.ephemeralManager.OnPeerConnected(ctx, peer)
|
||||
|
||||
s.secretsManager.SetupRefresh(ctx, accountID, peer.ID)
|
||||
|
||||
if s.appMetrics != nil {
|
||||
@@ -240,6 +279,76 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
|
||||
return s.handleUpdates(ctx, accountID, peerKey, peer, updates, srv)
|
||||
}
|
||||
|
||||
func (s *GRPCServer) handleHandshake(ctx context.Context, srv proto.ManagementService_JobServer) (wgtypes.Key, error) {
|
||||
hello, err := srv.Recv()
|
||||
if err != nil {
|
||||
return wgtypes.Key{}, status.Errorf(codes.InvalidArgument, "missing hello: %v", err)
|
||||
}
|
||||
|
||||
jobReq := &proto.JobRequest{}
|
||||
peerKey, err := s.parseRequest(ctx, hello, jobReq)
|
||||
if err != nil {
|
||||
return wgtypes.Key{}, err
|
||||
}
|
||||
|
||||
return peerKey, nil
|
||||
}
|
||||
|
||||
func (s *GRPCServer) startResponseReceiver(ctx context.Context, accountID string, srv proto.ManagementService_JobServer) {
|
||||
go func() {
|
||||
for {
|
||||
msg, err := srv.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
log.WithContext(ctx).Warnf("recv job response error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
jobResp := &proto.JobResponse{}
|
||||
if _, err := s.parseRequest(ctx, msg, jobResp); err != nil {
|
||||
log.WithContext(ctx).Warnf("invalid job response: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.jobManager.HandleResponse(ctx, accountID, jobResp); err != nil {
|
||||
log.WithContext(ctx).Errorf("handle job response failed: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *GRPCServer) sendJobsLoop(
|
||||
ctx context.Context,
|
||||
accountID string,
|
||||
peerKey wgtypes.Key,
|
||||
peer *nbpeer.Peer,
|
||||
updates <-chan *JobEvent,
|
||||
srv proto.ManagementService_JobServer,
|
||||
) error {
|
||||
for {
|
||||
select {
|
||||
case job, open := <-updates:
|
||||
if !open {
|
||||
log.WithContext(ctx).Debugf("jobs channel for peer %s was closed", peerKey.String())
|
||||
s.jobManager.CloseChannel(ctx, accountID, peer.ID)
|
||||
return nil
|
||||
}
|
||||
if err := s.sendJob(ctx, accountID, peerKey, peer, job, srv); err != nil {
|
||||
s.jobManager.CloseChannel(ctx, accountID, peer.ID)
|
||||
log.WithContext(ctx).Warnf("send job failed: %v", err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
// happens when connection drops, e.g. client disconnects
|
||||
log.WithContext(ctx).Debugf("stream of peer %s has been closed", peerKey.String())
|
||||
s.jobManager.CloseChannel(ctx, accountID, peer.ID)
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleUpdates sends updates to the connected peer until the updates channel is closed.
|
||||
func (s *GRPCServer) handleUpdates(ctx context.Context, accountID string, peerKey wgtypes.Key, peer *nbpeer.Peer, updates chan *UpdateMessage, srv proto.ManagementService_SyncServer) error {
|
||||
log.WithContext(ctx).Tracef("starting to handle updates for peer %s", peerKey.String())
|
||||
@@ -257,7 +366,6 @@ func (s *GRPCServer) handleUpdates(ctx context.Context, accountID string, peerKe
|
||||
return nil
|
||||
}
|
||||
log.WithContext(ctx).Debugf("received an update for peer %s", peerKey.String())
|
||||
|
||||
if err := s.sendUpdate(ctx, accountID, peerKey, peer, update, srv); err != nil {
|
||||
log.WithContext(ctx).Debugf("error while sending an update to peer %s: %v", peerKey.String(), err)
|
||||
return err
|
||||
@@ -281,7 +389,7 @@ func (s *GRPCServer) sendUpdate(ctx context.Context, accountID string, peerKey w
|
||||
s.cancelPeerRoutines(ctx, accountID, peer)
|
||||
return status.Errorf(codes.Internal, "failed processing update message")
|
||||
}
|
||||
err = srv.SendMsg(&proto.EncryptedMessage{
|
||||
err = srv.Send(&proto.EncryptedMessage{
|
||||
WgPubKey: s.wgKey.PublicKey().String(),
|
||||
Body: encryptedResp,
|
||||
})
|
||||
@@ -293,6 +401,27 @@ func (s *GRPCServer) sendUpdate(ctx context.Context, accountID string, peerKey w
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendJob encrypts the update message using the peer key and the server's wireguard key,
|
||||
// then sends the encrypted message to the connected peer via the sync server.
|
||||
func (s *GRPCServer) sendJob(ctx context.Context, accountID string, peerKey wgtypes.Key, peer *nbpeer.Peer, job *JobEvent, srv proto.ManagementService_JobServer) error {
|
||||
encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, job.Request)
|
||||
if err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to encrypt job for peer %s: %v", peerKey.String(), err)
|
||||
s.jobManager.CloseChannel(ctx, accountID, peer.ID)
|
||||
return nil
|
||||
}
|
||||
err = srv.Send(&proto.EncryptedMessage{
|
||||
WgPubKey: s.wgKey.PublicKey().String(),
|
||||
Body: encryptedResp,
|
||||
})
|
||||
if err != nil {
|
||||
s.jobManager.CloseChannel(ctx, accountID, peer.ID)
|
||||
return status.Errorf(codes.Internal, "failed sending job message")
|
||||
}
|
||||
log.WithContext(ctx).Debugf("sent an job to peer %s", peerKey.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GRPCServer) cancelPeerRoutines(ctx context.Context, accountID string, peer *nbpeer.Peer) {
|
||||
unlock := s.acquirePeerLockByUID(ctx, peer.Key)
|
||||
defer unlock()
|
||||
@@ -781,8 +910,8 @@ func (s *GRPCServer) IsHealthy(ctx context.Context, req *proto.Empty) (*proto.Em
|
||||
// sendInitialSync sends initial proto.SyncResponse to the peer requesting synchronization
|
||||
func (s *GRPCServer) sendInitialSync(ctx context.Context, peerKey wgtypes.Key, peer *nbpeer.Peer, networkMap *types.NetworkMap, postureChecks []*posture.Checks, srv proto.ManagementService_SyncServer) error {
|
||||
var err error
|
||||
|
||||
var turnToken *Token
|
||||
|
||||
if s.config.TURNConfig != nil && s.config.TURNConfig.TimeBasedCredentials {
|
||||
turnToken, err = s.secretsManager.GenerateTurnToken()
|
||||
if err != nil {
|
||||
|
||||
149
management/server/jobChannel.go
Normal file
149
management/server/jobChannel.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
"github.com/netbirdio/netbird/management/server/telemetry"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
const jobChannelBuffer = 100
|
||||
|
||||
type JobEvent struct {
|
||||
PeerID string
|
||||
Request *proto.JobRequest
|
||||
Response *proto.JobResponse
|
||||
Done chan struct{} // closed when response arrives
|
||||
}
|
||||
|
||||
type JobManager struct {
|
||||
mu *sync.RWMutex
|
||||
jobChannels map[string]chan *JobEvent // per-peer job streams
|
||||
pending map[string]*JobEvent // jobID → event
|
||||
responseWait time.Duration
|
||||
metrics telemetry.AppMetrics
|
||||
Store store.Store
|
||||
}
|
||||
|
||||
func NewJobManager(metrics telemetry.AppMetrics, store store.Store) *JobManager {
|
||||
|
||||
return &JobManager{
|
||||
jobChannels: make(map[string]chan *JobEvent),
|
||||
pending: make(map[string]*JobEvent),
|
||||
responseWait: 5 * time.Minute,
|
||||
metrics: metrics,
|
||||
mu: &sync.RWMutex{},
|
||||
Store: store,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateJobChannel creates or replaces a channel for a peer
|
||||
func (jm *JobManager) CreateJobChannel(peerID string) chan *JobEvent {
|
||||
// TODO: all pending jobs stored in db for this peer should be failed
|
||||
// jm.Store.MarkPendingJobsAsFailed(peerID)
|
||||
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
if ch, ok := jm.jobChannels[peerID]; ok {
|
||||
close(ch)
|
||||
delete(jm.jobChannels, peerID)
|
||||
}
|
||||
|
||||
ch := make(chan *JobEvent, jobChannelBuffer)
|
||||
jm.jobChannels[peerID] = ch
|
||||
return ch
|
||||
}
|
||||
|
||||
// SendJob sends a job to a peer and tracks it as pending
|
||||
func (jm *JobManager) SendJob(ctx context.Context, accountID, peerID string, req *proto.JobRequest) error {
|
||||
jm.mu.RLock()
|
||||
ch, ok := jm.jobChannels[peerID]
|
||||
jm.mu.RUnlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("peer %s has no channel", peerID)
|
||||
}
|
||||
|
||||
event := &JobEvent{
|
||||
PeerID: peerID,
|
||||
Request: req,
|
||||
Done: make(chan struct{}),
|
||||
}
|
||||
|
||||
jm.mu.Lock()
|
||||
jm.pending[string(req.ID)] = event
|
||||
jm.mu.Unlock()
|
||||
|
||||
select {
|
||||
case ch <- event:
|
||||
case <-time.After(5 * time.Second):
|
||||
jm.cleanup(ctx, accountID, string(req.ID), "timed out")
|
||||
return fmt.Errorf("job channel full for peer %s", peerID)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-event.Done:
|
||||
return nil
|
||||
case <-time.After(jm.responseWait):
|
||||
jm.cleanup(ctx, accountID, string(req.ID), "timed out")
|
||||
return fmt.Errorf("job %s timed out", req.ID)
|
||||
case <-ctx.Done():
|
||||
jm.cleanup(ctx, accountID, string(req.ID), ctx.Err().Error())
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// HandleResponse marks a job as finished and moves it to completed
|
||||
func (jm *JobManager) HandleResponse(ctx context.Context, accountID string, resp *proto.JobResponse) error {
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
event, ok := jm.pending[string(resp.ID)]
|
||||
if !ok {
|
||||
return fmt.Errorf("job %s not found", resp.ID)
|
||||
}
|
||||
|
||||
event.Response = resp
|
||||
//TODO: update the store for job response
|
||||
// jm.store.CompleteJob(ctx,accountID, string(resp.GetID()), string(resp.GetResult()),string(resp.GetReason()))
|
||||
close(event.Done)
|
||||
delete(jm.pending, string(resp.ID))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CloseChannel closes a peer’s channel and cleans up its jobs
|
||||
func (jm *JobManager) CloseChannel(ctx context.Context, accountID, peerID string) {
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
if ch, ok := jm.jobChannels[peerID]; ok {
|
||||
close(ch)
|
||||
jm.jobChannels[peerID] = nil
|
||||
delete(jm.jobChannels, peerID)
|
||||
}
|
||||
|
||||
for jobID, ev := range jm.pending {
|
||||
if ev.PeerID == peerID {
|
||||
// if the client disconnect and there is pending job then marke it as failed
|
||||
// jm.store.CompleteJob(ctx,accountID, jobID,"", "Time out ")
|
||||
delete(jm.pending, jobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup removes a pending job safely
|
||||
func (jm *JobManager) cleanup(ctx context.Context, accountID, jobID string, reason string) {
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
if ev, ok := jm.pending[jobID]; ok {
|
||||
close(ev.Done)
|
||||
// jm.store.CompleteJob(ctx, accountID, jobID, "", reason)
|
||||
delete(jm.pending, jobID)
|
||||
}
|
||||
}
|
||||
@@ -428,6 +428,7 @@ func startManagementForTest(t *testing.T, testFile string, config *config.Config
|
||||
}
|
||||
|
||||
peersUpdateManager := NewPeersUpdateManager(nil)
|
||||
jobManager := NewJobManager(nil, store)
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
|
||||
ctx := context.WithValue(context.Background(), hook.ExecutionContextKey, hook.SystemSource) //nolint:staticcheck
|
||||
@@ -451,7 +452,7 @@ func startManagementForTest(t *testing.T, testFile string, config *config.Config
|
||||
permissionsManager := permissions.NewManager(store)
|
||||
groupsManager := groups.NewManagerMock()
|
||||
|
||||
accountManager, err := BuildManager(ctx, store, peersUpdateManager, nil, "", "netbird.selfhosted",
|
||||
accountManager, err := BuildManager(ctx, store, peersUpdateManager, jobManager, nil, "", "netbird.selfhosted",
|
||||
eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
|
||||
if err != nil {
|
||||
@@ -462,7 +463,7 @@ func startManagementForTest(t *testing.T, testFile string, config *config.Config
|
||||
secretsManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay, settingsMockManager, groupsManager)
|
||||
|
||||
ephemeralMgr := manager.NewEphemeralManager(store, accountManager)
|
||||
mgmtServer, err := NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, secretsManager, nil, ephemeralMgr, nil, MockIntegratedValidator{})
|
||||
mgmtServer, err := NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, jobManager, secretsManager, nil, ephemeralMgr, nil, MockIntegratedValidator{})
|
||||
if err != nil {
|
||||
return nil, nil, "", cleanup, err
|
||||
}
|
||||
|
||||
@@ -177,6 +177,7 @@ func startServer(
|
||||
}
|
||||
|
||||
peersUpdateManager := server.NewPeersUpdateManager(nil)
|
||||
jobManager := server.NewJobManager(nil, str)
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
|
||||
metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
|
||||
@@ -203,6 +204,7 @@ func startServer(
|
||||
context.Background(),
|
||||
str,
|
||||
peersUpdateManager,
|
||||
jobManager,
|
||||
nil,
|
||||
"",
|
||||
"netbird.selfhosted",
|
||||
@@ -227,6 +229,7 @@ func startServer(
|
||||
accountManager,
|
||||
settingsMockManager,
|
||||
peersUpdateManager,
|
||||
jobManager,
|
||||
secretsManager,
|
||||
nil,
|
||||
&manager.EphemeralManager{},
|
||||
|
||||
@@ -130,6 +130,7 @@ type MockAccountManager struct {
|
||||
BufferUpdateAccountPeersFunc func(ctx context.Context, accountID string)
|
||||
}
|
||||
|
||||
|
||||
func (am *MockAccountManager) CreateGroup(ctx context.Context, accountID, userID string, group *types.Group) error {
|
||||
if am.SaveGroupFunc != nil {
|
||||
return am.SaveGroupFunc(ctx, accountID, userID, group, true)
|
||||
|
||||
@@ -785,7 +785,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) {
|
||||
AnyTimes()
|
||||
|
||||
permissionsManager := permissions.NewManager(store)
|
||||
return BuildManager(context.Background(), store, NewPeersUpdateManager(nil), nil, "", "netbird.selfhosted", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
return BuildManager(context.Background(), store, NewPeersUpdateManager(nil), NewJobManager(nil, store), nil, "", "netbird.selfhosted", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
}
|
||||
|
||||
func createNSStore(t *testing.T) (store.Store, error) {
|
||||
|
||||
@@ -1271,7 +1271,7 @@ func Test_RegisterPeerByUser(t *testing.T) {
|
||||
settingsMockManager := settings.NewMockManager(ctrl)
|
||||
permissionsManager := permissions.NewManager(s)
|
||||
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), NewJobManager(nil, s), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
assert.NoError(t, err)
|
||||
|
||||
existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b"
|
||||
@@ -1351,7 +1351,7 @@ func Test_RegisterPeerBySetupKey(t *testing.T) {
|
||||
AnyTimes()
|
||||
permissionsManager := permissions.NewManager(s)
|
||||
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), NewJobManager(nil, s), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
assert.NoError(t, err)
|
||||
|
||||
existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b"
|
||||
@@ -1499,7 +1499,7 @@ func Test_RegisterPeerRollbackOnFailure(t *testing.T) {
|
||||
|
||||
permissionsManager := permissions.NewManager(s)
|
||||
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), NewJobManager(nil, s), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
assert.NoError(t, err)
|
||||
|
||||
existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b"
|
||||
@@ -1573,7 +1573,7 @@ func Test_LoginPeer(t *testing.T) {
|
||||
AnyTimes()
|
||||
permissionsManager := permissions.NewManager(s)
|
||||
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
am, err := BuildManager(context.Background(), s, NewPeersUpdateManager(nil), NewJobManager(nil, s), nil, "", "netbird.cloud", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
assert.NoError(t, err)
|
||||
|
||||
existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b"
|
||||
|
||||
@@ -1285,7 +1285,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) {
|
||||
|
||||
permissionsManager := permissions.NewManager(store)
|
||||
|
||||
return BuildManager(context.Background(), store, NewPeersUpdateManager(nil), nil, "", "netbird.selfhosted", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
return BuildManager(context.Background(), store, NewPeersUpdateManager(nil), NewJobManager(nil, store), nil, "", "netbird.selfhosted", eventStore, nil, false, MockIntegratedValidator{}, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false)
|
||||
}
|
||||
|
||||
func createRouterStore(t *testing.T) (store.Store, error) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
type Client interface {
|
||||
io.Closer
|
||||
Sync(ctx context.Context, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error
|
||||
Job(ctx context.Context, msgHandler func(msg *proto.JobRequest) *proto.JobResponse) error
|
||||
GetServerPublicKey() (*wgtypes.Key, error)
|
||||
Register(serverKey wgtypes.Key, setupKey string, jwtToken string, sysInfo *system.Info, sshKey []byte, dnsLabels domain.List) (*proto.LoginResponse, error)
|
||||
Login(serverKey wgtypes.Key, sysInfo *system.Info, sshKey []byte, dnsLabels domain.List) (*proto.LoginResponse, error)
|
||||
|
||||
@@ -69,6 +69,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
|
||||
t.Cleanup(cleanUp)
|
||||
|
||||
peersUpdateManager := mgmt.NewPeersUpdateManager(nil)
|
||||
jobManager := mgmt.NewJobManager(nil, store)
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
@@ -111,7 +112,20 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
|
||||
Return(&types.ExtraSettings{}, nil).
|
||||
AnyTimes()
|
||||
|
||||
accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false)
|
||||
permissionsManagerMock := permissions.NewMockManager(ctrl)
|
||||
permissionsManagerMock.
|
||||
EXPECT().
|
||||
ValidateUserPermissions(
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
).
|
||||
Return(true, nil).
|
||||
AnyTimes()
|
||||
|
||||
accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, jobManager, nil, "", "netbird.selfhosted", eventStore, nil, false, ia, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -119,7 +133,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
|
||||
groupsManager := groups.NewManagerMock()
|
||||
|
||||
secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay, settingsMockManager, groupsManager)
|
||||
mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, secretsManager, nil, &manager.EphemeralManager{}, nil, mgmt.MockIntegratedValidator{})
|
||||
mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, settingsMockManager, peersUpdateManager, jobManager, secretsManager, nil, &manager.EphemeralManager{}, nil, mgmt.MockIntegratedValidator{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
gstatus "google.golang.org/grpc/status"
|
||||
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"github.com/google/uuid"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
|
||||
"google.golang.org/grpc"
|
||||
@@ -112,6 +113,25 @@ func (c *GrpcClient) ready() bool {
|
||||
// Sync wraps the real client's Sync endpoint call and takes care of retries and encryption/decryption of messages
|
||||
// Blocking request. The result will be sent via msgHandler callback function
|
||||
func (c *GrpcClient) Sync(ctx context.Context, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error {
|
||||
return c.withMgmtStream(ctx, func(ctx context.Context, serverPubKey wgtypes.Key) error {
|
||||
return c.handleSyncStream(ctx, serverPubKey, sysInfo, msgHandler)
|
||||
})
|
||||
}
|
||||
|
||||
// Job wraps the real client's Job endpoint call and takes care of retries and encryption/decryption of messages
|
||||
// Blocking request. The result will be sent via msgHandler callback function
|
||||
func (c *GrpcClient) Job(ctx context.Context, msgHandler func(msg *proto.JobRequest) *proto.JobResponse) error {
|
||||
return c.withMgmtStream(ctx, func(ctx context.Context, serverPubKey wgtypes.Key) error {
|
||||
return c.handleJobStream(ctx, serverPubKey, msgHandler)
|
||||
})
|
||||
}
|
||||
|
||||
// withMgmtStream runs a streaming operation against the ManagementService
|
||||
// It takes care of retries, connection readiness, and fetching server public key.
|
||||
func (c *GrpcClient) withMgmtStream(
|
||||
ctx context.Context,
|
||||
handler func(ctx context.Context, serverPubKey wgtypes.Key) error,
|
||||
) error {
|
||||
operation := func() error {
|
||||
log.Debugf("management connection state %v", c.conn.GetState())
|
||||
connState := c.conn.GetState()
|
||||
@@ -129,7 +149,7 @@ func (c *GrpcClient) Sync(ctx context.Context, sysInfo *system.Info, msgHandler
|
||||
return err
|
||||
}
|
||||
|
||||
return c.handleStream(ctx, *serverPubKey, sysInfo, msgHandler)
|
||||
return handler(ctx, *serverPubKey)
|
||||
}
|
||||
|
||||
err := backoff.Retry(operation, defaultBackoff(ctx))
|
||||
@@ -140,12 +160,132 @@ func (c *GrpcClient) Sync(ctx context.Context, sysInfo *system.Info, msgHandler
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *GrpcClient) handleStream(ctx context.Context, serverPubKey wgtypes.Key, sysInfo *system.Info,
|
||||
msgHandler func(msg *proto.SyncResponse) error) error {
|
||||
func (c *GrpcClient) handleJobStream(
|
||||
ctx context.Context,
|
||||
serverPubKey wgtypes.Key,
|
||||
msgHandler func(msg *proto.JobRequest) *proto.JobResponse,
|
||||
) error {
|
||||
stream, err := c.realClient.Job(ctx)
|
||||
if err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to open job stream: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Handshake with the server
|
||||
if err := c.sendHandshake(ctx, stream, serverPubKey); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.WithContext(ctx).Debug("job stream handshake sent successfully")
|
||||
|
||||
// Main loop: receive, process, respond
|
||||
for {
|
||||
jobReq, err := c.receiveJobRequest(ctx, stream, serverPubKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
|
||||
log.WithContext(ctx).Info("job stream closed by server")
|
||||
return nil
|
||||
}
|
||||
log.WithContext(ctx).Errorf("error receiving job request: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if jobReq == nil || len(jobReq.ID) == 0 {
|
||||
log.WithContext(ctx).Debug("received unknown or empty job request, skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
jobResp := c.processJobRequest(ctx, jobReq, msgHandler)
|
||||
if err := c.sendJobResponse(ctx, stream, serverPubKey, jobResp); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sendHandshake sends the initial handshake message
|
||||
func (c *GrpcClient) sendHandshake(ctx context.Context, stream proto.ManagementService_JobClient, serverPubKey wgtypes.Key) error {
|
||||
handshakeReq := &proto.JobRequest{
|
||||
ID: []byte(uuid.New().String()),
|
||||
}
|
||||
encHello, err := encryption.EncryptMessage(serverPubKey, c.key, handshakeReq)
|
||||
if err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to encrypt handshake message: %v", err)
|
||||
return err
|
||||
}
|
||||
return stream.Send(&proto.EncryptedMessage{
|
||||
WgPubKey: c.key.PublicKey().String(),
|
||||
Body: encHello,
|
||||
})
|
||||
}
|
||||
|
||||
// receiveJobRequest waits for and decrypts a job request
|
||||
func (c *GrpcClient) receiveJobRequest(
|
||||
ctx context.Context,
|
||||
stream proto.ManagementService_JobClient,
|
||||
serverPubKey wgtypes.Key,
|
||||
) (*proto.JobRequest, error) {
|
||||
encryptedMsg, err := stream.Recv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jobReq := &proto.JobRequest{}
|
||||
if err := encryption.DecryptMessage(serverPubKey, c.key, encryptedMsg.Body, jobReq); err != nil {
|
||||
log.WithContext(ctx).Warnf("failed to decrypt job request: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return jobReq, nil
|
||||
}
|
||||
|
||||
// processJobRequest executes the handler and ensures a valid response
|
||||
func (c *GrpcClient) processJobRequest(
|
||||
ctx context.Context,
|
||||
jobReq *proto.JobRequest,
|
||||
msgHandler func(msg *proto.JobRequest) *proto.JobResponse,
|
||||
) *proto.JobResponse {
|
||||
jobResp := msgHandler(jobReq)
|
||||
if jobResp == nil {
|
||||
jobResp = &proto.JobResponse{
|
||||
ID: jobReq.ID,
|
||||
Status: proto.JobStatus_failed,
|
||||
Reason: []byte("handler returned nil response"),
|
||||
}
|
||||
log.WithContext(ctx).Warnf("job handler returned nil for job %s", string(jobReq.ID))
|
||||
}
|
||||
return jobResp
|
||||
}
|
||||
|
||||
// sendJobResponse encrypts and sends a job response
|
||||
func (c *GrpcClient) sendJobResponse(
|
||||
ctx context.Context,
|
||||
stream proto.ManagementService_JobClient,
|
||||
serverPubKey wgtypes.Key,
|
||||
resp *proto.JobResponse,
|
||||
) error {
|
||||
encResp, err := encryption.EncryptMessage(serverPubKey, c.key, resp)
|
||||
if err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to encrypt job response for job %s: %v", string(resp.ID), err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stream.Send(&proto.EncryptedMessage{
|
||||
WgPubKey: c.key.PublicKey().String(),
|
||||
Body: encResp,
|
||||
}); err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to send job response for job %s: %v", string(resp.ID), err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.WithContext(ctx).Debugf("job response sent successfully for job %s", string(resp.ID))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *GrpcClient) handleSyncStream(ctx context.Context, serverPubKey wgtypes.Key, sysInfo *system.Info, msgHandler func(msg *proto.SyncResponse) error) error {
|
||||
ctx, cancelStream := context.WithCancel(ctx)
|
||||
defer cancelStream()
|
||||
|
||||
stream, err := c.connectToStream(ctx, serverPubKey, sysInfo)
|
||||
stream, err := c.connectToSyncStream(ctx, serverPubKey, sysInfo)
|
||||
if err != nil {
|
||||
log.Debugf("failed to open Management Service stream: %s", err)
|
||||
if s, ok := gstatus.FromError(err); ok && s.Code() == codes.PermissionDenied {
|
||||
@@ -158,7 +298,7 @@ func (c *GrpcClient) handleStream(ctx context.Context, serverPubKey wgtypes.Key,
|
||||
c.notifyConnected()
|
||||
|
||||
// blocking until error
|
||||
err = c.receiveEvents(stream, serverPubKey, msgHandler)
|
||||
err = c.receiveUpdatesEvents(stream, serverPubKey, msgHandler)
|
||||
if err != nil {
|
||||
c.notifyDisconnected(err)
|
||||
s, _ := gstatus.FromError(err)
|
||||
@@ -187,7 +327,7 @@ func (c *GrpcClient) GetNetworkMap(sysInfo *system.Info) (*proto.NetworkMap, err
|
||||
|
||||
ctx, cancelStream := context.WithCancel(c.ctx)
|
||||
defer cancelStream()
|
||||
stream, err := c.connectToStream(ctx, *serverPubKey, sysInfo)
|
||||
stream, err := c.connectToSyncStream(ctx, *serverPubKey, sysInfo)
|
||||
if err != nil {
|
||||
log.Debugf("failed to open Management Service stream: %s", err)
|
||||
return nil, err
|
||||
@@ -220,7 +360,7 @@ func (c *GrpcClient) GetNetworkMap(sysInfo *system.Info) (*proto.NetworkMap, err
|
||||
return decryptedResp.GetNetworkMap(), nil
|
||||
}
|
||||
|
||||
func (c *GrpcClient) connectToStream(ctx context.Context, serverPubKey wgtypes.Key, sysInfo *system.Info) (proto.ManagementService_SyncClient, error) {
|
||||
func (c *GrpcClient) connectToSyncStream(ctx context.Context, serverPubKey wgtypes.Key, sysInfo *system.Info) (proto.ManagementService_SyncClient, error) {
|
||||
req := &proto.SyncRequest{Meta: infoToMetaData(sysInfo)}
|
||||
|
||||
myPrivateKey := c.key
|
||||
@@ -239,7 +379,7 @@ func (c *GrpcClient) connectToStream(ctx context.Context, serverPubKey wgtypes.K
|
||||
return sync, nil
|
||||
}
|
||||
|
||||
func (c *GrpcClient) receiveEvents(stream proto.ManagementService_SyncClient, serverPubKey wgtypes.Key, msgHandler func(msg *proto.SyncResponse) error) error {
|
||||
func (c *GrpcClient) receiveUpdatesEvents(stream proto.ManagementService_SyncClient, serverPubKey wgtypes.Key, msgHandler func(msg *proto.SyncResponse) error) error {
|
||||
for {
|
||||
update, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
|
||||
@@ -20,6 +20,7 @@ type MockClient struct {
|
||||
GetPKCEAuthorizationFlowFunc func(serverKey wgtypes.Key) (*proto.PKCEAuthorizationFlow, error)
|
||||
SyncMetaFunc func(sysInfo *system.Info) error
|
||||
LogoutFunc func() error
|
||||
JobFunc func(ctx context.Context, msgHandler func(msg *proto.JobRequest) *proto.JobResponse) error
|
||||
}
|
||||
|
||||
func (m *MockClient) IsHealthy() bool {
|
||||
@@ -40,6 +41,13 @@ func (m *MockClient) Sync(ctx context.Context, sysInfo *system.Info, msgHandler
|
||||
return m.SyncFunc(ctx, sysInfo, msgHandler)
|
||||
}
|
||||
|
||||
func (m *MockClient) Job(ctx context.Context, msgHandler func(msg *proto.JobRequest) *proto.JobResponse) error {
|
||||
if m.JobFunc == nil {
|
||||
return nil
|
||||
}
|
||||
return m.JobFunc(ctx, msgHandler)
|
||||
}
|
||||
|
||||
func (m *MockClient) GetServerPublicKey() (*wgtypes.Key, error) {
|
||||
if m.GetServerPublicKeyFunc == nil {
|
||||
return nil, nil
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -48,6 +48,9 @@ service ManagementService {
|
||||
|
||||
// Logout logs out the peer and removes it from the management server
|
||||
rpc Logout(EncryptedMessage) returns (Empty) {}
|
||||
|
||||
// Executes a job on a target peer (e.g., debug bundle)
|
||||
rpc Job(stream EncryptedMessage) returns (stream EncryptedMessage) {}
|
||||
}
|
||||
|
||||
message EncryptedMessage {
|
||||
@@ -60,6 +63,42 @@ message EncryptedMessage {
|
||||
int32 version = 3;
|
||||
}
|
||||
|
||||
message JobRequest {
|
||||
bytes ID = 1;
|
||||
|
||||
oneof workload_parameters {
|
||||
BundleParameters bundle = 10;
|
||||
//OtherParameters other = 11;
|
||||
}
|
||||
}
|
||||
|
||||
enum JobStatus {
|
||||
unknown_status = 0; //placeholder
|
||||
succeeded = 1;
|
||||
failed = 2;
|
||||
}
|
||||
|
||||
message JobResponse{
|
||||
bytes ID = 1;
|
||||
JobStatus status=2;
|
||||
bytes Reason=3;
|
||||
oneof workload_results {
|
||||
BundleResult bundle = 10;
|
||||
//OtherResult other = 11;
|
||||
}
|
||||
}
|
||||
|
||||
message BundleParameters {
|
||||
bool bundle_for = 1;
|
||||
int64 bundle_for_time = 2;
|
||||
int32 log_file_count = 3;
|
||||
bool anonymize = 4;
|
||||
}
|
||||
|
||||
message BundleResult {
|
||||
string upload_key = 1;
|
||||
}
|
||||
|
||||
message SyncRequest {
|
||||
// Meta data of the peer
|
||||
PeerSystemMeta meta = 1;
|
||||
|
||||
@@ -50,6 +50,8 @@ type ManagementServiceClient interface {
|
||||
SyncMeta(ctx context.Context, in *EncryptedMessage, opts ...grpc.CallOption) (*Empty, error)
|
||||
// Logout logs out the peer and removes it from the management server
|
||||
Logout(ctx context.Context, in *EncryptedMessage, opts ...grpc.CallOption) (*Empty, error)
|
||||
// Executes a job on a target peer (e.g., debug bundle)
|
||||
Job(ctx context.Context, opts ...grpc.CallOption) (ManagementService_JobClient, error)
|
||||
}
|
||||
|
||||
type managementServiceClient struct {
|
||||
@@ -155,6 +157,37 @@ func (c *managementServiceClient) Logout(ctx context.Context, in *EncryptedMessa
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *managementServiceClient) Job(ctx context.Context, opts ...grpc.CallOption) (ManagementService_JobClient, error) {
|
||||
stream, err := c.cc.NewStream(ctx, &ManagementService_ServiceDesc.Streams[1], "/management.ManagementService/Job", opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &managementServiceJobClient{stream}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
type ManagementService_JobClient interface {
|
||||
Send(*EncryptedMessage) error
|
||||
Recv() (*EncryptedMessage, error)
|
||||
grpc.ClientStream
|
||||
}
|
||||
|
||||
type managementServiceJobClient struct {
|
||||
grpc.ClientStream
|
||||
}
|
||||
|
||||
func (x *managementServiceJobClient) Send(m *EncryptedMessage) error {
|
||||
return x.ClientStream.SendMsg(m)
|
||||
}
|
||||
|
||||
func (x *managementServiceJobClient) Recv() (*EncryptedMessage, error) {
|
||||
m := new(EncryptedMessage)
|
||||
if err := x.ClientStream.RecvMsg(m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ManagementServiceServer is the server API for ManagementService service.
|
||||
// All implementations must embed UnimplementedManagementServiceServer
|
||||
// for forward compatibility
|
||||
@@ -191,6 +224,8 @@ type ManagementServiceServer interface {
|
||||
SyncMeta(context.Context, *EncryptedMessage) (*Empty, error)
|
||||
// Logout logs out the peer and removes it from the management server
|
||||
Logout(context.Context, *EncryptedMessage) (*Empty, error)
|
||||
// Executes a job on a target peer (e.g., debug bundle)
|
||||
Job(ManagementService_JobServer) error
|
||||
mustEmbedUnimplementedManagementServiceServer()
|
||||
}
|
||||
|
||||
@@ -222,6 +257,9 @@ func (UnimplementedManagementServiceServer) SyncMeta(context.Context, *Encrypted
|
||||
func (UnimplementedManagementServiceServer) Logout(context.Context, *EncryptedMessage) (*Empty, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Logout not implemented")
|
||||
}
|
||||
func (UnimplementedManagementServiceServer) Job(ManagementService_JobServer) error {
|
||||
return status.Errorf(codes.Unimplemented, "method Job not implemented")
|
||||
}
|
||||
func (UnimplementedManagementServiceServer) mustEmbedUnimplementedManagementServiceServer() {}
|
||||
|
||||
// UnsafeManagementServiceServer may be embedded to opt out of forward compatibility for this service.
|
||||
@@ -382,6 +420,32 @@ func _ManagementService_Logout_Handler(srv interface{}, ctx context.Context, dec
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _ManagementService_Job_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
return srv.(ManagementServiceServer).Job(&managementServiceJobServer{stream})
|
||||
}
|
||||
|
||||
type ManagementService_JobServer interface {
|
||||
Send(*EncryptedMessage) error
|
||||
Recv() (*EncryptedMessage, error)
|
||||
grpc.ServerStream
|
||||
}
|
||||
|
||||
type managementServiceJobServer struct {
|
||||
grpc.ServerStream
|
||||
}
|
||||
|
||||
func (x *managementServiceJobServer) Send(m *EncryptedMessage) error {
|
||||
return x.ServerStream.SendMsg(m)
|
||||
}
|
||||
|
||||
func (x *managementServiceJobServer) Recv() (*EncryptedMessage, error) {
|
||||
m := new(EncryptedMessage)
|
||||
if err := x.ServerStream.RecvMsg(m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ManagementService_ServiceDesc is the grpc.ServiceDesc for ManagementService service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
@@ -424,6 +488,12 @@ var ManagementService_ServiceDesc = grpc.ServiceDesc{
|
||||
Handler: _ManagementService_Sync_Handler,
|
||||
ServerStreams: true,
|
||||
},
|
||||
{
|
||||
StreamName: "Job",
|
||||
Handler: _ManagementService_Job_Handler,
|
||||
ServerStreams: true,
|
||||
ClientStreams: true,
|
||||
},
|
||||
},
|
||||
Metadata: "management.proto",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user