Fix broken links in README.md (#992)

* Prevent sending nameserver configuration when peer is set as NS

* Add DNS filter tests

.github/workflows/test-docker-compose-linux.yml

@@ -53,6 +53,7 @@ jobs:
           CI_NETBIRD_MGMT_IDP: "none"
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
+          CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
 
       - name: check values
         working-directory: infrastructure_files

.gitignore

@@ -7,8 +7,15 @@ bin/
 conf.json
 http-cmds.sh
 infrastructure_files/management.json
+infrastructure_files/management-*.json
 infrastructure_files/docker-compose.yml
+infrastructure_files/openid-configuration.json
+infrastructure_files/turnserver.conf
+management/management
+client/client
+client/client.exe
 *.syso
 client/.distfiles/
 infrastructure_files/setup.env
+infrastructure_files/setup-*.env
 .vscode

README.md

@@ -70,9 +70,9 @@ For stable versions, see [releases](https://github.com/netbirdio/netbird/release
 
 ### Start using NetBird
 -  Hosted version: [https://app.netbird.io/](https://app.netbird.io/).
--  See our documentation for [Quickstart Guide](https://netbird.io/docs/getting-started/quickstart).
--  If you are looking to self-host NetBird, check our [Self-Hosting Guide](https://netbird.io/docs/getting-started/self-hosting).
--  Step-by-step [Installation Guide](https://netbird.io/docs/getting-started/installation) for different platforms.
+-  See our documentation for [Quickstart Guide](https://docs.netbird.io/how-to/getting-started).
+-  If you are looking to self-host NetBird, check our [Self-Hosting Guide](https://docs.netbird.io/selfhosted/selfhosted-guide).
+-  Step-by-step [Installation Guide](https://docs.netbird.io/how-to/getting-started#installation) for different platforms.
 -  Web UI [repository](https://github.com/netbirdio/dashboard).
 -  5 min [demo video](https://youtu.be/Tu9tPsUWaY0) on YouTube.
 
@@ -91,7 +91,7 @@ For stable versions, see [releases](https://github.com/netbirdio/netbird/release
   <img src="https://netbird.io/docs/img/architecture/high-level-dia.png" width="700"/>
 </p>
 
-See a complete [architecture overview](https://netbird.io/docs/overview/architecture) for details.
+See a complete [architecture overview](https://docs.netbird.io/about-netbird/how-netbird-works#architecture) for details.
 
 ### Roadmap
 -  [Public Roadmap](https://github.com/netbirdio/netbird/projects/2)

client/cmd/ssh.go

@@ -73,7 +73,8 @@ var sshCmd = &cobra.Command{
 		go func() {
 			// blocking
 			if err := runSSH(sshctx, host, []byte(config.SSHKey), cmd); err != nil {
-				log.Print(err)
+				log.Debug(err)
+				os.Exit(1)
 			}
 			cancel()
 		}()
@@ -92,12 +93,10 @@ func runSSH(ctx context.Context, addr string, pemKey []byte, cmd *cobra.Command)
 	c, err := nbssh.DialWithKey(fmt.Sprintf("%s:%d", addr, port), user, pemKey)
 	if err != nil {
 		cmd.Printf("Error: %v\n", err)
-		cmd.Printf("Couldn't connect. " +
-			"You might be disconnected from the NetBird network, or the NetBird agent isn't running.\n" +
-			"Run the status command: \n\n" +
-			" netbird status\n\n" +
-			"It might also be that the SSH server is disabled on the agent you are trying to connect to.\n")
-		return nil
+		cmd.Printf("Couldn't connect. Please check the connection status or if the ssh server is enabled on the other peer" +
+			"You can verify the connection by running:\n\n" +
+			" netbird status\n\n")
+		return err
 	}
 	go func() {
 		<-ctx.Done()

client/firewall/port.go

@@ -1,5 +1,9 @@
 package firewall
 
+import (
+	"strconv"
+)
+
 // Protocol is the protocol of the port
 type Protocol string
 
@@ -28,3 +32,15 @@ type Port struct {
 	// Values contains one value for single port, multiple values for the list of ports, or two values for the range of ports
 	Values []int
 }
+
+// String interface implementation
+func (p *Port) String() string {
+	var ports string
+	for _, port := range p.Values {
+		if ports != "" {
+			ports += ","
+		}
+		ports += strconv.Itoa(port)
+	}
+	return ports
+}

client/internal/acl/manager.go

@@ -1,10 +1,13 @@
 package acl
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"strconv"
 	"sync"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -42,6 +45,17 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) {
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
 
+	start := time.Now()
+	defer func() {
+		total := 0
+		for _, pairs := range d.rulesPairs {
+			total += len(pairs)
+		}
+		log.Infof(
+			"ACL rules processed in: %v, total rules count: %d",
+			time.Since(start), total)
+	}()
+
 	if d.manager == nil {
 		log.Debug("firewall manager is not supported, skipping firewall rules")
 		return
@@ -95,13 +109,13 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap) {
 	applyFailed := false
 	newRulePairs := make(map[string][]firewall.Rule)
 	for _, r := range rules {
-		rulePair, err := d.protoRuleToFirewallRule(r)
+		pairID, rulePair, err := d.protoRuleToFirewallRule(r)
 		if err != nil {
 			log.Errorf("failed to apply firewall rule: %+v, %v", r, err)
 			applyFailed = true
 			break
 		}
-		newRulePairs[rulePair[0].GetRuleID()] = rulePair
+		newRulePairs[pairID] = rulePair
 	}
 	if applyFailed {
 		log.Error("failed to apply firewall rules, rollback ACL to previous state")
@@ -140,33 +154,38 @@ func (d *DefaultManager) Stop() {
 	}
 }
 
-func (d *DefaultManager) protoRuleToFirewallRule(r *mgmProto.FirewallRule) ([]firewall.Rule, error) {
+func (d *DefaultManager) protoRuleToFirewallRule(r *mgmProto.FirewallRule) (string, []firewall.Rule, error) {
 	ip := net.ParseIP(r.PeerIP)
 	if ip == nil {
-		return nil, fmt.Errorf("invalid IP address, skipping firewall rule")
+		return "", nil, fmt.Errorf("invalid IP address, skipping firewall rule")
 	}
 
 	protocol := convertToFirewallProtocol(r.Protocol)
 	if protocol == firewall.ProtocolUnknown {
-		return nil, fmt.Errorf("invalid protocol type: %d, skipping firewall rule", r.Protocol)
+		return "", nil, fmt.Errorf("invalid protocol type: %d, skipping firewall rule", r.Protocol)
 	}
 
 	action := convertFirewallAction(r.Action)
 	if action == firewall.ActionUnknown {
-		return nil, fmt.Errorf("invalid action type: %d, skipping firewall rule", r.Action)
+		return "", nil, fmt.Errorf("invalid action type: %d, skipping firewall rule", r.Action)
 	}
 
 	var port *firewall.Port
 	if r.Port != "" {
 		value, err := strconv.Atoi(r.Port)
 		if err != nil {
-			return nil, fmt.Errorf("invalid port, skipping firewall rule")
+			return "", nil, fmt.Errorf("invalid port, skipping firewall rule")
 		}
 		port = &firewall.Port{
 			Values: []int{value},
 		}
 	}
 
+	ruleID := d.getRuleID(ip, protocol, int(r.Direction), port, action, "")
+	if rulesPair, ok := d.rulesPairs[ruleID]; ok {
+		return ruleID, rulesPair, nil
+	}
+
 	var rules []firewall.Rule
 	var err error
 	switch r.Direction {
@@ -175,15 +194,15 @@ func (d *DefaultManager) protoRuleToFirewallRule(r *mgmProto.FirewallRule) ([]fi
 	case mgmProto.FirewallRule_OUT:
 		rules, err = d.addOutRules(ip, protocol, port, action, "")
 	default:
-		return nil, fmt.Errorf("invalid direction, skipping firewall rule")
+		return "", nil, fmt.Errorf("invalid direction, skipping firewall rule")
 	}
 
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 
-	d.rulesPairs[rules[0].GetRuleID()] = rules
-	return rules, nil
+	d.rulesPairs[ruleID] = rules
+	return ruleID, rules, nil
 }
 
 func (d *DefaultManager) addInRules(ip net.IP, protocol firewall.Protocol, port *firewall.Port, action firewall.Action, comment string) ([]firewall.Rule, error) {
@@ -226,6 +245,23 @@ func (d *DefaultManager) addOutRules(ip net.IP, protocol firewall.Protocol, port
 	return append(rules, rule), nil
 }
 
+// getRuleID() returns unique ID for the rule based on its parameters.
+func (d *DefaultManager) getRuleID(
+	ip net.IP,
+	proto firewall.Protocol,
+	direction int,
+	port *firewall.Port,
+	action firewall.Action,
+	comment string,
+) string {
+	idStr := ip.String() + string(proto) + strconv.Itoa(direction) + strconv.Itoa(int(action)) + comment
+	if port != nil {
+		idStr += port.String()
+	}
+
+	return hex.EncodeToString(md5.New().Sum([]byte(idStr)))
+}
+
 // squashAcceptRules does complex logic to convert many rules which allows connection by traffic type
 // to all peers in the network map to one rule which just accepts that type of the traffic.
 //
@@ -235,7 +271,7 @@ func (d *DefaultManager) squashAcceptRules(
 	networkMap *mgmProto.NetworkMap,
 ) ([]*mgmProto.FirewallRule, map[mgmProto.FirewallRuleProtocol]struct{}) {
 	totalIPs := 0
-	for _, p := range networkMap.RemotePeers {
+	for _, p := range append(networkMap.RemotePeers, networkMap.OfflinePeers...) {
 		for range p.AllowedIps {
 			totalIPs++
 		}

client/internal/acl/manager_test.go

@@ -55,6 +55,11 @@ func TestDefaultManager(t *testing.T) {
 	})
 
 	t.Run("add extra rules", func(t *testing.T) {
+		existedPairs := map[string]struct{}{}
+		for id := range acl.rulesPairs {
+			existedPairs[id] = struct{}{}
+		}
+
 		// remove first rule
 		networkMap.FirewallRules = networkMap.FirewallRules[1:]
 		networkMap.FirewallRules = append(
@@ -67,11 +72,6 @@ func TestDefaultManager(t *testing.T) {
 			},
 		)
 
-		existedRulesID := map[string]struct{}{}
-		for id := range acl.rulesPairs {
-			existedRulesID[id] = struct{}{}
-		}
-
 		acl.ApplyFiltering(networkMap)
 
 		// we should have one old and one new rule in the existed rules
@@ -80,13 +80,16 @@ func TestDefaultManager(t *testing.T) {
 			return
 		}
 
-		// check that old rules was removed
-		for id := range existedRulesID {
-			if _, ok := acl.rulesPairs[id]; ok {
-				t.Errorf("old rule was not removed")
-				return
+		// check that old rule was removed
+		previousCount := 0
+		for id := range acl.rulesPairs {
+			if _, ok := existedPairs[id]; ok {
+				previousCount++
 			}
 		}
+		if previousCount != 1 {
+			t.Errorf("old rule was not removed")
+		}
 	})
 
 	t.Run("handle default rules", func(t *testing.T) {

client/internal/dns/mockServer.go

@@ -7,16 +7,17 @@ import (
 
 // MockServer is the mock instance of a dns server
 type MockServer struct {
-	StartFunc           func()
+	InitializeFunc      func() error
 	StopFunc            func()
 	UpdateDNSServerFunc func(serial uint64, update nbdns.Config) error
 }
 
-// Start mock implementation of Start from Server interface
-func (m *MockServer) Start() {
-	if m.StartFunc != nil {
-		m.StartFunc()
+// Initialize mock implementation of Initialize from Server interface
+func (m *MockServer) Initialize() error {
+	if m.InitializeFunc != nil {
+		return m.InitializeFunc()
 	}
+	return nil
 }
 
 // Stop mock implementation of Stop from Server interface

client/internal/dns/server.go

@@ -29,7 +29,7 @@ const (
 
 // Server is a dns server interface
 type Server interface {
-	Start()
+	Initialize() error
 	Stop()
 	DnsIP() string
 	UpdateDNSServer(serial uint64, update nbdns.Config) error
@@ -42,7 +42,7 @@ type DefaultServer struct {
 	ctx                context.Context
 	ctxCancel          context.CancelFunc
 	mux                sync.Mutex
-	fakeResolverWG     sync.WaitGroup
+	udpFilterHookID    string
 	server             *dns.Server
 	dnsMux             *dns.ServeMux
 	dnsMuxMap          registeredHandlerMap
@@ -82,11 +82,6 @@ func NewDefaultServer(ctx context.Context, wgInterface *iface.WGIface, customAdd
 		addrPort = &parsedAddrPort
 	}
 
-	hostManager, err := newHostManager(wgInterface)
-	if err != nil {
-		return nil, err
-	}
-
 	ctx, stop := context.WithCancel(ctx)
 
 	defaultServer := &DefaultServer{
@@ -104,32 +99,41 @@ func NewDefaultServer(ctx context.Context, wgInterface *iface.WGIface, customAdd
 		},
 		wgInterface:   wgInterface,
 		customAddress: addrPort,
-		hostManager:   hostManager,
 	}
 
 	if initialDnsCfg != nil {
 		defaultServer.enabled = hasValidDnsServer(initialDnsCfg)
 	}
 
-	defaultServer.evalRuntimeAddress()
+	if wgInterface.IsUserspaceBind() {
+		defaultServer.evelRuntimeAddressForUserspace()
+	}
+
 	return defaultServer, nil
 }
 
-// Start runs the listener in a go routine
-func (s *DefaultServer) Start() {
+// Initialize instantiate host manager. It required to be initialized wginterface
+func (s *DefaultServer) Initialize() (err error) {
+	s.mux.Lock()
+	defer s.mux.Unlock()
+
+	if s.hostManager != nil {
+		return nil
+	}
+
+	if !s.wgInterface.IsUserspaceBind() {
+		s.evalRuntimeAddress()
+	}
+	s.hostManager, err = newHostManager(s.wgInterface)
+	return
+}
+
+// listen runs the listener in a go routine
+func (s *DefaultServer) listen() {
 	// nil check required in unit tests
 	if s.wgInterface != nil && s.wgInterface.IsUserspaceBind() {
-		s.fakeResolverWG.Add(1)
-		go func() {
-			s.setListenerStatus(true)
-			defer s.setListenerStatus(false)
-
-			hookID := s.filterDNSTraffic()
-			s.fakeResolverWG.Wait()
-			if err := s.wgInterface.GetFilter().RemovePacketHook(hookID); err != nil {
-				log.Errorf("unable to remove DNS packet hook: %s", err)
-			}
-		}()
+		s.udpFilterHookID = s.filterDNSTraffic()
+		s.setListenerStatus(true)
 		return
 	}
 
@@ -146,6 +150,10 @@ func (s *DefaultServer) Start() {
 	}()
 }
 
+// DnsIP returns the DNS resolver server IP address
+//
+// When kernel space interface used it return real DNS server listener IP address
+// For bind interface, fake DNS resolver address returned (second last IP address from Nebird network)
 func (s *DefaultServer) DnsIP() string {
 	if !s.enabled {
 		return ""
@@ -187,22 +195,32 @@ func (s *DefaultServer) Stop() {
 	defer s.mux.Unlock()
 	s.ctxCancel()
 
-	err := s.hostManager.restoreHostDNS()
-	if err != nil {
-		log.Error(err)
+	if s.hostManager != nil {
+		err := s.hostManager.restoreHostDNS()
+		if err != nil {
+			log.Error(err)
+		}
 	}
 
-	if s.wgInterface != nil && s.wgInterface.IsUserspaceBind() && s.listenerIsRunning {
-		s.fakeResolverWG.Done()
-	}
-
-	err = s.stopListener()
+	err := s.stopListener()
 	if err != nil {
 		log.Error(err)
 	}
 }
 
 func (s *DefaultServer) stopListener() error {
+	if s.wgInterface != nil && s.wgInterface.IsUserspaceBind() && s.listenerIsRunning {
+		// udpFilterHookID here empty only in the unit tests
+		if filter := s.wgInterface.GetFilter(); filter != nil && s.udpFilterHookID != "" {
+			if err := filter.RemovePacketHook(s.udpFilterHookID); err != nil {
+				log.Errorf("unable to remove DNS packet hook: %s", err)
+			}
+		}
+		s.udpFilterHookID = ""
+		s.listenerIsRunning = false
+		return nil
+	}
+
 	if !s.listenerIsRunning {
 		return nil
 	}
@@ -231,6 +249,10 @@ func (s *DefaultServer) UpdateDNSServer(serial uint64, update nbdns.Config) erro
 		s.mux.Lock()
 		defer s.mux.Unlock()
 
+		if s.hostManager == nil {
+			return fmt.Errorf("dns service is not initialized yet")
+		}
+
 		hash, err := hashstructure.Hash(update, hashstructure.FormatV2, &hashstructure.HashOptions{
 			ZeroNil:         true,
 			IgnoreZeroValue: true,
@@ -262,15 +284,11 @@ func (s *DefaultServer) applyConfiguration(update nbdns.Config) error {
 	// is the service should be disabled, we stop the listener or fake resolver
 	// and proceed with a regular update to clean up the handlers and records
 	if !update.ServiceEnable {
-		if s.wgInterface != nil && s.wgInterface.IsUserspaceBind() {
-			s.fakeResolverWG.Done()
-		} else {
-			if err := s.stopListener(); err != nil {
-				log.Error(err)
-			}
+		if err := s.stopListener(); err != nil {
+			log.Error(err)
 		}
 	} else if !s.listenerIsRunning {
-		s.Start()
+		s.listen()
 	}
 
 	localMuxUpdates, localRecords, err := s.buildLocalHandlerUpdate(update.CustomZones)
@@ -542,17 +560,17 @@ func (s *DefaultServer) filterDNSTraffic() string {
 	return filter.AddUDPPacketHook(false, net.ParseIP(s.runtimeIP), uint16(s.runtimePort), hook)
 }
 
+func (s *DefaultServer) evelRuntimeAddressForUserspace() {
+	s.runtimeIP = getLastIPFromNetwork(s.wgInterface.Address().Network, 1)
+	s.runtimePort = defaultPort
+	s.server.Addr = fmt.Sprintf("%s:%d", s.runtimeIP, s.runtimePort)
+}
+
 func (s *DefaultServer) evalRuntimeAddress() {
 	defer func() {
 		s.server.Addr = fmt.Sprintf("%s:%d", s.runtimeIP, s.runtimePort)
 	}()
 
-	if s.wgInterface != nil && s.wgInterface.IsUserspaceBind() {
-		s.runtimeIP = getLastIPFromNetwork(s.wgInterface.Address().Network, 1)
-		s.runtimePort = defaultPort
-		return
-	}
-
 	if s.customAddress != nil {
 		s.runtimeIP = s.customAddress.Addr().String()
 		s.runtimePort = int(s.customAddress.Port())

client/internal/dns/server_test.go

@@ -5,15 +5,18 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/golang/mock/gomock"
 	"github.com/miekg/dns"
 
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/iface"
+	pfmock "github.com/netbirdio/netbird/iface/mocks"
 )
 
 var zoneRecords = []nbdns.SimpleRecord{
@@ -225,6 +228,10 @@ func TestUpdateDNSServer(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
+			err = dnsServer.Initialize()
+			if err != nil {
+				t.Fatal(err)
+			}
 			defer func() {
 				err = dnsServer.hostManager.restoreHostDNS()
 				if err != nil {
@@ -237,7 +244,6 @@ func TestUpdateDNSServer(t *testing.T) {
 			dnsServer.updateSerial = testCase.initSerial
 			// pretend we are running
 			dnsServer.listenerIsRunning = true
-			dnsServer.fakeResolverWG.Add(1)
 
 			err = dnsServer.UpdateDNSServer(testCase.inputSerial, testCase.inputUpdate)
 			if err != nil {
@@ -272,6 +278,133 @@ func TestUpdateDNSServer(t *testing.T) {
 	}
 }
 
+func TestDNSFakeResolverHandleUpdates(t *testing.T) {
+	ov := os.Getenv("NB_WG_KERNEL_DISABLED")
+	defer os.Setenv("NB_WG_KERNEL_DISABLED", ov)
+
+	os.Setenv("NB_WG_KERNEL_DISABLED", "true")
+	newNet, err := stdnet.NewNet(nil)
+	if err != nil {
+		t.Errorf("create stdnet: %v", err)
+		return
+	}
+
+	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", iface.DefaultMTU, nil, newNet)
+	if err != nil {
+		t.Errorf("build interface wireguard: %v", err)
+		return
+	}
+
+	err = wgIface.Create()
+	if err != nil {
+		t.Errorf("crate and init wireguard interface: %v", err)
+		return
+	}
+	defer func() {
+		if err = wgIface.Close(); err != nil {
+			t.Logf("close wireguard interface: %v", err)
+		}
+	}()
+
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	_, ipNet, err := net.ParseCIDR("100.66.100.1/32")
+	if err != nil {
+		t.Errorf("parse CIDR: %v", err)
+		return
+	}
+
+	packetfilter := pfmock.NewMockPacketFilter(ctrl)
+	packetfilter.EXPECT().SetNetwork(ipNet)
+	packetfilter.EXPECT().DropOutgoing(gomock.Any()).AnyTimes()
+	packetfilter.EXPECT().AddUDPPacketHook(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
+	packetfilter.EXPECT().RemovePacketHook(gomock.Any()).AnyTimes()
+
+	if err := wgIface.SetFilter(packetfilter); err != nil {
+		t.Errorf("set packet filter: %v", err)
+		return
+	}
+
+	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", nil)
+	if err != nil {
+		t.Errorf("create DNS server: %v", err)
+		return
+	}
+
+	err = dnsServer.Initialize()
+	if err != nil {
+		t.Errorf("run DNS server: %v", err)
+		return
+	}
+	defer func() {
+		if err = dnsServer.hostManager.restoreHostDNS(); err != nil {
+			t.Logf("restore DNS settings on the host: %v", err)
+			return
+		}
+	}()
+
+	dnsServer.dnsMuxMap = registeredHandlerMap{zoneRecords[0].Name: &localResolver{}}
+	dnsServer.localResolver.registeredMap = registrationMap{"netbird.cloud": struct{}{}}
+	dnsServer.updateSerial = 0
+
+	nameServers := []nbdns.NameServer{
+		{
+			IP:     netip.MustParseAddr("8.8.8.8"),
+			NSType: nbdns.UDPNameServerType,
+			Port:   53,
+		},
+		{
+			IP:     netip.MustParseAddr("8.8.4.4"),
+			NSType: nbdns.UDPNameServerType,
+			Port:   53,
+		},
+	}
+
+	update := nbdns.Config{
+		ServiceEnable: true,
+		CustomZones: []nbdns.CustomZone{
+			{
+				Domain:  "netbird.cloud",
+				Records: zoneRecords,
+			},
+		},
+		NameServerGroups: []*nbdns.NameServerGroup{
+			{
+				Domains:     []string{"netbird.io"},
+				NameServers: nameServers,
+			},
+			{
+				NameServers: nameServers,
+				Primary:     true,
+			},
+		},
+	}
+
+	// Start the server with regular configuration
+	if err := dnsServer.UpdateDNSServer(1, update); err != nil {
+		t.Fatalf("update dns server should not fail, got error: %v", err)
+		return
+	}
+
+	update2 := update
+	update2.ServiceEnable = false
+	// Disable the server, stop the listener
+	if err := dnsServer.UpdateDNSServer(2, update2); err != nil {
+		t.Fatalf("update dns server should not fail, got error: %v", err)
+		return
+	}
+
+	update3 := update2
+	update3.NameServerGroups = update3.NameServerGroups[:1]
+	// But service still get updates and we checking that we handle
+	// internal state in the right way
+	if err := dnsServer.UpdateDNSServer(3, update3); err != nil {
+		t.Fatalf("update dns server should not fail, got error: %v", err)
+		return
+	}
+}
+
 func TestDNSServerStartStop(t *testing.T) {
 	testCases := []struct {
 		name     string
@@ -291,7 +424,7 @@ func TestDNSServerStartStop(t *testing.T) {
 			dnsServer := getDefaultServerWithNoHostManager(t, testCase.addrPort)
 
 			dnsServer.hostManager = newNoopHostMocker()
-			dnsServer.Start()
+			dnsServer.listen()
 			time.Sleep(100 * time.Millisecond)
 			if !dnsServer.listenerIsRunning {
 				t.Fatal("dns server listener is not running")

client/internal/engine.go

@@ -259,6 +259,12 @@ func (e *Engine) Start() error {
 		e.acl = acl
 	}
 
+	err = e.dnsServer.Initialize()
+	if err != nil {
+		e.close()
+		return err
+	}
+
 	e.receiveSignalEvents()
 	e.receiveManagementEvents()
 
@@ -599,6 +605,7 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error {
 	// cleanup request, most likely our peer has been deleted
 	if networkMap.GetRemotePeersIsEmpty() {
 		err := e.removeAllPeers()
+		e.statusRecorder.FinishPeerListModifications()
 		if err != nil {
 			return err
 		}
@@ -618,6 +625,8 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error {
 			return err
 		}
 
+		e.statusRecorder.FinishPeerListModifications()
+
 		// update SSHServer by adding remote peer SSH keys
 		if !isNil(e.sshServer) {
 			for _, config := range networkMap.GetRemotePeers() {
@@ -753,17 +762,13 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error {
 		}
 		e.peerConns[peerKey] = conn
 
-		err = e.statusRecorder.AddPeer(peerKey)
+		err = e.statusRecorder.AddPeer(peerKey, peerConfig.Fqdn)
 		if err != nil {
 			log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err)
 		}
 
 		go e.connWorker(conn, peerKey)
 	}
-	err := e.statusRecorder.UpdatePeerFQDN(peerKey, peerConfig.Fqdn)
-	if err != nil {
-		log.Warnf("error updating peer's %s fqdn in the status recorder, got error: %v", peerKey, err)
-	}
 	return nil
 }
 

client/internal/peer/status.go

@@ -59,6 +59,11 @@ type Status struct {
 	mgmAddress      string
 	signalAddress   string
 	notifier        *notifier
+
+	// To reduce the number of notification invocation this bool will be true when need to call the notification
+	// Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events
+	// set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications()
+	peerListChangedForNotification bool
 }
 
 // NewRecorder returns a new Status instance
@@ -78,11 +83,13 @@ func (d *Status) ReplaceOfflinePeers(replacement []State) {
 	defer d.mux.Unlock()
 	d.offlinePeers = make([]State, len(replacement))
 	copy(d.offlinePeers, replacement)
-	d.notifyPeerListChanged()
+
+	// todo we should set to true in case if the list changed only
+	d.peerListChangedForNotification = true
 }
 
 // AddPeer adds peer to Daemon status map
-func (d *Status) AddPeer(peerPubKey string) error {
+func (d *Status) AddPeer(peerPubKey string, fqdn string) error {
 	d.mux.Lock()
 	defer d.mux.Unlock()
 
@@ -90,7 +97,12 @@ func (d *Status) AddPeer(peerPubKey string) error {
 	if ok {
 		return errors.New("peer already exist")
 	}
-	d.peers[peerPubKey] = State{PubKey: peerPubKey, ConnStatus: StatusDisconnected}
+	d.peers[peerPubKey] = State{
+		PubKey:     peerPubKey,
+		ConnStatus: StatusDisconnected,
+		FQDN:       fqdn,
+	}
+	d.peerListChangedForNotification = true
 	return nil
 }
 
@@ -112,13 +124,13 @@ func (d *Status) RemovePeer(peerPubKey string) error {
 	defer d.mux.Unlock()
 
 	_, ok := d.peers[peerPubKey]
-	if ok {
-		delete(d.peers, peerPubKey)
-		return nil
+	if !ok {
+		return errors.New("no peer with to remove")
 	}
 
-	d.notifyPeerListChanged()
-	return errors.New("no peer with to remove")
+	delete(d.peers, peerPubKey)
+	d.peerListChangedForNotification = true
+	return nil
 }
 
 // UpdatePeerState updates peer status
@@ -135,6 +147,8 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		peerState.IP = receivedState.IP
 	}
 
+	skipNotification := shouldSkipNotify(receivedState, peerState)
+
 	if receivedState.ConnStatus != peerState.ConnStatus {
 		peerState.ConnStatus = receivedState.ConnStatus
 		peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
@@ -146,7 +160,7 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 
 	d.peers[receivedState.PubKey] = peerState
 
-	if shouldSkipNotify(receivedState, peerState) {
+	if skipNotification {
 		return nil
 	}
 
@@ -186,10 +200,23 @@ func (d *Status) UpdatePeerFQDN(peerPubKey, fqdn string) error {
 	peerState.FQDN = fqdn
 	d.peers[peerPubKey] = peerState
 
-	d.notifyPeerListChanged()
 	return nil
 }
 
+// FinishPeerListModifications this event invoke the notification
+func (d *Status) FinishPeerListModifications() {
+	d.mux.Lock()
+
+	if !d.peerListChangedForNotification {
+		d.mux.Unlock()
+		return
+	}
+	d.peerListChangedForNotification = false
+	d.mux.Unlock()
+
+	d.notifyPeerListChanged()
+}
+
 // GetPeerStateChangeNotifier returns a change notifier channel for a peer
 func (d *Status) GetPeerStateChangeNotifier(peer string) <-chan struct{} {
 	d.mux.Lock()

client/internal/peer/status_test.go

@@ -9,13 +9,13 @@ import (
 func TestAddPeer(t *testing.T) {
 	key := "abc"
 	status := NewRecorder("https://mgm")
-	err := status.AddPeer(key)
+	err := status.AddPeer(key, "abc.netbird")
 	assert.NoError(t, err, "shouldn't return error")
 
 	_, exists := status.peers[key]
 	assert.True(t, exists, "value was found")
 
-	err = status.AddPeer(key)
+	err = status.AddPeer(key, "abc.netbird")
 
 	assert.Error(t, err, "should return error on duplicate")
 }
@@ -23,7 +23,7 @@ func TestAddPeer(t *testing.T) {
 func TestGetPeer(t *testing.T) {
 	key := "abc"
 	status := NewRecorder("https://mgm")
-	err := status.AddPeer(key)
+	err := status.AddPeer(key, "abc.netbird")
 	assert.NoError(t, err, "shouldn't return error")
 
 	peerStatus, err := status.GetPeer(key)

client/internal/routemanager/systemops_bsd.go

@@ -0,0 +1,82 @@
+//go:build darwin || dragonfly || freebsd || netbsd || openbsd
+// +build darwin dragonfly freebsd netbsd openbsd
+
+package routemanager
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"syscall"
+
+	"golang.org/x/net/route"
+)
+
+// selected BSD Route flags.
+const (
+	RTF_UP        = 0x1
+	RTF_GATEWAY   = 0x2
+	RTF_HOST      = 0x4
+	RTF_REJECT    = 0x8
+	RTF_DYNAMIC   = 0x10
+	RTF_MODIFIED  = 0x20
+	RTF_STATIC    = 0x800
+	RTF_BLACKHOLE = 0x1000
+	RTF_LOCAL     = 0x200000
+	RTF_BROADCAST = 0x400000
+	RTF_MULTICAST = 0x800000
+)
+
+func existsInRouteTable(prefix netip.Prefix) (bool, error) {
+	tab, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0)
+	if err != nil {
+		return false, err
+	}
+	msgs, err := route.ParseRIB(route.RIBTypeRoute, tab)
+	if err != nil {
+		return false, err
+	}
+
+	for _, msg := range msgs {
+		m := msg.(*route.RouteMessage)
+
+		if m.Version < 3 || m.Version > 5 {
+			return false, fmt.Errorf("unexpected RIB message version: %d", m.Version)
+		}
+		if m.Type != 4 /* RTM_GET */ {
+			return true, fmt.Errorf("unexpected RIB message type: %d", m.Type)
+		}
+
+		if m.Flags&RTF_UP == 0 ||
+			m.Flags&(RTF_REJECT|RTF_BLACKHOLE) != 0 {
+			continue
+		}
+
+		dst, err := toIPAddr(m.Addrs[0])
+		if err != nil {
+			return true, fmt.Errorf("unexpected RIB destination: %v", err)
+		}
+
+		mask, _ := toIPAddr(m.Addrs[2])
+		cidr, _ := net.IPMask(mask.To4()).Size()
+		if dst.String() == prefix.Addr().String() && cidr == prefix.Bits() {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func toIPAddr(a route.Addr) (net.IP, error) {
+	switch t := a.(type) {
+	case *route.Inet4Addr:
+		ip := net.IPv4(t.IP[0], t.IP[1], t.IP[2], t.IP[3])
+		return ip, nil
+	case *route.Inet6Addr:
+		ip := make(net.IP, net.IPv6len)
+		copy(ip, t.IP[:])
+		return ip, nil
+	default:
+		return net.IP{}, fmt.Errorf("unknown family: %v", t)
+	}
+}

client/internal/routemanager/systemops_linux.go

@@ -6,10 +6,28 @@ import (
 	"net"
 	"net/netip"
 	"os"
+	"syscall"
+	"unsafe"
 
 	"github.com/vishvananda/netlink"
 )
 
+// Pulled from http://man7.org/linux/man-pages/man7/rtnetlink.7.html
+// See the section on RTM_NEWROUTE, specifically 'struct rtmsg'.
+type routeInfoInMemory struct {
+	Family byte
+	DstLen byte
+	SrcLen byte
+	TOS    byte
+
+	Table    byte
+	Protocol byte
+	Scope    byte
+	Type     byte
+
+	Flags uint32
+}
+
 const ipv4ForwardingPath = "/proc/sys/net/ipv4/ip_forward"
 
 func addToRouteTable(prefix netip.Prefix, addr string) error {
@@ -61,6 +79,45 @@ func removeFromRouteTable(prefix netip.Prefix) error {
 	return nil
 }
 
+func existsInRouteTable(prefix netip.Prefix) (bool, error) {
+	tab, err := syscall.NetlinkRIB(syscall.RTM_GETROUTE, syscall.AF_UNSPEC)
+	if err != nil {
+		return true, err
+	}
+	msgs, err := syscall.ParseNetlinkMessage(tab)
+	if err != nil {
+		return true, err
+	}
+loop:
+	for _, m := range msgs {
+		switch m.Header.Type {
+		case syscall.NLMSG_DONE:
+			break loop
+		case syscall.RTM_NEWROUTE:
+			rt := (*routeInfoInMemory)(unsafe.Pointer(&m.Data[0]))
+			attrs, err := syscall.ParseNetlinkRouteAttr(&m)
+			if err != nil {
+				return true, err
+			}
+			if rt.Family != syscall.AF_INET {
+				continue loop
+			}
+
+			for _, attr := range attrs {
+				if attr.Attr.Type == syscall.RTA_DST {
+					ip := net.IP(attr.Value)
+					mask := net.CIDRMask(int(rt.DstLen), len(attr.Value)*8)
+					cidr, _ := mask.Size()
+					if ip.String() == prefix.Addr().String() && cidr == prefix.Bits() {
+						return true, nil
+					}
+				}
+			}
+		}
+	}
+	return false, nil
+}
+
 func enableIPForwarding() error {
 	bytes, err := os.ReadFile(ipv4ForwardingPath)
 	if err != nil {

client/internal/routemanager/systemops_nonandroid.go

@@ -14,19 +14,26 @@ import (
 var errRouteNotFound = fmt.Errorf("route not found")
 
 func addToRouteTableIfNoExists(prefix netip.Prefix, addr string) error {
-	gateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0"))
-	if err != nil && err != errRouteNotFound {
-		return err
-	}
-	prefixGateway, err := getExistingRIBRouteGateway(prefix)
+	defaultGateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0"))
 	if err != nil && err != errRouteNotFound {
 		return err
 	}
 
-	if prefixGateway != nil && !prefixGateway.Equal(gateway) {
-		log.Warnf("skipping adding a new route for network %s because it already exists and is pointing to the non default gateway: %s", prefix, prefixGateway)
+	gatewayIP := netip.MustParseAddr(defaultGateway.String())
+	if prefix.Contains(gatewayIP) {
+		log.Warnf("skipping adding a new route for network %s because it overlaps with the default gateway: %s", prefix, gatewayIP)
 		return nil
 	}
+
+	ok, err := existsInRouteTable(prefix)
+	if err != nil {
+		return err
+	}
+	if ok {
+		log.Warnf("skipping adding a new route for network %s because it already exists", prefix)
+		return nil
+	}
+
 	return addToRouteTable(prefix, addr)
 }
 
@@ -53,6 +60,7 @@ func getExistingRIBRouteGateway(prefix netip.Prefix) (net.IP, error) {
 		log.Errorf("getting routes returned an error: %v", err)
 		return nil, errRouteNotFound
 	}
+
 	if gateway == nil {
 		return preferredSrc, nil
 	}

client/internal/routemanager/systemops_nonandroid_test.go

@@ -1,13 +1,19 @@
 package routemanager
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/netbirdio/netbird/iface"
-	"github.com/pion/transport/v2/stdnet"
-	"github.com/stretchr/testify/require"
 	"net"
 	"net/netip"
+	"os"
+	"strings"
 	"testing"
+
+	"github.com/pion/transport/v2/stdnet"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 func TestAddRemoveRoutes(t *testing.T) {
@@ -114,3 +120,98 @@ func TestGetExistingRIBRouteGateway(t *testing.T) {
 		t.Fatalf("local ip should match with testing IP: want %s got %s", testingIP, localIP.String())
 	}
 }
+
+func TestAddExistAndRemoveRouteNonAndroid(t *testing.T) {
+	defaultGateway, err := getExistingRIBRouteGateway(netip.MustParsePrefix("0.0.0.0/0"))
+	fmt.Println("defaultGateway: ", defaultGateway)
+	if err != nil {
+		t.Fatal("shouldn't return error when fetching the gateway: ", err)
+	}
+	testCases := []struct {
+		name              string
+		prefix            netip.Prefix
+		preExistingPrefix netip.Prefix
+		shouldAddRoute    bool
+	}{
+		{
+			name:           "Should Add And Remove random Route",
+			prefix:         netip.MustParsePrefix("99.99.99.99/32"),
+			shouldAddRoute: true,
+		},
+		{
+			name:           "Should Not Add Route if overlaps with default gateway",
+			prefix:         netip.MustParsePrefix(defaultGateway.String() + "/31"),
+			shouldAddRoute: false,
+		},
+		{
+			name:              "Should Add Route if bigger network exists",
+			prefix:            netip.MustParsePrefix("100.100.100.0/24"),
+			preExistingPrefix: netip.MustParsePrefix("100.100.0.0/16"),
+			shouldAddRoute:    true,
+		},
+		{
+			name:              "Should Add Route if smaller network exists",
+			prefix:            netip.MustParsePrefix("100.100.0.0/16"),
+			preExistingPrefix: netip.MustParsePrefix("100.100.100.0/24"),
+			shouldAddRoute:    true,
+		},
+		{
+			name:              "Should Not Add Route if same network exists",
+			prefix:            netip.MustParsePrefix("100.100.0.0/16"),
+			preExistingPrefix: netip.MustParsePrefix("100.100.0.0/16"),
+			shouldAddRoute:    false,
+		},
+	}
+
+	for n, testCase := range testCases {
+		var buf bytes.Buffer
+		log.SetOutput(&buf)
+		defer func() {
+			log.SetOutput(os.Stderr)
+		}()
+		t.Run(testCase.name, func(t *testing.T) {
+			newNet, err := stdnet.NewNet()
+			if err != nil {
+				t.Fatal(err)
+			}
+			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", iface.DefaultMTU, nil, newNet)
+			require.NoError(t, err, "should create testing WGIface interface")
+			defer wgInterface.Close()
+
+			err = wgInterface.Create()
+			require.NoError(t, err, "should create testing wireguard interface")
+
+			MockAddr := wgInterface.Address().IP.String()
+
+			// Prepare the environment
+			if testCase.preExistingPrefix.IsValid() {
+				err := addToRouteTableIfNoExists(testCase.preExistingPrefix, MockAddr)
+				require.NoError(t, err, "should not return err when adding pre-existing route")
+			}
+
+			// Add the route
+			err = addToRouteTableIfNoExists(testCase.prefix, MockAddr)
+			require.NoError(t, err, "should not return err when adding route")
+
+			if testCase.shouldAddRoute {
+				// test if route exists after adding
+				ok, err := existsInRouteTable(testCase.prefix)
+				require.NoError(t, err, "should not return err")
+				require.True(t, ok, "route should exist")
+
+				// remove route again if added
+				err = removeFromRouteTableIfNonSystem(testCase.prefix, MockAddr)
+				require.NoError(t, err, "should not return err")
+			}
+
+			// route should either not have been added or should have been removed
+			// In case of already existing route, it should not have been added (but still exist)
+			ok, err := existsInRouteTable(testCase.prefix)
+			fmt.Println("Buffer string: ", buf.String())
+			require.NoError(t, err, "should not return err")
+			if !strings.Contains(buf.String(), "because it already exists") {
+				require.False(t, ok, "route should not exist")
+			}
+		})
+	}
+}

client/internal/routemanager/systemops_windows.go

@@ -0,0 +1,37 @@
+//go:build windows
+// +build windows
+
+package routemanager
+
+import (
+	"net"
+	"net/netip"
+
+	"github.com/yusufpapurcu/wmi"
+)
+
+type Win32_IP4RouteTable struct {
+	Destination string
+	Mask        string
+}
+
+func existsInRouteTable(prefix netip.Prefix) (bool, error) {
+	var routes []Win32_IP4RouteTable
+	query := "SELECT Destination, Mask FROM Win32_IP4RouteTable"
+
+	err := wmi.Query(query, &routes)
+	if err != nil {
+		return true, err
+	}
+
+	for _, route := range routes {
+		ip := net.ParseIP(route.Mask)
+		ip = ip.To4()
+		mask := net.IPv4Mask(ip[0], ip[1], ip[2], ip[3])
+		cidr, _ := mask.Size()
+		if route.Destination == prefix.Addr().String() && cidr == prefix.Bits() {
+			return true, nil
+		}
+	}
+	return false, nil
+}

client/ssh/server.go

@@ -2,9 +2,6 @@ package ssh
 
 import (
 	"fmt"
-	"github.com/creack/pty"
-	"github.com/gliderlabs/ssh"
-	log "github.com/sirupsen/logrus"
 	"io"
 	"net"
 	"os"
@@ -13,11 +10,22 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"time"
+
+	"github.com/creack/pty"
+	"github.com/gliderlabs/ssh"
+	log "github.com/sirupsen/logrus"
 )
 
 // DefaultSSHPort is the default SSH port of the NetBird's embedded SSH server
 const DefaultSSHPort = 44338
 
+// TerminalTimeout is the timeout for terminal session to be ready
+const TerminalTimeout = 10 * time.Second
+
+// TerminalBackoffDelay is the delay between terminal session readiness checks
+const TerminalBackoffDelay = 500 * time.Millisecond
+
 // DefaultSSHServer is a function that creates DefaultServer
 func DefaultSSHServer(hostKeyPEM []byte, addr string) (Server, error) {
 	return newDefaultServer(hostKeyPEM, addr)
@@ -137,6 +145,8 @@ func (srv *DefaultServer) sessionHandler(session ssh.Session) {
 		}
 	}()
 
+	log.Infof("Establishing SSH session for %s from host %s", session.User(), session.RemoteAddr().String())
+
 	localUser, err := userNameLookup(session.User())
 	if err != nil {
 		_, err = fmt.Fprintf(session, "remote SSH server couldn't find local user %s\n", session.User()) //nolint
@@ -172,6 +182,7 @@ func (srv *DefaultServer) sessionHandler(session ssh.Session) {
 			}
 		}
 
+		log.Debugf("Login command: %s", cmd.String())
 		file, err := pty.Start(cmd)
 		if err != nil {
 			log.Errorf("failed starting SSH server %v", err)
@@ -199,6 +210,7 @@ func (srv *DefaultServer) sessionHandler(session ssh.Session) {
 			return
 		}
 	}
+	log.Debugf("SSH session ended")
 }
 
 func (srv *DefaultServer) stdInOut(file *os.File, session ssh.Session) {
@@ -206,17 +218,29 @@ func (srv *DefaultServer) stdInOut(file *os.File, session ssh.Session) {
 		// stdin
 		_, err := io.Copy(file, session)
 		if err != nil {
+			_ = session.Exit(1)
 			return
 		}
 	}()
 
-	go func() {
-		// stdout
-		_, err := io.Copy(session, file)
-		if err != nil {
+	// AWS Linux 2 machines need some time to open the terminal so we need to wait for it
+	timer := time.NewTimer(TerminalTimeout)
+	for {
+		select {
+		case <-timer.C:
+			_, _ = session.Write([]byte("Reached timeout while opening connection\n"))
+			_ = session.Exit(1)
 			return
+		default:
+			// stdout
+			writtenBytes, err := io.Copy(session, file)
+			if err != nil && writtenBytes != 0 {
+				_ = session.Exit(0)
+				return
+			}
+			time.Sleep(TerminalBackoffDelay)
 		}
-	}()
+	}
 }
 
 // Start starts SSH server. Blocking

go.mod

@@ -17,12 +17,12 @@ require (
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/vishvananda/netlink v1.1.0
-	golang.org/x/crypto v0.7.0
+	golang.org/x/crypto v0.9.0
 	golang.org/x/sys v0.8.0
 	golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211215182854-7a385b3431de
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/grpc v1.52.3
+	google.golang.org/grpc v1.55.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
@@ -57,6 +57,7 @@ require (
 	github.com/rs/xid v1.3.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/stretchr/testify v1.8.1
+	github.com/yusufpapurcu/wmi v1.2.3
 	go.opentelemetry.io/otel v1.11.1
 	go.opentelemetry.io/otel/exporters/prometheus v0.33.0
 	go.opentelemetry.io/otel/metric v0.33.0
@@ -65,18 +66,22 @@ require (
 	golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf
 	golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028
 	golang.org/x/net v0.10.0
-	golang.org/x/sync v0.1.0
+	golang.org/x/oauth2 v0.8.0
+	golang.org/x/sync v0.2.0
 	golang.org/x/term v0.8.0
+	google.golang.org/api v0.126.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	cloud.google.com/go/compute v1.19.3 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -92,9 +97,14 @@ require (
 	github.com/go-gl/glfw/v3.3/glfw v0.0.0-20211024062804-40e447a793be // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.10.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/native v1.0.0 // indirect
@@ -120,17 +130,17 @@ require (
 	github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9 // indirect
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
 	github.com/yuin/goldmark v1.4.13 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.11.1 // indirect
 	go.opentelemetry.io/otel/trace v1.11.1 // indirect
 	golang.org/x/image v0.5.0 // indirect
 	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect

go.sum

@@ -20,7 +20,11 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
+cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@@ -87,8 +91,9 @@ github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -101,6 +106,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -163,6 +169,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -219,6 +226,7 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -250,12 +258,14 @@ github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff/go.mod h1:wfqRWLHRBs
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -321,13 +331,19 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
+github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
 github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
+github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gax-go/v2 v2.10.0 h1:ebSgKfMxynOdxw8QQuFOKMgomqeLGPqNLQox2bo42zg=
+github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
@@ -661,6 +677,8 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
+github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -668,6 +686,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4=
 go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE=
 go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60=
@@ -697,10 +717,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -831,8 +852,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -949,6 +971,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
@@ -1041,6 +1064,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o=
+google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1082,8 +1107,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c=
-google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
+google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1102,9 +1129,10 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.51.0-dev/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ=
-google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
+google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
+google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

iface/tun_windows.go

@@ -56,7 +56,7 @@ func (c *tunDevice) createWithUserspace() (NetInterface, error) {
 	c.wrapper = newDeviceWrapper(tunIface)
 
 	// We need to create a wireguard-go device and listen to configuration requests
-	tunDev := device.NewDevice(tunIface, c.iceBind, device.NewLogger(device.LogLevelSilent, "[netbird] "))
+	tunDev := device.NewDevice(c.wrapper, c.iceBind, device.NewLogger(device.LogLevelSilent, "[netbird] "))
 	err = tunDev.Up()
 	if err != nil {
 		_ = tunIface.Close()

infrastructure_files/configure.sh

@@ -97,16 +97,9 @@ curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o openid-configuration.js
 
 export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' openid-configuration.json)
 export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' openid-configuration.json)
-export NETBIRD_AUTH_SUPPORTED_SCOPES=$(jq -r '.scopes_supported | join(" ")' openid-configuration.json)
 export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' openid-configuration.json)
 export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' openid-configuration.json)
 
-if [ "$NETBIRD_USE_AUTH0" == "true" ]; then
-  export NETBIRD_AUTH_SUPPORTED_SCOPES="openid profile email offline_access api email_verified"
-else
-  export NETBIRD_AUTH_SUPPORTED_SCOPES="openid profile email offline_access api"
-fi
-
 if [[ ! -z "${NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID}" ]]; then
   # user enabled Device Authorization Grant feature
   export NETBIRD_AUTH_DEVICE_AUTH_PROVIDER="hosted"

infrastructure_files/setup.env.example

@@ -11,7 +11,10 @@ NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT=""
 NETBIRD_AUTH_AUDIENCE=""
 # e.g. netbird-client
 NETBIRD_AUTH_CLIENT_ID=""
-NETBIRD_AUTH_CLIENT_SECRET=""
+# indicates the scopes that will be requested to the IDP
+NETBIRD_AUTH_SUPPORTED_SCOPES=""
+# NETBIRD_AUTH_CLIENT_SECRET is required only by Google workspace.
+# NETBIRD_AUTH_CLIENT_SECRET=""
 # if you want to use a custom claim for the user ID instead of 'sub', set it here
 # NETBIRD_AUTH_USER_ID_CLAIM=""
 # indicates whether to use Auth0 or not: true or false

infrastructure_files/tests/setup.env

@@ -6,6 +6,7 @@ NETBIRD_DOMAIN=$CI_NETBIRD_DOMAIN
 NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT="https://example.eu.auth0.com/.well-known/openid-configuration"
 # e.g. netbird-client
 NETBIRD_AUTH_CLIENT_ID=$CI_NETBIRD_AUTH_CLIENT_ID
+NETBIRD_AUTH_SUPPORTED_SCOPES=$CI_NETBIRD_AUTH_SUPPORTED_SCOPES
 NETBIRD_AUTH_CLIENT_SECRET=$CI_NETBIRD_AUTH_CLIENT_SECRET
 # indicates whether to use Auth0 or not: true or false
 NETBIRD_USE_AUTH0=$CI_NETBIRD_USE_AUTH0

management/server/account.go

@@ -34,6 +34,8 @@ const (
 	PublicCategory             = "public"
 	PrivateCategory            = "private"
 	UnknownCategory            = "unknown"
+	GroupIssuedAPI             = "api"
+	GroupIssuedJWT             = "jwt"
 	CacheExpirationMax         = 7 * 24 * 3600 * time.Second // 7 days
 	CacheExpirationMin         = 3 * 24 * 3600 * time.Second // 3 days
 	DefaultPeerLoginExpiration = 24 * time.Hour
@@ -139,6 +141,13 @@ type Settings struct {
 	// PeerLoginExpiration is a setting that indicates when peer login expires.
 	// Applies to all peers that have Peer.LoginExpirationEnabled set to true.
 	PeerLoginExpiration time.Duration
+
+	// JWTGroupsEnabled allows extract groups from JWT claim, which name defined in the JWTGroupsClaimName
+	// and add it to account groups.
+	JWTGroupsEnabled bool
+
+	// JWTGroupsClaimName from which we extract groups name to add it to account groups
+	JWTGroupsClaimName string
 }
 
 // Copy copies the Settings struct
@@ -146,6 +155,8 @@ func (s *Settings) Copy() *Settings {
 	return &Settings{
 		PeerLoginExpirationEnabled: s.PeerLoginExpirationEnabled,
 		PeerLoginExpiration:        s.PeerLoginExpiration,
+		JWTGroupsEnabled:           s.JWTGroupsEnabled,
+		JWTGroupsClaimName:         s.JWTGroupsClaimName,
 	}
 }
 
@@ -612,6 +623,28 @@ func (a *Account) GetPeer(peerID string) *Peer {
 	return a.Peers[peerID]
 }
 
+// AddJWTGroups to existed groups if they does not exists
+func (a *Account) AddJWTGroups(groups []string) (int, error) {
+	existedGroups := make(map[string]*Group)
+	for _, g := range a.Groups {
+		existedGroups[g.Name] = g
+	}
+
+	var count int
+	for _, name := range groups {
+		if _, ok := existedGroups[name]; !ok {
+			id := xid.New().String()
+			a.Groups[id] = &Group{
+				ID:     id,
+				Name:   name,
+				Issued: GroupIssuedJWT,
+			}
+			count++
+		}
+	}
+	return count, nil
+}
+
 // BuildManager creates a new DefaultAccountManager with a provided Store
 func BuildManager(store Store, peersUpdateManager *PeersUpdateManager, idpManager idp.Manager,
 	singleAccountModeDomain string, dnsDomain string, eventStore activity.Store,
@@ -1241,6 +1274,38 @@ func (am *DefaultAccountManager) GetAccountFromToken(claims jwtclaims.Authorizat
 		}
 	}
 
+	if account.Settings.JWTGroupsEnabled {
+		if account.Settings.JWTGroupsClaimName == "" {
+			log.Errorf("JWT groups are enabled but no claim name is set")
+			return account, user, nil
+		}
+		if claim, ok := claims.Raw[account.Settings.JWTGroupsClaimName]; ok {
+			if slice, ok := claim.([]interface{}); ok {
+				var groups []string
+				for _, item := range slice {
+					if g, ok := item.(string); ok {
+						groups = append(groups, g)
+					} else {
+						log.Errorf("JWT claim %q is not a string: %v", account.Settings.JWTGroupsClaimName, item)
+					}
+				}
+				n, err := account.AddJWTGroups(groups)
+				if err != nil {
+					log.Errorf("failed to add JWT groups: %v", err)
+				}
+				if n > 0 {
+					if err := am.Store.SaveAccount(account); err != nil {
+						log.Errorf("failed to save account: %v", err)
+					}
+				}
+			} else {
+				log.Debugf("JWT claim %q is not a string array", account.Settings.JWTGroupsClaimName)
+			}
+		} else {
+			log.Debugf("JWT claim %q not found", account.Settings.JWTGroupsClaimName)
+		}
+	}
+
 	return account, user, nil
 }
 
@@ -1344,8 +1409,9 @@ func (am *DefaultAccountManager) GetDNSDomain() string {
 func addAllGroup(account *Account) error {
 	if len(account.Groups) == 0 {
 		allGroup := &Group{
-			ID:   xid.New().String(),
-			Name: "All",
+			ID:     xid.New().String(),
+			Name:   "All",
+			Issued: GroupIssuedAPI,
 		}
 		for _, peer := range account.Peers {
 			allGroup.Peers = append(allGroup.Peers, peer.ID)
@@ -1373,33 +1439,28 @@ func addAllGroup(account *Account) error {
 }
 
 // newAccountWithId creates a new Account with a default SetupKey (doesn't store in a Store) and provided id
-func newAccountWithId(accountId, userId, domain string) *Account {
+func newAccountWithId(accountID, userID, domain string) *Account {
 	log.Debugf("creating new account")
 
-	setupKeys := make(map[string]*SetupKey)
-	defaultKey := GenerateDefaultSetupKey()
-	oneOffKey := GenerateSetupKey("One-off key", SetupKeyOneOff, DefaultSetupKeyDuration, []string{},
-		SetupKeyUnlimitedUsage)
-	setupKeys[defaultKey.Key] = defaultKey
-	setupKeys[oneOffKey.Key] = oneOffKey
 	network := NewNetwork()
 	peers := make(map[string]*Peer)
 	users := make(map[string]*User)
 	routes := make(map[string]*route.Route)
+	setupKeys := map[string]*SetupKey{}
 	nameServersGroups := make(map[string]*nbdns.NameServerGroup)
-	users[userId] = NewAdminUser(userId)
+	users[userID] = NewAdminUser(userID)
 	dnsSettings := &DNSSettings{
 		DisabledManagementGroups: make([]string, 0),
 	}
-	log.Debugf("created new account %s with setup key %s", accountId, defaultKey.Key)
+	log.Debugf("created new account %s", accountID)
 
 	acc := &Account{
-		Id:               accountId,
+		Id:               accountID,
 		SetupKeys:        setupKeys,
 		Network:          network,
 		Peers:            peers,
 		Users:            users,
-		CreatedBy:        userId,
+		CreatedBy:        userID,
 		Domain:           domain,
 		Routes:           routes,
 		NameServerGroups: nameServersGroups,

management/server/account_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang-jwt/jwt"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/route"
@@ -53,7 +54,7 @@ func verifyNewAccountHasDefaultFields(t *testing.T, account *Account, createdBy
 		t.Errorf("expected account to have len(Peers) = %v, got %v", 0, len(account.Peers))
 	}
 
-	if len(account.SetupKeys) != 2 {
+	if len(account.SetupKeys) != 0 {
 		t.Errorf("expected account to have len(SetupKeys) = %v, got %v", 2, len(account.SetupKeys))
 	}
 
@@ -460,6 +461,69 @@ func TestDefaultAccountManager_GetAccountFromToken(t *testing.T) {
 	}
 }
 
+func TestDefaultAccountManager_GetGroupsFromTheToken(t *testing.T) {
+	userId := "user-id"
+	domain := "test.domain"
+
+	initAccount := newAccountWithId("", userId, domain)
+	manager, err := createManager(t)
+	require.NoError(t, err, "unable to create account manager")
+
+	accountID := initAccount.Id
+	_, err = manager.GetAccountByUserOrAccountID(userId, accountID, domain)
+	require.NoError(t, err, "create init user failed")
+
+	claims := jwtclaims.AuthorizationClaims{
+		AccountId:      accountID,
+		Domain:         domain,
+		UserId:         userId,
+		DomainCategory: "test-category",
+		Raw:            jwt.MapClaims{"idp-groups": []interface{}{"group1", "group2"}},
+	}
+
+	t.Run("JWT groups disabled", func(t *testing.T) {
+		account, _, err := manager.GetAccountFromToken(claims)
+		require.NoError(t, err, "get account by token failed")
+		require.Len(t, account.Groups, 1, "only ALL group should exists")
+	})
+
+	t.Run("JWT groups enabled without claim name", func(t *testing.T) {
+		initAccount.Settings.JWTGroupsEnabled = true
+		err := manager.Store.SaveAccount(initAccount)
+		require.NoError(t, err, "save account failed")
+
+		account, _, err := manager.GetAccountFromToken(claims)
+		require.NoError(t, err, "get account by token failed")
+		require.Len(t, account.Groups, 1, "if group claim is not set no group added from JWT")
+	})
+
+	t.Run("JWT groups enabled", func(t *testing.T) {
+		initAccount.Settings.JWTGroupsEnabled = true
+		initAccount.Settings.JWTGroupsClaimName = "idp-groups"
+		err := manager.Store.SaveAccount(initAccount)
+		require.NoError(t, err, "save account failed")
+
+		account, _, err := manager.GetAccountFromToken(claims)
+		require.NoError(t, err, "get account by token failed")
+		require.Len(t, account.Groups, 3, "groups should be added to the account")
+
+		groupsByNames := map[string]*Group{}
+		for _, g := range account.Groups {
+			groupsByNames[g.Name] = g
+		}
+
+		g1, ok := groupsByNames["group1"]
+		require.True(t, ok, "group1 should be added to the account")
+		require.Equal(t, g1.Name, "group1", "group1 name should match")
+		require.Equal(t, g1.Issued, GroupIssuedJWT, "group1 issued should match")
+
+		g2, ok := groupsByNames["group2"]
+		require.True(t, ok, "group2 should be added to the account")
+		require.Equal(t, g2.Name, "group2", "group2 name should match")
+		require.Equal(t, g2.Issued, GroupIssuedJWT, "group2 issued should match")
+	})
+}
+
 func TestAccountManager_GetAccountFromPAT(t *testing.T) {
 	store := newStore(t)
 	account := newAccountWithId("account_id", "testuser", "")
@@ -704,20 +768,21 @@ func TestAccountManager_AddPeer(t *testing.T) {
 		return
 	}
 
-	account, err := createAccount(manager, "test_account", "account_creator", "netbird.cloud")
+	userID := "account_creator"
+	account, err := createAccount(manager, "test_account", userID, "netbird.cloud")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	serial := account.Network.CurrentSerial() // should be 0
 
-	var setupKey *SetupKey
-	for _, key := range account.SetupKeys {
-		setupKey = key
+	setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userID)
+	if err != nil {
+		return
 	}
 
-	if setupKey == nil {
-		t.Errorf("expecting account to have a default setup key")
+	if err != nil {
+		t.Fatal("error creating setup key")
 		return
 	}
 
@@ -858,16 +923,13 @@ func TestAccountManager_NetworkUpdates(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	var setupKey *SetupKey
-	for _, key := range account.SetupKeys {
-		setupKey = key
-		if setupKey.Type == SetupKeyReusable {
-			break
-		}
+	setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userID)
+	if err != nil {
+		return
 	}
 
-	if setupKey == nil {
-		t.Errorf("expecting account to have a default setup key")
+	if err != nil {
+		t.Fatal("error creating setup key")
 		return
 	}
 
@@ -1042,9 +1104,14 @@ func TestAccountManager_DeletePeer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	var setupKey *SetupKey
-	for _, key := range account.SetupKeys {
-		setupKey = key
+	setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userID)
+	if err != nil {
+		return
+	}
+
+	if err != nil {
+		t.Fatal("error creating setup key")
+		return
 	}
 
 	key, err := wgtypes.GenerateKey()

management/server/dns.go

@@ -2,13 +2,15 @@ package server
 
 import (
 	"fmt"
+	"strconv"
+
 	"github.com/miekg/dns"
+	log "github.com/sirupsen/logrus"
+
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/management/proto"
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/status"
-	log "github.com/sirupsen/logrus"
-	"strconv"
 )
 
 const defaultTTL = 300
@@ -199,8 +201,10 @@ func getPeerNSGroups(account *Account, peerID string) []*nbdns.NameServerGroup {
 		for _, gID := range nsGroup.Groups {
 			_, found := groupList[gID]
 			if found {
-				peerNSGroups = append(peerNSGroups, nsGroup.Copy())
-				break
+				if !peerIsNameserver(account.GetPeer(peerID), nsGroup) {
+					peerNSGroups = append(peerNSGroups, nsGroup.Copy())
+					break
+				}
 			}
 		}
 	}
@@ -208,6 +212,16 @@ func getPeerNSGroups(account *Account, peerID string) []*nbdns.NameServerGroup {
 	return peerNSGroups
 }
 
+// peerIsNameserver returns true if the peer is a nameserver for a nsGroup
+func peerIsNameserver(peer *Peer, nsGroup *nbdns.NameServerGroup) bool {
+	for _, ns := range nsGroup.NameServers {
+		if peer.IP.Equal(ns.IP.AsSlice()) {
+			return true
+		}
+	}
+	return false
+}
+
 func addPeerLabelsToAccount(account *Account, peerLabels lookupMap) {
 	for _, peer := range account.Peers {
 		label, err := getPeerHostLabel(peer.Name, peerLabels)

management/server/dns_test.go

@@ -1,10 +1,12 @@
 package server
 
 import (
+	"net/netip"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/status"
 )
@@ -17,6 +19,7 @@ const (
 	dnsAccountID     = "testingAcc"
 	dnsAdminUserID   = "testingAdminUser"
 	dnsRegularUserID = "testingRegularUser"
+	dnsNSGroup1      = "ns1"
 )
 
 func TestGetDNSSettings(t *testing.T) {
@@ -163,6 +166,7 @@ func TestGetNetworkMap_DNSConfigSync(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, newAccountDNSConfig.DNSConfig.CustomZones, 1, "default DNS config should have one custom zone for peers")
 	require.True(t, newAccountDNSConfig.DNSConfig.ServiceEnable, "default DNS config should have local DNS service enabled")
+	require.Len(t, newAccountDNSConfig.DNSConfig.NameServerGroups, 0, "updated DNS config should have no nameserver groups since peer 1 is NS for the only existing NS group")
 
 	dnsSettings := account.DNSSettings.Copy()
 	dnsSettings.DisabledManagementGroups = append(dnsSettings.DisabledManagementGroups, dnsGroup1ID)
@@ -174,11 +178,11 @@ func TestGetNetworkMap_DNSConfigSync(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, updatedAccountDNSConfig.DNSConfig.CustomZones, 0, "updated DNS config should have no custom zone when peer belongs to a disabled group")
 	require.False(t, updatedAccountDNSConfig.DNSConfig.ServiceEnable, "updated DNS config should have local DNS service disabled when peer belongs to a disabled group")
-
 	peer2AccountDNSConfig, err := am.GetNetworkMap(peer2.ID)
 	require.NoError(t, err)
 	require.Len(t, peer2AccountDNSConfig.DNSConfig.CustomZones, 1, "DNS config should have one custom zone for peers not in the disabled group")
 	require.True(t, peer2AccountDNSConfig.DNSConfig.ServiceEnable, "DNS config should have DNS service enabled for peers not in the disabled group")
+	require.Len(t, peer2AccountDNSConfig.DNSConfig.NameServerGroups, 1, "updated DNS config should have 1 nameserver groups since peer 2 is part of the group All")
 }
 
 func createDNSManager(t *testing.T) (*DefaultAccountManager, error) {
@@ -246,7 +250,7 @@ func initTestDNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, erro
 		return nil, err
 	}
 
-	_, _, err = am.AddPeer("", dnsAdminUserID, peer1)
+	savedPeer1, _, err := am.AddPeer("", dnsAdminUserID, peer1)
 	if err != nil {
 		return nil, err
 	}
@@ -284,6 +288,24 @@ func initTestDNSAccount(t *testing.T, am *DefaultAccountManager) (*Account, erro
 	account.Groups[newGroup1.ID] = newGroup1
 	account.Groups[newGroup2.ID] = newGroup2
 
+	allGroup, err := account.GetGroupAll()
+	if err != nil {
+		return nil, err
+	}
+
+	account.NameServerGroups[dnsNSGroup1] = &dns.NameServerGroup{
+		ID:   dnsNSGroup1,
+		Name: "ns-group-1",
+		NameServers: []dns.NameServer{{
+			IP:     netip.MustParseAddr(savedPeer1.IP.String()),
+			NSType: dns.UDPNameServerType,
+			Port:   dns.DefaultDNSPort,
+		}},
+		Primary: true,
+		Enabled: true,
+		Groups:  []string{allGroup.ID},
+	}
+
 	err = am.Store.SaveAccount(account)
 	if err != nil {
 		return nil, err

management/server/file_store.go

@@ -157,6 +157,14 @@ func restore(file string) (*FileStore, error) {
 			addPeerLabelsToAccount(account, existingLabels)
 		}
 
+		// TODO: delete this block after migration
+		// Set API as issuer for groups which has not this field
+		for _, group := range account.Groups {
+			if group.Issued == "" {
+				group.Issued = GroupIssuedAPI
+			}
+		}
+
 		allGroup, err := account.GetGroupAll()
 		if err != nil {
 			log.Errorf("unable to find the All group, this should happen only when migrate from a version that didn't support groups. Error: %v", err)

management/server/file_store_test.go

@@ -262,6 +262,7 @@ func TestRestore(t *testing.T) {
 	require.Len(t, store.TokenID2UserID, 1, "failed to restore a FileStore wrong TokenID2UserID mapping length")
 }
 
+// TODO: outdated, delete this
 func TestRestorePolicies_Migration(t *testing.T) {
 	storeDir := t.TempDir()
 
@@ -296,6 +297,40 @@ func TestRestorePolicies_Migration(t *testing.T) {
 		"failed to restore a FileStore file - missing Account Policies Sources")
 }
 
+func TestRestoreGroups_Migration(t *testing.T) {
+	storeDir := t.TempDir()
+
+	err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store, err := NewFileStore(storeDir, nil)
+	if err != nil {
+		return
+	}
+
+	// create default group
+	account := store.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"]
+	account.Groups = map[string]*Group{
+		"cfefqs706sqkneg59g3g": {
+			ID:   "cfefqs706sqkneg59g3g",
+			Name: "All",
+		},
+	}
+	err = store.SaveAccount(account)
+	require.NoError(t, err, "failed to save account")
+
+	// restore account with default group with empty Issue field
+	if store, err = NewFileStore(storeDir, nil); err != nil {
+		return
+	}
+	account = store.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"]
+
+	require.Contains(t, account.Groups, "cfefqs706sqkneg59g3g", "failed to restore a FileStore file - missing Account Groups")
+	require.Equal(t, GroupIssuedAPI, account.Groups["cfefqs706sqkneg59g3g"].Issued, "default group should has API issued mark")
+}
+
 func TestGetAccountByPrivateDomain(t *testing.T) {
 	storeDir := t.TempDir()
 

management/server/group.go

@@ -14,6 +14,9 @@ type Group struct {
 	// Name visible in the UI
 	Name string
 
+	// Issued of the group
+	Issued string
+
 	// Peers list of the group
 	Peers []string
 }
@@ -45,9 +48,10 @@ func (g *Group) EventMeta() map[string]any {
 
 func (g *Group) Copy() *Group {
 	return &Group{
-		ID:    g.ID,
-		Name:  g.Name,
-		Peers: g.Peers[:],
+		ID:     g.ID,
+		Name:   g.Name,
+		Issued: g.Issued,
+		Peers:  g.Peers[:],
 	}
 }
 

management/server/http/accounts_handler.go

@@ -72,10 +72,19 @@ func (h *AccountsHandler) UpdateAccount(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	updatedAccount, err := h.accountManager.UpdateAccountSettings(accountID, user.Id, &server.Settings{
+	settings := &server.Settings{
 		PeerLoginExpirationEnabled: req.Settings.PeerLoginExpirationEnabled,
 		PeerLoginExpiration:        time.Duration(float64(time.Second.Nanoseconds()) * float64(req.Settings.PeerLoginExpiration)),
-	})
+	}
+
+	if req.Settings.JwtGroupsEnabled != nil {
+		settings.JWTGroupsEnabled = *req.Settings.JwtGroupsEnabled
+	}
+	if req.Settings.JwtGroupsClaimName != nil {
+		settings.JWTGroupsClaimName = *req.Settings.JwtGroupsClaimName
+	}
+
+	updatedAccount, err := h.accountManager.UpdateAccountSettings(accountID, user.Id, settings)
 
 	if err != nil {
 		util.WriteError(err, w)
@@ -93,6 +102,8 @@ func toAccountResponse(account *server.Account) *api.Account {
 		Settings: api.AccountSettings{
 			PeerLoginExpiration:        int(account.Settings.PeerLoginExpiration.Seconds()),
 			PeerLoginExpirationEnabled: account.Settings.PeerLoginExpirationEnabled,
+			JwtGroupsEnabled:           &account.Settings.JWTGroupsEnabled,
+			JwtGroupsClaimName:         &account.Settings.JWTGroupsClaimName,
 		},
 	}
 }

management/server/http/accounts_handler_test.go

@@ -58,6 +58,9 @@ func TestAccounts_AccountsHandler(t *testing.T) {
 	accountID := "test_account"
 	adminUser := server.NewAdminUser("test_user")
 
+	sr := func(v string) *string { return &v }
+	br := func(v bool) *bool { return &v }
+
 	handler := initAccountsTestData(&server.Account{
 		Id:      accountID,
 		Domain:  "hotmail.com",
@@ -91,6 +94,8 @@ func TestAccounts_AccountsHandler(t *testing.T) {
 			expectedSettings: api.AccountSettings{
 				PeerLoginExpiration:        int(time.Hour.Seconds()),
 				PeerLoginExpirationEnabled: false,
+				JwtGroupsClaimName:         sr(""),
+				JwtGroupsEnabled:           br(false),
 			},
 			expectedArray: true,
 			expectedID:    accountID,
@@ -105,6 +110,24 @@ func TestAccounts_AccountsHandler(t *testing.T) {
 			expectedSettings: api.AccountSettings{
 				PeerLoginExpiration:        15552000,
 				PeerLoginExpirationEnabled: true,
+				JwtGroupsClaimName:         sr(""),
+				JwtGroupsEnabled:           br(false),
+			},
+			expectedArray: false,
+			expectedID:    accountID,
+		},
+		{
+			name:           "PutAccount OK wiht JWT",
+			expectedBody:   true,
+			requestType:    http.MethodPut,
+			requestPath:    "/api/accounts/" + accountID,
+			requestBody:    bytes.NewBufferString("{\"settings\": {\"peer_login_expiration\": 15552000,\"peer_login_expiration_enabled\": false,\"jwt_groups_enabled\":true,\"jwt_groups_claim_name\":\"roles\"}}"),
+			expectedStatus: http.StatusOK,
+			expectedSettings: api.AccountSettings{
+				PeerLoginExpiration:        15552000,
+				PeerLoginExpirationEnabled: false,
+				JwtGroupsClaimName:         sr("roles"),
+				JwtGroupsEnabled:           br(true),
 			},
 			expectedArray: false,
 			expectedID:    accountID,

management/server/http/api/openapi.yml

@@ -54,6 +54,14 @@ components:
           description: Period of time after which peer login expires (seconds).
           type: integer
           example: 43200
+        jwt_groups_enabled:
+          description: Allows extract groups from JWT claim and add it to account groups.
+          type: boolean
+          example: true
+        jwt_groups_claim_name:
+          description: Name of the claim from which we extract groups names to add it to account groups.
+          type: string
+          example: "roles"
       required:
         - peer_login_expiration_enabled
         - peer_login_expiration
@@ -462,6 +470,10 @@ components:
           description: Count of peers associated to the group
           type: integer
           example: 2
+        issued:
+          description: How group was issued by API or from JWT token
+          type: string
+          example: api
       required:
         - id
         - name
@@ -1102,8 +1114,8 @@ paths:
         '500':
           "$ref": "#/components/responses/internal_error"
     delete:
-      summary: Delete a User
-      description: Delete a user
+      summary: Block a User
+      description: This method blocks a user from accessing the system, but leaves the IDP user intact.
       tags: [ Users ]
       security:
         - BearerAuth: [ ]

management/server/http/api/types.gen.go

@@ -129,6 +129,12 @@ type AccountRequest struct {
 
 // AccountSettings defines model for AccountSettings.
 type AccountSettings struct {
+	// JwtGroupsClaimName Name of the claim from which we extract groups names to add it to account groups.
+	JwtGroupsClaimName *string `json:"jwt_groups_claim_name,omitempty"`
+
+	// JwtGroupsEnabled Allows extract groups from JWT claim and add it to account groups.
+	JwtGroupsEnabled *bool `json:"jwt_groups_enabled,omitempty"`
+
 	// PeerLoginExpiration Period of time after which peer login expires (seconds).
 	PeerLoginExpiration int `json:"peer_login_expiration"`
 
@@ -174,6 +180,9 @@ type Group struct {
 	// Id Group ID
 	Id string `json:"id"`
 
+	// Issued How group was issued by API or from JWT token
+	Issued *string `json:"issued,omitempty"`
+
 	// Name Group Name identifier
 	Name string `json:"name"`
 
@@ -189,6 +198,9 @@ type GroupMinimum struct {
 	// Id Group ID
 	Id string `json:"id"`
 
+	// Issued How group was issued by API or from JWT token
+	Issued *string `json:"issued,omitempty"`
+
 	// Name Group Name identifier
 	Name string `json:"name"`
 

management/server/http/groups_handler.go

@@ -72,7 +72,7 @@ func (h *GroupsHandler) UpdateGroup(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	_, ok = account.Groups[groupID]
+	eg, ok := account.Groups[groupID]
 	if !ok {
 		util.WriteError(status.Errorf(status.NotFound, "couldn't find group with ID %s", groupID), w)
 		return
@@ -107,9 +107,10 @@ func (h *GroupsHandler) UpdateGroup(w http.ResponseWriter, r *http.Request) {
 		peers = *req.Peers
 	}
 	group := server.Group{
-		ID:    groupID,
-		Name:  req.Name,
-		Peers: peers,
+		ID:     groupID,
+		Name:   req.Name,
+		Peers:  peers,
+		Issued: eg.Issued,
 	}
 
 	if err := h.accountManager.SaveGroup(account.Id, user.Id, &group); err != nil {
@@ -149,9 +150,10 @@ func (h *GroupsHandler) CreateGroup(w http.ResponseWriter, r *http.Request) {
 		peers = *req.Peers
 	}
 	group := server.Group{
-		ID:    xid.New().String(),
-		Name:  req.Name,
-		Peers: peers,
+		ID:     xid.New().String(),
+		Name:   req.Name,
+		Peers:  peers,
+		Issued: server.GroupIssuedAPI,
 	}
 
 	err = h.accountManager.SaveGroup(account.Id, user.Id, &group)
@@ -237,6 +239,7 @@ func toGroupResponse(account *server.Account, group *server.Group) *api.Group {
 		Id:         group.ID,
 		Name:       group.Name,
 		PeersCount: len(group.Peers),
+		Issued:     &group.Issued,
 	}
 
 	for _, pid := range group.Peers {

management/server/http/groups_handler_test.go

@@ -42,9 +42,17 @@ func initGroupTestData(user *server.User, groups ...*server.Group) *GroupsHandle
 				if groupID != "idofthegroup" {
 					return nil, status.Errorf(status.NotFound, "not found")
 				}
+				if groupID == "id-jwt-group" {
+					return &server.Group{
+						ID:     "id-jwt-group",
+						Name:   "Default Group",
+						Issued: server.GroupIssuedJWT,
+					}, nil
+				}
 				return &server.Group{
-					ID:   "idofthegroup",
-					Name: "Group",
+					ID:     "idofthegroup",
+					Name:   "Group",
+					Issued: server.GroupIssuedAPI,
 				}, nil
 			},
 			UpdateGroupFunc: func(_ string, groupID string, operations []server.GroupUpdateOperation) (*server.Group, error) {
@@ -80,8 +88,9 @@ func initGroupTestData(user *server.User, groups ...*server.Group) *GroupsHandle
 						user.Id: user,
 					},
 					Groups: map[string]*server.Group{
-						"id-existed": {ID: "id-existed", Peers: []string{"A", "B"}},
-						"id-all":     {ID: "id-all", Name: "All"},
+						"id-jwt-group": {ID: "id-jwt-group", Name: "From JWT", Issued: server.GroupIssuedJWT},
+						"id-existed":   {ID: "id-existed", Peers: []string{"A", "B"}, Issued: server.GroupIssuedAPI},
+						"id-all":       {ID: "id-all", Name: "All", Issued: server.GroupIssuedAPI},
 					},
 				}, user, nil
 			},
@@ -169,6 +178,8 @@ func TestGetGroup(t *testing.T) {
 }
 
 func TestWriteGroup(t *testing.T) {
+	groupIssuedAPI := "api"
+	groupIssuedJWT := "jwt"
 	tt := []struct {
 		name           string
 		expectedStatus int
@@ -187,8 +198,9 @@ func TestWriteGroup(t *testing.T) {
 			expectedStatus: http.StatusOK,
 			expectedBody:   true,
 			expectedGroup: &api.Group{
-				Id:   "id-was-set",
-				Name: "Default POSTed Group",
+				Id:     "id-was-set",
+				Name:   "Default POSTed Group",
+				Issued: &groupIssuedAPI,
 			},
 		},
 		{
@@ -208,8 +220,9 @@ func TestWriteGroup(t *testing.T) {
 				[]byte(`{"Name":"Default POSTed Group"}`)),
 			expectedStatus: http.StatusOK,
 			expectedGroup: &api.Group{
-				Id:   "id-existed",
-				Name: "Default POSTed Group",
+				Id:     "id-existed",
+				Name:   "Default POSTed Group",
+				Issued: &groupIssuedAPI,
 			},
 		},
 		{
@@ -230,6 +243,19 @@ func TestWriteGroup(t *testing.T) {
 			expectedStatus: http.StatusUnprocessableEntity,
 			expectedBody:   false,
 		},
+		{
+			name:        "Write Group PUT not not change Issue",
+			requestType: http.MethodPut,
+			requestPath: "/api/groups/id-jwt-group",
+			requestBody: bytes.NewBuffer(
+				[]byte(`{"Name":"changed","Issued":"api"}`)),
+			expectedStatus: http.StatusOK,
+			expectedGroup: &api.Group{
+				Id:     "id-jwt-group",
+				Name:   "changed",
+				Issued: &groupIssuedJWT,
+			},
+		},
 	}
 
 	adminUser := server.NewAdminUser("test_user")

management/server/idp/google_workspace.go

@@ -0,0 +1,344 @@
+package idp
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/oauth2/google"
+	admin "google.golang.org/api/admin/directory/v1"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// GoogleWorkspaceManager Google Workspace manager client instance.
+type GoogleWorkspaceManager struct {
+	usersService *admin.UsersService
+	CustomerID   string
+	httpClient   ManagerHTTPClient
+	credentials  ManagerCredentials
+	helper       ManagerHelper
+	appMetrics   telemetry.AppMetrics
+}
+
+// GoogleWorkspaceClientConfig Google Workspace manager client configurations.
+type GoogleWorkspaceClientConfig struct {
+	ServiceAccountKey string
+	CustomerID        string
+}
+
+// GoogleWorkspaceCredentials Google Workspace authentication information.
+type GoogleWorkspaceCredentials struct {
+	clientConfig GoogleWorkspaceClientConfig
+	helper       ManagerHelper
+	httpClient   ManagerHTTPClient
+	appMetrics   telemetry.AppMetrics
+}
+
+func (gc *GoogleWorkspaceCredentials) Authenticate() (JWTToken, error) {
+	return JWTToken{}, nil
+}
+
+// NewGoogleWorkspaceManager creates a new instance of the GoogleWorkspaceManager.
+func NewGoogleWorkspaceManager(config GoogleWorkspaceClientConfig, appMetrics telemetry.AppMetrics) (*GoogleWorkspaceManager, error) {
+	httpTransport := http.DefaultTransport.(*http.Transport).Clone()
+	httpTransport.MaxIdleConns = 5
+
+	httpClient := &http.Client{
+		Timeout:   10 * time.Second,
+		Transport: httpTransport,
+	}
+	helper := JsonParser{}
+
+	if config.CustomerID == "" {
+		return nil, fmt.Errorf("google IdP configuration is incomplete, CustomerID is missing")
+	}
+
+	credentials := &GoogleWorkspaceCredentials{
+		clientConfig: config,
+		httpClient:   httpClient,
+		helper:       helper,
+		appMetrics:   appMetrics,
+	}
+
+	// Create a new Admin SDK Directory service client
+	adminCredentials, err := getGoogleCredentials(config.ServiceAccountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	service, err := admin.NewService(context.Background(),
+		option.WithScopes(admin.AdminDirectoryUserScope, admin.AdminDirectoryUserschemaScope),
+		option.WithCredentials(adminCredentials),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = configureAppMetadataSchema(service, config.CustomerID); err != nil {
+		return nil, err
+	}
+
+	return &GoogleWorkspaceManager{
+		usersService: service.Users,
+		CustomerID:   config.CustomerID,
+		httpClient:   httpClient,
+		credentials:  credentials,
+		helper:       helper,
+		appMetrics:   appMetrics,
+	}, nil
+}
+
+// UpdateUserAppMetadata updates user app metadata based on userID and metadata map.
+func (gm *GoogleWorkspaceManager) UpdateUserAppMetadata(userID string, appMetadata AppMetadata) error {
+	metadata, err := gm.helper.Marshal(appMetadata)
+	if err != nil {
+		return err
+	}
+
+	user := &admin.User{
+		CustomSchemas: map[string]googleapi.RawMessage{
+			"app_metadata": metadata,
+		},
+	}
+
+	_, err = gm.usersService.Update(userID, user).Do()
+	if err != nil {
+		return err
+	}
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountUpdateUserAppMetadata()
+	}
+
+	return nil
+}
+
+// GetUserDataByID requests user data from Google Workspace via ID.
+func (gm *GoogleWorkspaceManager) GetUserDataByID(userID string, appMetadata AppMetadata) (*UserData, error) {
+	user, err := gm.usersService.Get(userID).Projection("full").Do()
+	if err != nil {
+		return nil, err
+	}
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountGetUserDataByID()
+	}
+
+	return parseGoogleWorkspaceUser(user)
+}
+
+// GetAccount returns all the users for a given profile.
+func (gm *GoogleWorkspaceManager) GetAccount(accountID string) ([]*UserData, error) {
+	query := fmt.Sprintf("app_metadata.wt_account_id=\"%s\"", accountID)
+	usersList, err := gm.usersService.List().Customer(gm.CustomerID).Query(query).Projection("full").Do()
+	if err != nil {
+		return nil, err
+	}
+
+	usersData := make([]*UserData, 0)
+	for _, user := range usersList.Users {
+		userData, err := parseGoogleWorkspaceUser(user)
+		if err != nil {
+			return nil, err
+		}
+
+		usersData = append(usersData, userData)
+	}
+
+	return usersData, nil
+}
+
+// GetAllAccounts gets all registered accounts with corresponding user data.
+// It returns a list of users indexed by accountID.
+func (gm *GoogleWorkspaceManager) GetAllAccounts() (map[string][]*UserData, error) {
+	usersList, err := gm.usersService.List().Customer(gm.CustomerID).Projection("full").Do()
+	if err != nil {
+		return nil, err
+	}
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountGetAllAccounts()
+	}
+
+	indexedUsers := make(map[string][]*UserData)
+	for _, user := range usersList.Users {
+		userData, err := parseGoogleWorkspaceUser(user)
+		if err != nil {
+			return nil, err
+		}
+
+		accountID := userData.AppMetadata.WTAccountID
+		if accountID != "" {
+			if _, ok := indexedUsers[accountID]; !ok {
+				indexedUsers[accountID] = make([]*UserData, 0)
+			}
+			indexedUsers[accountID] = append(indexedUsers[accountID], userData)
+		}
+	}
+
+	return indexedUsers, nil
+}
+
+// CreateUser creates a new user in Google Workspace and sends an invitation.
+func (gm *GoogleWorkspaceManager) CreateUser(email string, name string, accountID string) (*UserData, error) {
+	invite := true
+	metadata := AppMetadata{
+		WTAccountID:     accountID,
+		WTPendingInvite: &invite,
+	}
+
+	username := &admin.UserName{}
+	fields := strings.Fields(name)
+	if n := len(fields); n > 0 {
+		username.GivenName = strings.Join(fields[:n-1], " ")
+		username.FamilyName = fields[n-1]
+	}
+
+	payload, err := gm.helper.Marshal(metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	user := &admin.User{
+		Name:         username,
+		PrimaryEmail: email,
+		CustomSchemas: map[string]googleapi.RawMessage{
+			"app_metadata": payload,
+		},
+		Password: GeneratePassword(8, 1, 1, 1),
+	}
+	user, err = gm.usersService.Insert(user).Do()
+	if err != nil {
+		return nil, err
+	}
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountCreateUser()
+	}
+
+	return parseGoogleWorkspaceUser(user)
+}
+
+// GetUserByEmail searches users with a given email.
+// If no users have been found, this function returns an empty list.
+func (gm *GoogleWorkspaceManager) GetUserByEmail(email string) ([]*UserData, error) {
+	user, err := gm.usersService.Get(email).Projection("full").Do()
+	if err != nil {
+		return nil, err
+	}
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountGetUserByEmail()
+	}
+
+	userData, err := parseGoogleWorkspaceUser(user)
+	if err != nil {
+		return nil, err
+	}
+
+	users := make([]*UserData, 0)
+	users = append(users, userData)
+
+	return users, nil
+}
+
+// getGoogleCredentials retrieves Google credentials based on the provided serviceAccountKey.
+// It decodes the base64-encoded serviceAccountKey and attempts to obtain credentials using it.
+// If that fails, it falls back to using the default Google credentials path.
+// It returns the retrieved credentials or an error if unsuccessful.
+func getGoogleCredentials(serviceAccountKey string) (*google.Credentials, error) {
+	log.Debug("retrieving google credentials from the base64 encoded service account key")
+	decodeKey, err := base64.StdEncoding.DecodeString(serviceAccountKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode service account key: %w", err)
+	}
+
+	creds, err := google.CredentialsFromJSON(
+		context.Background(),
+		decodeKey,
+		admin.AdminDirectoryUserschemaScope,
+		admin.AdminDirectoryUserScope,
+	)
+	if err == nil {
+		return creds, err
+	}
+
+	log.Debugf("failed to retrieve Google credentials from ServiceAccountKey: %v", err)
+	log.Debug("falling back to default google credentials location")
+
+	creds, err = google.FindDefaultCredentials(
+		context.Background(),
+		admin.AdminDirectoryUserschemaScope,
+		admin.AdminDirectoryUserScope,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return creds, nil
+}
+
+// configureAppMetadataSchema create a custom schema for managing app metadata fields in Google Workspace.
+func configureAppMetadataSchema(service *admin.Service, customerID string) error {
+	schemaList, err := service.Schemas.List(customerID).Do()
+	if err != nil {
+		return err
+	}
+
+	// checks if app_metadata schema is already created
+	for _, schema := range schemaList.Schemas {
+		if schema.SchemaName == "app_metadata" {
+			return nil
+		}
+	}
+
+	// create new app_metadata schema
+	appMetadataSchema := &admin.Schema{
+		SchemaName: "app_metadata",
+		Fields: []*admin.SchemaFieldSpec{
+			{
+				FieldName:   "wt_account_id",
+				FieldType:   "STRING",
+				MultiValued: false,
+			},
+			{
+				FieldName:   "wt_pending_invite",
+				FieldType:   "BOOL",
+				MultiValued: false,
+			},
+		},
+	}
+	_, err = service.Schemas.Insert(customerID, appMetadataSchema).Do()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// parseGoogleWorkspaceUser parse google user to UserData.
+func parseGoogleWorkspaceUser(user *admin.User) (*UserData, error) {
+	var appMetadata AppMetadata
+
+	// Get app metadata from custom schemas
+	if user.CustomSchemas != nil {
+		rawMessage := user.CustomSchemas["app_metadata"]
+		helper := JsonParser{}
+
+		if err := helper.Unmarshal(rawMessage, &appMetadata); err != nil {
+			return nil, err
+		}
+	}
+
+	return &UserData{
+		ID:          user.Id,
+		Email:       user.PrimaryEmail,
+		Name:        user.Name.FullName,
+		AppMetadata: appMetadata,
+	}, nil
+}

management/server/idp/idp.go

@@ -162,6 +162,12 @@ func NewManager(config Config, appMetrics telemetry.AppMetrics) (Manager, error)
 			APIToken:      config.ExtraConfig["ApiToken"],
 		}
 		return NewOktaManager(oktaClientConfig, appMetrics)
+	case "google":
+		googleClientConfig := GoogleWorkspaceClientConfig{
+			ServiceAccountKey: config.ExtraConfig["ServiceAccountKey"],
+			CustomerID:        config.ExtraConfig["CustomerId"],
+		}
+		return NewGoogleWorkspaceManager(googleClientConfig, appMetrics)
 
 	default:
 		return nil, fmt.Errorf("invalid manager type: %s", config.ManagerType)

management/server/jwtclaims/claims.go

@@ -1,9 +1,15 @@
 package jwtclaims
 
+import (
+	"github.com/golang-jwt/jwt"
+)
+
 // AuthorizationClaims stores authorization information from JWTs
 type AuthorizationClaims struct {
 	UserId         string
 	AccountId      string
 	Domain         string
 	DomainCategory string
+
+	Raw jwt.MapClaims
 }

management/server/jwtclaims/extractor.go

@@ -73,7 +73,9 @@ func NewClaimsExtractor(options ...ClaimsExtractorOption) *ClaimsExtractor {
 // FromToken extracts claims from the token (after auth)
 func (c *ClaimsExtractor) FromToken(token *jwt.Token) AuthorizationClaims {
 	claims := token.Claims.(jwt.MapClaims)
-	jwtClaims := AuthorizationClaims{}
+	jwtClaims := AuthorizationClaims{
+		Raw: claims,
+	}
 	userID, ok := claims[c.userIDClaim].(string)
 	if !ok {
 		return jwtClaims

management/server/jwtclaims/extractor_test.go

@@ -48,6 +48,12 @@ func TestExtractClaimsFromRequestContext(t *testing.T) {
 			Domain:         "test.com",
 			AccountId:      "testAcc",
 			DomainCategory: "public",
+			Raw: jwt.MapClaims{
+				"https://login/wt_account_domain":          "test.com",
+				"https://login/wt_account_domain_category": "public",
+				"https://login/wt_account_id":              "testAcc",
+				"sub":                                      "test",
+			},
 		},
 		testingFunc: require.EqualValues,
 		expectedMSG: "extracted claims should match input claims",
@@ -59,6 +65,10 @@ func TestExtractClaimsFromRequestContext(t *testing.T) {
 		inputAuthorizationClaims: AuthorizationClaims{
 			UserId:    "test",
 			AccountId: "testAcc",
+			Raw: jwt.MapClaims{
+				"https://login/wt_account_id": "testAcc",
+				"sub":                         "test",
+			},
 		},
 		testingFunc: require.EqualValues,
 		expectedMSG: "extracted claims should match input claims",
@@ -70,6 +80,10 @@ func TestExtractClaimsFromRequestContext(t *testing.T) {
 		inputAuthorizationClaims: AuthorizationClaims{
 			UserId: "test",
 			Domain: "test.com",
+			Raw: jwt.MapClaims{
+				"https://login/wt_account_domain": "test.com",
+				"sub":                             "test",
+			},
 		},
 		testingFunc: require.EqualValues,
 		expectedMSG: "extracted claims should match input claims",
@@ -82,6 +96,11 @@ func TestExtractClaimsFromRequestContext(t *testing.T) {
 			UserId:    "test",
 			Domain:    "test.com",
 			AccountId: "testAcc",
+			Raw: jwt.MapClaims{
+				"https://login/wt_account_domain": "test.com",
+				"https://login/wt_account_id":     "testAcc",
+				"sub":                             "test",
+			},
 		},
 		testingFunc: require.EqualValues,
 		expectedMSG: "extracted claims should match input claims",
@@ -92,6 +111,9 @@ func TestExtractClaimsFromRequestContext(t *testing.T) {
 		inputAudiance: "https://login/",
 		inputAuthorizationClaims: AuthorizationClaims{
 			UserId: "test",
+			Raw: jwt.MapClaims{
+				"sub": "test",
+			},
 		},
 		testingFunc: require.EqualValues,
 		expectedMSG: "extracted claims should match input claims",

management/server/peer_test.go

@@ -78,11 +78,14 @@ func TestAccountManager_GetNetworkMap(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	var setupKey *SetupKey
-	for _, key := range account.SetupKeys {
-		if key.Type == SetupKeyReusable {
-			setupKey = key
-		}
+	setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userId)
+	if err != nil {
+		return
+	}
+
+	if err != nil {
+		t.Fatal("error creating setup key")
+		return
 	}
 
 	peerKey1, err := wgtypes.GeneratePrivateKey()
@@ -328,7 +331,15 @@ func TestAccountManager_GetPeerNetwork(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	setupKey := getSetupKey(account, SetupKeyReusable)
+	setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, userId)
+	if err != nil {
+		return
+	}
+
+	if err != nil {
+		t.Fatal("error creating setup key")
+		return
+	}
 
 	peerKey1, err := wgtypes.GeneratePrivateKey()
 	if err != nil {
@@ -394,7 +405,15 @@ func TestDefaultAccountManager_GetPeer(t *testing.T) {
 	}
 
 	// two peers one added by a regular user and one with a setup key
-	setupKey := getSetupKey(account, SetupKeyReusable)
+	setupKey, err := manager.CreateSetupKey(account.Id, "test-key", SetupKeyReusable, time.Hour, nil, 999, adminUser)
+	if err != nil {
+		return
+	}
+
+	if err != nil {
+		t.Fatal("error creating setup key")
+		return
+	}
 	peerKey1, err := wgtypes.GeneratePrivateKey()
 	if err != nil {
 		t.Fatal(err)
@@ -470,13 +489,3 @@ func TestDefaultAccountManager_GetPeer(t *testing.T) {
 	}
 	assert.NotNil(t, peer)
 }
-
-func getSetupKey(account *Account, keyType SetupKeyType) *SetupKey {
-	var setupKey *SetupKey
-	for _, key := range account.SetupKeys {
-		if key.Type == keyType {
-			setupKey = key
-		}
-	}
-	return setupKey
-}