Fix codacy issues

README.md

@@ -16,14 +16,7 @@
      <a href="https://github.com/netbirdio/netbird/blob/main/LICENSE">
        <img src="https://img.shields.io/badge/license-BSD--3-blue" />
      </a> 
-     <a href="https://hub.docker.com/r/wiretrustee/wiretrustee/tags">
-        <img src="https://img.shields.io/docker/pulls/wiretrustee/wiretrustee" />
-     </a> 
-    <br>
-    <a href="https://www.codacy.com/gh/wiretrustee/wiretrustee/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=wiretrustee/wiretrustee&amp;utm_campaign=Badge_Grade"><img src="https://app.codacy.com/project/badge/Grade/d366de2c9d8b4cf982da27f8f5831809"/></a>
-     <a href="https://goreportcard.com/report/wiretrustee/wiretrustee">
-        <img src="https://goreportcard.com/badge/github.com/wiretrustee/wiretrustee?style=flat-square" />
-     </a>
+   <a href="https://www.codacy.com/gh/netbirdio/netbird/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=netbirdio/netbird&amp;utm_campaign=Badge_Grade"><img src="https://app.codacy.com/project/badge/Grade/e3013d046aec44cdb7462c8673b00976"/></a>
     <br>
     <a href="https://join.slack.com/t/wiretrustee/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">
         <img src="https://img.shields.io/badge/slack-@wiretrustee-red.svg?logo=slack"/>

management/server/account.go

@@ -13,6 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"math/rand"
 	"reflect"
 	"strings"
 	"sync"
@@ -20,48 +21,50 @@ import (
 )
 
 const (
-	PublicCategory  = "public"
-	PrivateCategory = "private"
-	UnknownCategory = "unknown"
+	PublicCategory     = "public"
+	PrivateCategory    = "private"
+	UnknownCategory    = "unknown"
+	CacheExpirationMax = 7 * 24 * 3600 * time.Second // 7 days
+	CacheExpirationMin = 3 * 24 * 3600 * time.Second // 3 days
 )
 
 type AccountManager interface {
-	GetOrCreateAccountByUser(userId, domain string) (*Account, error)
-	GetAccountByUser(userId string) (*Account, error)
+	GetOrCreateAccountByUser(userID, domain string) (*Account, error)
+	GetAccountByUser(userID string) (*Account, error)
 	AddSetupKey(
-		accountId string,
+		accountID string,
 		keyName string,
 		keyType SetupKeyType,
 		expiresIn *util.Duration,
 	) (*SetupKey, error)
-	RevokeSetupKey(accountId string, keyId string) (*SetupKey, error)
-	RenameSetupKey(accountId string, keyId string, newName string) (*SetupKey, error)
-	GetAccountById(accountId string) (*Account, error)
-	GetAccountByUserOrAccountId(userId, accountId, domain string) (*Account, error)
+	RevokeSetupKey(accountID string, keyID string) (*SetupKey, error)
+	RenameSetupKey(accountID string, keyID string, newName string) (*SetupKey, error)
+	GetAccountById(accountID string) (*Account, error)
+	GetAccountByUserOrAccountId(userID, accountID, domain string) (*Account, error)
 	GetAccountWithAuthorizationClaims(claims jwtclaims.AuthorizationClaims) (*Account, error)
 	IsUserAdmin(claims jwtclaims.AuthorizationClaims) (bool, error)
-	AccountExists(accountId string) (*bool, error)
-	AddAccount(accountId, userId, domain string) (*Account, error)
+	AccountExists(accountID string) (*bool, error)
+	AddAccount(accountID, userID, domain string) (*Account, error)
 	GetPeer(peerKey string) (*Peer, error)
 	MarkPeerConnected(peerKey string, connected bool) error
-	RenamePeer(accountId string, peerKey string, newName string) (*Peer, error)
-	DeletePeer(accountId string, peerKey string) (*Peer, error)
-	GetPeerByIP(accountId string, peerIP string) (*Peer, error)
+	RenamePeer(accountID string, peerKey string, newName string) (*Peer, error)
+	DeletePeer(accountID string, peerKey string) (*Peer, error)
+	GetPeerByIP(accountID string, peerIP string) (*Peer, error)
 	GetNetworkMap(peerKey string) (*NetworkMap, error)
-	AddPeer(setupKey string, userId string, peer *Peer) (*Peer, error)
+	AddPeer(setupKey string, userID string, peer *Peer) (*Peer, error)
 	UpdatePeerMeta(peerKey string, meta PeerSystemMeta) error
-	GetUsersFromAccount(accountId string) ([]*UserInfo, error)
-	GetGroup(accountId, groupID string) (*Group, error)
-	SaveGroup(accountId string, group *Group) error
-	DeleteGroup(accountId, groupID string) error
-	ListGroups(accountId string) ([]*Group, error)
-	GroupAddPeer(accountId, groupID, peerKey string) error
-	GroupDeletePeer(accountId, groupID, peerKey string) error
-	GroupListPeers(accountId, groupID string) ([]*Peer, error)
-	GetRule(accountId, ruleID string) (*Rule, error)
+	GetUsersFromAccount(accountID string) ([]*UserInfo, error)
+	GetGroup(accountID, groupID string) (*Group, error)
+	SaveGroup(accountID string, group *Group) error
+	DeleteGroup(accountID, groupID string) error
+	ListGroups(accountID string) ([]*Group, error)
+	GroupAddPeer(accountID, groupID, peerKey string) error
+	GroupDeletePeer(accountID, groupID, peerKey string) error
+	GroupListPeers(accountID, groupID string) ([]*Peer, error)
+	GetRule(accountID, ruleID string) (*Rule, error)
 	SaveRule(accountID string, rule *Rule) error
-	DeleteRule(accountId, ruleID string) error
-	ListRules(accountId string) ([]*Rule, error)
+	DeleteRule(accountID, ruleID string) error
+	ListRules(accountID string) ([]*Rule, error)
 }
 
 type DefaultAccountManager struct {
@@ -98,9 +101,9 @@ type UserInfo struct {
 }
 
 // NewAccount creates a new Account with a generated ID and generated default setup keys
-func NewAccount(userId, domain string) *Account {
-	accountId := xid.New().String()
-	return newAccountWithId(accountId, userId, domain)
+func NewAccount(userID, domain string) *Account {
+	accountID := xid.New().String()
+	return newAccountWithId(accountID, userID, domain)
 }
 
 func (a *Account) Copy() *Account {
@@ -162,28 +165,62 @@ func BuildManager(
 		ctx:                context.Background(),
 	}
 
-	// if account has not default account
-	// we build 'all' group and add all peers into it
-	// also we create default rule with source an destination
-	// groups 'all'
+	// if account has not default group
+	// we create 'all' group and add all peers into it
+	// also we create default rule with source as destination
 	for _, account := range store.GetAllAccounts() {
-		am.addAllGroup(account)
-		if err := store.SaveAccount(account); err != nil {
-			return nil, err
+		_, err := account.GetGroupAll()
+		if err != nil {
+			am.addAllGroup(account)
+			if err := store.SaveAccount(account); err != nil {
+				return nil, err
+			}
 		}
 	}
 
-	gocacheClient := gocache.New(7*24*time.Hour, 30*time.Minute)
+	gocacheClient := gocache.New(CacheExpirationMax, 30*time.Minute)
 	gocacheStore := cacheStore.NewGoCache(gocacheClient, nil)
 
 	am.cacheManager = cache.NewLoadable(am.loadFromCache, cache.New(gocacheStore))
+
+	if !isNil(am.idpManager) {
+		go func() {
+			err := am.warmupIDPCache()
+			if err != nil {
+				log.Warnf("failed warming up cache due to error: %v", err)
+				//todo retry?
+				return
+			}
+		}()
+	}
+
 	return am, nil
 
 }
 
+func (am *DefaultAccountManager) warmupIDPCache() error {
+	userData, err := am.idpManager.GetAllAccounts()
+	if err != nil {
+		return err
+	}
+
+	for accountID, users := range userData {
+		rand.Seed(time.Now().UnixNano())
+
+		r := rand.Intn(int(CacheExpirationMax.Milliseconds()-CacheExpirationMin.Milliseconds())) + int(CacheExpirationMin.Milliseconds())
+		expiration := time.Duration(r) * time.Millisecond
+		err = am.cacheManager.Set(am.ctx, accountID, users, &cacheStore.Options{Expiration: expiration})
+		if err != nil {
+			return err
+		}
+	}
+	log.Infof("warmed up IDP cache with %d entries", len(userData))
+	return nil
+}
+
 // AddSetupKey generates a new setup key with a given name and type, and adds it to the specified account
 func (am *DefaultAccountManager) AddSetupKey(
-	accountId string,
+	accountID string,
 	keyName string,
 	keyType SetupKeyType,
 	expiresIn *util.Duration,
@@ -196,7 +233,7 @@ func (am *DefaultAccountManager) AddSetupKey(
 		keyDuration = expiresIn.Duration
 	}
 
-	account, err := am.Store.GetAccount(accountId)
+	account, err := am.Store.GetAccount(accountID)
 	if err != nil {
 		return nil, status.Errorf(codes.NotFound, "account not found")
 	}
@@ -213,18 +250,18 @@ func (am *DefaultAccountManager) AddSetupKey(
 }
 
 // RevokeSetupKey marks SetupKey as revoked - becomes not valid anymore
-func (am *DefaultAccountManager) RevokeSetupKey(accountId string, keyId string) (*SetupKey, error) {
+func (am *DefaultAccountManager) RevokeSetupKey(accountID string, keyID string) (*SetupKey, error) {
 	am.mux.Lock()
 	defer am.mux.Unlock()
 
-	account, err := am.Store.GetAccount(accountId)
+	account, err := am.Store.GetAccount(accountID)
 	if err != nil {
 		return nil, status.Errorf(codes.NotFound, "account not found")
 	}
 
-	setupKey := getAccountSetupKeyById(account, keyId)
+	setupKey := getAccountSetupKeyById(account, keyID)
 	if setupKey == nil {
-		return nil, status.Errorf(codes.NotFound, "unknown setupKey %s", keyId)
+		return nil, status.Errorf(codes.NotFound, "unknown setupKey %s", keyID)
 	}
 
 	keyCopy := setupKey.Copy()
@@ -240,21 +277,21 @@ func (am *DefaultAccountManager) RevokeSetupKey(accountId string, keyId string)
 
 // RenameSetupKey renames existing setup key of the specified account.
 func (am *DefaultAccountManager) RenameSetupKey(
-	accountId string,
-	keyId string,
+	accountID string,
+	keyID string,
 	newName string,
 ) (*SetupKey, error) {
 	am.mux.Lock()
 	defer am.mux.Unlock()
 
-	account, err := am.Store.GetAccount(accountId)
+	account, err := am.Store.GetAccount(accountID)
 	if err != nil {
 		return nil, status.Errorf(codes.NotFound, "account not found")
 	}
 
-	setupKey := getAccountSetupKeyById(account, keyId)
+	setupKey := getAccountSetupKeyById(account, keyID)
 	if setupKey == nil {
-		return nil, status.Errorf(codes.NotFound, "unknown setupKey %s", keyId)
+		return nil, status.Errorf(codes.NotFound, "unknown setupKey %s", keyID)
 	}
 
 	keyCopy := setupKey.Copy()
@@ -269,11 +306,11 @@ func (am *DefaultAccountManager) RenameSetupKey(
 }
 
 // GetAccountById returns an existing account using its ID or error (NotFound) if doesn't exist
-func (am *DefaultAccountManager) GetAccountById(accountId string) (*Account, error) {
+func (am *DefaultAccountManager) GetAccountById(accountID string) (*Account, error) {
 	am.mux.Lock()
 	defer am.mux.Unlock()
 
-	account, err := am.Store.GetAccount(accountId)
+	account, err := am.Store.GetAccount(accountID)
 	if err != nil {
 		return nil, status.Errorf(codes.NotFound, "account not found")
 	}
@@ -284,16 +321,16 @@ func (am *DefaultAccountManager) GetAccountById(accountId string) (*Account, err
 // GetAccountByUserOrAccountId look for an account by user or account Id, if no account is provided and
 // user id doesn't have an account associated with it, one account is created
 func (am *DefaultAccountManager) GetAccountByUserOrAccountId(
-	userId, accountId, domain string,
+	userID, accountID, domain string,
 ) (*Account, error) {
-	if accountId != "" {
-		return am.GetAccountById(accountId)
-	} else if userId != "" {
-		account, err := am.GetOrCreateAccountByUser(userId, domain)
+	if accountID != "" {
+		return am.GetAccountById(accountID)
+	} else if userID != "" {
+		account, err := am.GetOrCreateAccountByUser(userID, domain)
 		if err != nil {
-			return nil, status.Errorf(codes.NotFound, "account not found using user id: %s", userId)
+			return nil, status.Errorf(codes.NotFound, "account not found using user id: %s", userID)
 		}
-		err = am.updateIDPMetadata(userId, account.Id)
+		err = am.updateIDPMetadata(userID, account.Id)
 		if err != nil {
 			return nil, err
 		}
@@ -308,9 +345,9 @@ func isNil(i idp.Manager) bool {
 }
 
 // updateIDPMetadata update user's  app metadata in idp manager
-func (am *DefaultAccountManager) updateIDPMetadata(userId, accountID string) error {
+func (am *DefaultAccountManager) updateIDPMetadata(userID, accountID string) error {
 	if !isNil(am.idpManager) {
-		err := am.idpManager.UpdateUserAppMetadata(userId, idp.AppMetadata{WTAccountId: accountID})
+		err := am.idpManager.UpdateUserAppMetadata(userID, idp.AppMetadata{WTAccountId: accountID})
 		if err != nil {
 			return status.Errorf(
 				codes.Internal,
@@ -332,7 +369,7 @@ func mergeLocalAndQueryUser(queried idp.UserData, local User) *UserInfo {
 }
 
 func (am *DefaultAccountManager) loadFromCache(ctx context.Context, accountID interface{}) (interface{}, error) {
-	return am.idpManager.GetBatchedUserData(fmt.Sprintf("%v", accountID))
+	return am.idpManager.GetAccount(fmt.Sprintf("%v", accountID))
 }
 
 func (am *DefaultAccountManager) lookupCache(accountUsers map[string]*User, accountID string) ([]*idp.UserData, error) {
@@ -487,6 +524,7 @@ func (am *DefaultAccountManager) handleNewUserAccount(
 		}
 	} else {
 		account = NewAccount(claims.UserId, lowerDomain)
+		am.addAllGroup(account)
 		account.Users[claims.UserId] = NewAdminUser(claims.UserId)
 		err = am.updateAccountDomainAttributes(account, claims, true)
 		if err != nil {
@@ -565,12 +603,12 @@ func (am *DefaultAccountManager) GetAccountWithAuthorizationClaims(
 }
 
 // AccountExists checks whether account exists (returns true) or not (returns false)
-func (am *DefaultAccountManager) AccountExists(accountId string) (*bool, error) {
+func (am *DefaultAccountManager) AccountExists(accountID string) (*bool, error) {
 	am.mux.Lock()
 	defer am.mux.Unlock()
 
 	var res bool
-	_, err := am.Store.GetAccount(accountId)
+	_, err := am.Store.GetAccount(accountID)
 	if err != nil {
 		if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound {
 			res = false
@@ -584,16 +622,16 @@ func (am *DefaultAccountManager) AccountExists(accountId string) (*bool, error)
 	return &res, nil
 }
 
-// AddAccount generates a new Account with a provided accountId and userId, saves to the Store
-func (am *DefaultAccountManager) AddAccount(accountId, userId, domain string) (*Account, error) {
+// AddAccount generates a new Account with a provided accountID and userID, saves to the Store
+func (am *DefaultAccountManager) AddAccount(accountID, userID, domain string) (*Account, error) {
 	am.mux.Lock()
 	defer am.mux.Unlock()
 
-	return am.createAccount(accountId, userId, domain)
+	return am.createAccountWithID(accountID, userID, domain)
 }
 
-func (am *DefaultAccountManager) createAccount(accountId, userId, domain string) (*Account, error) {
-	account := newAccountWithId(accountId, userId, domain)
+func (am *DefaultAccountManager) createAccountWithID(accountID, userID, domain string) (*Account, error) {
+	account := newAccountWithId(accountID, userID, domain)
 
 	am.addAllGroup(account)
 
@@ -605,7 +643,7 @@ func (am *DefaultAccountManager) createAccount(accountId, userId, domain string)
 	return account, nil
 }
 
-// addAllGroup to account object it it doesn't exists
+// addAllGroup to account object if it doesn't exists
 func (am *DefaultAccountManager) addAllGroup(account *Account) {
 	if len(account.Groups) == 0 {
 		allGroup := &Group{
@@ -628,7 +666,7 @@ func (am *DefaultAccountManager) addAllGroup(account *Account) {
 }
 
 // newAccountWithId creates a new Account with a default SetupKey (doesn't store in a Store) and provided id
-func newAccountWithId(accountId, userId, domain string) *Account {
+func newAccountWithId(accountID, userID, domain string) *Account {
 	log.Debugf("creating new account")
 
 	setupKeys := make(map[string]*SetupKey)
@@ -640,22 +678,22 @@ func newAccountWithId(accountId, userId, domain string) *Account {
 	peers := make(map[string]*Peer)
 	users := make(map[string]*User)
 
-	log.Debugf("created new account %s with setup key %s", accountId, defaultKey.Key)
+	log.Debugf("created new account %s with setup key %s", accountID, defaultKey.Key)
 
 	return &Account{
-		Id:        accountId,
+		Id:        accountID,
 		SetupKeys: setupKeys,
 		Network:   network,
 		Peers:     peers,
 		Users:     users,
-		CreatedBy: userId,
+		CreatedBy: userID,
 		Domain:    domain,
 	}
 }
 
-func getAccountSetupKeyById(acc *Account, keyId string) *SetupKey {
+func getAccountSetupKeyById(acc *Account, keyID string) *SetupKey {
 	for _, k := range acc.SetupKeys {
-		if keyId == k.Id {
+		if keyID == k.Id {
 			return k
 		}
 	}

management/server/idp/auth0.go

@@ -1,6 +1,9 @@
 package idp
 
 import (
+	"bytes"
+	"compress/gzip"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -51,6 +54,47 @@ type Auth0Credentials struct {
 	mux          sync.Mutex
 }
 
+// userExportJobRequest is a user export request struct
+type userExportJobRequest struct {
+	Format string              `json:"format"`
+	Fields []map[string]string `json:"fields"`
+}
+
+// userExportJobResponse is a user export response struct
+type userExportJobResponse struct {
+	Type         string    `json:"type"`
+	Status       string    `json:"status"`
+	ConnectionID string    `json:"connection_id"`
+	Format       string    `json:"format"`
+	Limit        int       `json:"limit"`
+	Connection   string    `json:"connection"`
+	CreatedAt    time.Time `json:"created_at"`
+	ID           string    `json:"id"`
+}
+
+// userExportJobStatusResponse is a user export status response struct
+type userExportJobStatusResponse struct {
+	Type         string    `json:"type"`
+	Status       string    `json:"status"`
+	ConnectionID string    `json:"connection_id"`
+	Format       string    `json:"format"`
+	Limit        int       `json:"limit"`
+	Location     string    `json:"location"`
+	Connection   string    `json:"connection"`
+	CreatedAt    time.Time `json:"created_at"`
+	ID           string    `json:"id"`
+}
+
+// auth0Profile represents an Auth0 user profile response
+type auth0Profile struct {
+	AccountID string `json:"wt_account_id"`
+	UserID    string `json:"user_id"`
+	Name      string `json:"name"`
+	Email     string `json:"email"`
+	CreatedAt string `json:"created_at"`
+	LastLogin string `json:"last_login"`
+}
+
 // NewAuth0Manager creates a new instance of the Auth0Manager
 func NewAuth0Manager(config Auth0ClientConfig) (*Auth0Manager, error) {
 
@@ -186,7 +230,7 @@ func (c *Auth0Credentials) Authenticate() (JWTToken, error) {
 	return c.jwtToken, nil
 }
 
-func batchRequestUsersUrl(authIssuer, accountId string, page int) (string, url.Values, error) {
+func batchRequestUsersURL(authIssuer, accountID string, page int) (string, url.Values, error) {
 	u, err := url.Parse(authIssuer + "/api/v2/users")
 	if err != nil {
 		return "", nil, err
@@ -194,18 +238,18 @@ func batchRequestUsersUrl(authIssuer, accountId string, page int) (string, url.V
 	q := u.Query()
 	q.Set("page", strconv.Itoa(page))
 	q.Set("search_engine", "v3")
-	q.Set("q", "app_metadata.wt_account_id:"+accountId)
+	q.Set("q", "app_metadata.wt_account_id:"+accountID)
 	u.RawQuery = q.Encode()
 
 	return u.String(), q, nil
 }
 
-func requestByUserIdUrl(authIssuer, userId string) string {
-	return authIssuer + "/api/v2/users/" + userId
+func requestByUserIDURL(authIssuer, userID string) string {
+	return authIssuer + "/api/v2/users/" + userID
 }
 
-// GetBatchedUserData requests users in batches from Auth0
-func (am *Auth0Manager) GetBatchedUserData(accountId string) ([]*UserData, error) {
+// GetAccount returns all the users for a given profile. Calls Auth0 API.
+func (am *Auth0Manager) GetAccount(accountID string) ([]*UserData, error) {
 	jwtToken, err := am.credentials.Authenticate()
 	if err != nil {
 		return nil, err
@@ -216,7 +260,7 @@ func (am *Auth0Manager) GetBatchedUserData(accountId string) ([]*UserData, error
 	// https://auth0.com/docs/manage-users/user-search/retrieve-users-with-get-users-endpoint#limitations
 	// auth0 limitation of 1000 users via this endpoint
 	for page := 0; page < 20; page++ {
-		reqURL, query, err := batchRequestUsersUrl(am.authIssuer, accountId, page)
+		reqURL, query, err := batchRequestUsersURL(am.authIssuer, accountID, page)
 		if err != nil {
 			return nil, err
 		}
@@ -269,13 +313,13 @@ func (am *Auth0Manager) GetBatchedUserData(accountId string) ([]*UserData, error
 }
 
 // GetUserDataByID requests user data from auth0 via ID
-func (am *Auth0Manager) GetUserDataByID(userId string, appMetadata AppMetadata) (*UserData, error) {
+func (am *Auth0Manager) GetUserDataByID(userID string, appMetadata AppMetadata) (*UserData, error) {
 	jwtToken, err := am.credentials.Authenticate()
 	if err != nil {
 		return nil, err
 	}
 
-	reqURL := requestByUserIdUrl(am.authIssuer, userId)
+	reqURL := requestByUserIDURL(am.authIssuer, userID)
 	req, err := http.NewRequest(http.MethodGet, reqURL, nil)
 	if err != nil {
 		return nil, err
@@ -314,14 +358,14 @@ func (am *Auth0Manager) GetUserDataByID(userId string, appMetadata AppMetadata)
 }
 
 // UpdateUserAppMetadata updates user app metadata based on userId and metadata map
-func (am *Auth0Manager) UpdateUserAppMetadata(userId string, appMetadata AppMetadata) error {
+func (am *Auth0Manager) UpdateUserAppMetadata(userID string, appMetadata AppMetadata) error {
 
 	jwtToken, err := am.credentials.Authenticate()
 	if err != nil {
 		return err
 	}
 
-	reqURL := am.authIssuer + "/api/v2/users/" + userId
+	reqURL := am.authIssuer + "/api/v2/users/" + userID
 
 	data, err := am.helper.Marshal(appMetadata)
 	if err != nil {
@@ -339,7 +383,7 @@ func (am *Auth0Manager) UpdateUserAppMetadata(userId string, appMetadata AppMeta
 	req.Header.Add("authorization", "Bearer "+jwtToken.AccessToken)
 	req.Header.Add("content-type", "application/json")
 
-	log.Debugf("updating metadata for user %s", userId)
+	log.Debugf("updating metadata for user %s", userID)
 
 	res, err := am.httpClient.Do(req)
 	if err != nil {
@@ -359,3 +403,211 @@ func (am *Auth0Manager) UpdateUserAppMetadata(userId string, appMetadata AppMeta
 
 	return nil
 }
+
+func buildUserExportRequest() (string, error) {
+	req := &userExportJobRequest{}
+	fields := make([]map[string]string, 0)
+
+	for _, field := range []string{"created_at", "last_login", "user_id", "email", "name"} {
+		fields = append(fields, map[string]string{"name": field})
+	}
+
+	fields = append(fields, map[string]string{
+		"name":      "app_metadata.wt_account_id",
+		"export_as": "wt_account_id",
+	})
+
+	req.Format = "json"
+	req.Fields = fields
+
+	str, err := json.Marshal(req)
+	if err != nil {
+		return "", err
+	}
+
+	return string(str), nil
+}
+
+// GetAllAccounts gets all registered accounts with corresponding user data.
+// It returns a list of users indexed by accountID.
+func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) {
+	jwtToken, err := am.credentials.Authenticate()
+	if err != nil {
+		return nil, err
+	}
+
+	reqURL := am.authIssuer + "/api/v2/jobs/users-exports"
+
+	payloadString, err := buildUserExportRequest()
+	if err != nil {
+		return nil, err
+	}
+	payload := strings.NewReader(payloadString)
+
+	exportJobReq, err := http.NewRequest("POST", reqURL, payload)
+	if err != nil {
+		return nil, err
+	}
+	exportJobReq.Header.Add("authorization", "Bearer "+jwtToken.AccessToken)
+	exportJobReq.Header.Add("content-type", "application/json")
+
+	jobResp, err := am.httpClient.Do(exportJobReq)
+	if err != nil {
+		log.Debugf("Couldn't get job response %v", err)
+		return nil, err
+	}
+
+	defer func() {
+		err = jobResp.Body.Close()
+		if err != nil {
+			log.Errorf("error while closing update user app metadata response body: %v", err)
+		}
+	}()
+	if jobResp.StatusCode != 200 {
+		return nil, fmt.Errorf("unable to update the appMetadata, statusCode %d", jobResp.StatusCode)
+	}
+
+	var exportJobResp userExportJobResponse
+
+	body, err := ioutil.ReadAll(jobResp.Body)
+	if err != nil {
+		log.Debugf("Coudln't read export job response; %v", err)
+		return nil, err
+	}
+
+	err = am.helper.Unmarshal(body, &exportJobResp)
+	if err != nil {
+		log.Debugf("Coudln't unmarshal export job response; %v", err)
+		return nil, err
+	}
+
+	if exportJobResp.ID == "" {
+		return nil, fmt.Errorf("couldn't get an batch id status %d, %s, response body: %v", jobResp.StatusCode, jobResp.Status, exportJobResp)
+	}
+
+	log.Debugf("batch id status %d, %s, response body: %v", jobResp.StatusCode, jobResp.Status, exportJobResp)
+
+	done, downloadLink, err := am.checkExportJobStatus(exportJobResp.ID)
+	if err != nil {
+		log.Debugf("Failed at getting status checks from exportJob; %v", err)
+		return nil, err
+	}
+
+	if done {
+		return am.downloadProfileExport(downloadLink)
+	}
+
+	return nil, fmt.Errorf("failed extracting user profiles from auth0")
+}
+
+// checkExportJobStatus checks the status of the job created at CreateExportUsersJob.
+// If the status is "completed", then return the downloadLink
+func (am *Auth0Manager) checkExportJobStatus(jobID string) (bool, string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	retry := time.NewTicker(10 * time.Second)
+	for {
+		select {
+		case <-ctx.Done():
+			log.Debugf("Export job status stopped...\n")
+			return false, "", ctx.Err()
+		case <-retry.C:
+			jwtToken, err := am.credentials.Authenticate()
+			if err != nil {
+				return false, "", err
+			}
+
+			statusURL := am.authIssuer + "/api/v2/jobs/" + jobID
+			body, err := doGetReq(am.httpClient, statusURL, jwtToken.AccessToken)
+			if err != nil {
+				return false, "", err
+			}
+
+			var status userExportJobStatusResponse
+			err = am.helper.Unmarshal(body, &status)
+			if err != nil {
+				return false, "", err
+			}
+
+			log.Debugf("current export job status is %v", status.Status)
+
+			if status.Status != "completed" {
+				continue
+			}
+
+			return true, status.Location, nil
+		}
+	}
+}
+
+// downloadProfileExport downloads user profiles from auth0 batch job
+func (am *Auth0Manager) downloadProfileExport(location string) (map[string][]*UserData, error) {
+	body, err := doGetReq(am.httpClient, location, "")
+	if err != nil {
+		return nil, err
+	}
+
+	bodyReader := bytes.NewReader(body)
+
+	gzipReader, err := gzip.NewReader(bodyReader)
+	if err != nil {
+		return nil, err
+	}
+
+	decoder := json.NewDecoder(gzipReader)
+
+	res := make(map[string][]*UserData)
+	for decoder.More() {
+		profile := auth0Profile{}
+		err = decoder.Decode(&profile)
+		if err != nil {
+			return nil, err
+		}
+		if profile.AccountID != "" {
+			if _, ok := res[profile.AccountID]; !ok {
+				res[profile.AccountID] = []*UserData{}
+			}
+			res[profile.AccountID] = append(res[profile.AccountID],
+				&UserData{
+					ID:    profile.UserID,
+					Name:  profile.Name,
+					Email: profile.Email,
+				})
+		}
+	}
+
+	return res, nil
+}
+
+// Boilerplate implementation for Get Requests.
+func doGetReq(client ManagerHTTPClient, url, accessToken string) ([]byte, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if accessToken != "" {
+		req.Header.Add("authorization", "Bearer "+accessToken)
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		err = res.Body.Close()
+		if err != nil {
+			log.Errorf("error while closing body for url %s: %v", url, err)
+		}
+	}()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("unable to get %s, statusCode %d", url, res.StatusCode)
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	return body, nil
+}

management/server/idp/idp.go

@@ -11,7 +11,8 @@ import (
 type Manager interface {
 	UpdateUserAppMetadata(userId string, appMetadata AppMetadata) error
 	GetUserDataByID(userId string, appMetadata AppMetadata) (*UserData, error)
-	GetBatchedUserData(accountId string) ([]*UserData, error)
+	GetAccount(accountId string) ([]*UserData, error)
+	GetAllAccounts() (map[string][]*UserData, error)
 }
 
 // Config an idp configuration struct to be loaded from management server's config file