Add Pagination for IdP Users Fetch (#1210)

* Retrieve all workspace users via pagination, excluding custom user attributes

* Retrieve all authentik users via pagination

* Retrieve all Azure AD users via pagination

* Simplify user data appending operation

Reduced unnecessary iteration and used an efficient way to append all users to 'indexedUsers'

* Fix ineffectual assignment to reqURL

* Retrieve all Okta users via pagination

* Add missing GetAccount metrics

* Refactor

* minimize memory allocation

Refactored the memory allocation for the 'users' slice in the Okta IDP code. Previously, the slice was only initialized but not given a size. Now the size of userList is utilized to optimize memory allocation, reducing potential slice resizing and memory re-allocation costs while appending users.

* Add logging for entries received from IdP management

Added informative and debug logging statements in account.go file. Logging has been added to identify the number of entries received from Identity Provider (IdP) management. This will aid in tracking and debugging any potential data ingestion issues.

management/server/idp/google_workspace.go

@@ -96,7 +96,7 @@ func (gm *GoogleWorkspaceManager) UpdateUserAppMetadata(_ string, _ AppMetadata)
 
 // GetUserDataByID requests user data from Google Workspace via ID.
 func (gm *GoogleWorkspaceManager) GetUserDataByID(userID string, appMetadata AppMetadata) (*UserData, error) {
-	user, err := gm.usersService.Get(userID).Projection("full").Do()
+	user, err := gm.usersService.Get(userID).Do()
 	if err != nil {
 		return nil, err
 	}
@@ -113,43 +113,69 @@ func (gm *GoogleWorkspaceManager) GetUserDataByID(userID string, appMetadata App
 
 // GetAccount returns all the users for a given profile.
 func (gm *GoogleWorkspaceManager) GetAccount(accountID string) ([]*UserData, error) {
-	usersList, err := gm.usersService.List().Customer(gm.CustomerID).Projection("full").Do()
-	if err != nil {
-		return nil, err
-	}
-
-	usersData := make([]*UserData, 0)
-	for _, user := range usersList.Users {
-		userData := parseGoogleWorkspaceUser(user)
-		userData.AppMetadata.WTAccountID = accountID
-
-		usersData = append(usersData, userData)
-	}
-
-	return usersData, nil
-}
-
-// GetAllAccounts gets all registered accounts with corresponding user data.
-// It returns a list of users indexed by accountID.
-func (gm *GoogleWorkspaceManager) GetAllAccounts() (map[string][]*UserData, error) {
-	usersList, err := gm.usersService.List().Customer(gm.CustomerID).Projection("full").Do()
+	users, err := gm.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
 
 	if gm.appMetrics != nil {
-		gm.appMetrics.IDPMetrics().CountGetAllAccounts()
+		gm.appMetrics.IDPMetrics().CountGetAccount()
+	}
+
+	for index, user := range users {
+		user.AppMetadata.WTAccountID = accountID
+		users[index] = user
+	}
+
+	return users, nil
+}
+
+// GetAllAccounts gets all registered accounts with corresponding user data.
+// It returns a list of users indexed by accountID.
+func (gm *GoogleWorkspaceManager) GetAllAccounts() (map[string][]*UserData, error) {
+	users, err := gm.getAllUsers()
+	if err != nil {
+		return nil, err
 	}
 
 	indexedUsers := make(map[string][]*UserData)
-	for _, user := range usersList.Users {
-		userData := parseGoogleWorkspaceUser(user)
-		indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], userData)
+	indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], users...)
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountGetAllAccounts()
 	}
 
 	return indexedUsers, nil
 }
 
+// getAllUsers returns all users in a Google Workspace account filtered by customer ID.
+func (gm *GoogleWorkspaceManager) getAllUsers() ([]*UserData, error) {
+	users := make([]*UserData, 0)
+	pageToken := ""
+	for {
+		call := gm.usersService.List().Customer(gm.CustomerID).MaxResults(500)
+		if pageToken != "" {
+			call.PageToken(pageToken)
+		}
+
+		resp, err := call.Do()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, user := range resp.Users {
+			users = append(users, parseGoogleWorkspaceUser(user))
+		}
+
+		pageToken = resp.NextPageToken
+		if pageToken == "" {
+			break
+		}
+	}
+
+	return users, nil
+}
+
 // CreateUser creates a new user in Google Workspace and sends an invitation.
 func (gm *GoogleWorkspaceManager) CreateUser(_, _, _, _ string) (*UserData, error) {
 	return nil, fmt.Errorf("method CreateUser not implemented")
@@ -158,7 +184,7 @@ func (gm *GoogleWorkspaceManager) CreateUser(_, _, _, _ string) (*UserData, erro
 // GetUserByEmail searches users with a given email.
 // If no users have been found, this function returns an empty list.
 func (gm *GoogleWorkspaceManager) GetUserByEmail(email string) ([]*UserData, error) {
-	user, err := gm.usersService.Get(email).Projection("full").Do()
+	user, err := gm.usersService.Get(email).Do()
 	if err != nil {
 		return nil, err
 	}