feat: Implement draft version of pagination with user tweet trigger

packages/backend/src/apps/twitter/client/endpoints/get-user-tweets.ts

@@ -1,5 +1,8 @@
 import { IJSONObject } from '@automatisch/types';
+import { URLSearchParams } from 'url';
 import TwitterClient from '../index';
+import omitBy from 'lodash/omitBy';
+import isEmpty from 'lodash/isEmpty';
 
 export default class GetUserTweets {
   client: TwitterClient;
@@ -8,26 +11,52 @@ export default class GetUserTweets {
     this.client = client;
   }
 
-  async run(userId: string) {
+  async run(userId: string, lastInternalId?: string) {
     const token = {
       key: this.client.connection.formattedData.accessToken as string,
       secret: this.client.connection.formattedData.accessSecret as string,
     };
 
-    const requestPath = `/2/users/${userId}/tweets`;
+    let response;
+    const tweets: IJSONObject[] = [];
 
-    const requestData = {
-      url: `${TwitterClient.baseUrl}${requestPath}`,
-      method: 'GET',
-    };
+    do {
+      const params: IJSONObject = {
+        since_id: lastInternalId,
+        pagination_token: response?.data?.meta?.next_token,
+      };
 
-    const authHeader = this.client.oauthClient.toHeader(
-      this.client.oauthClient.authorize(requestData, token)
-    );
+      const queryParams = new URLSearchParams({
+        ...omitBy(params, isEmpty),
+      });
 
-    const response = await this.client.httpClient.get(requestPath, {
-      headers: { ...authHeader },
-    });
+      const requestPath = `/2/users/${userId}/tweets${
+        queryParams.toString() ? `?${queryParams.toString()}` : ''
+      }`;
+
+      const requestData = {
+        url: `${TwitterClient.baseUrl}${requestPath}`,
+        method: 'GET',
+      };
+
+      const authHeader = this.client.oauthClient.toHeader(
+        this.client.oauthClient.authorize(requestData, token)
+      );
+
+      response = await this.client.httpClient.get(requestPath, {
+        headers: { ...authHeader },
+      });
+
+      if (response.data.meta.result_count > 0) {
+        response.data.data.forEach((tweet: IJSONObject) => {
+          if (!lastInternalId || Number(tweet.id) > Number(lastInternalId)) {
+            tweets.push(tweet);
+          } else {
+            return;
+          }
+        });
+      }
+    } while (response.data.meta.next_token && lastInternalId);
 
     if (response.data?.errors) {
       const errorMessages = response.data.errors
@@ -39,6 +68,6 @@ export default class GetUserTweets {
       );
     }
 
-    return response;
+    return tweets;
   }
 }