feat: Implement draft version of pagination with user tweet trigger

packages/backend/src/apps/twitter/client/endpoints/get-user-by-username.ts

@@ -39,6 +39,6 @@ export default class GetUserByUsername {
       );
     }
 
-    return response;
+    return response.data.data;
   }
 }

packages/backend/src/apps/twitter/client/endpoints/get-user-tweets.ts

@@ -1,5 +1,8 @@
 import { IJSONObject } from '@automatisch/types';
+import { URLSearchParams } from 'url';
 import TwitterClient from '../index';
+import omitBy from 'lodash/omitBy';
+import isEmpty from 'lodash/isEmpty';
 
 export default class GetUserTweets {
   client: TwitterClient;
@@ -8,26 +11,52 @@ export default class GetUserTweets {
     this.client = client;
   }
 
-  async run(userId: string) {
+  async run(userId: string, lastInternalId?: string) {
     const token = {
       key: this.client.connection.formattedData.accessToken as string,
       secret: this.client.connection.formattedData.accessSecret as string,
     };
 
-    const requestPath = `/2/users/${userId}/tweets`;
+    let response;
+    const tweets: IJSONObject[] = [];
 
-    const requestData = {
-      url: `${TwitterClient.baseUrl}${requestPath}`,
-      method: 'GET',
-    };
+    do {
+      const params: IJSONObject = {
+        since_id: lastInternalId,
+        pagination_token: response?.data?.meta?.next_token,
+      };
 
-    const authHeader = this.client.oauthClient.toHeader(
-      this.client.oauthClient.authorize(requestData, token)
-    );
+      const queryParams = new URLSearchParams({
+        ...omitBy(params, isEmpty),
+      });
 
-    const response = await this.client.httpClient.get(requestPath, {
-      headers: { ...authHeader },
-    });
+      const requestPath = `/2/users/${userId}/tweets${
+        queryParams.toString() ? `?${queryParams.toString()}` : ''
+      }`;
+
+      const requestData = {
+        url: `${TwitterClient.baseUrl}${requestPath}`,
+        method: 'GET',
+      };
+
+      const authHeader = this.client.oauthClient.toHeader(
+        this.client.oauthClient.authorize(requestData, token)
+      );
+
+      response = await this.client.httpClient.get(requestPath, {
+        headers: { ...authHeader },
+      });
+
+      if (response.data.meta.result_count > 0) {
+        response.data.data.forEach((tweet: IJSONObject) => {
+          if (!lastInternalId || Number(tweet.id) > Number(lastInternalId)) {
+            tweets.push(tweet);
+          } else {
+            return;
+          }
+        });
+      }
+    } while (response.data.meta.next_token && lastInternalId);
 
     if (response.data?.errors) {
       const errorMessages = response.data.errors
@@ -39,6 +68,6 @@ export default class GetUserTweets {
       );
     }
 
-    return response;
+    return tweets;
   }
 }

packages/backend/src/apps/twitter/triggers/user-tweet.ts

@@ -7,24 +7,19 @@ export default class UserTweet {
     this.client = client;
   }
 
-  async run() {
-    return this.getTweets();
+  async run(lastInternalId: string) {
+    return this.getTweets(lastInternalId);
   }
 
   async testRun() {
     return this.getTweets();
   }
 
-  async getTweets() {
-    const userResponse = await this.client.getUserByUsername.run(
+  async getTweets(lastInternalId?: string) {
+    const user = await this.client.getUserByUsername.run(
       this.client.step.parameters.username as string
     );
 
-    const userId = userResponse.data.data.id;
-
-    const tweetsResponse = await this.client.getUserTweets.run(userId);
-    const tweets = tweetsResponse.data.data;
-
-    return tweets;
+    return await this.client.getUserTweets.run(user.id, lastInternalId);
   }
 }