refactor: Restructure twitter app with beforeRequest hook

packages/backend/src/apps/twitter/common/add-auth-header.ts

@@ -0,0 +1,28 @@
+import { Token } from 'oauth-1.0a';
+import { TBeforeRequest } from '@automatisch/types';
+import oauthClient from './oauth-client';
+
+const addAuthHeader: TBeforeRequest = ($, requestConfig) => {
+  const { url, method, data } = requestConfig;
+
+  const token: Token = {
+    key: $.auth.data?.accessToken as string,
+    secret: $.auth.data?.accessSecret as string,
+  };
+
+  const requestData = {
+    url: `${requestConfig.baseURL}${url}`,
+    method,
+    data,
+  };
+
+  const authHeader = oauthClient($).toHeader(
+    oauthClient($).authorize(requestData, token)
+  );
+
+  requestConfig.headers.Authorization = authHeader.Authorization;
+
+  return requestConfig;
+};
+
+export default addAuthHeader;

packages/backend/src/apps/twitter/common/generate-request.ts

@@ -1,44 +0,0 @@
-import { Token } from 'oauth-1.0a';
-import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-import oauthClient from './oauth-client';
-import { Method } from 'axios';
-
-type IGenereateRequestOptons = {
-  requestPath: string;
-  method: string;
-  data?: IJSONObject;
-};
-
-const generateRequest = async (
-  $: IGlobalVariable,
-  options: IGenereateRequestOptons
-) => {
-  const { requestPath, method, data } = options;
-
-  const token: Token = {
-    key: $.auth.data.accessToken as string,
-    secret: $.auth.data.accessSecret as string,
-  };
-
-  const requestData = {
-    url: `${$.app.apiBaseUrl}${requestPath}`,
-    method,
-    data,
-  };
-
-  const authHeader = oauthClient($).toHeader(
-    oauthClient($).authorize(requestData, token)
-  );
-
-  const response = await $.http.request({
-    url: requestData.url,
-    method: requestData.method as Method,
-    headers: {
-      ...authHeader,
-    },
-  });
-
-  return response;
-};
-
-export default generateRequest;

packages/backend/src/apps/twitter/common/get-current-user.ts

@@ -1,13 +1,9 @@
 import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-import generateRequest from './generate-request';
 
 const getCurrentUser = async ($: IGlobalVariable): Promise<IJSONObject> => {
-  const response = await generateRequest($, {
-    requestPath: '/2/users/me',
-    method: 'GET',
-  });
-
+  const response = await $.http.get('/2/users/me');
   const currentUser = response.data.data;
+
   return currentUser;
 };
 

packages/backend/src/apps/twitter/common/get-user-by-username.ts

@@ -1,11 +1,7 @@
 import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-import generateRequest from './generate-request';
 
 const getUserByUsername = async ($: IGlobalVariable, username: string) => {
-  const response = await generateRequest($, {
-    requestPath: `/2/users/by/username/${username}`,
-    method: 'GET',
-  });
+  const response = await $.http.get(`/2/users/by/username/${username}`);
 
   if (response.data.errors) {
     const errorMessages = response.data.errors

packages/backend/src/apps/twitter/common/get-user-followers.ts

@@ -5,11 +5,9 @@ import {
 } from '@automatisch/types';
 import { URLSearchParams } from 'url';
 import { omitBy, isEmpty } from 'lodash';
-import generateRequest from './generate-request';
 
 type GetUserFollowersOptions = {
   userId: string;
-  lastInternalId?: string;
 };
 
 const getUserFollowers = async (
@@ -33,10 +31,7 @@ const getUserFollowers = async (
       queryParams.toString() ? `?${queryParams.toString()}` : ''
     }`;
 
-    response = await generateRequest($, {
-      requestPath,
-      method: 'GET',
-    });
+    response = await $.http.get(requestPath);
 
     if (response.integrationError) {
       followers.error = response.integrationError;
@@ -49,18 +44,18 @@ const getUserFollowers = async (
     }
 
     if (response.data.meta.result_count > 0) {
-      response.data.data.forEach((follower: IJSONObject) => {
+      for (const follower of response.data.data) {
+        if ($.flow.isAlreadyProcessed(follower.id as string)) {
+          return followers;
+        }
+
         followers.data.push({
           raw: follower,
           meta: { internalId: follower.id as string },
         });
-      });
+      }
     }
-  } while (response.data.meta.next_token && options.lastInternalId);
-
-  followers.data.sort((follower, nextFollower) => {
-    return (follower.raw.id as number) - (nextFollower.raw.id as number);
-  });
+  } while (response.data.meta.next_token && !$.execution.testRun);
 
   return followers;
 };

packages/backend/src/apps/twitter/common/get-user-tweets.ts

@@ -6,7 +6,6 @@ import {
 import { URLSearchParams } from 'url';
 import omitBy from 'lodash/omitBy';
 import isEmpty from 'lodash/isEmpty';
-import generateRequest from './generate-request';
 import getCurrentUser from './get-current-user';
 import getUserByUsername from './get-user-by-username';
 
@@ -14,19 +13,7 @@ type IGetUserTweetsOptions = {
   currentUser: boolean;
 };
 
-const getUserTweets = async (
-  $: IGlobalVariable,
-  options: IGetUserTweetsOptions
-) => {
-  let username: string;
-
-  if (options.currentUser) {
-    const currentUser = await getCurrentUser($);
-    username = currentUser.username as string;
-  } else {
-    username = $.step.parameters.username as string;
-  }
-
+const fetchTweets = async ($: IGlobalVariable, username: string) => {
   const user = await getUserByUsername($, username);
 
   let response;
@@ -47,10 +34,7 @@ const getUserTweets = async (
       queryParams.toString() ? `?${queryParams.toString()}` : ''
     }`;
 
-    response = await generateRequest($, {
-      requestPath,
-      method: 'GET',
-    });
+    response = await $.http.get(requestPath);
 
     if (response.integrationError) {
       tweets.error = response.integrationError;
@@ -72,4 +56,26 @@ const getUserTweets = async (
   return tweets;
 };
 
+const getUserTweets = async (
+  $: IGlobalVariable,
+  options: IGetUserTweetsOptions
+) => {
+  let username: string;
+
+  if (options.currentUser) {
+    const currentUser = await getCurrentUser($);
+    username = currentUser.username as string;
+  } else {
+    username = $.step.parameters.username as string;
+  }
+
+  const tweets = await fetchTweets($, username);
+
+  tweets.data.sort((tweet, nextTweet) => {
+    return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId);
+  });
+
+  return tweets;
+};
+
 export default getUserTweets;