refactor: Restructure twitter app with beforeRequest hook

packages/backend/src/apps/slack/index.ts

@@ -4,5 +4,6 @@ export default {
   iconUrl: '{BASE_URL}/apps/slack/assets/favicon.svg',
   authDocUrl: 'https://automatisch.io/docs/connections/slack',
   supportsConnections: true,
-  baseUrl: 'https://slack.com/api',
+  baseUrl: 'https://slack.com',
+  apiBaseUrl: 'https://slack.com/api',
 };

packages/backend/src/apps/twitter/auth/create-auth-data.ts

@@ -1,4 +1,3 @@
-import generateRequest from '../common/generate-request';
 import { IJSONObject, IField, IGlobalVariable } from '@automatisch/types';
 import { URLSearchParams } from 'url';
 
@@ -9,13 +8,10 @@ export default async function createAuthData($: IGlobalVariable) {
     );
 
     const callbackUrl = oauthRedirectUrlField.value;
+    const requestPath = '/oauth/request_token';
+    const data = { oauth_callback: callbackUrl };
 
-    const response = await generateRequest($, {
-      requestPath: '/oauth/request_token',
-      method: 'POST',
-      data: { oauth_callback: callbackUrl },
-    });
-
+    const response = await $.http.post(requestPath, data);
     const responseData = Object.fromEntries(new URLSearchParams(response.data));
 
     await $.auth.set({

packages/backend/src/apps/twitter/common/add-auth-header.ts

@@ -0,0 +1,28 @@
+import { Token } from 'oauth-1.0a';
+import { TBeforeRequest } from '@automatisch/types';
+import oauthClient from './oauth-client';
+
+const addAuthHeader: TBeforeRequest = ($, requestConfig) => {
+  const { url, method, data } = requestConfig;
+
+  const token: Token = {
+    key: $.auth.data?.accessToken as string,
+    secret: $.auth.data?.accessSecret as string,
+  };
+
+  const requestData = {
+    url: `${requestConfig.baseURL}${url}`,
+    method,
+    data,
+  };
+
+  const authHeader = oauthClient($).toHeader(
+    oauthClient($).authorize(requestData, token)
+  );
+
+  requestConfig.headers.Authorization = authHeader.Authorization;
+
+  return requestConfig;
+};
+
+export default addAuthHeader;

packages/backend/src/apps/twitter/common/generate-request.ts

@@ -1,44 +0,0 @@
-import { Token } from 'oauth-1.0a';
-import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-import oauthClient from './oauth-client';
-import { Method } from 'axios';
-
-type IGenereateRequestOptons = {
-  requestPath: string;
-  method: string;
-  data?: IJSONObject;
-};
-
-const generateRequest = async (
-  $: IGlobalVariable,
-  options: IGenereateRequestOptons
-) => {
-  const { requestPath, method, data } = options;
-
-  const token: Token = {
-    key: $.auth.data.accessToken as string,
-    secret: $.auth.data.accessSecret as string,
-  };
-
-  const requestData = {
-    url: `${$.app.apiBaseUrl}${requestPath}`,
-    method,
-    data,
-  };
-
-  const authHeader = oauthClient($).toHeader(
-    oauthClient($).authorize(requestData, token)
-  );
-
-  const response = await $.http.request({
-    url: requestData.url,
-    method: requestData.method as Method,
-    headers: {
-      ...authHeader,
-    },
-  });
-
-  return response;
-};
-
-export default generateRequest;

packages/backend/src/apps/twitter/common/get-current-user.ts

@@ -1,13 +1,9 @@
 import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-import generateRequest from './generate-request';
 
 const getCurrentUser = async ($: IGlobalVariable): Promise<IJSONObject> => {
-  const response = await generateRequest($, {
-    requestPath: '/2/users/me',
-    method: 'GET',
-  });
-
+  const response = await $.http.get('/2/users/me');
   const currentUser = response.data.data;
+
   return currentUser;
 };
 

packages/backend/src/apps/twitter/common/get-user-by-username.ts

@@ -1,11 +1,7 @@
 import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-import generateRequest from './generate-request';
 
 const getUserByUsername = async ($: IGlobalVariable, username: string) => {
-  const response = await generateRequest($, {
-    requestPath: `/2/users/by/username/${username}`,
-    method: 'GET',
-  });
+  const response = await $.http.get(`/2/users/by/username/${username}`);
 
   if (response.data.errors) {
     const errorMessages = response.data.errors

packages/backend/src/apps/twitter/common/get-user-followers.ts

@@ -5,11 +5,9 @@ import {
 } from '@automatisch/types';
 import { URLSearchParams } from 'url';
 import { omitBy, isEmpty } from 'lodash';
-import generateRequest from './generate-request';
 
 type GetUserFollowersOptions = {
   userId: string;
-  lastInternalId?: string;
 };
 
 const getUserFollowers = async (
@@ -33,10 +31,7 @@ const getUserFollowers = async (
       queryParams.toString() ? `?${queryParams.toString()}` : ''
     }`;
 
-    response = await generateRequest($, {
-      requestPath,
-      method: 'GET',
-    });
+    response = await $.http.get(requestPath);
 
     if (response.integrationError) {
       followers.error = response.integrationError;
@@ -49,18 +44,18 @@ const getUserFollowers = async (
     }
 
     if (response.data.meta.result_count > 0) {
-      response.data.data.forEach((follower: IJSONObject) => {
+      for (const follower of response.data.data) {
+        if ($.flow.isAlreadyProcessed(follower.id as string)) {
+          return followers;
+        }
+
         followers.data.push({
           raw: follower,
           meta: { internalId: follower.id as string },
         });
-      });
       }
-  } while (response.data.meta.next_token && options.lastInternalId);
-
-  followers.data.sort((follower, nextFollower) => {
-    return (follower.raw.id as number) - (nextFollower.raw.id as number);
-  });
+    }
+  } while (response.data.meta.next_token && !$.execution.testRun);
 
   return followers;
 };

packages/backend/src/apps/twitter/common/get-user-tweets.ts

@@ -6,7 +6,6 @@ import {
 import { URLSearchParams } from 'url';
 import omitBy from 'lodash/omitBy';
 import isEmpty from 'lodash/isEmpty';
-import generateRequest from './generate-request';
 import getCurrentUser from './get-current-user';
 import getUserByUsername from './get-user-by-username';
 
@@ -14,19 +13,7 @@ type IGetUserTweetsOptions = {
   currentUser: boolean;
 };
 
-const getUserTweets = async (
-  $: IGlobalVariable,
-  options: IGetUserTweetsOptions
-) => {
-  let username: string;
-
-  if (options.currentUser) {
-    const currentUser = await getCurrentUser($);
-    username = currentUser.username as string;
-  } else {
-    username = $.step.parameters.username as string;
-  }
-
+const fetchTweets = async ($: IGlobalVariable, username: string) => {
   const user = await getUserByUsername($, username);
 
   let response;
@@ -47,10 +34,7 @@ const getUserTweets = async (
       queryParams.toString() ? `?${queryParams.toString()}` : ''
     }`;
 
-    response = await generateRequest($, {
-      requestPath,
-      method: 'GET',
-    });
+    response = await $.http.get(requestPath);
 
     if (response.integrationError) {
       tweets.error = response.integrationError;
@@ -72,4 +56,26 @@ const getUserTweets = async (
   return tweets;
 };
 
+const getUserTweets = async (
+  $: IGlobalVariable,
+  options: IGetUserTweetsOptions
+) => {
+  let username: string;
+
+  if (options.currentUser) {
+    const currentUser = await getCurrentUser($);
+    username = currentUser.username as string;
+  } else {
+    username = $.step.parameters.username as string;
+  }
+
+  const tweets = await fetchTweets($, username);
+
+  tweets.data.sort((tweet, nextTweet) => {
+    return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId);
+  });
+
+  return tweets;
+};
+
 export default getUserTweets;

packages/backend/src/apps/twitter/index.ts

@@ -1,4 +1,7 @@
-export default {
+import defineApp from '../../helpers/define-app';
+import addAuthHeader from './common/add-auth-header';
+
+export default defineApp({
   name: 'Twitter',
   key: 'twitter',
   iconUrl: '{BASE_URL}/apps/twitter/assets/favicon.svg',
@@ -6,4 +9,6 @@ export default {
   supportsConnections: true,
   baseUrl: 'https://twitter.com',
   apiBaseUrl: 'https://api.twitter.com',
-};
+  primaryColor: '1da1f2',
+  beforeRequest: [addAuthHeader],
+});

packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts

@@ -6,6 +6,7 @@ export default defineTrigger({
   key: 'myFollowers',
   pollInterval: 15,
   description: 'Will be triggered when you have a new follower.',
+  dedupeStrategy: 'unique',
   substeps: [
     {
       key: 'chooseConnection',
@@ -18,6 +19,6 @@ export default defineTrigger({
   ],
 
   async run($) {
-    return await myFollowers($, $.flow.lastInternalId);
+    return await myFollowers($);
   },
 });

packages/backend/src/apps/twitter/triggers/new-follower-of-me/my-followers.ts

@@ -3,13 +3,12 @@ import getCurrentUser from '../../common/get-current-user';
 import getUserByUsername from '../../common/get-user-by-username';
 import getUserFollowers from '../../common/get-user-followers';
 
-const myFollowers = async ($: IGlobalVariable, lastInternalId?: string) => {
+const myFollowers = async ($: IGlobalVariable) => {
   const { username } = await getCurrentUser($);
   const user = await getUserByUsername($, username as string);
 
   const tweets = await getUserFollowers($, {
     userId: user.id,
-    lastInternalId,
   });
   return tweets;
 };

packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts

@@ -4,10 +4,9 @@ import {
   ITriggerOutput,
 } from '@automatisch/types';
 import qs from 'qs';
-import generateRequest from '../../common/generate-request';
 import { omitBy, isEmpty } from 'lodash';
 
-const searchTweets = async ($: IGlobalVariable) => {
+const fetchTweets = async ($: IGlobalVariable) => {
   const searchTerm = $.step.parameters.searchTerm as string;
 
   let response;
@@ -29,10 +28,7 @@ const searchTweets = async ($: IGlobalVariable) => {
       queryParams.toString() ? `?${queryParams.toString()}` : ''
     }`;
 
-    response = await generateRequest($, {
-      requestPath,
-      method: 'GET',
-    });
+    response = await $.http.get(requestPath);
 
     if (response.integrationError) {
       tweets.error = response.integrationError;
@@ -58,8 +54,14 @@ const searchTweets = async ($: IGlobalVariable) => {
     }
   } while (response.data.meta.next_token && !$.execution.testRun);
 
+  return tweets;
+};
+
+const searchTweets = async ($: IGlobalVariable) => {
+  const tweets = await fetchTweets($);
+
   tweets.data.sort((tweet, nextTweet) => {
-    return (tweet.raw.id as number) - (nextTweet.raw.id as number);
+    return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId);
   });
 
   return tweets;

packages/backend/src/helpers/global-variable.ts

@@ -68,9 +68,11 @@ const globalVariable = async (
   });
 
   if (trigger && trigger.dedupeStrategy === 'unique') {
-    const lastInternalIds = await flow?.lastInternalIds();
+    const lastInternalIds = testRun ? [] : await flow?.lastInternalIds();
 
     const isAlreadyProcessed = (internalId: string) => {
+      if (testRun) return false;
+
       return lastInternalIds?.includes(internalId);
     };
 

packages/backend/src/helpers/http-client/index.ts

@@ -1,17 +1,40 @@
 import axios, { AxiosRequestConfig } from 'axios';
 export { AxiosInstance as IHttpClient } from 'axios';
 import { IHttpClientParams } from '@automatisch/types';
+import { URL } from 'url';
 
-export default function createHttpClient({ $, baseURL, beforeRequest = [] }: IHttpClientParams) {
+const removeBaseUrlForAbsoluteUrls = (
+  requestConfig: AxiosRequestConfig
+): AxiosRequestConfig => {
+  try {
+    const url = new URL(requestConfig.url);
+    requestConfig.baseURL = url.origin;
+    requestConfig.url = url.pathname + url.search;
+
+    return requestConfig;
+  } catch {
+    return requestConfig;
+  }
+};
+
+export default function createHttpClient({
+  $,
+  baseURL,
+  beforeRequest = [],
+}: IHttpClientParams) {
   const instance = axios.create({
     baseURL,
   });
 
-  instance.interceptors.request.use((requestConfig: AxiosRequestConfig): AxiosRequestConfig => {
+  instance.interceptors.request.use(
+    (requestConfig: AxiosRequestConfig): AxiosRequestConfig => {
+      const newRequestConfig = removeBaseUrlForAbsoluteUrls(requestConfig);
+
       return beforeRequest.reduce((newConfig, beforeRequestFunc) => {
         return beforeRequestFunc($, newConfig);
-    }, requestConfig);
-  });
+      }, newRequestConfig);
+    }
+  );
 
   instance.interceptors.response.use(
     (response) => response,