diff --git a/packages/backend/src/apps/slack/index.ts b/packages/backend/src/apps/slack/index.ts index 644c8135..97652ba8 100644 --- a/packages/backend/src/apps/slack/index.ts +++ b/packages/backend/src/apps/slack/index.ts @@ -4,5 +4,6 @@ export default { iconUrl: '{BASE_URL}/apps/slack/assets/favicon.svg', authDocUrl: 'https://automatisch.io/docs/connections/slack', supportsConnections: true, - baseUrl: 'https://slack.com/api', + baseUrl: 'https://slack.com', + apiBaseUrl: 'https://slack.com/api', }; diff --git a/packages/backend/src/apps/twitter/auth/create-auth-data.ts b/packages/backend/src/apps/twitter/auth/create-auth-data.ts index b8fe115f..effbd8a6 100644 --- a/packages/backend/src/apps/twitter/auth/create-auth-data.ts +++ b/packages/backend/src/apps/twitter/auth/create-auth-data.ts @@ -1,4 +1,3 @@ -import generateRequest from '../common/generate-request'; import { IJSONObject, IField, IGlobalVariable } from '@automatisch/types'; import { URLSearchParams } from 'url'; @@ -9,13 +8,10 @@ export default async function createAuthData($: IGlobalVariable) { ); const callbackUrl = oauthRedirectUrlField.value; + const requestPath = '/oauth/request_token'; + const data = { oauth_callback: callbackUrl }; - const response = await generateRequest($, { - requestPath: '/oauth/request_token', - method: 'POST', - data: { oauth_callback: callbackUrl }, - }); - + const response = await $.http.post(requestPath, data); const responseData = Object.fromEntries(new URLSearchParams(response.data)); await $.auth.set({ diff --git a/packages/backend/src/apps/twitter/common/add-auth-header.ts b/packages/backend/src/apps/twitter/common/add-auth-header.ts new file mode 100644 index 00000000..86b35f00 --- /dev/null +++ b/packages/backend/src/apps/twitter/common/add-auth-header.ts @@ -0,0 +1,28 @@ +import { Token } from 'oauth-1.0a'; +import { TBeforeRequest } from '@automatisch/types'; +import oauthClient from './oauth-client'; + +const addAuthHeader: TBeforeRequest = ($, requestConfig) => { + const { url, method, data } = requestConfig; + + const token: Token = { + key: $.auth.data?.accessToken as string, + secret: $.auth.data?.accessSecret as string, + }; + + const requestData = { + url: `${requestConfig.baseURL}${url}`, + method, + data, + }; + + const authHeader = oauthClient($).toHeader( + oauthClient($).authorize(requestData, token) + ); + + requestConfig.headers.Authorization = authHeader.Authorization; + + return requestConfig; +}; + +export default addAuthHeader; diff --git a/packages/backend/src/apps/twitter/common/generate-request.ts b/packages/backend/src/apps/twitter/common/generate-request.ts deleted file mode 100644 index 2827754e..00000000 --- a/packages/backend/src/apps/twitter/common/generate-request.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Token } from 'oauth-1.0a'; -import { IGlobalVariable, IJSONObject } from '@automatisch/types'; -import oauthClient from './oauth-client'; -import { Method } from 'axios'; - -type IGenereateRequestOptons = { - requestPath: string; - method: string; - data?: IJSONObject; -}; - -const generateRequest = async ( - $: IGlobalVariable, - options: IGenereateRequestOptons -) => { - const { requestPath, method, data } = options; - - const token: Token = { - key: $.auth.data.accessToken as string, - secret: $.auth.data.accessSecret as string, - }; - - const requestData = { - url: `${$.app.apiBaseUrl}${requestPath}`, - method, - data, - }; - - const authHeader = oauthClient($).toHeader( - oauthClient($).authorize(requestData, token) - ); - - const response = await $.http.request({ - url: requestData.url, - method: requestData.method as Method, - headers: { - ...authHeader, - }, - }); - - return response; -}; - -export default generateRequest; diff --git a/packages/backend/src/apps/twitter/common/get-current-user.ts b/packages/backend/src/apps/twitter/common/get-current-user.ts index 0823192f..3ce130a0 100644 --- a/packages/backend/src/apps/twitter/common/get-current-user.ts +++ b/packages/backend/src/apps/twitter/common/get-current-user.ts @@ -1,13 +1,9 @@ import { IGlobalVariable, IJSONObject } from '@automatisch/types'; -import generateRequest from './generate-request'; const getCurrentUser = async ($: IGlobalVariable): Promise => { - const response = await generateRequest($, { - requestPath: '/2/users/me', - method: 'GET', - }); - + const response = await $.http.get('/2/users/me'); const currentUser = response.data.data; + return currentUser; }; diff --git a/packages/backend/src/apps/twitter/common/get-user-by-username.ts b/packages/backend/src/apps/twitter/common/get-user-by-username.ts index 45e108be..4240fff4 100644 --- a/packages/backend/src/apps/twitter/common/get-user-by-username.ts +++ b/packages/backend/src/apps/twitter/common/get-user-by-username.ts @@ -1,11 +1,7 @@ import { IGlobalVariable, IJSONObject } from '@automatisch/types'; -import generateRequest from './generate-request'; const getUserByUsername = async ($: IGlobalVariable, username: string) => { - const response = await generateRequest($, { - requestPath: `/2/users/by/username/${username}`, - method: 'GET', - }); + const response = await $.http.get(`/2/users/by/username/${username}`); if (response.data.errors) { const errorMessages = response.data.errors diff --git a/packages/backend/src/apps/twitter/common/get-user-followers.ts b/packages/backend/src/apps/twitter/common/get-user-followers.ts index 1874f61a..f63baabb 100644 --- a/packages/backend/src/apps/twitter/common/get-user-followers.ts +++ b/packages/backend/src/apps/twitter/common/get-user-followers.ts @@ -5,11 +5,9 @@ import { } from '@automatisch/types'; import { URLSearchParams } from 'url'; import { omitBy, isEmpty } from 'lodash'; -import generateRequest from './generate-request'; type GetUserFollowersOptions = { userId: string; - lastInternalId?: string; }; const getUserFollowers = async ( @@ -33,10 +31,7 @@ const getUserFollowers = async ( queryParams.toString() ? `?${queryParams.toString()}` : '' }`; - response = await generateRequest($, { - requestPath, - method: 'GET', - }); + response = await $.http.get(requestPath); if (response.integrationError) { followers.error = response.integrationError; @@ -49,18 +44,18 @@ const getUserFollowers = async ( } if (response.data.meta.result_count > 0) { - response.data.data.forEach((follower: IJSONObject) => { + for (const follower of response.data.data) { + if ($.flow.isAlreadyProcessed(follower.id as string)) { + return followers; + } + followers.data.push({ raw: follower, meta: { internalId: follower.id as string }, }); - }); + } } - } while (response.data.meta.next_token && options.lastInternalId); - - followers.data.sort((follower, nextFollower) => { - return (follower.raw.id as number) - (nextFollower.raw.id as number); - }); + } while (response.data.meta.next_token && !$.execution.testRun); return followers; }; diff --git a/packages/backend/src/apps/twitter/common/get-user-tweets.ts b/packages/backend/src/apps/twitter/common/get-user-tweets.ts index 6a2f5360..7e7004bd 100644 --- a/packages/backend/src/apps/twitter/common/get-user-tweets.ts +++ b/packages/backend/src/apps/twitter/common/get-user-tweets.ts @@ -6,7 +6,6 @@ import { import { URLSearchParams } from 'url'; import omitBy from 'lodash/omitBy'; import isEmpty from 'lodash/isEmpty'; -import generateRequest from './generate-request'; import getCurrentUser from './get-current-user'; import getUserByUsername from './get-user-by-username'; @@ -14,19 +13,7 @@ type IGetUserTweetsOptions = { currentUser: boolean; }; -const getUserTweets = async ( - $: IGlobalVariable, - options: IGetUserTweetsOptions -) => { - let username: string; - - if (options.currentUser) { - const currentUser = await getCurrentUser($); - username = currentUser.username as string; - } else { - username = $.step.parameters.username as string; - } - +const fetchTweets = async ($: IGlobalVariable, username: string) => { const user = await getUserByUsername($, username); let response; @@ -47,10 +34,7 @@ const getUserTweets = async ( queryParams.toString() ? `?${queryParams.toString()}` : '' }`; - response = await generateRequest($, { - requestPath, - method: 'GET', - }); + response = await $.http.get(requestPath); if (response.integrationError) { tweets.error = response.integrationError; @@ -72,4 +56,26 @@ const getUserTweets = async ( return tweets; }; +const getUserTweets = async ( + $: IGlobalVariable, + options: IGetUserTweetsOptions +) => { + let username: string; + + if (options.currentUser) { + const currentUser = await getCurrentUser($); + username = currentUser.username as string; + } else { + username = $.step.parameters.username as string; + } + + const tweets = await fetchTweets($, username); + + tweets.data.sort((tweet, nextTweet) => { + return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId); + }); + + return tweets; +}; + export default getUserTweets; diff --git a/packages/backend/src/apps/twitter/index.ts b/packages/backend/src/apps/twitter/index.ts index d97c5d4b..f2c39a8e 100644 --- a/packages/backend/src/apps/twitter/index.ts +++ b/packages/backend/src/apps/twitter/index.ts @@ -1,4 +1,7 @@ -export default { +import defineApp from '../../helpers/define-app'; +import addAuthHeader from './common/add-auth-header'; + +export default defineApp({ name: 'Twitter', key: 'twitter', iconUrl: '{BASE_URL}/apps/twitter/assets/favicon.svg', @@ -6,4 +9,6 @@ export default { supportsConnections: true, baseUrl: 'https://twitter.com', apiBaseUrl: 'https://api.twitter.com', -}; + primaryColor: '1da1f2', + beforeRequest: [addAuthHeader], +}); diff --git a/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts b/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts index e4d875d8..8f5fe4f6 100644 --- a/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts +++ b/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts @@ -6,6 +6,7 @@ export default defineTrigger({ key: 'myFollowers', pollInterval: 15, description: 'Will be triggered when you have a new follower.', + dedupeStrategy: 'unique', substeps: [ { key: 'chooseConnection', @@ -18,6 +19,6 @@ export default defineTrigger({ ], async run($) { - return await myFollowers($, $.flow.lastInternalId); + return await myFollowers($); }, }); diff --git a/packages/backend/src/apps/twitter/triggers/new-follower-of-me/my-followers.ts b/packages/backend/src/apps/twitter/triggers/new-follower-of-me/my-followers.ts index a4d0fa0e..69185bf7 100644 --- a/packages/backend/src/apps/twitter/triggers/new-follower-of-me/my-followers.ts +++ b/packages/backend/src/apps/twitter/triggers/new-follower-of-me/my-followers.ts @@ -3,13 +3,12 @@ import getCurrentUser from '../../common/get-current-user'; import getUserByUsername from '../../common/get-user-by-username'; import getUserFollowers from '../../common/get-user-followers'; -const myFollowers = async ($: IGlobalVariable, lastInternalId?: string) => { +const myFollowers = async ($: IGlobalVariable) => { const { username } = await getCurrentUser($); const user = await getUserByUsername($, username as string); const tweets = await getUserFollowers($, { userId: user.id, - lastInternalId, }); return tweets; }; diff --git a/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts b/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts index 475403b5..431b0071 100644 --- a/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts +++ b/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts @@ -4,10 +4,9 @@ import { ITriggerOutput, } from '@automatisch/types'; import qs from 'qs'; -import generateRequest from '../../common/generate-request'; import { omitBy, isEmpty } from 'lodash'; -const searchTweets = async ($: IGlobalVariable) => { +const fetchTweets = async ($: IGlobalVariable) => { const searchTerm = $.step.parameters.searchTerm as string; let response; @@ -29,10 +28,7 @@ const searchTweets = async ($: IGlobalVariable) => { queryParams.toString() ? `?${queryParams.toString()}` : '' }`; - response = await generateRequest($, { - requestPath, - method: 'GET', - }); + response = await $.http.get(requestPath); if (response.integrationError) { tweets.error = response.integrationError; @@ -58,8 +54,14 @@ const searchTweets = async ($: IGlobalVariable) => { } } while (response.data.meta.next_token && !$.execution.testRun); + return tweets; +}; + +const searchTweets = async ($: IGlobalVariable) => { + const tweets = await fetchTweets($); + tweets.data.sort((tweet, nextTweet) => { - return (tweet.raw.id as number) - (nextTweet.raw.id as number); + return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId); }); return tweets; diff --git a/packages/backend/src/helpers/global-variable.ts b/packages/backend/src/helpers/global-variable.ts index e2c08e74..314ed368 100644 --- a/packages/backend/src/helpers/global-variable.ts +++ b/packages/backend/src/helpers/global-variable.ts @@ -68,9 +68,11 @@ const globalVariable = async ( }); if (trigger && trigger.dedupeStrategy === 'unique') { - const lastInternalIds = await flow?.lastInternalIds(); + const lastInternalIds = testRun ? [] : await flow?.lastInternalIds(); const isAlreadyProcessed = (internalId: string) => { + if (testRun) return false; + return lastInternalIds?.includes(internalId); }; diff --git a/packages/backend/src/helpers/http-client/index.ts b/packages/backend/src/helpers/http-client/index.ts index 2924383f..a98820a0 100644 --- a/packages/backend/src/helpers/http-client/index.ts +++ b/packages/backend/src/helpers/http-client/index.ts @@ -1,17 +1,40 @@ import axios, { AxiosRequestConfig } from 'axios'; export { AxiosInstance as IHttpClient } from 'axios'; import { IHttpClientParams } from '@automatisch/types'; +import { URL } from 'url'; -export default function createHttpClient({ $, baseURL, beforeRequest = [] }: IHttpClientParams) { +const removeBaseUrlForAbsoluteUrls = ( + requestConfig: AxiosRequestConfig +): AxiosRequestConfig => { + try { + const url = new URL(requestConfig.url); + requestConfig.baseURL = url.origin; + requestConfig.url = url.pathname + url.search; + + return requestConfig; + } catch { + return requestConfig; + } +}; + +export default function createHttpClient({ + $, + baseURL, + beforeRequest = [], +}: IHttpClientParams) { const instance = axios.create({ baseURL, }); - instance.interceptors.request.use((requestConfig: AxiosRequestConfig): AxiosRequestConfig => { - return beforeRequest.reduce((newConfig, beforeRequestFunc) => { - return beforeRequestFunc($, newConfig); - }, requestConfig); - }); + instance.interceptors.request.use( + (requestConfig: AxiosRequestConfig): AxiosRequestConfig => { + const newRequestConfig = removeBaseUrlForAbsoluteUrls(requestConfig); + + return beforeRequest.reduce((newConfig, beforeRequestFunc) => { + return beforeRequestFunc($, newConfig); + }, newRequestConfig); + } + ); instance.interceptors.response.use( (response) => response,