From abaad7cb822078fa32264399837e10a831d48fcc Mon Sep 17 00:00:00 2001 From: Faruk AYDIN Date: Wed, 31 Aug 2022 15:28:53 +0300 Subject: [PATCH] feat: Implement search tweets trigger --- .../twitter/client/endpoints/search-tweets.ts | 73 +++++++++++++++++++ .../backend/src/apps/twitter/client/index.ts | 3 + packages/backend/src/apps/twitter/info.json | 4 +- packages/backend/src/apps/twitter/triggers.ts | 3 + .../src/apps/twitter/triggers/search-tweet.ts | 58 --------------- .../apps/twitter/triggers/search-tweets.ts | 26 +++++++ 6 files changed, 107 insertions(+), 60 deletions(-) create mode 100644 packages/backend/src/apps/twitter/client/endpoints/search-tweets.ts delete mode 100644 packages/backend/src/apps/twitter/triggers/search-tweet.ts create mode 100644 packages/backend/src/apps/twitter/triggers/search-tweets.ts diff --git a/packages/backend/src/apps/twitter/client/endpoints/search-tweets.ts b/packages/backend/src/apps/twitter/client/endpoints/search-tweets.ts new file mode 100644 index 00000000..634c66cf --- /dev/null +++ b/packages/backend/src/apps/twitter/client/endpoints/search-tweets.ts @@ -0,0 +1,73 @@ +import { IJSONObject } from '@automatisch/types'; +import { URLSearchParams } from 'url'; +import TwitterClient from '../index'; +import omitBy from 'lodash/omitBy'; +import isEmpty from 'lodash/isEmpty'; +import qs from 'qs'; + +export default class SearchTweets { + client: TwitterClient; + + constructor(client: TwitterClient) { + this.client = client; + } + + async run(searchTerm: string, lastInternalId?: string) { + const token = { + key: this.client.connection.formattedData.accessToken as string, + secret: this.client.connection.formattedData.accessSecret as string, + }; + + let response; + const tweets: IJSONObject[] = []; + + do { + const params: IJSONObject = { + query: searchTerm, + since_id: lastInternalId, + pagination_token: response?.data?.meta?.next_token, + }; + + const queryParams = qs.stringify(omitBy(params, isEmpty)); + + const requestPath = `/2/tweets/search/recent${ + queryParams.toString() ? `?${queryParams.toString()}` : '' + }`; + + const requestData = { + url: `${TwitterClient.baseUrl}${requestPath}`, + method: 'GET', + }; + + const authHeader = this.client.oauthClient.toHeader( + this.client.oauthClient.authorize(requestData, token) + ); + + response = await this.client.httpClient.get(requestPath, { + headers: { ...authHeader }, + }); + + if (response.data.meta.result_count > 0) { + response.data.data.forEach((tweet: IJSONObject) => { + if (!lastInternalId || Number(tweet.id) > Number(lastInternalId)) { + tweets.push(tweet); + } else { + return; + } + }); + } + } while (response.data.meta.next_token && lastInternalId); + + if (response.data?.errors) { + const errorMessages = response.data.errors + .map((error: IJSONObject) => error.detail) + .join(' '); + + throw new Error( + `Error occured while fetching user data: ${errorMessages}` + ); + } + + return tweets; + } +} diff --git a/packages/backend/src/apps/twitter/client/index.ts b/packages/backend/src/apps/twitter/client/index.ts index c3544d9f..f27e3f44 100644 --- a/packages/backend/src/apps/twitter/client/index.ts +++ b/packages/backend/src/apps/twitter/client/index.ts @@ -8,6 +8,7 @@ import GetCurrentUser from './endpoints/get-current-user'; import GetUserByUsername from './endpoints/get-user-by-username'; import GetUserTweets from './endpoints/get-user-tweets'; import CreateTweet from './endpoints/create-tweet'; +import SearchTweets from './endpoints/search-tweets'; export default class TwitterClient { flow: IFlow; @@ -22,6 +23,7 @@ export default class TwitterClient { getUserByUsername: GetUserByUsername; getUserTweets: GetUserTweets; createTweet: CreateTweet; + searchTweets: SearchTweets; static baseUrl = 'https://api.twitter.com'; @@ -54,5 +56,6 @@ export default class TwitterClient { this.getUserByUsername = new GetUserByUsername(this); this.getUserTweets = new GetUserTweets(this); this.createTweet = new CreateTweet(this); + this.searchTweets = new SearchTweets(this); } } diff --git a/packages/backend/src/apps/twitter/info.json b/packages/backend/src/apps/twitter/info.json index 2b383999..973ceba8 100644 --- a/packages/backend/src/apps/twitter/info.json +++ b/packages/backend/src/apps/twitter/info.json @@ -260,8 +260,8 @@ ] }, { - "name": "Search Tweet", - "key": "searchTweet", + "name": "Search Tweets", + "key": "searchTweets", "description": "Will be triggered when any user tweet something containing a specific keyword, phrase, username or hashtag.", "substeps": [ { diff --git a/packages/backend/src/apps/twitter/triggers.ts b/packages/backend/src/apps/twitter/triggers.ts index 40ea39bd..b8ddf879 100644 --- a/packages/backend/src/apps/twitter/triggers.ts +++ b/packages/backend/src/apps/twitter/triggers.ts @@ -1,12 +1,15 @@ import TwitterClient from './client'; import UserTweet from './triggers/user-tweet'; +import SearchTweets from './triggers/search-tweets'; export default class Triggers { client: TwitterClient; userTweet: UserTweet; + searchTweets: SearchTweets; constructor(client: TwitterClient) { this.client = client; this.userTweet = new UserTweet(client); + this.searchTweets = new SearchTweets(client); } } diff --git a/packages/backend/src/apps/twitter/triggers/search-tweet.ts b/packages/backend/src/apps/twitter/triggers/search-tweet.ts deleted file mode 100644 index 7d3b7af9..00000000 --- a/packages/backend/src/apps/twitter/triggers/search-tweet.ts +++ /dev/null @@ -1,58 +0,0 @@ -import TwitterApi, { TwitterApiTokens } from 'twitter-api-v2'; -import { IJSONObject } from '@automatisch/types'; - -export default class SearchTweet { - client: TwitterApi; - parameters: IJSONObject; - - constructor(connectionData: IJSONObject, parameters: IJSONObject) { - this.client = new TwitterApi({ - appKey: connectionData.consumerKey, - appSecret: connectionData.consumerSecret, - accessToken: connectionData.accessToken, - accessSecret: connectionData.accessSecret, - } as TwitterApiTokens); - - this.parameters = parameters; - } - - async run(startTime: Date) { - const tweets = []; - - const response = await this.client.v2.search( - this.parameters.searchTerm as string, - { - max_results: 50, - 'tweet.fields': 'created_at', - } - ); - - for await (const tweet of response.data.data) { - if (new Date(tweet.created_at).getTime() <= startTime.getTime()) { - break; - } - - tweets.push(tweet); - - if (response.data.meta.next_token) { - await response.fetchNext(); - } - } - - return tweets; - } - - async testRun() { - const response = await this.client.v2.search( - this.parameters.searchTerm as string, - { - max_results: 10, - 'tweet.fields': 'created_at', - } - ); - - const mostRecentTweet = response.data.data[0]; - - return [mostRecentTweet]; - } -} diff --git a/packages/backend/src/apps/twitter/triggers/search-tweets.ts b/packages/backend/src/apps/twitter/triggers/search-tweets.ts new file mode 100644 index 00000000..44ccb5df --- /dev/null +++ b/packages/backend/src/apps/twitter/triggers/search-tweets.ts @@ -0,0 +1,26 @@ +import TwitterClient from '../client'; + +export default class SearchTweets { + client: TwitterClient; + + constructor(client: TwitterClient) { + this.client = client; + } + + async run(lastInternalId: string) { + return this.getTweets(lastInternalId); + } + + async testRun() { + return this.getTweets(); + } + + async getTweets(lastInternalId?: string) { + const tweets = await this.client.searchTweets.run( + this.client.step.parameters.searchTerm as string, + lastInternalId + ); + + return tweets; + } +}