From baebec2270f7fa8cdb05df95e9779f4139868f56 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:18:50 +0200
Subject: [PATCH 01/18] feat: Add base and early exit error classes

---
 packages/backend/src/errors/base.ts       | 17 +++++++++++++++++
 packages/backend/src/errors/early-exit.ts |  3 +++
 2 files changed, 20 insertions(+)
 create mode 100644 packages/backend/src/errors/base.ts
 create mode 100644 packages/backend/src/errors/early-exit.ts

diff --git a/packages/backend/src/errors/base.ts b/packages/backend/src/errors/base.ts
new file mode 100644
index 00000000..9a871982
--- /dev/null
+++ b/packages/backend/src/errors/base.ts
@@ -0,0 +1,17 @@
+import { IJSONObject } from '@automatisch/types';
+
+export default class BaseError extends Error {
+  error = {};
+
+  constructor(error?: string | IJSONObject) {
+    super();
+
+    try {
+      this.error = JSON.parse(error as string);
+    } catch {
+      this.error = typeof error === 'string' ? { error } : error;
+    }
+
+    this.name = this.constructor.name;
+  }
+}
diff --git a/packages/backend/src/errors/early-exit.ts b/packages/backend/src/errors/early-exit.ts
new file mode 100644
index 00000000..c0d92d2c
--- /dev/null
+++ b/packages/backend/src/errors/early-exit.ts
@@ -0,0 +1,3 @@
+import BaseError from './base';
+
+export default class EarlyExitError extends BaseError {}

From 11dc650e0eff149b8df110e2d786b7fe6da9d335 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:20:03 +0200
Subject: [PATCH 02/18] refactor: Remove dedupe strategies, use only dedupe
 list

---
 .../backend/src/helpers/global-variable.ts    | 37 +++++++------------
 packages/backend/src/services/flow.ts         | 17 +++++----
 2 files changed, 24 insertions(+), 30 deletions(-)

diff --git a/packages/backend/src/helpers/global-variable.ts b/packages/backend/src/helpers/global-variable.ts
index 69097354..095cf6b8 100644
--- a/packages/backend/src/helpers/global-variable.ts
+++ b/packages/backend/src/helpers/global-variable.ts
@@ -10,6 +10,7 @@ import {
   ITriggerItem,
   IActionItem,
 } from '@automatisch/types';
+import EarlyExitError from '../errors/early-exit';
 
 type GlobalVariableOptions = {
   connection?: Connection;
@@ -25,9 +26,7 @@ const globalVariable = async (
 ): Promise<IGlobalVariable> => {
   const { connection, app, flow, step, execution, testRun = false } = options;
 
-  const lastInternalId = await flow?.lastInternalId();
-
-  const trigger = await step?.getTriggerCommand();
+  const lastInternalId = testRun ? undefined : await flow?.lastInternalId();
   const nextStep = await step?.getNextStep();
 
   const $: IGlobalVariable = {
@@ -75,6 +74,13 @@ const globalVariable = async (
     },
     pushTriggerItem: (triggerItem: ITriggerItem) => {
       $.triggerOutput.data.push(triggerItem);
+
+      if (
+        $.execution.testRun ||
+        isAlreadyProcessed(triggerItem.meta.internalId)
+      ) {
+        throw new EarlyExitError();
+      }
     },
     setActionItem: (actionItem: IActionItem) => {
       $.actionOutput.data = actionItem;
@@ -87,27 +93,12 @@ const globalVariable = async (
     beforeRequest: app.beforeRequest,
   });
 
-  if (trigger) {
-    if (trigger.dedupeStrategy === 'unique') {
-      const lastInternalIds = testRun ? [] : await flow?.lastInternalIds();
+  const lastInternalIds =
+    testRun || (flow && step.isAction) ? [] : await flow?.lastInternalIds();
 
-      const isAlreadyProcessed = (internalId: string) => {
-        if (testRun) return false;
-
-        return lastInternalIds?.includes(internalId);
-      };
-
-      $.flow.isAlreadyProcessed = isAlreadyProcessed;
-    } else if (trigger.dedupeStrategy === 'greatest') {
-      const isAlreadyProcessed = (internalId: string) => {
-        if (testRun) return false;
-
-        return Number(internalId) <= Number($.flow.lastInternalId);
-      };
-
-      $.flow.isAlreadyProcessed = isAlreadyProcessed;
-    }
-  }
+  const isAlreadyProcessed = (internalId: string) => {
+    return lastInternalIds?.includes(internalId);
+  };
 
   return $;
 };
diff --git a/packages/backend/src/services/flow.ts b/packages/backend/src/services/flow.ts
index 4cde0fd7..8e481444 100644
--- a/packages/backend/src/services/flow.ts
+++ b/packages/backend/src/services/flow.ts
@@ -1,5 +1,6 @@
 import Flow from '../models/flow';
 import globalVariable from '../helpers/global-variable';
+import EarlyExitError from '../errors/early-exit';
 
 type ProcessFlowOptions = {
   flowId: string;
@@ -23,13 +24,15 @@ export const processFlow = async (options: ProcessFlowOptions) => {
   try {
     await triggerCommand.run($);
   } catch (error) {
-    if (error?.response?.httpError) {
-      $.triggerOutput.error = error.response.httpError;
-    } else {
-      try {
-        $.triggerOutput.error = JSON.parse(error.message);
-      } catch {
-        $.triggerOutput.error = { error: error.message };
+    if (error instanceof EarlyExitError === false) {
+      if (error?.response?.httpError) {
+        $.triggerOutput.error = error.response.httpError;
+      } else {
+        try {
+          $.triggerOutput.error = JSON.parse(error.message);
+        } catch {
+          $.triggerOutput.error = { error: error.message };
+        }
       }
     }
   }

From 5189e5b1312b119f16e6b7d70550ea38756fc646 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:22:30 +0200
Subject: [PATCH 03/18] refactor: Adjust pagination and test run for search
 tweets

---
 .../backend/src/apps/twitter/triggers/search-tweets/index.ts  | 4 ----
 .../src/apps/twitter/triggers/search-tweets/search-tweets.ts  | 4 ++--
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/backend/src/apps/twitter/triggers/search-tweets/index.ts b/packages/backend/src/apps/twitter/triggers/search-tweets/index.ts
index 506bbe83..d97e565d 100644
--- a/packages/backend/src/apps/twitter/triggers/search-tweets/index.ts
+++ b/packages/backend/src/apps/twitter/triggers/search-tweets/index.ts
@@ -33,8 +33,4 @@ export default defineTrigger({
   async run($) {
     await searchTweets($);
   },
-
-  sort(tweet, nextTweet) {
-    return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId);
-  },
 });
diff --git a/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts b/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts
index b79481e6..929bfac3 100644
--- a/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts
+++ b/packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts
@@ -10,7 +10,7 @@ const searchTweets = async ($: IGlobalVariable) => {
   do {
     const params: IJSONObject = {
       query: searchTerm,
-      since_id: $.execution.testRun ? null : $.flow.lastInternalId,
+      since_id: $.flow.lastInternalId,
       pagination_token: response?.data?.meta?.next_token,
     };
 
@@ -38,7 +38,7 @@ const searchTweets = async ($: IGlobalVariable) => {
         $.pushTriggerItem(dataItem);
       });
     }
-  } while (response.data.meta.next_token && !$.execution.testRun);
+  } while (response.data.meta.next_token);
 };
 
 export default searchTweets;

From 96007a1f4818c452a477c53cb562adb2fe4e5682 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:25:33 +0200
Subject: [PATCH 04/18] refactor: Remove pagination and test run checks from
 user tweets

---
 packages/backend/src/apps/twitter/common/get-user-tweets.ts   | 4 ++--
 packages/backend/src/apps/twitter/triggers/my-tweets/index.ts | 4 ----
 .../backend/src/apps/twitter/triggers/user-tweets/index.ts    | 4 ----
 3 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/packages/backend/src/apps/twitter/common/get-user-tweets.ts b/packages/backend/src/apps/twitter/common/get-user-tweets.ts
index 6d0f2807..2eb2aa11 100644
--- a/packages/backend/src/apps/twitter/common/get-user-tweets.ts
+++ b/packages/backend/src/apps/twitter/common/get-user-tweets.ts
@@ -16,7 +16,7 @@ const fetchTweets = async ($: IGlobalVariable, username: string) => {
 
   do {
     const params: IJSONObject = {
-      since_id: $.execution.testRun ? null : $.flow.lastInternalId,
+      since_id: $.flow.lastInternalId,
       pagination_token: response?.data?.meta?.next_token,
     };
 
@@ -40,7 +40,7 @@ const fetchTweets = async ($: IGlobalVariable, username: string) => {
         $.pushTriggerItem(dataItem);
       });
     }
-  } while (response.data.meta.next_token && !$.execution.testRun);
+  } while (response.data.meta.next_token);
 
   return $.triggerOutput;
 };
diff --git a/packages/backend/src/apps/twitter/triggers/my-tweets/index.ts b/packages/backend/src/apps/twitter/triggers/my-tweets/index.ts
index cfcd13f2..134e9084 100644
--- a/packages/backend/src/apps/twitter/triggers/my-tweets/index.ts
+++ b/packages/backend/src/apps/twitter/triggers/my-tweets/index.ts
@@ -20,8 +20,4 @@ export default defineTrigger({
   async run($) {
     await getUserTweets($, { currentUser: true });
   },
-
-  sort(tweet, nextTweet) {
-    return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId);
-  },
 });
diff --git a/packages/backend/src/apps/twitter/triggers/user-tweets/index.ts b/packages/backend/src/apps/twitter/triggers/user-tweets/index.ts
index dab530ae..4e07959b 100644
--- a/packages/backend/src/apps/twitter/triggers/user-tweets/index.ts
+++ b/packages/backend/src/apps/twitter/triggers/user-tweets/index.ts
@@ -32,8 +32,4 @@ export default defineTrigger({
   async run($) {
     await getUserTweets($, { currentUser: false });
   },
-
-  sort(tweet, nextTweet) {
-    return Number(nextTweet.meta.internalId) - Number(tweet.meta.internalId);
-  },
 });

From 8e0553147bf1812db3b0802b7c7af558e1f9e0ef Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:33:33 +0200
Subject: [PATCH 05/18] refactor: Remove testRun logic from get user followers

---
 .../backend/src/apps/twitter/common/get-user-followers.ts   | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/backend/src/apps/twitter/common/get-user-followers.ts b/packages/backend/src/apps/twitter/common/get-user-followers.ts
index cc174311..406ee8b6 100644
--- a/packages/backend/src/apps/twitter/common/get-user-followers.ts
+++ b/packages/backend/src/apps/twitter/common/get-user-followers.ts
@@ -31,17 +31,13 @@ const getUserFollowers = async (
 
     if (response.data.meta.result_count > 0) {
       for (const follower of response.data.data) {
-        if ($.flow.isAlreadyProcessed(follower.id as string)) {
-          return;
-        }
-
         $.pushTriggerItem({
           raw: follower,
           meta: { internalId: follower.id as string },
         });
       }
     }
-  } while (response.data.meta.next_token && !$.execution.testRun);
+  } while (response.data.meta.next_token);
 };
 
 export default getUserFollowers;

From 796121acfc9c8a602a28d9e70b02a86d67870d80 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:35:54 +0200
Subject: [PATCH 06/18] chore: Remove redundant IActionOutput type import

---
 .../backend/src/apps/slack/actions/find-message/find-message.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend/src/apps/slack/actions/find-message/find-message.ts b/packages/backend/src/apps/slack/actions/find-message/find-message.ts
index a98874f0..b910e01b 100644
--- a/packages/backend/src/apps/slack/actions/find-message/find-message.ts
+++ b/packages/backend/src/apps/slack/actions/find-message/find-message.ts
@@ -1,4 +1,4 @@
-import { IGlobalVariable, IActionOutput } from '@automatisch/types';
+import { IGlobalVariable } from '@automatisch/types';
 
 type FindMessageOptions = {
   query: string;

From 3308a44c2b2e294595e9070d258b446d055261aa Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:41:23 +0200
Subject: [PATCH 07/18] refactor: Remove testRun and completion check from new
 stargazers

---
 .../src/apps/github/triggers/new-stargazers/index.ts        | 6 ------
 .../apps/github/triggers/new-stargazers/new-stargazers.ts   | 5 +----
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/packages/backend/src/apps/github/triggers/new-stargazers/index.ts b/packages/backend/src/apps/github/triggers/new-stargazers/index.ts
index 5584e022..7ebfda63 100644
--- a/packages/backend/src/apps/github/triggers/new-stargazers/index.ts
+++ b/packages/backend/src/apps/github/triggers/new-stargazers/index.ts
@@ -43,10 +43,4 @@ export default defineTrigger({
   async run($) {
     await newStargazers($);
   },
-
-  sort(stargazerA, stargazerB) {
-    return (
-      Number(stargazerB.meta.internalId) - Number(stargazerA.meta.internalId)
-    );
-  },
 });
diff --git a/packages/backend/src/apps/github/triggers/new-stargazers/new-stargazers.ts b/packages/backend/src/apps/github/triggers/new-stargazers/new-stargazers.ts
index e34e819e..2dab3f55 100644
--- a/packages/backend/src/apps/github/triggers/new-stargazers/new-stargazers.ts
+++ b/packages/backend/src/apps/github/triggers/new-stargazers/new-stargazers.ts
@@ -48,9 +48,6 @@ const newStargazers = async ($: IGlobalVariable) => {
         const { starred_at, user } = starEntry;
         const timestamp = DateTime.fromISO(starred_at).toMillis();
 
-        if (timestamp <= Number($.flow.lastInternalId) && !$.execution.testRun)
-          return;
-
         const dataItem = {
           raw: user,
           meta: {
@@ -61,7 +58,7 @@ const newStargazers = async ($: IGlobalVariable) => {
         $.pushTriggerItem(dataItem);
       }
     }
-  } while (pathname && !$.execution.testRun);
+  } while (pathname);
 };
 
 export default newStargazers;

From 87fdc02224e053f698e2c10ab6ee89eca55b2136 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 00:46:32 +0200
Subject: [PATCH 08/18] refactor: Remove test run and completion logic from new
 watchers

---
 .../backend/src/apps/github/triggers/new-watchers/index.ts | 4 ----
 .../src/apps/github/triggers/new-watchers/new-watchers.ts  | 7 ++-----
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/packages/backend/src/apps/github/triggers/new-watchers/index.ts b/packages/backend/src/apps/github/triggers/new-watchers/index.ts
index a7f744d1..03204e7f 100644
--- a/packages/backend/src/apps/github/triggers/new-watchers/index.ts
+++ b/packages/backend/src/apps/github/triggers/new-watchers/index.ts
@@ -44,8 +44,4 @@ export default defineTrigger({
   async run($) {
     await newWatchers($);
   },
-
-  sort() {
-    return -1;
-  },
 });
diff --git a/packages/backend/src/apps/github/triggers/new-watchers/new-watchers.ts b/packages/backend/src/apps/github/triggers/new-watchers/new-watchers.ts
index 69753514..f7e13035 100644
--- a/packages/backend/src/apps/github/triggers/new-watchers/new-watchers.ts
+++ b/packages/backend/src/apps/github/triggers/new-watchers/new-watchers.ts
@@ -1,4 +1,4 @@
-import { IGlobalVariable, ITriggerOutput } from '@automatisch/types';
+import { IGlobalVariable } from '@automatisch/types';
 import getRepoOwnerAndRepo from '../../common/get-repo-owner-and-repo';
 import parseLinkHeader from '../../../../helpers/parse-header-link';
 
@@ -34,9 +34,6 @@ const newWatchers = async ($: IGlobalVariable) => {
       for (const watcher of response.data) {
         const watcherId = watcher.id.toString();
 
-        if ($.flow.isAlreadyProcessed(watcherId) && !$.execution.testRun)
-          return;
-
         const dataItem = {
           raw: watcher,
           meta: {
@@ -47,7 +44,7 @@ const newWatchers = async ($: IGlobalVariable) => {
         $.pushTriggerItem(dataItem);
       }
     }
-  } while (pathname && !$.execution.testRun);
+  } while (pathname);
 };
 
 export default newWatchers;

From 031e175e9f6aeb8782da39e70789e34ea4f382c3 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 01:10:15 +0200
Subject: [PATCH 09/18] refactor: Remove test run and completion check from new
 issues

---
 .../src/apps/github/triggers/new-issues/new-issues.ts        | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/backend/src/apps/github/triggers/new-issues/new-issues.ts b/packages/backend/src/apps/github/triggers/new-issues/new-issues.ts
index 8a52fabc..4d1b4978 100644
--- a/packages/backend/src/apps/github/triggers/new-issues/new-issues.ts
+++ b/packages/backend/src/apps/github/triggers/new-issues/new-issues.ts
@@ -34,9 +34,6 @@ const newIssues = async ($: IGlobalVariable) => {
       for (const issue of response.data) {
         const issueId = issue.id;
 
-        if (issueId <= Number($.flow.lastInternalId) && !$.execution.testRun)
-          return;
-
         const dataItem = {
           raw: issue,
           meta: {
@@ -47,7 +44,7 @@ const newIssues = async ($: IGlobalVariable) => {
         $.pushTriggerItem(dataItem);
       }
     }
-  } while (links.next && !$.execution.testRun);
+  } while (links.next);
 };
 
 export default newIssues;

From 232ba230eba002e861d6de9ff5bd70c81e4ca129 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 01:10:33 +0200
Subject: [PATCH 10/18] refactor: Remove test run and completion check from new
 pull requests

---
 .../triggers/new-pull-requests/new-pull-requests.ts       | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/backend/src/apps/github/triggers/new-pull-requests/new-pull-requests.ts b/packages/backend/src/apps/github/triggers/new-pull-requests/new-pull-requests.ts
index ad26c9d9..a19ef398 100644
--- a/packages/backend/src/apps/github/triggers/new-pull-requests/new-pull-requests.ts
+++ b/packages/backend/src/apps/github/triggers/new-pull-requests/new-pull-requests.ts
@@ -26,12 +26,6 @@ const newPullRequests = async ($: IGlobalVariable) => {
       for (const pullRequest of response.data) {
         const pullRequestId = pullRequest.id;
 
-        if (
-          pullRequestId <= Number($.flow.lastInternalId) &&
-          !$.execution.testRun
-        )
-          return;
-
         const dataItem = {
           raw: pullRequest,
           meta: {
@@ -42,7 +36,7 @@ const newPullRequests = async ($: IGlobalVariable) => {
         $.pushTriggerItem(dataItem);
       }
     }
-  } while (links.next && !$.execution.testRun);
+  } while (links.next);
 };
 
 export default newPullRequests;

From 74d7e472af52db48900a915697da48d22166687c Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 01:11:29 +0200
Subject: [PATCH 11/18] refactor: Remove test run and completion check from new
 photos

---
 .../src/apps/flickr/triggers/new-photos/new-photos.ts | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/packages/backend/src/apps/flickr/triggers/new-photos/new-photos.ts b/packages/backend/src/apps/flickr/triggers/new-photos/new-photos.ts
index 3f8850ef..a27a9185 100644
--- a/packages/backend/src/apps/flickr/triggers/new-photos/new-photos.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-photos/new-photos.ts
@@ -48,17 +48,14 @@ const newPhotos = async ($: IGlobalVariable) => {
     pages = photos.pages;
 
     for (const photo of photos.photo) {
-      if ($.flow.isAlreadyProcessed(photo.id) && !$.execution.testRun)
-        return;
-
       $.pushTriggerItem({
         raw: photo,
         meta: {
-          internalId: photo.id as string
-        }
-      })
+          internalId: photo.id as string,
+        },
+      });
     }
-  } while (page <= pages && !$.execution.testRun);
+  } while (page <= pages);
 };
 
 export default newPhotos;

From 18ccb25b29c592b8888e079eedee21e7b80a93ce Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 01:14:19 +0200
Subject: [PATCH 12/18] refactor: Remove is already processed from new feed
 items

---
 .../triggers/new-items-in-feed/new-items-in-feed.ts   | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/packages/backend/src/apps/rss/triggers/new-items-in-feed/new-items-in-feed.ts b/packages/backend/src/apps/rss/triggers/new-items-in-feed/new-items-in-feed.ts
index 6111929d..cdc76791 100644
--- a/packages/backend/src/apps/rss/triggers/new-items-in-feed/new-items-in-feed.ts
+++ b/packages/backend/src/apps/rss/triggers/new-items-in-feed/new-items-in-feed.ts
@@ -1,5 +1,5 @@
 import { IGlobalVariable } from '@automatisch/types';
-import { XMLParser } from 'fast-xml-parser';[]
+import { XMLParser } from 'fast-xml-parser';
 
 const newItemsInFeed = async ($: IGlobalVariable) => {
   const { data } = await $.http.get($.step.parameters.feedUrl as string);
@@ -7,15 +7,12 @@ const newItemsInFeed = async ($: IGlobalVariable) => {
   const parsedData = parser.parse(data);
 
   for (const item of parsedData.rss.channel.item) {
-    if ($.flow.isAlreadyProcessed(item.guid))
-      return;
-
     const dataItem = {
       raw: item,
       meta: {
-        internalId: item.guid
-      }
-    }
+        internalId: item.guid,
+      },
+    };
 
     $.pushTriggerItem(dataItem);
   }

From ae58e629ea4fdafe0d2f3fc8125704175c4f8c55 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 13:22:15 +0200
Subject: [PATCH 13/18] fix: Do not push recent trigger item in case of already
 processed

---
 packages/backend/src/helpers/global-variable.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/backend/src/helpers/global-variable.ts b/packages/backend/src/helpers/global-variable.ts
index 095cf6b8..12d8bb59 100644
--- a/packages/backend/src/helpers/global-variable.ts
+++ b/packages/backend/src/helpers/global-variable.ts
@@ -73,14 +73,16 @@ const globalVariable = async (
       },
     },
     pushTriggerItem: (triggerItem: ITriggerItem) => {
-      $.triggerOutput.data.push(triggerItem);
-
-      if (
-        $.execution.testRun ||
-        isAlreadyProcessed(triggerItem.meta.internalId)
-      ) {
+      if ($.execution.testRun) {
+        $.triggerOutput.data.push(triggerItem);
         throw new EarlyExitError();
       }
+
+      if (isAlreadyProcessed(triggerItem.meta.internalId)) {
+        throw new EarlyExitError();
+      }
+
+      $.triggerOutput.data.push(triggerItem);
     },
     setActionItem: (actionItem: IActionItem) => {
       $.actionOutput.data = actionItem;

From 2b6ff7de417cb16e9c512be1f203b30df1694465 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 13:23:51 +0200
Subject: [PATCH 14/18] refactor: Remove dedupeStrategy field from triggers

---
 .../apps/flickr/triggers/new-albums/index.ts  |  7 +++----
 .../triggers/new-favorite-photos/index.ts     |  7 +++----
 .../triggers/new-photos-in-album/index.ts     | 19 +++++++++----------
 .../apps/flickr/triggers/new-photos/index.ts  |  7 +++----
 .../github/triggers/new-watchers/index.ts     |  1 -
 .../triggers/new-follower-of-me/index.ts      |  1 -
 packages/types/index.d.ts                     |  1 -
 7 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/packages/backend/src/apps/flickr/triggers/new-albums/index.ts b/packages/backend/src/apps/flickr/triggers/new-albums/index.ts
index 57737f27..fd92a9c5 100644
--- a/packages/backend/src/apps/flickr/triggers/new-albums/index.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-albums/index.ts
@@ -6,16 +6,15 @@ export default defineTrigger({
   pollInterval: 15,
   key: 'new-albums',
   description: 'Triggers when you create a new album.',
-  dedupeStrategy: 'greatest',
   substeps: [
     {
       key: 'chooseConnection',
-      name: 'Choose connection'
+      name: 'Choose connection',
     },
     {
       key: 'testStep',
-      name: 'Test trigger'
-    }
+      name: 'Test trigger',
+    },
   ],
 
   async run($) {
diff --git a/packages/backend/src/apps/flickr/triggers/new-favorite-photos/index.ts b/packages/backend/src/apps/flickr/triggers/new-favorite-photos/index.ts
index 2647c910..3ae20938 100644
--- a/packages/backend/src/apps/flickr/triggers/new-favorite-photos/index.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-favorite-photos/index.ts
@@ -6,16 +6,15 @@ export default defineTrigger({
   pollInterval: 15,
   key: 'newFavoritePhotos',
   description: 'Triggers when you favorite a photo.',
-  dedupeStrategy: 'unique',
   substeps: [
     {
       key: 'chooseConnection',
-      name: 'Choose connection'
+      name: 'Choose connection',
     },
     {
       key: 'testStep',
-      name: 'Test trigger'
-    }
+      name: 'Test trigger',
+    },
   ],
 
   async run($) {
diff --git a/packages/backend/src/apps/flickr/triggers/new-photos-in-album/index.ts b/packages/backend/src/apps/flickr/triggers/new-photos-in-album/index.ts
index 340b83c2..b5332df1 100644
--- a/packages/backend/src/apps/flickr/triggers/new-photos-in-album/index.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-photos-in-album/index.ts
@@ -6,11 +6,10 @@ export default defineTrigger({
   pollInterval: 15,
   key: 'newPhotosInAlbum',
   description: 'Triggers when you add a new photo in an album.',
-  dedupeStrategy: 'greatest',
   substeps: [
     {
       key: 'chooseConnection',
-      name: 'Choose connection'
+      name: 'Choose connection',
     },
     {
       key: 'chooseTrigger',
@@ -28,17 +27,17 @@ export default defineTrigger({
             arguments: [
               {
                 name: 'key',
-                value: 'listAlbums'
-              }
-            ]
-          }
-        }
-      ]
+                value: 'listAlbums',
+              },
+            ],
+          },
+        },
+      ],
     },
     {
       key: 'testStep',
-      name: 'Test trigger'
-    }
+      name: 'Test trigger',
+    },
   ],
 
   async run($) {
diff --git a/packages/backend/src/apps/flickr/triggers/new-photos/index.ts b/packages/backend/src/apps/flickr/triggers/new-photos/index.ts
index 41d7e5e3..663bf8a2 100644
--- a/packages/backend/src/apps/flickr/triggers/new-photos/index.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-photos/index.ts
@@ -6,16 +6,15 @@ export default defineTrigger({
   pollInterval: 15,
   key: 'newPhotos',
   description: 'Triggers when you add a new photo.',
-  dedupeStrategy: 'greatest',
   substeps: [
     {
       key: 'chooseConnection',
-      name: 'Choose connection'
+      name: 'Choose connection',
     },
     {
       key: 'testStep',
-      name: 'Test trigger'
-    }
+      name: 'Test trigger',
+    },
   ],
 
   async run($) {
diff --git a/packages/backend/src/apps/github/triggers/new-watchers/index.ts b/packages/backend/src/apps/github/triggers/new-watchers/index.ts
index 03204e7f..a583c1f2 100644
--- a/packages/backend/src/apps/github/triggers/new-watchers/index.ts
+++ b/packages/backend/src/apps/github/triggers/new-watchers/index.ts
@@ -5,7 +5,6 @@ export default defineTrigger({
   name: 'New watchers',
   key: 'newWatchers',
   pollInterval: 15,
-  dedupeStrategy: 'unique',
   description: 'Triggers when a user watches a repository',
   substeps: [
     {
diff --git a/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts b/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts
index 19adfd6c..4ce76592 100644
--- a/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts
+++ b/packages/backend/src/apps/twitter/triggers/new-follower-of-me/index.ts
@@ -6,7 +6,6 @@ export default defineTrigger({
   key: 'myFollowers',
   pollInterval: 15,
   description: 'Will be triggered when you have a new follower.',
-  dedupeStrategy: 'unique',
   substeps: [
     {
       key: 'chooseConnection',
diff --git a/packages/types/index.d.ts b/packages/types/index.d.ts
index d2e337f3..16736dfa 100644
--- a/packages/types/index.d.ts
+++ b/packages/types/index.d.ts
@@ -210,7 +210,6 @@ export interface ITrigger {
   key: string;
   pollInterval?: number;
   description: string;
-  dedupeStrategy?: 'greatest' | 'unique' | 'last';
   substeps: ISubstep[];
   getInterval?(parameters: IStep['parameters']): string;
   run($: IGlobalVariable): Promise<void>;

From c3a54e0c6916b21f95353c26c84712a107523b74 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 13:36:28 +0200
Subject: [PATCH 15/18] feat: Introduce http error class

---
 packages/backend/src/errors/http-error.ts         | 3 +++
 packages/backend/src/helpers/http-client/index.ts | 4 ++--
 packages/backend/src/services/flow.ts             | 5 +++--
 3 files changed, 8 insertions(+), 4 deletions(-)
 create mode 100644 packages/backend/src/errors/http-error.ts

diff --git a/packages/backend/src/errors/http-error.ts b/packages/backend/src/errors/http-error.ts
new file mode 100644
index 00000000..2e71ebeb
--- /dev/null
+++ b/packages/backend/src/errors/http-error.ts
@@ -0,0 +1,3 @@
+import BaseError from './base';
+
+export default class HttpError extends BaseError {}
diff --git a/packages/backend/src/helpers/http-client/index.ts b/packages/backend/src/helpers/http-client/index.ts
index c6346e9f..611d5d9b 100644
--- a/packages/backend/src/helpers/http-client/index.ts
+++ b/packages/backend/src/helpers/http-client/index.ts
@@ -2,6 +2,7 @@ import axios, { AxiosRequestConfig } from 'axios';
 export { AxiosInstance as IHttpClient } from 'axios';
 import { IHttpClientParams } from '@automatisch/types';
 import { URL } from 'url';
+import HttpError from '../../errors/http-error';
 
 const removeBaseUrlForAbsoluteUrls = (
   requestConfig: AxiosRequestConfig
@@ -39,8 +40,7 @@ export default function createHttpClient({
   instance.interceptors.response.use(
     (response) => response,
     (error) => {
-      error.response.httpError = error.response.data;
-      throw error;
+      throw new HttpError(error.response.data);
     }
   );
 
diff --git a/packages/backend/src/services/flow.ts b/packages/backend/src/services/flow.ts
index 8e481444..82cc8aa3 100644
--- a/packages/backend/src/services/flow.ts
+++ b/packages/backend/src/services/flow.ts
@@ -1,6 +1,7 @@
 import Flow from '../models/flow';
 import globalVariable from '../helpers/global-variable';
 import EarlyExitError from '../errors/early-exit';
+import HttpError from '../errors/http-error';
 
 type ProcessFlowOptions = {
   flowId: string;
@@ -25,8 +26,8 @@ export const processFlow = async (options: ProcessFlowOptions) => {
     await triggerCommand.run($);
   } catch (error) {
     if (error instanceof EarlyExitError === false) {
-      if (error?.response?.httpError) {
-        $.triggerOutput.error = error.response.httpError;
+      if (error instanceof HttpError) {
+        $.triggerOutput.error = error.error;
       } else {
         try {
           $.triggerOutput.error = JSON.parse(error.message);

From d8e1f6df81ed6351e633bc3e8620054826924ae7 Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 13:43:01 +0200
Subject: [PATCH 16/18] refactor: Remove test run and completion check for
 flick triggers

---
 .../apps/flickr/triggers/new-albums/new-albums.ts   | 13 +++++--------
 .../new-favorite-photos/new-favorite-photos.ts      | 11 ++++-------
 .../new-photos-in-album/new-photos-in-album.ts      | 11 ++++-------
 3 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/packages/backend/src/apps/flickr/triggers/new-albums/new-albums.ts b/packages/backend/src/apps/flickr/triggers/new-albums/new-albums.ts
index 08779cda..918ac412 100644
--- a/packages/backend/src/apps/flickr/triggers/new-albums/new-albums.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-albums/new-albums.ts
@@ -19,7 +19,7 @@ const extraFields = [
   'url_t',
   'url_s',
   'url_m',
-  'url_o'
+  'url_o',
 ].join(',');
 
 const newAlbums = async ($: IGlobalVariable) => {
@@ -42,17 +42,14 @@ const newAlbums = async ($: IGlobalVariable) => {
     pages = photosets.pages;
 
     for (const photoset of photosets.photoset) {
-      if ($.flow.isAlreadyProcessed(photoset.id) && !$.execution.testRun)
-        return;
-
       $.pushTriggerItem({
         raw: photoset,
         meta: {
-          internalId: photoset.id as string
-        }
-      })
+          internalId: photoset.id as string,
+        },
+      });
     }
-  } while (page <= pages && !$.execution.testRun);
+  } while (page <= pages);
 };
 
 export default newAlbums;
diff --git a/packages/backend/src/apps/flickr/triggers/new-favorite-photos/new-favorite-photos.ts b/packages/backend/src/apps/flickr/triggers/new-favorite-photos/new-favorite-photos.ts
index 61feb645..4e7f9842 100644
--- a/packages/backend/src/apps/flickr/triggers/new-favorite-photos/new-favorite-photos.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-favorite-photos/new-favorite-photos.ts
@@ -48,17 +48,14 @@ const newPhotos = async ($: IGlobalVariable) => {
     pages = photos.pages;
 
     for (const photo of photos.photo) {
-      if ($.flow.isAlreadyProcessed(photo.date_faved) && !$.execution.testRun)
-        return;
-
       $.pushTriggerItem({
         raw: photo,
         meta: {
-          internalId: photo.date_faved as string
-        }
-      })
+          internalId: photo.date_faved as string,
+        },
+      });
     }
-  } while (page <= pages && !$.execution.testRun);
+  } while (page <= pages);
 };
 
 export default newPhotos;
diff --git a/packages/backend/src/apps/flickr/triggers/new-photos-in-album/new-photos-in-album.ts b/packages/backend/src/apps/flickr/triggers/new-photos-in-album/new-photos-in-album.ts
index 0ac6c601..98ba9ff3 100644
--- a/packages/backend/src/apps/flickr/triggers/new-photos-in-album/new-photos-in-album.ts
+++ b/packages/backend/src/apps/flickr/triggers/new-photos-in-album/new-photos-in-album.ts
@@ -43,17 +43,14 @@ const newPhotosInAlbum = async ($: IGlobalVariable) => {
     pages = photoset.pages;
 
     for (const photo of photoset.photo) {
-      if ($.flow.isAlreadyProcessed(photo.id) && !$.execution.testRun)
-        return;
-
       $.pushTriggerItem({
         raw: photo,
         meta: {
-          internalId: photo.id as string
-        }
-      })
+          internalId: photo.id as string,
+        },
+      });
     }
-  } while (page <= pages && !$.execution.testRun);
+  } while (page <= pages);
 };
 
 export default newPhotosInAlbum;

From 2b38e5a0ddd3ba7a61b9e7dd9aea799783a1902d Mon Sep 17 00:00:00 2001
From: Faruk AYDIN <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 13:46:08 +0200
Subject: [PATCH 17/18] chore: Use last 2000 records as deduplication list

---
 packages/backend/src/helpers/global-variable.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend/src/helpers/global-variable.ts b/packages/backend/src/helpers/global-variable.ts
index 12d8bb59..0307591c 100644
--- a/packages/backend/src/helpers/global-variable.ts
+++ b/packages/backend/src/helpers/global-variable.ts
@@ -96,7 +96,7 @@ const globalVariable = async (
   });
 
   const lastInternalIds =
-    testRun || (flow && step.isAction) ? [] : await flow?.lastInternalIds();
+    testRun || (flow && step.isAction) ? [] : await flow?.lastInternalIds(2000);
 
   const isAlreadyProcessed = (internalId: string) => {
     return lastInternalIds?.includes(internalId);

From 964a38db2cba225d4d4033cafafae91900182adf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=96mer=20Faruk=20Ayd=C4=B1n?= <omerfaruk26@gmail.com>
Date: Fri, 28 Oct 2022 20:07:31 +0200
Subject: [PATCH 18/18] chore: Add explanations to early exit conditions of
 global variable

Co-authored-by: Ali BARIN <ali.barin53@gmail.com>
---
 packages/backend/src/helpers/global-variable.ts | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/packages/backend/src/helpers/global-variable.ts b/packages/backend/src/helpers/global-variable.ts
index 0307591c..bd7bdfe9 100644
--- a/packages/backend/src/helpers/global-variable.ts
+++ b/packages/backend/src/helpers/global-variable.ts
@@ -73,16 +73,17 @@ const globalVariable = async (
       },
     },
     pushTriggerItem: (triggerItem: ITriggerItem) => {
-      if ($.execution.testRun) {
-        $.triggerOutput.data.push(triggerItem);
-        throw new EarlyExitError();
-      }
-
-      if (isAlreadyProcessed(triggerItem.meta.internalId)) {
+      if (isAlreadyProcessed(triggerItem.meta.internalId) && !$.execution.testRun) {
+        // early exit as we do not want to process duplicate items in actual executions
         throw new EarlyExitError();
       }
 
       $.triggerOutput.data.push(triggerItem);
+
+      if ($.execution.testRun) {
+        // early exit after receiving one item as it is enough for test execution
+        throw new EarlyExitError();
+      }
     },
     setActionItem: (actionItem: IActionItem) => {
       $.actionOutput.data = actionItem;