feat: Capture unhandled errors by restructuring apps

packages/backend/src/apps/github/common/paginate-all.ts

@@ -1,13 +1,16 @@
-import { IGlobalVariable, IJSONObject } from "@automatisch/types";
+import { IGlobalVariable, IJSONObject } from '@automatisch/types';
 import type { AxiosResponse } from 'axios';
 import parseLinkHeader from '../../../helpers/parse-header-link';
 
 type TResponse = {
-  data: IJSONObject[],
+  data: IJSONObject[];
-  error?: IJSONObject,
+  error?: IJSONObject;
-}
+};
 
-export default async function paginateAll($: IGlobalVariable, request: Promise<AxiosResponse>) {
+export default async function paginateAll(
+  $: IGlobalVariable,
+  request: Promise<AxiosResponse>
+) {
   const response = await request;
   const aggregatedResponse: TResponse = {
     data: [...response.data],
@@ -21,8 +24,8 @@ export default async function paginateAll($: IGlobalVariable, request: Promise<A
       url: links.next.uri,
     });
 
-    if (nextPageResponse.integrationError) {
+    if (nextPageResponse.httpError) {
-      aggregatedResponse.error = nextPageResponse.integrationError;
+      aggregatedResponse.error = nextPageResponse.httpError;
 
       links = null;
     } else {

packages/backend/src/apps/github/triggers/new-issues/new-issues.ts

@@ -1,12 +1,11 @@
-import {
+import { IGlobalVariable, ITriggerOutput } from '@automatisch/types';
-  IGlobalVariable,
-  ITriggerOutput,
-} from '@automatisch/types';
 import getRepoOwnerAndRepo from '../../common/get-repo-owner-and-repo';
 import parseLinkHeader from '../../../../helpers/parse-header-link';
 
 function getPathname($: IGlobalVariable) {
-  const { repoOwner, repo } = getRepoOwnerAndRepo($.step.parameters.repo as string);
+  const { repoOwner, repo } = getRepoOwnerAndRepo(
+    $.step.parameters.repo as string
+  );
 
   if (repoOwner && repo) {
     return `/repos/${repoOwner}/${repo}/issues`;
@@ -35,8 +34,8 @@ const newIssues = async ($: IGlobalVariable) => {
     const response = await $.http.get(pathname, { params });
     links = parseLinkHeader(response.headers.link);
 
-    if (response.integrationError) {
+    if (response.httpError) {
-      issues.error = response.integrationError;
+      issues.error = response.httpError;
       return issues;
     }
 
@@ -44,7 +43,8 @@ const newIssues = async ($: IGlobalVariable) => {
       for (const issue of response.data) {
         const issueId = issue.id;
 
-        if (issueId <= Number($.flow.lastInternalId) && !$.execution.testRun) return issues;
+        if (issueId <= Number($.flow.lastInternalId) && !$.execution.testRun)
+          return issues;
 
         const dataItem = {
           raw: issue,

packages/backend/src/apps/github/triggers/new-stargazers/new-stargazers.ts

@@ -10,10 +10,12 @@ import parseLinkHeader from '../../../../helpers/parse-header-link';
 type TResponseDataItem = {
   starred_at: string;
   user: IJSONObject;
-}
+};
 
 const fetchStargazers = async ($: IGlobalVariable) => {
-  const { repoOwner, repo } = getRepoOwnerAndRepo($.step.parameters.repo as string);
+  const { repoOwner, repo } = getRepoOwnerAndRepo(
+    $.step.parameters.repo as string
+  );
   const firstPagePathname = `/repos/${repoOwner}/${repo}/stargazers`;
   const requestConfig = {
     params: {
@@ -22,10 +24,13 @@ const fetchStargazers = async ($: IGlobalVariable) => {
     headers: {
       // needed to get `starred_at` time
       Accept: 'application/vnd.github.star+json',
-    }
+    },
-  }
+  };
 
-  const firstPageResponse = await $.http.get<TResponseDataItem[]>(firstPagePathname, requestConfig);
+  const firstPageResponse = await $.http.get<TResponseDataItem[]>(
+    firstPagePathname,
+    requestConfig
+  );
   const firstPageLinks = parseLinkHeader(firstPageResponse.headers.link);
 
   // in case there is only single page to fetch
@@ -36,12 +41,15 @@ const fetchStargazers = async ($: IGlobalVariable) => {
   };
 
   do {
-    const response = await $.http.get<TResponseDataItem[]>(pathname, requestConfig);
+    const response = await $.http.get<TResponseDataItem[]>(
+      pathname,
+      requestConfig
+    );
     const links = parseLinkHeader(response.headers.link);
     pathname = links.prev?.uri;
 
-    if (response.integrationError) {
+    if (response.httpError) {
-      stargazers.error = response.integrationError;
+      stargazers.error = response.httpError;
       return stargazers;
     }
 
@@ -50,7 +58,8 @@ const fetchStargazers = async ($: IGlobalVariable) => {
         const { starred_at, user } = starEntry;
         const timestamp = DateTime.fromISO(starred_at).toMillis();
 
-        if (timestamp <= Number($.flow.lastInternalId) && !$.execution.testRun) return stargazers;
+        if (timestamp <= Number($.flow.lastInternalId) && !$.execution.testRun)
+          return stargazers;
 
         const dataItem = {
           raw: user,
@@ -65,13 +74,15 @@ const fetchStargazers = async ($: IGlobalVariable) => {
   } while (pathname && !$.execution.testRun);
 
   return stargazers;
-}
+};
 
 const newStargazers = async ($: IGlobalVariable) => {
   const stargazers = await fetchStargazers($);
 
   stargazers.data.sort((stargazerA, stargazerB) => {
-    return Number(stargazerA.meta.internalId) - Number(stargazerB.meta.internalId);
+    return (
+      Number(stargazerA.meta.internalId) - Number(stargazerB.meta.internalId)
+    );
   });
 
   return stargazers;

packages/backend/src/apps/slack/actions/find-message/find-message.ts

@@ -25,7 +25,7 @@ const findMessage = async ($: IGlobalVariable, options: FindMessageOptions) => {
     data: {
       raw: data?.messages.matches[0],
     },
-    error: response?.integrationError || (!data.ok && data),
+    error: response?.httpError || (!data.ok && data),
   };
 
   return message;

packages/backend/src/apps/slack/actions/send-a-message-to-channel/post-message.ts

@@ -16,7 +16,7 @@ const postMessage = async (
     data: {
       raw: response?.data?.message,
     },
-    error: response?.integrationError,
+    error: response?.httpError,
   };
 
   if (response.data.ok === false) {

packages/backend/src/apps/slack/data/list-channels/index.ts

@@ -15,8 +15,8 @@ export default {
 
     const response = await $.http.get('/conversations.list');
 
-    if (response.integrationError) {
+    if (response.httpError) {
-      channels.error = response.integrationError;
+      channels.error = response.httpError;
       return channels;
     }
 

packages/backend/src/apps/twitter/actions/create-tweet/index.ts

@@ -40,7 +40,7 @@ export default defineAction({
       data: {
         raw: response.data,
       },
-      error: response?.integrationError,
+      error: response?.httpError,
     };
 
     return tweet;

packages/backend/src/apps/twitter/common/get-user-followers.ts

@@ -33,8 +33,8 @@ const getUserFollowers = async (
 
     response = await $.http.get(requestPath);
 
-    if (response.integrationError) {
+    if (response.httpError) {
-      followers.error = response.integrationError;
+      followers.error = response.httpError;
       return followers;
     }
 

packages/backend/src/apps/twitter/common/get-user-tweets.ts

@@ -36,8 +36,8 @@ const fetchTweets = async ($: IGlobalVariable, username: string) => {
 
     response = await $.http.get(requestPath);
 
-    if (response.integrationError) {
+    if (response.httpError) {
-      tweets.error = response.integrationError;
+      tweets.error = response.httpError;
       return tweets;
     }
 

packages/backend/src/apps/twitter/triggers/search-tweets/index.ts

@@ -33,4 +33,10 @@ export default defineTrigger({
   async run($) {
     return await searchTweets($);
   },
+
+  sort($) {
+    $.output.data.sort((tweet, nextTweet) => {
+      return Number(tweet.meta.internalId) - Number(nextTweet.meta.internalId);
+    });
+  },
 });

packages/backend/src/apps/twitter/triggers/search-tweets/search-tweets.ts

@@ -1,20 +1,12 @@
-import {
+import { IGlobalVariable, IJSONObject } from '@automatisch/types';
-  IGlobalVariable,
-  IJSONObject,
-  ITriggerOutput,
-} from '@automatisch/types';
 import qs from 'qs';
 import { omitBy, isEmpty } from 'lodash';
 
-const fetchTweets = async ($: IGlobalVariable) => {
+const searchTweets = async ($: IGlobalVariable) => {
   const searchTerm = $.step.parameters.searchTerm as string;
 
   let response;
 
-  const tweets: ITriggerOutput = {
-    data: [],
-  };
-
   do {
     const params: IJSONObject = {
       query: searchTerm,
@@ -30,14 +22,8 @@ const fetchTweets = async ($: IGlobalVariable) => {
 
     response = await $.http.get(requestPath);
 
-    if (response.integrationError) {
-      tweets.error = response.integrationError;
-      return tweets;
-    }
-
     if (response.data.errors) {
-      tweets.error = response.data.errors;
+      throw new Error(JSON.stringify(response.data.errors));
-      return tweets;
     }
 
     if (response.data.meta.result_count > 0) {
@@ -49,22 +35,10 @@ const fetchTweets = async ($: IGlobalVariable) => {
           },
         };
 
-        tweets.data.push(dataItem);
+        $.output.data.push(dataItem);
       });
     }
   } while (response.data.meta.next_token && !$.execution.testRun);
-
-  return tweets;
-};
-
-const searchTweets = async ($: IGlobalVariable) => {
-  const tweets = await fetchTweets($);
-
-  tweets.data.sort((tweet, nextTweet) => {
-    return Number(tweet.meta.internalId) - Number(nextTweet.meta.internalId);
-  });
-
-  return tweets;
 };
 
 export default searchTweets;

packages/backend/src/helpers/global-variable.ts

@@ -59,6 +59,10 @@ const globalVariable = async (
       id: execution?.id,
       testRun,
     },
+    output: {
+      data: [],
+      error: null,
+    },
   };
 
   $.http = createHttpClient({

packages/backend/src/helpers/http-client/index.ts

@@ -39,8 +39,8 @@ export default function createHttpClient({
   instance.interceptors.response.use(
     (response) => response,
     (error) => {
-      error.response.integrationError = error.response.data;
+      error.response.httpError = error.response.data;
-      return error.response;
+      throw error;
     }
   );
 

packages/backend/src/models/app.ts

@@ -10,12 +10,7 @@ class App {
 
   // Temporaryly restrict the apps we expose until
   // their actions/triggers are implemented!
-  static temporaryList = [
+  static temporaryList = ['github', 'scheduler', 'slack', 'twitter'];
-    'github',
-    'scheduler',
-    'slack',
-    'twitter',
-  ];
 
   static async findAll(name?: string, stripFuncs = true): Promise<IApp[]> {
     if (!name)

packages/backend/src/services/flow.ts

@@ -20,5 +20,23 @@ export const processFlow = async (options: ProcessFlowOptions) => {
     testRun: options.testRun,
   });
 
-  return await triggerCommand.run($);
+  try {
+    await triggerCommand.run($);
+  } catch (error) {
+    if (error?.response?.httpError) {
+      $.output.error = error.response.httpError;
+    } else {
+      try {
+        $.output.error = JSON.parse(error.message);
+      } catch {
+        $.output.error = { error: error.message };
+      }
+    }
+  }
+
+  if (triggerCommand?.sort) {
+    triggerCommand.sort($);
+  }
+
+  return $.output;
 };

packages/types/index.d.ts

@@ -213,7 +213,8 @@ export interface ITrigger {
   dedupeStrategy?: 'greatest' | 'unique' | 'last';
   substeps: ISubstep[];
   getInterval?(parameters: IGlobalVariable['step']['parameters']): string;
-  run($: IGlobalVariable): Promise<ITriggerOutput>;
+  run($: IGlobalVariable): Promise<void | ITriggerOutput>;
+  sort?($: IGlobalVariable): void | ITriggerOutput;
 }
 
 export interface IActionOutput {
@@ -279,11 +280,15 @@ export type IGlobalVariable = {
     id: string;
     testRun: boolean;
   };
+  output: {
+    data: ITriggerDataItem[];
+    error?: IJSONObject;
+  }
   process?: (triggerDataItem: ITriggerDataItem) => Promise<void>;
 };
 
 declare module 'axios' {
   interface AxiosResponse {
-    integrationError?: IJSONObject;
+    httpError?: IJSONObject;
   }
 }