test: ntfy

fix(axios): update order of interceptors

packages/backend/src/apps/formatter/actions/text/transformers/replace.js

@@ -1,8 +1,26 @@
 const replace = ($) => {
   const input = $.step.parameters.input;
-
   const find = $.step.parameters.find;
   const replace = $.step.parameters.replace;
+  const useRegex = $.step.parameters.useRegex;
+
+  if (useRegex) {
+    const ignoreCase = $.step.parameters.ignoreCase;
+
+    const flags = [ignoreCase && 'i', 'g'].filter(Boolean).join('');
+
+    const timeoutId = setTimeout(() => {
+      $.execution.exit();
+    }, 100);
+
+    const regex = new RegExp(find, flags);
+
+    const replacedValue = input.replaceAll(regex, replace);
+
+    clearTimeout(timeoutId);
+
+    return replacedValue;
+  }
 
   return input.replaceAll(find, replace);
 };

packages/backend/src/apps/formatter/dynamic-fields/index.js

@@ -1,3 +1,4 @@
 import listTransformOptions from './list-transform-options/index.js';
+import listReplaceRegexOptions from './list-replace-regex-options/index.js';
 
-export default [listTransformOptions];
+export default [listTransformOptions, listReplaceRegexOptions];

packages/backend/src/apps/formatter/dynamic-fields/list-replace-regex-options/index.js

@@ -0,0 +1,23 @@
+export default {
+  name: 'List replace regex options',
+  key: 'listReplaceRegexOptions',
+
+  async run($) {
+    if (!$.step.parameters.useRegex) return [];
+
+    return [
+      {
+        label: 'Ignore case',
+        key: 'ignoreCase',
+        type: 'dropdown',
+        required: true,
+        description: 'Ignore case sensitivity.',
+        variables: true,
+        options: [
+          { label: 'Yes', value: true },
+          { label: 'No', value: false },
+        ],
+      },
+    ];
+  },
+};

packages/backend/src/apps/formatter/dynamic-fields/list-transform-options/text/replace.js

@@ -23,6 +23,33 @@ const replace = [
     description: 'Text that will replace the found text.',
     variables: true,
   },
+  {
+    label: 'Use Regular Expression',
+    key: 'useRegex',
+    type: 'dropdown',
+    required: true,
+    description: 'Use regex to search values.',
+    variables: true,
+    value: false,
+    options: [
+      { label: 'Yes', value: true },
+      { label: 'No', value: false },
+    ],
+    additionalFields: {
+      type: 'query',
+      name: 'getDynamicFields',
+      arguments: [
+        {
+          name: 'key',
+          value: 'listReplaceRegexOptions',
+        },
+        {
+          name: 'parameters.useRegex',
+          value: '{parameters.useRegex}',
+        },
+      ],
+    },
+  },
 ];
 
 export default replace;

packages/backend/src/apps/ntfy/actions/send-message/index.js

@@ -89,6 +89,8 @@ export default defineAction({
 
     const response = await $.http.post('/', payload);
 
+    console.log(response.config.additionalProperties.extraData);
+
     $.setActionItem({
       raw: response.data,
     });

packages/backend/src/apps/ntfy/common/add-auth-header.js

@@ -1,4 +1,7 @@
 const addAuthHeader = ($, requestConfig) => {
+  console.log('requestConfig', requestConfig)
+  if (requestConfig.additionalProperties?.skip) return requestConfig;
+
   if ($.auth.data.serverUrl) {
     requestConfig.baseURL = $.auth.data.serverUrl;
   }

packages/backend/src/apps/ntfy/common/async-before-request.js

@@ -0,0 +1,23 @@
+const asyncBeforeRequest = async ($, requestConfig) => {
+  if (requestConfig.additionalProperties?.skip)
+    return requestConfig;
+
+  const response = await $.http.post(
+    'http://localhost:3000/webhooks/flows/8a040f4e-817f-4076-80ba-3c1c0af7e65e/sync',
+    null,
+    {
+      additionalProperties: {
+        skip: true,
+      },
+    }
+  );
+
+  console.log(response);
+  requestConfig.additionalProperties = {
+    extraData: response.data
+  }
+
+  return requestConfig;
+};
+
+export default asyncBeforeRequest;

packages/backend/src/apps/ntfy/index.js

@@ -1,5 +1,6 @@
 import defineApp from '../../helpers/define-app.js';
 import addAuthHeader from './common/add-auth-header.js';
+import asyncBeforeRequest from './common/async-before-request.js';
 import auth from './auth/index.js';
 import actions from './actions/index.js';
 
@@ -12,7 +13,7 @@ export default defineApp({
   baseUrl: 'https://ntfy.sh',
   apiBaseUrl: 'https://ntfy.sh',
   primaryColor: '56bda8',
-  beforeRequest: [addAuthHeader],
+  beforeRequest: [asyncBeforeRequest, addAuthHeader],
   auth,
   actions,
 });

packages/backend/src/config/app.js

@@ -100,6 +100,9 @@ const appConfig = {
   additionalDrawerLinkIcon: process.env.ADDITIONAL_DRAWER_LINK_ICON,
   additionalDrawerLinkText: process.env.ADDITIONAL_DRAWER_LINK_TEXT,
   disableSeedUser: process.env.DISABLE_SEED_USER === 'true',
+  httpProxy: process.env.http_proxy,
+  httpsProxy: process.env.https_proxy,
+  noProxy: process.env.no_proxy,
 };
 
 if (!appConfig.encryptionKey) {

packages/backend/src/helpers/axios-with-proxy.js

@@ -1,43 +1,102 @@
 import axios from 'axios';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import { HttpProxyAgent } from 'http-proxy-agent';
+import appConfig from '../config/app.js';
 
-const config = axios.defaults;
+export function createInstance(customConfig = {}, { requestInterceptor, responseErrorInterceptor } = {}) {
-const httpProxyUrl = process.env.http_proxy;
+  const config = {
-const httpsProxyUrl = process.env.https_proxy;
+    ...axios.defaults,
-const supportsProxy = httpProxyUrl || httpsProxyUrl;
+    ...customConfig
-const noProxyEnv = process.env.no_proxy;
+  };
-const noProxyHosts = noProxyEnv ? noProxyEnv.split(',').map(host => host.trim()) : [];
+  const httpProxyUrl = appConfig.httpProxy;
+  const httpsProxyUrl = appConfig.httpsProxy;
+  const supportsProxy = httpProxyUrl || httpsProxyUrl;
+  const noProxyEnv = appConfig.noProxy;
+  const noProxyHosts = noProxyEnv ? noProxyEnv.split(',').map(host => host.trim()) : [];
 
-if (supportsProxy) {
+  if (supportsProxy) {
-  if (httpProxyUrl) {
+    if (httpProxyUrl) {
-    config.httpAgent = new HttpProxyAgent(httpProxyUrl);
+      config.httpAgent = new HttpProxyAgent(httpProxyUrl);
+    }
+
+    if (httpsProxyUrl) {
+      config.httpsAgent = new HttpsProxyAgent(httpsProxyUrl);
+    }
+
+    config.proxy = false;
   }
 
-  if (httpsProxyUrl) {
+  const instance = axios.create(config);
-    config.httpsAgent = new HttpsProxyAgent(httpsProxyUrl);
+
+  function shouldSkipProxy(hostname) {
+    return noProxyHosts.some(noProxyHost => {
+      return hostname.endsWith(noProxyHost) || hostname === noProxyHost;
+    });
+  };
+
+  /**
+   * The interceptors are executed in the reverse order they are added.
+   */
+  instance.interceptors.request.use(
+    function skipProxyIfInNoProxy(requestConfig) {
+      const hostname = new URL(requestConfig.baseURL).hostname;
+
+      if (supportsProxy && shouldSkipProxy(hostname)) {
+        requestConfig.httpAgent = undefined;
+        requestConfig.httpsAgent = undefined;
+      }
+
+      return requestConfig;
+    },
+    (error) => Promise.reject(error)
+  );
+
+  // not always we have custom request interceptors
+  if (requestInterceptor) {
+    instance.interceptors.request.use(
+      async function customInterceptor(requestConfig) {
+        let newRequestConfig = requestConfig;
+
+        for (const interceptor of requestInterceptor) {
+          newRequestConfig = await interceptor(newRequestConfig);
+        }
+
+        return newRequestConfig;
+      }
+    );
   }
 
-  config.proxy = false;
+  instance.interceptors.request.use(
+    function removeBaseUrlForAbsoluteUrls(requestConfig) {
+      /**
+       * If the URL is an absolute URL, we remove its origin out of the URL
+       * and set it as baseURL. This lets us streamlines the requests made by Automatisch
+       * and requests made by app integrations.
+       */
+      try {
+        const url = new URL(requestConfig.url);
+        requestConfig.baseURL = url.origin;
+        requestConfig.url = url.pathname + url.search;
+
+        return requestConfig;
+      } catch (err) {
+        return requestConfig;
+      }
+    },
+    (error) => Promise.reject(error)
+  );
+
+  // not always we have custom response error interceptor
+  if (responseErrorInterceptor) {
+    instance.interceptors.response.use(
+      (response) => response,
+      responseErrorInterceptor
+    );
+  }
+
+  return instance;
 }
 
-const axiosWithProxyInstance = axios.create(config);
+const defaultInstance = createInstance();
 
-function shouldSkipProxy(hostname) {
+export default defaultInstance;
-  return noProxyHosts.some(noProxyHost => {
-    return hostname.endsWith(noProxyHost) || hostname === noProxyHost;
-  });
-};
-
-axiosWithProxyInstance.interceptors.request.use(function skipProxyIfInNoProxy(requestConfig) {
-  const hostname = new URL(requestConfig.url).hostname;
-
-  if (supportsProxy && shouldSkipProxy(hostname)) {
-    requestConfig.httpAgent = undefined;
-    requestConfig.httpsAgent = undefined;
-  }
-
-  return requestConfig;
-});
-
-export default axiosWithProxyInstance;

packages/backend/src/helpers/axios-with-proxy.test.js

@@ -0,0 +1,169 @@
+import { beforeEach, describe, it, expect, vi } from 'vitest';
+
+describe('Custom default axios with proxy', () => {
+  beforeEach(() => {
+    vi.resetModules();
+  });
+
+  it('should have two interceptors by default', async () => {
+    const axios = (await import('./axios-with-proxy.js')).default;
+    const requestInterceptors = axios.interceptors.request.handlers;
+
+    expect(requestInterceptors.length).toBe(2);
+  });
+
+  it('should have default interceptors in a certain order', async () => {
+    const axios = (await import('./axios-with-proxy.js')).default;
+
+    const requestInterceptors = axios.interceptors.request.handlers;
+    const firstRequestInterceptor = requestInterceptors[0];
+    const secondRequestInterceptor = requestInterceptors[1];
+
+    expect(firstRequestInterceptor.fulfilled.name).toBe('skipProxyIfInNoProxy');
+    expect(secondRequestInterceptor.fulfilled.name).toBe('removeBaseUrlForAbsoluteUrls');
+  });
+
+  it('should throw with invalid url (consisting of path alone)', async () => {
+    const axios = (await import('./axios-with-proxy.js')).default;
+
+    await expect(() => axios('/just-a-path')).rejects.toThrowError('Invalid URL');
+  });
+
+  describe('with skipProxyIfInNoProxy interceptor', () => {
+    let appConfig, axios;
+    beforeEach(async() => {
+      appConfig = (await import('../config/app.js')).default;
+
+      vi.spyOn(appConfig, 'httpProxy', 'get').mockReturnValue('http://proxy.automatisch.io');
+      vi.spyOn(appConfig, 'httpsProxy', 'get').mockReturnValue('http://proxy.automatisch.io');
+      vi.spyOn(appConfig, 'noProxy', 'get').mockReturnValue('name.tld,automatisch.io');
+
+      axios = (await import('./axios-with-proxy.js')).default;
+    });
+
+    it('should skip proxy for hosts in no_proxy environment variable', async () => {
+      const skipProxyIfInNoProxy = axios.interceptors.request.handlers[0].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        baseURL: 'https://automatisch.io'
+      };
+
+      const interceptedRequestConfig = skipProxyIfInNoProxy(mockRequestConfig);
+
+      expect(interceptedRequestConfig.httpAgent).toBeUndefined();
+      expect(interceptedRequestConfig.httpsAgent).toBeUndefined();
+      expect(interceptedRequestConfig.proxy).toBe(false);
+    });
+
+    it('should not skip proxy for hosts not in no_proxy environment variable', async () => {
+      const skipProxyIfInNoProxy = axios.interceptors.request.handlers[0].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        // beware the intentional typo!
+        baseURL: 'https://automatish.io'
+      };
+
+      const interceptedRequestConfig = skipProxyIfInNoProxy(mockRequestConfig);
+
+      expect(interceptedRequestConfig.httpAgent).toBeDefined();
+      expect(interceptedRequestConfig.httpsAgent).toBeDefined();
+      expect(interceptedRequestConfig.proxy).toBe(false);
+    });
+  });
+
+  describe('with removeBaseUrlForAbsoluteUrls interceptor', () => {
+    let axios;
+    beforeEach(async() => {
+      axios = (await import('./axios-with-proxy.js')).default;
+    });
+
+    it('should trim the baseUrl from absolute urls', async () => {
+      const removeBaseUrlForAbsoluteUrls = axios.interceptors.request.handlers[1].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        url: 'https://automatisch.io/path'
+      };
+
+      const interceptedRequestConfig = removeBaseUrlForAbsoluteUrls(mockRequestConfig);
+
+      expect(interceptedRequestConfig.baseURL).toBe('https://automatisch.io');
+      expect(interceptedRequestConfig.url).toBe('/path');
+    });
+
+    it('should not mutate separate baseURL and urls', async () => {
+      const removeBaseUrlForAbsoluteUrls = axios.interceptors.request.handlers[1].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        baseURL: 'https://automatisch.io',
+        url: '/path?query=1'
+      };
+
+      const interceptedRequestConfig = removeBaseUrlForAbsoluteUrls(mockRequestConfig);
+
+      expect(interceptedRequestConfig.baseURL).toBe('https://automatisch.io');
+      expect(interceptedRequestConfig.url).toBe('/path?query=1');
+    });
+
+    it('should not strip querystring from url', async () => {
+      const removeBaseUrlForAbsoluteUrls = axios.interceptors.request.handlers[1].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        url: 'https://automatisch.io/path?query=1'
+      };
+
+      const interceptedRequestConfig = removeBaseUrlForAbsoluteUrls(mockRequestConfig);
+
+      expect(interceptedRequestConfig.baseURL).toBe('https://automatisch.io');
+      expect(interceptedRequestConfig.url).toBe('/path?query=1');
+    });
+  });
+
+  describe('with extra requestInterceptors', () => {
+    it('should apply extra request interceptors in the middle', async () => {
+      const { createInstance } = await import('./axios-with-proxy.js');
+
+      const interceptor = (config) => {
+        config.test = true;
+        return config;
+      }
+
+      const instance = createInstance({}, {
+        requestInterceptor: [
+          interceptor
+        ]
+      });
+      const requestInterceptors = instance.interceptors.request.handlers;
+      const customInterceptor = requestInterceptors[1].fulfilled;
+
+      expect(requestInterceptors.length).toBe(3);
+      await expect(customInterceptor({})).resolves.toStrictEqual({ test: true });
+    });
+
+    it('should work with a custom interceptor setting a baseURL and a request to path', async () => {
+      const { createInstance } = await import('./axios-with-proxy.js');
+
+      const interceptor = (config) => {
+        config.baseURL = 'http://localhost';
+        return config;
+      }
+
+      const instance = createInstance({}, {
+        requestInterceptor: [
+          interceptor
+        ]
+      });
+
+      try {
+        await instance.get('/just-a-path');
+      } catch (error) {
+        expect(error.config.baseURL).toBe('http://localhost');
+        expect(error.config.url).toBe('/just-a-path');
+      }
+    })
+  });
+});

packages/backend/src/helpers/http-client/index.js

@@ -1,68 +1,43 @@
-import { URL } from 'node:url';
 import HttpError from '../../errors/http.js';
-import axios from '../axios-with-proxy.js';
+import { createInstance } from '../axios-with-proxy.js';
-
-const removeBaseUrlForAbsoluteUrls = (requestConfig) => {
-  try {
-    const url = new URL(requestConfig.url);
-    requestConfig.baseURL = url.origin;
-    requestConfig.url = url.pathname + url.search;
-
-    return requestConfig;
-  } catch {
-    return requestConfig;
-  }
-};
 
 export default function createHttpClient({ $, baseURL, beforeRequest = [] }) {
-  const instance = axios.create({
+  async function interceptResponseError(error) {
-    baseURL,
+    const { config, response } = error;
-  });
+    // Do not destructure `status` from `error.response` because it might not exist
+    const status = response?.status;
 
-  instance.interceptors.request.use((requestConfig) => {
+    if (
-    const newRequestConfig = removeBaseUrlForAbsoluteUrls(requestConfig);
+      // TODO: provide a `shouldRefreshToken` function in the app
+      (status === 401 || status === 403) &&
+      $.app.auth &&
+      $.app.auth.refreshToken &&
+      !$.app.auth.isRefreshTokenRequested
+    ) {
+      $.app.auth.isRefreshTokenRequested = true;
+      await $.app.auth.refreshToken($);
 
-    const result = beforeRequest.reduce((newConfig, beforeRequestFunc) => {
+      // retry the previous request before the expired token error
-      return beforeRequestFunc($, newConfig);
+      const newResponse = await instance.request(config);
-    }, newRequestConfig);
+      $.app.auth.isRefreshTokenRequested = false;
 
-    /**
+      return newResponse;
-     * axios seems to want InternalAxiosRequestConfig returned not AxioRequestConfig
-     * anymore even though requests do require AxiosRequestConfig.
-     *
-     * Since both interfaces are very similar (InternalAxiosRequestConfig
-     * extends AxiosRequestConfig), we can utilize an assertion below
-     **/
-    return result;
-  });
-
-  instance.interceptors.response.use(
-    (response) => response,
-    async (error) => {
-      const { config, response } = error;
-      // Do not destructure `status` from `error.response` because it might not exist
-      const status = response?.status;
-
-      if (
-        // TODO: provide a `shouldRefreshToken` function in the app
-        (status === 401 || status === 403) &&
-        $.app.auth &&
-        $.app.auth.refreshToken &&
-        !$.app.auth.isRefreshTokenRequested
-      ) {
-        $.app.auth.isRefreshTokenRequested = true;
-        await $.app.auth.refreshToken($);
-
-        // retry the previous request before the expired token error
-        const newResponse = await instance.request(config);
-        $.app.auth.isRefreshTokenRequested = false;
-
-        return newResponse;
-      }
-
-      throw new HttpError(error);
     }
-  );
+
+    throw new HttpError(error);
+  };
+
+  const instance = createInstance(
+    {
+      baseURL,
+    },
+    {
+      requestInterceptor: beforeRequest.map((originalBeforeRequest) => {
+        return async (requestConfig) => await originalBeforeRequest($, requestConfig);
+      }),
+      responseErrorInterceptor: interceptResponseError,
+    }
+  )
 
   return instance;
 }

packages/backend/src/helpers/webhook-handler-sync.js

@@ -63,6 +63,8 @@ export default async (flowId, request, response) => {
     });
 
     if (testRun) {
+      response.status(204).end();
+
       // in case of testing, we do not process the whole process.
       continue;
     }
@@ -74,6 +76,12 @@ export default async (flowId, request, response) => {
         executionId,
       });
 
+      if (actionStep.appKey === 'filter' && !actionExecutionStep.dataOut) {
+        response.status(422).end();
+
+        break;
+      }
+
       if (actionStep.key === 'respondWith' && !response.headersSent) {
         const { headers, statusCode, body } = actionExecutionStep.dataOut;
 

packages/docs/pages/apps/appwrite/connection.md

@@ -14,7 +14,7 @@ connection in Automatisch. If any of the steps are outdated, please let us know!
 7. Click on the **Select all** and then click on the **Create** button.
 8. Now, copy your **API key secret** and paste the key into the **API Key** field in Automatisch.
 9. Write any screen name to be displayed in Automatisch.
-10. You can find your project ID next to your project name. Paste the id into **Project ID** field in Automatsich.
+10. You can find your project ID next to your project name. Paste the id into **Project ID** field in Automatisch.
-11. If you are using self-hosted Appwrite project, you can paste the instace url into **Appwrite instance URL** field in Automatisch.
+11. If you are using self-hosted Appwrite project, you can paste the instance url into **Appwrite instance URL** field in Automatisch.
 12. Fill the host name field with the hostname of your instance URL. It's either `cloud.appwrite.io` or hostname of your instance URL.
 13. Start using Appwrite integration with Automatisch!

packages/docs/pages/apps/appwrite/triggers.md

@@ -1,7 +1,7 @@
 ---
 favicon: /favicons/appwrite.svg
 items:
-  - name: New documets
+  - name: New documents
     desc: Triggers when a new document is created.
 ---