refactor(http-client): inherit interceptors from parent instance

packages/backend/src/apps/formatter/actions/text/transformers/replace.js

@@ -1,8 +1,26 @@
 const replace = ($) => {
   const input = $.step.parameters.input;
-
   const find = $.step.parameters.find;
   const replace = $.step.parameters.replace;
+  const useRegex = $.step.parameters.useRegex;
+
+  if (useRegex) {
+    const ignoreCase = $.step.parameters.ignoreCase;
+
+    const flags = [ignoreCase && 'i', 'g'].filter(Boolean).join('');
+
+    const timeoutId = setTimeout(() => {
+      $.execution.exit();
+    }, 100);
+
+    const regex = new RegExp(find, flags);
+
+    const replacedValue = input.replaceAll(regex, replace);
+
+    clearTimeout(timeoutId);
+
+    return replacedValue;
+  }
 
   return input.replaceAll(find, replace);
 };

packages/backend/src/apps/formatter/dynamic-fields/index.js

@@ -1,3 +1,4 @@
 import listTransformOptions from './list-transform-options/index.js';
+import listReplaceRegexOptions from './list-replace-regex-options/index.js';
 
-export default [listTransformOptions];
+export default [listTransformOptions, listReplaceRegexOptions];

packages/backend/src/apps/formatter/dynamic-fields/list-replace-regex-options/index.js

@@ -0,0 +1,23 @@
+export default {
+  name: 'List replace regex options',
+  key: 'listReplaceRegexOptions',
+
+  async run($) {
+    if (!$.step.parameters.useRegex) return [];
+
+    return [
+      {
+        label: 'Ignore case',
+        key: 'ignoreCase',
+        type: 'dropdown',
+        required: true,
+        description: 'Ignore case sensitivity.',
+        variables: true,
+        options: [
+          { label: 'Yes', value: true },
+          { label: 'No', value: false },
+        ],
+      },
+    ];
+  },
+};

packages/backend/src/apps/formatter/dynamic-fields/list-transform-options/text/replace.js

@@ -23,6 +23,33 @@ const replace = [
     description: 'Text that will replace the found text.',
     variables: true,
   },
+  {
+    label: 'Use Regular Expression',
+    key: 'useRegex',
+    type: 'dropdown',
+    required: true,
+    description: 'Use regex to search values.',
+    variables: true,
+    value: false,
+    options: [
+      { label: 'Yes', value: true },
+      { label: 'No', value: false },
+    ],
+    additionalFields: {
+      type: 'query',
+      name: 'getDynamicFields',
+      arguments: [
+        {
+          name: 'key',
+          value: 'listReplaceRegexOptions',
+        },
+        {
+          name: 'parameters.useRegex',
+          value: '{parameters.useRegex}',
+        },
+      ],
+    },
+  },
 ];
 
 export default replace;

packages/backend/src/config/app.js

@@ -100,6 +100,9 @@ const appConfig = {
   additionalDrawerLinkIcon: process.env.ADDITIONAL_DRAWER_LINK_ICON,
   additionalDrawerLinkText: process.env.ADDITIONAL_DRAWER_LINK_TEXT,
   disableSeedUser: process.env.DISABLE_SEED_USER === 'true',
+  httpProxy: process.env.http_proxy,
+  httpsProxy: process.env.https_proxy,
+  noProxy: process.env.no_proxy,
 };
 
 if (!appConfig.encryptionKey) {

packages/backend/src/helpers/axios-with-proxy.js

@@ -1,12 +1,13 @@
 import axios from 'axios';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import { HttpProxyAgent } from 'http-proxy-agent';
+import appConfig from '../config/app.js';
 
 const config = axios.defaults;
-const httpProxyUrl = process.env.http_proxy;
+const httpProxyUrl = appConfig.httpProxy;
-const httpsProxyUrl = process.env.https_proxy;
+const httpsProxyUrl = appConfig.httpsProxy;
 const supportsProxy = httpProxyUrl || httpsProxyUrl;
-const noProxyEnv = process.env.no_proxy;
+const noProxyEnv = appConfig.noProxy;
 const noProxyHosts = noProxyEnv ? noProxyEnv.split(',').map(host => host.trim()) : [];
 
 if (supportsProxy) {
@@ -29,15 +30,43 @@ function shouldSkipProxy(hostname) {
   });
 };
 
-axiosWithProxyInstance.interceptors.request.use(function skipProxyIfInNoProxy(requestConfig) {
+/**
-  const hostname = new URL(requestConfig.url).hostname;
+ * The interceptors are executed in the reverse order they are added.
+ */
+axiosWithProxyInstance.interceptors.request.use(
+  function skipProxyIfInNoProxy(requestConfig) {
+    const hostname = new URL(requestConfig.baseURL).hostname;
 
-  if (supportsProxy && shouldSkipProxy(hostname)) {
+    if (supportsProxy && shouldSkipProxy(hostname)) {
-    requestConfig.httpAgent = undefined;
+      requestConfig.httpAgent = undefined;
-    requestConfig.httpsAgent = undefined;
+      requestConfig.httpsAgent = undefined;
-  }
+    }
 
-  return requestConfig;
+    return requestConfig;
-});
+  },
+  undefined,
+  { synchronous: true }
+);
+
+axiosWithProxyInstance.interceptors.request.use(
+  function removeBaseUrlForAbsoluteUrls(requestConfig) {
+    /**
+     * If the URL is an absolute URL, we remove its origin out of the URL
+     * and set it as baseURL. This lets us streamlines the requests made by Automatisch
+     * and requests made by app integrations.
+     */
+    try {
+      const url = new URL(requestConfig.url);
+      requestConfig.baseURL = url.origin;
+      requestConfig.url = url.pathname + url.search;
+
+      return requestConfig;
+    } catch {
+      return requestConfig;
+    }
+  },
+  undefined,
+  { synchronous: true}
+);
 
 export default axiosWithProxyInstance;

packages/backend/src/helpers/axios-with-proxy.test.js

@@ -0,0 +1,119 @@
+import { beforeEach, describe, it, expect, vi } from 'vitest';
+
+describe('Axios with proxy', () => {
+  beforeEach(() => {
+    vi.resetModules();
+  });
+
+  it('should have two interceptors by default', async () => {
+    const axios = (await import('./axios-with-proxy.js')).default;
+    const requestInterceptors = axios.interceptors.request.handlers;
+
+    expect(requestInterceptors.length).toBe(2);
+  });
+
+  it('should have default interceptors in a certain order', async () => {
+    const axios = (await import('./axios-with-proxy.js')).default;
+
+    const requestInterceptors = axios.interceptors.request.handlers;
+    const firstRequestInterceptor = requestInterceptors[0];
+    const secondRequestInterceptor = requestInterceptors[1];
+
+    expect(firstRequestInterceptor.fulfilled.name).toBe('skipProxyIfInNoProxy');
+    expect(secondRequestInterceptor.fulfilled.name).toBe('removeBaseUrlForAbsoluteUrls');
+  });
+
+  describe('skipProxyIfInNoProxy', () => {
+    let appConfig, axios;
+    beforeEach(async() => {
+      appConfig = (await import('../config/app.js')).default;
+
+      vi.spyOn(appConfig, 'httpProxy', 'get').mockReturnValue('http://proxy.automatisch.io');
+      vi.spyOn(appConfig, 'httpsProxy', 'get').mockReturnValue('http://proxy.automatisch.io');
+      vi.spyOn(appConfig, 'noProxy', 'get').mockReturnValue('name.tld,automatisch.io');
+
+      axios = (await import('./axios-with-proxy.js')).default;
+    });
+
+    it('should skip proxy for hosts in no_proxy environment variable', async () => {
+      const skipProxyIfInNoProxy = axios.interceptors.request.handlers[0].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        baseURL: 'https://automatisch.io'
+      };
+
+      const interceptedRequestConfig = skipProxyIfInNoProxy(mockRequestConfig);
+
+      expect(interceptedRequestConfig.httpAgent).toBeUndefined();
+      expect(interceptedRequestConfig.httpsAgent).toBeUndefined();
+      expect(interceptedRequestConfig.proxy).toBe(false);
+    });
+
+    it('should not skip proxy for hosts not in no_proxy environment variable', async () => {
+      const skipProxyIfInNoProxy = axios.interceptors.request.handlers[0].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        // beware the intentional typo!
+        baseURL: 'https://automatish.io'
+      };
+
+      const interceptedRequestConfig = skipProxyIfInNoProxy(mockRequestConfig);
+
+      expect(interceptedRequestConfig.httpAgent).toBeDefined();
+      expect(interceptedRequestConfig.httpsAgent).toBeDefined();
+      expect(interceptedRequestConfig.proxy).toBe(false);
+    });
+  });
+
+  describe('removeBaseUrlForAbsoluteUrls', () => {
+    let axios;
+    beforeEach(async() => {
+      axios = (await import('./axios-with-proxy.js')).default;
+    });
+
+    it('should trim the baseUrl from absolute urls', async () => {
+      const removeBaseUrlForAbsoluteUrls = axios.interceptors.request.handlers[1].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        url: 'https://automatisch.io/path'
+      };
+
+      const interceptedRequestConfig = removeBaseUrlForAbsoluteUrls(mockRequestConfig);
+
+      expect(interceptedRequestConfig.baseURL).toBe('https://automatisch.io');
+      expect(interceptedRequestConfig.url).toBe('/path');
+    });
+
+    it('should not mutate separate baseURL and urls', async () => {
+      const removeBaseUrlForAbsoluteUrls = axios.interceptors.request.handlers[1].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        baseURL: 'https://automatisch.io',
+        url: '/path?query=1'
+      };
+
+      const interceptedRequestConfig = removeBaseUrlForAbsoluteUrls(mockRequestConfig);
+
+      expect(interceptedRequestConfig.baseURL).toBe('https://automatisch.io');
+      expect(interceptedRequestConfig.url).toBe('/path?query=1');
+    });
+
+    it('should not strip querystring from url', async () => {
+      const removeBaseUrlForAbsoluteUrls = axios.interceptors.request.handlers[1].fulfilled;
+
+      const mockRequestConfig = {
+        ...axios.defaults,
+        url: 'https://automatisch.io/path?query=1'
+      };
+
+      const interceptedRequestConfig = removeBaseUrlForAbsoluteUrls(mockRequestConfig);
+
+      expect(interceptedRequestConfig.baseURL).toBe('https://automatisch.io');
+      expect(interceptedRequestConfig.url).toBe('/path?query=1');
+    });
+  });
+});

packages/backend/src/helpers/http-client/index.js

@@ -1,41 +1,38 @@
-import { URL } from 'node:url';
 import HttpError from '../../errors/http.js';
 import axios from '../axios-with-proxy.js';
 
-const removeBaseUrlForAbsoluteUrls = (requestConfig) => {
+// Mutates the `toInstance` by copying the request interceptors from `fromInstance`
-  try {
+const copyRequestInterceptors = (fromInstance, toInstance) => {
-    const url = new URL(requestConfig.url);
+  // Copy request interceptors
-    requestConfig.baseURL = url.origin;
+  fromInstance.interceptors.request.forEach(interceptor => {
-    requestConfig.url = url.pathname + url.search;
+    toInstance.interceptors.request.use(
-
+      interceptor.fulfilled,
-    return requestConfig;
+      interceptor.rejected,
-  } catch {
+      {
-    return requestConfig;
+        synchronous: interceptor.synchronous,
-  }
+        runWhen: interceptor.runWhen
-};
+      }
+    );
+  });
+}
 
 export default function createHttpClient({ $, baseURL, beforeRequest = [] }) {
   const instance = axios.create({
     baseURL,
   });
 
+  // 1. apply the beforeRequest functions from the app
   instance.interceptors.request.use((requestConfig) => {
-    const newRequestConfig = removeBaseUrlForAbsoluteUrls(requestConfig);
-
     const result = beforeRequest.reduce((newConfig, beforeRequestFunc) => {
       return beforeRequestFunc($, newConfig);
-    }, newRequestConfig);
+    }, requestConfig);
 
-    /**
-     * axios seems to want InternalAxiosRequestConfig returned not AxioRequestConfig
-     * anymore even though requests do require AxiosRequestConfig.
-     *
-     * Since both interfaces are very similar (InternalAxiosRequestConfig
-     * extends AxiosRequestConfig), we can utilize an assertion below
-     **/
     return result;
   });
 
+  // 2. inherit the request inceptors from the parent instance
+  copyRequestInterceptors(axios, instance);
+
   instance.interceptors.response.use(
     (response) => response,
     async (error) => {