feat(cli): run migrations and app in start command (#284)

* feat: perform pending migrations in start command

* feat: log error when DB connection is refused

* feat: log error when Redis connection is refused

* refactor: fix type errors

* fix: correct server and copy graphql schema

* fix: differentiate migrations by env

* chore: remove dev executable

* chore: fix typo in default postgresUsername

* fix: copy json files into dist folder

* chore(cli): add dev script

* chore: pull non-dev logs to info level

* feat(cli): run app in start command

* fix(backend): remove default count in Connection

* fix(cli): remove .eslintrc usage in lint script

* refactor: remove disableMigrationsListValidation

* refactor: make Step optional in ExecutionStep

* refactor: make Flow optional in Step

packages/backend/bin/database/seed-user.ts

@@ -1,5 +1,5 @@
 import User from '../../src/models/user';
-import '../../src/config/database';
+import '../../src/config/orm';
 import logger from '../../src/helpers/logger';
 
 const userParams = {

packages/backend/knexfile.ts

@@ -1,5 +1,7 @@
 import appConfig from './src/config/app';
 
+const fileExtension = appConfig.isDev ? 'ts' : 'js';
+
 const knexConfig = {
   client: 'pg',
   connection: {
@@ -13,6 +15,8 @@ const knexConfig = {
   pool: { min: 0, max: 20 },
   migrations: {
     directory: __dirname + '/src/db/migrations',
+    extension: fileExtension,
+    loadExtensions: [`.${fileExtension}`],
   },
   seeds: {
     directory: __dirname + '/src/db/seeds',

packages/backend/package.json

@@ -3,10 +3,10 @@
   "version": "0.1.0",
   "description": "> TODO: description",
   "scripts": {
-    "dev": "nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/app.ts",
+    "dev": "nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/server.ts",
     "worker": "nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/worker.ts",
-    "build": "tsc",
-    "start": "node dist/src/app.js",
+    "build": "tsc && yarn copy-statics",
+    "start": "node dist/src/server.js",
     "test": "ava",
     "lint": "eslint . --ignore-path ../../.eslintignore",
     "db:create": "ts-node ./bin/database/create.ts",
@@ -14,7 +14,8 @@
     "db:drop": "ts-node ./bin/database/drop.ts",
     "db:migration:create": "knex migrate:make",
     "db:rollback": "knex migrate:rollback",
-    "db:migrate": "knex migrate:latest"
+    "db:migrate": "knex migrate:latest",
+    "copy-statics": "copyfiles src/**/*.{graphql,json} dist"
   },
   "dependencies": {
     "@automatisch/web": "0.1.0",
@@ -27,6 +28,7 @@
     "axios": "0.24.0",
     "bcrypt": "^5.0.1",
     "bullmq": "^1.76.1",
+    "copyfiles": "^2.4.1",
     "cors": "^2.8.5",
     "crypto-js": "^4.1.1",
     "debug": "~2.6.9",
@@ -60,7 +62,7 @@
     }
   ],
   "homepage": "https://github.com/automatisch/automatisch#readme",
-  "main": "src/app.ts",
+  "main": "dist/src/app",
   "directories": {
     "src": "src",
     "test": "__tests__"
@@ -78,6 +80,7 @@
   "devDependencies": {
     "@automatisch/types": "0.1.0",
     "@types/bcrypt": "^5.0.0",
+    "@types/bull": "^3.15.8",
     "@types/cors": "^2.8.12",
     "@types/crypto-js": "^4.0.2",
     "@types/express": "^4.17.13",

packages/backend/src/app.ts

@@ -4,12 +4,11 @@ import express, { Request, Response, NextFunction } from 'express';
 import cors from 'cors';
 import corsOptions from './config/cors-options';
 import graphQLInstance from './helpers/graphql-instance';
-import logger from './helpers/logger';
 import morgan from './helpers/morgan';
 import appAssetsHandler from './helpers/app-assets-handler';
 import webUIHandler from './helpers/web-ui-handler';
 import errorHandler from './helpers/error-handler';
-import './config/database';
+import './config/orm';
 import {
   createBullBoardHandler,
   serverAdapter,
@@ -21,7 +20,6 @@ if (appConfig.appEnv === 'development') {
 }
 
 const app = express();
-const port = appConfig.port;
 
 if (appConfig.appEnv === 'development') {
   injectBullBoardHandler(app, serverAdapter);
@@ -44,6 +42,4 @@ app.use(function (req: Request, res: Response, next: NextFunction) {
 
 app.use(errorHandler);
 
-app.listen(port, () => {
-  logger.info(`Server is listening on ${port}`);
-});
+export default app;

packages/backend/src/config/app.ts

@@ -5,15 +5,16 @@ type AppConfig = {
   host: string;
   protocol: string;
   port: string;
-  webAppUrl?: string;
+  webAppUrl: string;
   appEnv: string;
+  isDev: boolean;
   postgresDatabase: string;
   postgresPort: number;
   postgresHost: string;
   postgresUsername: string;
-  postgresPassword: string;
+  postgresPassword?: string;
   postgresEnableSsl: boolean;
-  baseUrl?: string;
+  baseUrl: string;
   encryptionKey: string;
   appSecretKey: string;
   serveWebAppSeparately: boolean;
@@ -21,37 +22,44 @@ type AppConfig = {
   redisPort: number;
 };
 
+const host = process.env.HOST || 'localhost';
+const protocol = process.env.PROTOCOL || 'http';
+const port = process.env.PORT || '3000';
+const serveWebAppSeparately = process.env.SERVE_WEB_APP_SEPARATELY === 'true' ? true : false;
+
+let webAppUrl = `${protocol}://${host}:${port}`;
+if (serveWebAppSeparately) {
+  webAppUrl = process.env.WEB_APP_URL || 'http://localhost:3001';
+}
+
+const baseUrl = `${protocol}://${host}:${port}`;
+
+const appEnv = process.env.APP_ENV || 'development';
+
 const appConfig: AppConfig = {
-  host: process.env.HOST || 'localhost',
-  protocol: process.env.PROTOCOL || 'http',
-  port: process.env.PORT || '3000',
-  appEnv: process.env.APP_ENV || 'development',
+  host,
+  protocol,
+  port,
+  appEnv: appEnv,
+  isDev: appEnv === 'development',
   postgresDatabase: process.env.POSTGRES_DATABASE || 'automatisch_development',
-  postgresPort: parseInt(process.env.POSTGRES_PORT) || 5432,
+  postgresPort: parseInt(process.env.POSTGRES_PORT|| '5432'),
   postgresHost: process.env.POSTGRES_HOST || 'localhost',
   postgresUsername:
-    process.env.POSTGRES_USERNAME || 'automatish_development_user',
+    process.env.POSTGRES_USERNAME || 'automatisch_development_user',
   postgresPassword: process.env.POSTGRES_PASSWORD,
   postgresEnableSsl: process.env.POSTGRES_ENABLE_SSL === 'true' ? true : false,
-  encryptionKey: process.env.ENCRYPTION_KEY,
-  appSecretKey: process.env.APP_SECRET_KEY,
-  serveWebAppSeparately:
-    process.env.SERVE_WEB_APP_SEPARATELY === 'true' ? true : false,
+  encryptionKey: process.env.ENCRYPTION_KEY || '',
+  appSecretKey: process.env.APP_SECRET_KEY || '',
+  serveWebAppSeparately,
   redisHost: process.env.REDIS_HOST || '127.0.0.1',
-  redisPort: parseInt(process.env.REDIS_PORT) || 6379,
+  redisPort: parseInt(process.env.REDIS_PORT || '6379'),
+  baseUrl,
+  webAppUrl,
 };
 
-if (appConfig.serveWebAppSeparately) {
-  appConfig.webAppUrl = process.env.WEB_APP_URL || 'http://localhost:3001';
-} else {
-  appConfig.webAppUrl = `${appConfig.protocol}://${appConfig.host}:${appConfig.port}`;
-}
-
 if (!appConfig.encryptionKey) {
   throw new Error('ENCRYPTION_KEY environment variable needs to be set!');
 }
 
-const baseUrl = `${appConfig.protocol}://${appConfig.host}:${appConfig.port}`;
-appConfig.baseUrl = baseUrl;
-
 export default appConfig;

packages/backend/src/config/database.ts

@@ -1,6 +1,19 @@
-import { Model } from 'objection';
-import knexInstance from 'knex';
+import process from 'process';
+import knex from 'knex';
+import type { Knex } from 'knex';
 import knexConfig from '../../knexfile';
+import logger from '../helpers/logger';
 
-const knex = knexInstance(knexConfig)
-Model.knex(knex)
+const knexInstance: Knex = knex(knexConfig);
+
+const CONNECTION_REFUSED = 'ECONNREFUSED';
+
+knexInstance.raw('SELECT 1')
+  .catch((err) => {
+    if (err.code === CONNECTION_REFUSED) {
+      logger.error('Make sure you have installed PostgreSQL and it is running.', err);
+      process.exit();
+    }
+  });
+
+export default knexInstance;

packages/backend/src/config/orm.ts

@@ -0,0 +1,4 @@
+import { Model } from 'objection';
+import database from './database';
+
+Model.knex(database)

packages/backend/src/config/redis.ts

@@ -3,6 +3,7 @@ import appConfig from './app';
 const redisConfig = {
   host: appConfig.redisHost,
   port: appConfig.redisPort,
+  enableOfflineQueue: false,
 };
 
 export default redisConfig;

packages/backend/src/graphql/mutations/create-auth-data.ts

@@ -22,6 +22,8 @@ const createAuthData = async (
   const appClass = (await import(`../../apps/${connection.key}`)).default;
   const appData = App.findOneByKey(connection.key);
 
+  if (!connection.formattedData) { return null; }
+
   const appInstance = new appClass(appData, {
     consumerKey: connection.formattedData.consumerKey,
     consumerSecret: connection.formattedData.consumerSecret,

packages/backend/src/graphql/mutations/delete-step.ts

@@ -19,6 +19,8 @@ const deleteStep = async (
     })
     .throwIfNotFound();
 
+  if (!step) return;
+
   await step.$query().delete();
 
   const nextSteps = await step.flow

packages/backend/src/graphql/mutations/reset-connection.ts

@@ -18,6 +18,8 @@ const resetConnection = async (
     })
     .throwIfNotFound();
 
+  if (!connection.formattedData) { return null; }
+
   connection = await connection.$query().patchAndFetch({
     formattedData: { screenName: connection.formattedData.screenName },
   });

packages/backend/src/graphql/queries/get-connected-apps.ts

@@ -26,10 +26,13 @@ const getConnectedApps = async (
     .filter((app: IApp) => connectionKeys.includes(app.key))
     .map((app: IApp) => {
       const connection = connections.find(
-        (connection: IConnection) => connection.key === app.key
+        (connection) => (connection as IConnection).key === app.key
       );
 
-      app.connectionCount = connection.count;
+      if (connection) {
+        app.connectionCount = connection.count;
+      }
+
       return app;
     });
 

packages/backend/src/graphql/queries/get-data.ts

@@ -12,8 +12,12 @@ const getData = async (_parent: unknown, params: Params, context: Context) => {
     .withGraphFetched('connection')
     .findById(params.stepId);
 
+  if (!step) return null;
+
   const connection = step.connection;
 
+  if (!connection || !step.appKey) return null;
+
   const appData = App.findOneByKey(step.appKey);
   const AppClass = (await import(`../../apps/${step.appKey}`)).default;
 

packages/backend/src/helpers/logger.ts

@@ -10,7 +10,7 @@ const levels = {
 }
 
 const level = () => {
-  return appConfig.appEnv === 'development' ? 'debug' : 'warn'
+  return appConfig.appEnv === 'development' ? 'debug' : 'info'
 }
 
 const colors = {

packages/backend/src/models/app.ts

@@ -1,5 +1,6 @@
 import fs from 'fs';
 import { dirname, join } from 'path';
+import { IApp } from '@automatisch/types';
 import appInfoConverter from '../helpers/app-info-converter';
 
 class App {
@@ -7,7 +8,7 @@ class App {
   static folderPath = join(dirname(this.backendPath), 'apps');
   static list = fs.readdirSync(this.folderPath);
 
-  static findAll(name?: string): object[] {
+  static findAll(name?: string): IApp[] {
     if (!name) return this.list.map((name) => this.findOneByName(name));
 
     return this.list
@@ -15,7 +16,7 @@ class App {
       .map((name) => this.findOneByName(name));
   }
 
-  static findOneByName(name: string): object {
+  static findOneByName(name: string): IApp {
     const rawAppData = fs.readFileSync(
       this.folderPath + `/${name}/info.json`,
       'utf-8'
@@ -23,7 +24,7 @@ class App {
     return appInfoConverter(rawAppData);
   }
 
-  static findOneByKey(key: string): object {
+  static findOneByKey(key: string): IApp {
     const rawAppData = fs.readFileSync(
       this.folderPath + `/${key}/info.json`,
       'utf-8'

packages/backend/src/models/connection.ts

@@ -9,10 +9,10 @@ import { IJSONObject } from '@automatisch/types';
 class Connection extends Base {
   id!: string;
   key!: string;
-  data: string;
-  formattedData!: IJSONObject;
+  data = '';
+  formattedData?: IJSONObject;
   userId!: string;
-  verified: boolean;
+  verified = false;
   count: number;
 
   static tableName = 'connections';
@@ -50,7 +50,7 @@ class Connection extends Base {
       appConfig.encryptionKey
     ).toString();
 
-    delete this['formattedData'];
+    delete this.formattedData;
   }
 
   decryptData(): void {

packages/backend/src/models/execution-step.ts

@@ -8,7 +8,7 @@ class ExecutionStep extends Base {
   stepId!: string;
   dataIn!: Record<string, unknown>;
   dataOut!: Record<string, unknown>;
-  status: string;
+  status = 'failure';
   step: Step;
 
   static tableName = 'execution_steps';

packages/backend/src/models/execution.ts

@@ -5,8 +5,8 @@ import ExecutionStep from './execution-step';
 class Execution extends Base {
   id!: string;
   flowId!: string;
-  testRun: boolean;
-  executionSteps: ExecutionStep[];
+  testRun = false;
+  executionSteps: ExecutionStep[] = [];
 
   static tableName = 'executions';
 

packages/backend/src/models/flow.ts

@@ -6,9 +6,9 @@ import Execution from './execution';
 
 class Flow extends Base {
   id!: string;
-  name: string;
+  name!: string;
   userId!: string;
-  active: boolean;
+  active = false;
   steps?: [Step];
 
   static tableName = 'flows';

packages/backend/src/models/step.ts

@@ -7,15 +7,15 @@ import type { IStep } from '@automatisch/types';
 class Step extends Base {
   id!: string;
   flowId!: string;
-  key: string;
-  appKey: string;
+  key?: string;
+  appKey?: string;
   type!: IStep["type"];
   connectionId?: string;
-  status: string;
-  position: number;
-  parameters: Record<string, unknown>;
+  status = 'incomplete';
+  position!: number;
+  parameters: Record<string, unknown> = {};
   connection?: Connection;
-  flow?: Flow;
+  flow: Flow;
   executionSteps?: [ExecutionStep];
 
   static tableName = 'steps';

packages/backend/src/queues/processor.ts

@@ -1,11 +1,22 @@
+import process from 'process';
 import { Queue, QueueScheduler } from 'bullmq';
 import redisConfig from '../config/redis';
+import logger from '../helpers/logger';
+
+const CONNECTION_REFUSED = 'ECONNREFUSED';
 
 const redisConnection = {
   connection: redisConfig,
 };
 
-new QueueScheduler('processor', redisConnection);
 const processorQueue = new Queue('processor', redisConnection);
+new QueueScheduler('processor', redisConnection);
+
+processorQueue.on('error', (err) => {
+  if ((err as any).code === CONNECTION_REFUSED) {
+    logger.error('Make sure you have installed Redis and it is running.', err);
+    process.exit();
+  }
+});
 
 export default processorQueue;

packages/backend/src/server.ts

@@ -0,0 +1,9 @@
+import app from './app';
+import appConfig from './config/app';
+import logger from './helpers/logger';
+
+const port = appConfig.port;
+
+app.listen(port, () => {
+  logger.info(`Server is listening on ${port}`);
+});

packages/backend/src/services/processor.ts

@@ -14,8 +14,8 @@ type ProcessorOptions = {
 
 class Processor {
   flow: Flow;
-  untilStep: Step;
-  testRun: boolean;
+  untilStep?: Step;
+  testRun?: boolean;
 
   static variableRegExp = /({{step\..+\..+}})/g;
 
@@ -32,7 +32,8 @@ class Processor {
       .orderBy('position', 'asc');
 
     const triggerStep = steps.find((step) => step.type === 'trigger');
-    let initialTriggerData = await this.getInitialTriggerData(triggerStep);
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    let initialTriggerData = await this.getInitialTriggerData(triggerStep!);
 
     if (this.testRun) {
       initialTriggerData = [initialTriggerData[0]];
@@ -53,6 +54,8 @@ class Processor {
       let fetchedActionData = {};
 
       for await (const step of steps) {
+        if (!step.appKey) continue;
+
         const appData = App.findOneByKey(step.appKey);
 
         const {
@@ -74,11 +77,11 @@ class Processor {
 
         const appInstance = new AppClass(
           appData,
-          connection.formattedData,
+          connection?.formattedData,
           computedParameters
         );
 
-        if (!isTrigger) {
+        if (!isTrigger && key) {
           const command = appInstance.actions[key];
           fetchedActionData = await command.run();
         }
@@ -107,17 +110,21 @@ class Processor {
       .orderBy('created_at', 'desc')
       .first();
 
-    return lastExecutionStepFromFirstExecution.dataOut;
+    return lastExecutionStepFromFirstExecution?.dataOut;
   }
 
   async getInitialTriggerData(step: Step) {
+    if (!step.appKey) return null;
+
     const appData = App.findOneByKey(step.appKey);
     const { appKey, connection, key, parameters: rawParameters = {} } = step;
 
+    if (!key) return null;
+
     const AppClass = (await import(`../apps/${appKey}`)).default;
     const appInstance = new AppClass(
       appData,
-      connection.formattedData,
+      connection?.formattedData,
       rawParameters
     );
 
@@ -149,30 +156,34 @@ class Processor {
     executionSteps: ExecutionSteps
   ): Step['parameters'] {
     const entries = Object.entries(parameters);
-    return entries.reduce((result, [key, value]: [string, string]) => {
-      const parts = value.split(Processor.variableRegExp);
+    return entries.reduce((result, [key, value]: [string, unknown]) => {
+      if (typeof value === 'string') {
+        const parts = value.split(Processor.variableRegExp);
 
-      const computedValue = parts
-        .map((part: string) => {
-          const isVariable = part.match(Processor.variableRegExp);
-          if (isVariable) {
-            const stepIdAndKeyPath = part.replace(/{{step.|}}/g, '') as string;
-            const [stepId, ...keyPaths] = stepIdAndKeyPath.split('.');
-            const keyPath = keyPaths.join('.');
-            const executionStep = executionSteps[stepId.toString() as string];
-            const data = executionStep?.dataOut;
-            const dataValue = get(data, keyPath);
-            return dataValue;
-          }
+        const computedValue = parts
+          .map((part: string) => {
+            const isVariable = part.match(Processor.variableRegExp);
+            if (isVariable) {
+              const stepIdAndKeyPath = part.replace(/{{step.|}}/g, '') as string;
+              const [stepId, ...keyPaths] = stepIdAndKeyPath.split('.');
+              const keyPath = keyPaths.join('.');
+              const executionStep = executionSteps[stepId.toString() as string];
+              const data = executionStep?.dataOut;
+              const dataValue = get(data, keyPath);
+              return dataValue;
+            }
 
-          return part;
-        })
-        .join('');
+            return part;
+          })
+          .join('');
 
-      return {
-        ...result,
-        [key]: computedValue,
-      };
+        return {
+          ...result,
+          [key]: computedValue,
+        };
+      }
+
+      return result;
     }, {});
   }
 }

packages/backend/src/types/interfaces/authentication-interface.ts

@@ -1,10 +1,9 @@
-import type { IApp } from '@automatisch/types';
-import JSONObject from './json-object';
+import type { IApp, IJSONObject } from '@automatisch/types';
 
 export default interface AuthenticationInterface {
   appData: IApp;
   connectionData: IJSONObject;
   client: unknown;
-  verifyCredentials(): Promise<JSONObject>;
+  verifyCredentials(): Promise<IJSONObject>;
   isStillVerified(): Promise<boolean>;
 }

packages/backend/src/worker.ts

@@ -1,2 +1,2 @@
-import './config/database';
+import './config/orm';
 import './workers/processor';

packages/backend/tsconfig.json

@@ -1,19 +1,22 @@
 {
   "compilerOptions": {
-    "module": "commonjs",
-    "esModuleInterop": true,
-    "target": "es6",
-    "noImplicitAny": true,
-    "moduleResolution": "node",
-    "sourceMap": true,
-    "outDir": "dist",
     "baseUrl": ".",
+    "declaration": true,
+    "esModuleInterop": true,
+    "lib": ["es2021"],
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "noImplicitAny": true,
+    "outDir": "dist",
     "paths": {
       "*": [
         "node_modules/*",
         "src/types/*"
       ]
     },
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "target": "es2021",
     "typeRoots": [
       "node_modules/@types",
       "./src/types",
@@ -21,6 +24,12 @@
     ]
   },
   "include": [
-    "src/**/*"
-  ]
+    "src/**/*",
+    "bin/**/*"
+  ],
+  "ts-node": {
+    "compilerOptions": {
+      "module": "commonjs"
+    }
+  }
 }