diff --git a/docker-compose.yml b/docker-compose.yml index 48c633ab..fbbb0a1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,8 @@ services: main: build: context: ./docker - image: automatischio/automatisch + dockerfile: Dockerfile.compose + entrypoint: /compose-entrypoint.sh ports: - '3000:3000' depends_on: @@ -28,7 +29,8 @@ services: worker: build: context: ./docker - image: automatischio/automatisch + dockerfile: Dockerfile.compose + entrypoint: /compose-entrypoint.sh depends_on: - main environment: @@ -52,7 +54,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + test: ['CMD-SHELL', 'pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}'] interval: 10s timeout: 5s retries: 5 diff --git a/docker/Dockerfile b/docker/Dockerfile index 901a1242..afff33c9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,11 +1,8 @@ # syntax=docker/dockerfile:1 -FROM node:16 +FROM node:16-alpine WORKDIR /automatisch -RUN apt-get update && apt-get install -y dos2unix - COPY ./entrypoint.sh /entrypoint.sh -RUN dos2unix /entrypoint.sh && apt-get --purge remove -y dos2unix && rm -rf /var/lib/apt/lists/* RUN yarn global add @automatisch/cli@0.2.0 diff --git a/docker/Dockerfile.compose b/docker/Dockerfile.compose new file mode 100644 index 00000000..12e4bb92 --- /dev/null +++ b/docker/Dockerfile.compose @@ -0,0 +1,11 @@ +# syntax=docker/dockerfile:1 +FROM automatischio/automatisch:0.2.0 +WORKDIR /automatisch + +RUN apk add --no-cache openssl dos2unix + +COPY ./compose-entrypoint.sh /compose-entrypoint.sh +RUN dos2unix /compose-entrypoint.sh + +EXPOSE 3000 +ENTRYPOINT ["sh", "/compose-entrypoint.sh"] diff --git a/docker/compose-entrypoint.sh b/docker/compose-entrypoint.sh new file mode 100755 index 00000000..3a5448c6 --- /dev/null +++ b/docker/compose-entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +if [ ! -f /automatisch/storage/.env ]; then + >&2 echo "Saving environment variables" + ENCRYPTION_KEY="${ENCRYPTION_KEY:-$(openssl rand -base64 36)}" + APP_SECRET_KEY="${APP_SECRET_KEY:-$(openssl rand -base64 36)}" + echo "ENCRYPTION_KEY=$ENCRYPTION_KEY" >> /automatisch/storage/.env + echo "APP_SECRET_KEY=$APP_SECRET_KEY" >> /automatisch/storage/.env +fi + +# initiate env. vars. from /automatisch/storage/.env file +export $(grep -v '^#' /automatisch/storage/.env | xargs) + +echo "Environment variables have been set!" + +sh /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 5f1f6f24..60de858c 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,16 +2,8 @@ set -e -if [ ! -f /automatisch/storage/.env ]; then - >&2 echo "Saving environment variables" - ENCRYPTION_KEY="${ENCRYPTION_KEY:-$(openssl rand -base64 36)}" - APP_SECRET_KEY="${APP_SECRET_KEY:-$(openssl rand -base64 36)}" - echo "ENCRYPTION_KEY=$ENCRYPTION_KEY" >> /automatisch/storage/.env - echo "APP_SECRET_KEY=$APP_SECRET_KEY" >> /automatisch/storage/.env -fi - if [ -n "$WORKER" ]; then - automatisch start-worker --env-file /automatisch/storage/.env + automatisch start-worker else - automatisch start --env-file /automatisch/storage/.env + automatisch start fi diff --git a/docker/images/main/wait-for-postgres.sh b/docker/images/main/wait-for-postgres.sh deleted file mode 100644 index ae7e32c9..00000000 --- a/docker/images/main/wait-for-postgres.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -until psql -h "$POSTGRES_HOST" -U "$POSTGRES_USERNAME" -d "$POSTGRES_DATABASE" -c '\q'; do - >&2 echo "Waiting for Postgres to be ready..." - sleep 1 -done - ->&2 echo "Postgres is up - executing command" -exec "$@" diff --git a/docker/images/worker/wait-for-postgres.sh b/docker/images/worker/wait-for-postgres.sh deleted file mode 100644 index ae7e32c9..00000000 --- a/docker/images/worker/wait-for-postgres.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -until psql -h "$POSTGRES_HOST" -U "$POSTGRES_USERNAME" -d "$POSTGRES_DATABASE" -c '\q'; do - >&2 echo "Waiting for Postgres to be ready..." - sleep 1 -done - ->&2 echo "Postgres is up - executing command" -exec "$@" diff --git a/packages/backend/.env-example b/packages/backend/.env-example index e86e6ebf..53505520 100644 --- a/packages/backend/.env-example +++ b/packages/backend/.env-example @@ -13,5 +13,8 @@ ENCRYPTION_KEY=sample-encryption-key APP_SECRET_KEY=sample-app-secret-key REDIS_PORT=6379 REDIS_HOST=127.0.0.1 +REDIS_USERNAME=redis_username +REDIS_PASSWORD=redis_password +REDIS_TLS=true ENABLE_BULLMQ_DASHBOARD=false SERVE_WEB_APP_SEPARATELY=true diff --git a/packages/backend/src/config/app.ts b/packages/backend/src/config/app.ts index 2b8ef999..6d41956b 100644 --- a/packages/backend/src/config/app.ts +++ b/packages/backend/src/config/app.ts @@ -21,6 +21,9 @@ type AppConfig = { serveWebAppSeparately: boolean; redisHost: string; redisPort: number; + redisUsername: string; + redisPassword: string; + redisTls: boolean; enableBullMQDashboard: boolean; bullMQDashboardUsername: string; bullMQDashboardPassword: string; @@ -55,14 +58,17 @@ const appConfig: AppConfig = { postgresUsername: process.env.POSTGRES_USERNAME || 'automatisch_development_user', postgresPassword: process.env.POSTGRES_PASSWORD, - postgresEnableSsl: process.env.POSTGRES_ENABLE_SSL === 'true' ? true : false, + postgresEnableSsl: process.env.POSTGRES_ENABLE_SSL === 'true', encryptionKey: process.env.ENCRYPTION_KEY || '', appSecretKey: process.env.APP_SECRET_KEY || '', serveWebAppSeparately, redisHost: process.env.REDIS_HOST || '127.0.0.1', redisPort: parseInt(process.env.REDIS_PORT || '6379'), + redisUsername: process.env.REDIS_USERNAME, + redisPassword: process.env.REDIS_PASSWORD, + redisTls: process.env.REDIS_TLS === 'true', enableBullMQDashboard: - process.env.ENABLE_BULLMQ_DASHBOARD === 'true' ? true : false, + process.env.ENABLE_BULLMQ_DASHBOARD === 'true', bullMQDashboardUsername: process.env.BULLMQ_DASHBOARD_USERNAME, bullMQDashboardPassword: process.env.BULLMQ_DASHBOARD_PASSWORD, baseUrl, diff --git a/packages/backend/src/config/redis.ts b/packages/backend/src/config/redis.ts index dd816d0f..fedb5be0 100644 --- a/packages/backend/src/config/redis.ts +++ b/packages/backend/src/config/redis.ts @@ -1,9 +1,24 @@ import appConfig from './app'; -const redisConfig = { +type TRedisConfig = { + host: string, + port: number, + username?: string, + password?: string, + tls?: Record, + enableOfflineQueue: boolean, +} + +const redisConfig: TRedisConfig = { host: appConfig.redisHost, port: appConfig.redisPort, + username: appConfig.redisUsername, + password: appConfig.redisPassword, enableOfflineQueue: false, }; +if (appConfig.redisTls) { + redisConfig.tls = {}; +} + export default redisConfig; diff --git a/packages/backend/src/errors/http.ts b/packages/backend/src/errors/http.ts index d4396f50..d7a38e13 100644 --- a/packages/backend/src/errors/http.ts +++ b/packages/backend/src/errors/http.ts @@ -1,12 +1,17 @@ +import type { AxiosResponse, AxiosError } from 'axios'; import { IJSONObject } from '@automatisch/types'; import BaseError from './base'; export default class HttpError extends BaseError { - constructor(error: IJSONObject) { + response: AxiosResponse; + + constructor(error: AxiosError) { const computedError = - ((error.response as IJSONObject)?.data as IJSONObject) || - (error.message as string); + error.response?.data as IJSONObject || + error.message as string; super(computedError); + + this.response = error.response; } } diff --git a/packages/backend/src/graphql/mutations/update-flow-status.ts b/packages/backend/src/graphql/mutations/update-flow-status.ts index 1fc783ed..10e2f9f5 100644 --- a/packages/backend/src/graphql/mutations/update-flow-status.ts +++ b/packages/backend/src/graphql/mutations/update-flow-status.ts @@ -1,5 +1,6 @@ import Context from '../../types/express/context'; import flowQueue from '../../queues/flow'; +import { REMOVE_AFTER_30_DAYS_OR_150_JOBS, REMOVE_AFTER_7_DAYS_OR_50_JOBS } from '../../helpers/remove-job-configuration'; type Params = { input: { @@ -51,6 +52,8 @@ const updateFlowStatus = async ( { repeat: repeatOptions, jobId: flow.id, + removeOnComplete: REMOVE_AFTER_7_DAYS_OR_50_JOBS, + removeOnFail: REMOVE_AFTER_30_DAYS_OR_150_JOBS } ); } else { diff --git a/packages/backend/src/helpers/remove-job-configuration.ts b/packages/backend/src/helpers/remove-job-configuration.ts new file mode 100644 index 00000000..e6e01443 --- /dev/null +++ b/packages/backend/src/helpers/remove-job-configuration.ts @@ -0,0 +1,10 @@ +export const REMOVE_AFTER_30_DAYS_OR_150_JOBS = { + age: 30 * 24 * 3600, + count: 150, +}; + +export const REMOVE_AFTER_7_DAYS_OR_50_JOBS = { + age: 7 * 24 * 3600, + count: 50, +}; + diff --git a/packages/backend/src/workers/action.ts b/packages/backend/src/workers/action.ts index 75d9bd86..80dd2849 100644 --- a/packages/backend/src/workers/action.ts +++ b/packages/backend/src/workers/action.ts @@ -4,6 +4,7 @@ import logger from '../helpers/logger'; import Step from '../models/step'; import actionQueue from '../queues/action'; import { processAction } from '../services/action'; +import { REMOVE_AFTER_30_DAYS_OR_150_JOBS, REMOVE_AFTER_7_DAYS_OR_50_JOBS } from '../helpers/remove-job-configuration'; type JobData = { flowId: string; @@ -31,7 +32,12 @@ export const worker = new Worker( stepId: nextStep.id, }; - await actionQueue.add(jobName, jobPayload); + const jobOptions = { + removeOnComplete: REMOVE_AFTER_7_DAYS_OR_50_JOBS, + removeOnFail: REMOVE_AFTER_30_DAYS_OR_150_JOBS, + } + + await actionQueue.add(jobName, jobPayload, jobOptions); }, { connection: redisConfig } ); @@ -42,7 +48,7 @@ worker.on('completed', (job) => { worker.on('failed', (job, err) => { logger.info( - `JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has failed22 to start with ${err.message}` + `JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has failed to start with ${err.message}` ); }); diff --git a/packages/backend/src/workers/flow.ts b/packages/backend/src/workers/flow.ts index d8dc670a..7ca05782 100644 --- a/packages/backend/src/workers/flow.ts +++ b/packages/backend/src/workers/flow.ts @@ -4,6 +4,7 @@ import logger from '../helpers/logger'; import triggerQueue from '../queues/trigger'; import { processFlow } from '../services/flow'; import Flow from '../models/flow'; +import { REMOVE_AFTER_30_DAYS_OR_150_JOBS, REMOVE_AFTER_7_DAYS_OR_50_JOBS } from '../helpers/remove-job-configuration'; export const worker = new Worker( 'flow', @@ -17,6 +18,11 @@ export const worker = new Worker( const reversedData = data.reverse(); + const jobOptions = { + removeOnComplete: REMOVE_AFTER_7_DAYS_OR_50_JOBS, + removeOnFail: REMOVE_AFTER_30_DAYS_OR_150_JOBS, + } + for (const triggerItem of reversedData) { const jobName = `${triggerStep.id}-${triggerItem.meta.internalId}`; @@ -26,7 +32,7 @@ export const worker = new Worker( triggerItem, }; - await triggerQueue.add(jobName, jobPayload); + await triggerQueue.add(jobName, jobPayload, jobOptions); } if (error) { @@ -38,7 +44,7 @@ export const worker = new Worker( error, }; - await triggerQueue.add(jobName, jobPayload); + await triggerQueue.add(jobName, jobPayload, jobOptions); } }, { connection: redisConfig } diff --git a/packages/backend/src/workers/trigger.ts b/packages/backend/src/workers/trigger.ts index 8d64e032..a7492328 100644 --- a/packages/backend/src/workers/trigger.ts +++ b/packages/backend/src/workers/trigger.ts @@ -5,6 +5,7 @@ import { IJSONObject, ITriggerItem } from '@automatisch/types'; import actionQueue from '../queues/action'; import Step from '../models/step'; import { processTrigger } from '../services/trigger'; +import { REMOVE_AFTER_30_DAYS_OR_150_JOBS, REMOVE_AFTER_7_DAYS_OR_50_JOBS } from '../helpers/remove-job-configuration'; type JobData = { flowId: string; @@ -32,7 +33,12 @@ export const worker = new Worker( stepId: nextStep.id, }; - await actionQueue.add(jobName, jobPayload); + const jobOptions = { + removeOnComplete: REMOVE_AFTER_7_DAYS_OR_50_JOBS, + removeOnFail: REMOVE_AFTER_30_DAYS_OR_150_JOBS, + } + + await actionQueue.add(jobName, jobPayload, jobOptions); }, { connection: redisConfig } ); diff --git a/packages/docs/pages/advanced/configuration.md b/packages/docs/pages/advanced/configuration.md index 592ee31f..88d90ba1 100644 --- a/packages/docs/pages/advanced/configuration.md +++ b/packages/docs/pages/advanced/configuration.md @@ -29,6 +29,9 @@ Please be careful with the `ENCRYPTION_KEY` environment variable. It is used to | `APP_SECRET_KEY` | string | | Secret Key to authenticate the user | | `REDIS_HOST` | string | `redis` | Redis Host | | `REDIS_PORT` | number | `6379` | Redis Port | +| `REDIS_USERNAME` | string | `` | Redis Username | +| `REDIS_PASSWORD` | string | `` | Redis Password | +| `REDIS_TLS` | boolean | `false` | Redis TLS | | `TELEMETRY_ENABLED` | boolean | `true` | Enable/Disable Telemetry | | `ENABLE_BULLMQ_DASHBOARD` | boolean | `false` | Enable BullMQ Dashboard | | `BULLMQ_DASHBOARD_USERNAME` | string | | Username to login BullMQ Dashboard |