Compare commits

...

15 Commits

Author SHA1 Message Date
8f9041301c docker-compose.yml aktualisiert
All checks were successful
release-tag / release-image (push) Successful in 1m20s
2024-12-12 20:14:47 +00:00
3573fc8895 .gitea/workflows/registry.yml aktualisiert
All checks were successful
release-tag / release-image (push) Successful in 1m24s
2024-12-12 20:10:07 +00:00
253d4ab6bd .gitea/workflows/registry.yml aktualisiert
All checks were successful
release-tag / release-image (push) Successful in 1m25s
2024-12-12 20:04:42 +00:00
1c9d30b688 .gitea/workflows/registry.yml aktualisiert
Some checks failed
release-tag / release-image (push) Failing after 16s
2024-12-12 20:01:14 +00:00
feff219994 .gitea/workflows/registry.yml aktualisiert
Some checks failed
release-tag / release-image (push) Failing after 14s
2024-12-12 19:57:41 +00:00
6bc2fe0f46 .gitea/workflows/registry.yml hinzugefügt
Some checks failed
release-tag / release-image (push) Failing after 14s
2024-12-12 19:52:35 +00:00
Ömer Faruk Aydın
3d62fabaac Merge pull request #2246 from automatisch/restructure-queues
refactor: Abstract queue generation and configuration
2024-12-12 14:57:03 +01:00
Faruk AYDIN
e41a331ad7 refactor: Abstract queue generation and configuration 2024-12-12 14:51:13 +01:00
Ömer Faruk Aydın
978ceaadb6 Merge pull request #2238 from automatisch/workers-refactoring
refactor: Workers and queues and eliminate redundant process listeners
2024-12-11 14:57:51 +03:00
Faruk AYDIN
770b07179f refactor: Workers and queues and eliminate redundant process listeners 2024-12-11 12:53:24 +01:00
Ömer Faruk Aydın
6d15167ad9 Merge pull request #2243 from automatisch/fix-tests
Fix failing API endpoint tests
2024-12-11 14:52:34 +03:00
Faruk AYDIN
39cba6bc74 test: Fix tests for create dynamic data and fields endpoints 2024-12-11 12:45:50 +01:00
Faruk AYDIN
9558e66abf test: Fix tests for get apps connection 2024-12-11 12:27:15 +01:00
Ömer Faruk Aydın
ff7908955e Merge pull request #2239 from automatisch/fix-flaky-user-tests
test(user): use relative future dates
2024-12-11 14:06:33 +03:00
Ali BARIN
26b095b835 test(user): use relative future dates 2024-12-05 11:46:38 +00:00
22 changed files with 208 additions and 266 deletions

View File

@@ -0,0 +1,52 @@
name: release-tag
on:
push:
branches:
- 'main'
jobs:
release-image:
runs-on: ubuntu-latest
env:
DOCKER_ORG: groot
DOCKER_LATEST: latest
RUNNER_TOOL_CACHE: /toolcache
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@v2
with: # replace it with your local IP
config-inline: |
[registry."git.send.nrw"]
http = true
insecure = true
- name: Login to DockerHub
uses: docker/login-action@v2
with:
registry: git.send.nrw # replace it with your local IP
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Get Meta
id: meta
run: |
echo REPO_NAME=$(echo ${GITHUB_REPOSITORY} | awk -F"/" '{print $2}') >> $GITHUB_OUTPUT
echo REPO_VERSION=$(git describe --tags --always | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@v4
with:
context: ./docker
file: ./docker/Dockerfile.compose
entrypoint: ./docker/compose-entrypoint.sh
platforms: |
linux/amd64
push: true
tags: | # replace it with your local IP and tags
git.send.nrw/${{ env.DOCKER_ORG }}/${{ steps.meta.outputs.REPO_NAME }}:${{ steps.meta.outputs.REPO_VERSION }}
git.send.nrw/${{ env.DOCKER_ORG }}/${{ steps.meta.outputs.REPO_NAME }}:${{ env.DOCKER_LATEST }}

View File

@@ -1,10 +1,7 @@
version: '3.9'
services:
main:
build:
context: ./docker
dockerfile: Dockerfile.compose
entrypoint: /compose-entrypoint.sh
image: git.send.nrw/groot/automatisch:latest
ports:
- '3000:3000'
depends_on:
@@ -28,10 +25,7 @@ services:
volumes:
- automatisch_storage:/automatisch/storage
worker:
build:
context: ./docker
dockerfile: Dockerfile.compose
entrypoint: /compose-entrypoint.sh
image: git.send.nrw/groot/automatisch:latest
depends_on:
- main
environment:

View File

@@ -87,14 +87,14 @@ describe('GET /api/v1/apps/:appKey/connections', () => {
it('should return not found response for invalid connection UUID', async () => {
await createPermission({
action: 'update',
action: 'read',
subject: 'Connection',
roleId: currentUserRole.id,
conditions: ['isCreator'],
});
await request(app)
.get('/api/v1/connections/invalid-connection-id/connections')
.get('/api/v1/apps/invalid-connection-id/connections')
.set('Authorization', token)
.expect(404);
});

View File

@@ -193,7 +193,7 @@ describe('POST /api/v1/steps/:stepId/dynamic-data', () => {
const notExistingStepUUID = Crypto.randomUUID();
await request(app)
.get(`/api/v1/steps/${notExistingStepUUID}/dynamic-data`)
.post(`/api/v1/steps/${notExistingStepUUID}/dynamic-data`)
.set('Authorization', token)
.expect(404);
});
@@ -216,7 +216,7 @@ describe('POST /api/v1/steps/:stepId/dynamic-data', () => {
const step = await createStep({ appKey: null });
await request(app)
.get(`/api/v1/steps/${step.id}/dynamic-data`)
.post(`/api/v1/steps/${step.id}/dynamic-data`)
.set('Authorization', token)
.expect(404);
});

View File

@@ -118,7 +118,7 @@ describe('POST /api/v1/steps/:stepId/dynamic-fields', () => {
const notExistingStepUUID = Crypto.randomUUID();
await request(app)
.get(`/api/v1/steps/${notExistingStepUUID}/dynamic-fields`)
.post(`/api/v1/steps/${notExistingStepUUID}/dynamic-fields`)
.set('Authorization', token)
.expect(404);
});
@@ -138,10 +138,11 @@ describe('POST /api/v1/steps/:stepId/dynamic-fields', () => {
conditions: [],
});
const step = await createStep({ appKey: null });
const step = await createStep();
await step.$query().patch({ appKey: null });
await request(app)
.get(`/api/v1/steps/${step.id}/dynamic-fields`)
.post(`/api/v1/steps/${step.id}/dynamic-fields`)
.set('Authorization', token)
.expect(404);
});

View File

@@ -996,21 +996,9 @@ describe('User model', () => {
const user = await createUser();
const presentDate = DateTime.fromObject(
{ year: 2024, month: 11, day: 17, hour: 11, minute: 30 },
{ zone: 'UTC+0' }
);
vi.setSystemTime(presentDate);
await user.startTrialPeriod();
const futureDate = DateTime.fromObject(
{ year: 2025, month: 1, day: 1 },
{ zone: 'UTC+0' }
);
vi.setSystemTime(futureDate);
vi.setSystemTime(DateTime.now().plus({ month: 1 }));
const refetchedUser = await user.$query();
@@ -1118,7 +1106,9 @@ describe('User model', () => {
const user = await createUser();
expect(() => user.getPlanAndUsage()).rejects.toThrow('NotFoundError');
await expect(() => user.getPlanAndUsage()).rejects.toThrow(
'NotFoundError'
);
});
});
@@ -1189,7 +1179,7 @@ describe('User model', () => {
});
it('should throw not found error when user role does not exist', async () => {
expect(() =>
await expect(() =>
User.registerUser({
fullName: 'Sample user',
email: 'user@automatisch.io',
@@ -1257,6 +1247,8 @@ describe('User model', () => {
describe('createUsageData', () => {
it('should create usage data if Automatisch is a cloud installation', async () => {
vi.useFakeTimers();
vi.spyOn(appConfig, 'isCloud', 'get').mockReturnValue(true);
const user = await createUser({
@@ -1264,10 +1256,14 @@ describe('User model', () => {
email: 'user@automatisch.io',
});
vi.setSystemTime(DateTime.now().plus({ month: 1 }));
const usageData = await user.createUsageData();
const currentUsageData = await user.$relatedQuery('currentUsageData');
expect(usageData).toStrictEqual(currentUsageData);
vi.useRealTimers();
});
it('should not create usage data if Automatisch is not a cloud installation', async () => {

View File

@@ -1,31 +1,4 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
const actionQueue = new Queue('action', redisConnection);
process.on('SIGTERM', async () => {
await actionQueue.close();
});
actionQueue.on('error', (error) => {
if (error.code === CONNECTION_REFUSED) {
logger.error(
'Make sure you have installed Redis and it is running.',
error
);
process.exit();
}
logger.error('Error happened in action queue!', error);
});
import { generateQueue } from './queue.js';
const actionQueue = generateQueue('action');
export default actionQueue;

View File

@@ -1,31 +1,4 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
const deleteUserQueue = new Queue('delete-user', redisConnection);
process.on('SIGTERM', async () => {
await deleteUserQueue.close();
});
deleteUserQueue.on('error', (error) => {
if (error.code === CONNECTION_REFUSED) {
logger.error(
'Make sure you have installed Redis and it is running.',
error
);
process.exit();
}
logger.error('Error happened in delete user queue!', error);
});
import { generateQueue } from './queue.js';
const deleteUserQueue = generateQueue('delete-user');
export default deleteUserQueue;

View File

@@ -1,31 +1,4 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
const emailQueue = new Queue('email', redisConnection);
process.on('SIGTERM', async () => {
await emailQueue.close();
});
emailQueue.on('error', (error) => {
if (error.code === CONNECTION_REFUSED) {
logger.error(
'Make sure you have installed Redis and it is running.',
error
);
process.exit();
}
logger.error('Error happened in email queue!', error);
});
import { generateQueue } from './queue.js';
const emailQueue = generateQueue('email');
export default emailQueue;

View File

@@ -1,31 +1,4 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
const flowQueue = new Queue('flow', redisConnection);
process.on('SIGTERM', async () => {
await flowQueue.close();
});
flowQueue.on('error', (error) => {
if (error.code === CONNECTION_REFUSED) {
logger.error(
'Make sure you have installed Redis and it is running.',
error
);
process.exit();
}
logger.error('Error happened in flow queue!', error);
});
import { generateQueue } from './queue.js';
const flowQueue = generateQueue('flow');
export default flowQueue;

View File

@@ -0,0 +1,21 @@
import appConfig from '../config/app.js';
import actionQueue from './action.js';
import emailQueue from './email.js';
import flowQueue from './flow.js';
import triggerQueue from './trigger.js';
import deleteUserQueue from './delete-user.ee.js';
import removeCancelledSubscriptionsQueue from './remove-cancelled-subscriptions.ee.js';
const queues = [
actionQueue,
emailQueue,
flowQueue,
triggerQueue,
deleteUserQueue,
];
if (appConfig.isCloud) {
queues.push(removeCancelledSubscriptionsQueue);
}
export default queues;

View File

@@ -0,0 +1,44 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
export const generateQueue = (queueName, options) => {
const queue = new Queue(queueName, redisConnection);
queue.on('error', (error) => queueOnError(error, queueName));
if (options?.runDaily) addScheduler(queueName, queue);
return queue;
};
const queueOnError = (error, queueName) => {
if (error.code === CONNECTION_REFUSED) {
const errorMessage =
'Make sure you have installed Redis and it is running.';
logger.error(errorMessage, error);
process.exit();
}
logger.error(`Error happened in ${queueName} queue!`, error);
};
const addScheduler = (queueName, queue) => {
const everydayAtOneOclock = '0 1 * * *';
queue.add(queueName, null, {
jobId: queueName,
repeat: {
pattern: everydayAtOneOclock,
},
});
};

View File

@@ -1,44 +1,8 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
import { generateQueue } from './queue.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
const removeCancelledSubscriptionsQueue = new Queue(
const removeCancelledSubscriptionsQueue = generateQueue(
'remove-cancelled-subscriptions',
redisConnection
{ runDaily: true }
);
process.on('SIGTERM', async () => {
await removeCancelledSubscriptionsQueue.close();
});
removeCancelledSubscriptionsQueue.on('error', (error) => {
if (error.code === CONNECTION_REFUSED) {
logger.error(
'Make sure you have installed Redis and it is running.',
error
);
process.exit();
}
logger.error(
'Error happened in remove cancelled subscriptions queue!',
error
);
});
removeCancelledSubscriptionsQueue.add('remove-cancelled-subscriptions', null, {
jobId: 'remove-cancelled-subscriptions',
repeat: {
pattern: '0 1 * * *',
},
});
export default removeCancelledSubscriptionsQueue;

View File

@@ -1,31 +1,4 @@
import process from 'process';
import { Queue } from 'bullmq';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
const CONNECTION_REFUSED = 'ECONNREFUSED';
const redisConnection = {
connection: redisConfig,
};
const triggerQueue = new Queue('trigger', redisConnection);
process.on('SIGTERM', async () => {
await triggerQueue.close();
});
triggerQueue.on('error', (error) => {
if (error.code === CONNECTION_REFUSED) {
logger.error(
'Make sure you have installed Redis and it is running.',
error
);
process.exit();
}
logger.error('Error happened in trigger queue!', error);
});
import { generateQueue } from './queue.js';
const triggerQueue = generateQueue('trigger');
export default triggerQueue;

View File

@@ -1,20 +1,22 @@
import * as Sentry from './helpers/sentry.ee.js';
import appConfig from './config/app.js';
import process from 'node:process';
Sentry.init();
import './config/orm.js';
import './helpers/check-worker-readiness.js';
import './workers/flow.js';
import './workers/trigger.js';
import './workers/action.js';
import './workers/email.js';
import './workers/delete-user.ee.js';
import queues from './queues/index.js';
import workers from './workers/index.js';
if (appConfig.isCloud) {
import('./workers/remove-cancelled-subscriptions.ee.js');
import('./queues/remove-cancelled-subscriptions.ee.js');
}
process.on('SIGTERM', async () => {
for (const queue of queues) {
await queue.close();
}
for (const worker of workers) {
await worker.close();
}
});
import telemetry from './helpers/telemetry/index.js';

View File

@@ -1,5 +1,4 @@
import { Worker } from 'bullmq';
import process from 'node:process';
import * as Sentry from '../helpers/sentry.ee.js';
import redisConfig from '../config/redis.js';
@@ -15,7 +14,7 @@ import delayAsMilliseconds from '../helpers/delay-as-milliseconds.js';
const DEFAULT_DELAY_DURATION = 0;
export const worker = new Worker(
const actionWorker = new Worker(
'action',
async (job) => {
const { stepId, flowId, executionId, computedParameters, executionStep } =
@@ -55,11 +54,11 @@ export const worker = new Worker(
{ connection: redisConfig }
);
worker.on('completed', (job) => {
actionWorker.on('completed', (job) => {
logger.info(`JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has started!`);
});
worker.on('failed', (job, err) => {
actionWorker.on('failed', (job, err) => {
const errorMessage = `
JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has failed to start with ${err.message}
\n ${err.stack}
@@ -74,6 +73,4 @@ worker.on('failed', (job, err) => {
});
});
process.on('SIGTERM', async () => {
await worker.close();
});
export default actionWorker;

View File

@@ -1,5 +1,4 @@
import { Worker } from 'bullmq';
import process from 'node:process';
import * as Sentry from '../helpers/sentry.ee.js';
import redisConfig from '../config/redis.js';
@@ -8,7 +7,7 @@ import appConfig from '../config/app.js';
import User from '../models/user.js';
import ExecutionStep from '../models/execution-step.js';
export const worker = new Worker(
const deleteUserWorker = new Worker(
'delete-user',
async (job) => {
const { id } = job.data;
@@ -46,13 +45,13 @@ export const worker = new Worker(
{ connection: redisConfig }
);
worker.on('completed', (job) => {
deleteUserWorker.on('completed', (job) => {
logger.info(
`JOB ID: ${job.id} - The user with the ID of '${job.data.id}' has been deleted!`
);
});
worker.on('failed', (job, err) => {
deleteUserWorker.on('failed', (job, err) => {
const errorMessage = `
JOB ID: ${job.id} - The user with the ID of '${job.data.id}' has failed to be deleted! ${err.message}
\n ${err.stack}
@@ -67,6 +66,4 @@ worker.on('failed', (job, err) => {
});
});
process.on('SIGTERM', async () => {
await worker.close();
});
export default deleteUserWorker;

View File

@@ -1,5 +1,4 @@
import { Worker } from 'bullmq';
import process from 'node:process';
import * as Sentry from '../helpers/sentry.ee.js';
import redisConfig from '../config/redis.js';
@@ -16,7 +15,7 @@ const isAutomatischEmail = (email) => {
return email.endsWith('@automatisch.io');
};
export const worker = new Worker(
const emailWorker = new Worker(
'email',
async (job) => {
const { email, subject, template, params } = job.data;
@@ -39,13 +38,13 @@ export const worker = new Worker(
{ connection: redisConfig }
);
worker.on('completed', (job) => {
emailWorker.on('completed', (job) => {
logger.info(
`JOB ID: ${job.id} - ${job.data.subject} email sent to ${job.data.email}!`
);
});
worker.on('failed', (job, err) => {
emailWorker.on('failed', (job, err) => {
const errorMessage = `
JOB ID: ${job.id} - ${job.data.subject} email to ${job.data.email} has failed to send with ${err.message}
\n ${err.stack}
@@ -60,6 +59,4 @@ worker.on('failed', (job, err) => {
});
});
process.on('SIGTERM', async () => {
await worker.close();
});
export default emailWorker;

View File

@@ -1,5 +1,4 @@
import { Worker } from 'bullmq';
import process from 'node:process';
import * as Sentry from '../helpers/sentry.ee.js';
import redisConfig from '../config/redis.js';
@@ -13,7 +12,7 @@ import {
REMOVE_AFTER_7_DAYS_OR_50_JOBS,
} from '../helpers/remove-job-configuration.js';
export const worker = new Worker(
const flowWorker = new Worker(
'flow',
async (job) => {
const { flowId } = job.data;
@@ -64,11 +63,11 @@ export const worker = new Worker(
{ connection: redisConfig }
);
worker.on('completed', (job) => {
flowWorker.on('completed', (job) => {
logger.info(`JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has started!`);
});
worker.on('failed', async (job, err) => {
flowWorker.on('failed', async (job, err) => {
const errorMessage = `
JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has failed to start with ${err.message}
\n ${err.stack}
@@ -95,6 +94,4 @@ worker.on('failed', async (job, err) => {
});
});
process.on('SIGTERM', async () => {
await worker.close();
});
export default flowWorker;

View File

@@ -0,0 +1,21 @@
import appConfig from '../config/app.js';
import actionWorker from './action.js';
import emailWorker from './email.js';
import flowWorker from './flow.js';
import triggerWorker from './trigger.js';
import deleteUserWorker from './delete-user.ee.js';
import removeCancelledSubscriptionsWorker from './remove-cancelled-subscriptions.ee.js';
const workers = [
actionWorker,
emailWorker,
flowWorker,
triggerWorker,
deleteUserWorker,
];
if (appConfig.isCloud) {
workers.push(removeCancelledSubscriptionsWorker);
}
export default workers;

View File

@@ -1,12 +1,11 @@
import { Worker } from 'bullmq';
import process from 'node:process';
import { DateTime } from 'luxon';
import * as Sentry from '../helpers/sentry.ee.js';
import redisConfig from '../config/redis.js';
import logger from '../helpers/logger.js';
import Subscription from '../models/subscription.ee.js';
export const worker = new Worker(
const removeCancelledSubscriptionsWorker = new Worker(
'remove-cancelled-subscriptions',
async () => {
await Subscription.query()
@@ -23,13 +22,13 @@ export const worker = new Worker(
{ connection: redisConfig }
);
worker.on('completed', (job) => {
removeCancelledSubscriptionsWorker.on('completed', (job) => {
logger.info(
`JOB ID: ${job.id} - The cancelled subscriptions have been removed!`
);
});
worker.on('failed', (job, err) => {
removeCancelledSubscriptionsWorker.on('failed', (job, err) => {
const errorMessage = `
JOB ID: ${job.id} - ERROR: The cancelled subscriptions can not be removed! ${err.message}
\n ${err.stack}
@@ -42,6 +41,4 @@ worker.on('failed', (job, err) => {
});
});
process.on('SIGTERM', async () => {
await worker.close();
});
export default removeCancelledSubscriptionsWorker;

View File

@@ -1,5 +1,4 @@
import { Worker } from 'bullmq';
import process from 'node:process';
import * as Sentry from '../helpers/sentry.ee.js';
import redisConfig from '../config/redis.js';
@@ -12,7 +11,7 @@ import {
REMOVE_AFTER_7_DAYS_OR_50_JOBS,
} from '../helpers/remove-job-configuration.js';
export const worker = new Worker(
const triggerWorker = new Worker(
'trigger',
async (job) => {
const { flowId, executionId, stepId, executionStep } = await processTrigger(
@@ -41,11 +40,11 @@ export const worker = new Worker(
{ connection: redisConfig }
);
worker.on('completed', (job) => {
triggerWorker.on('completed', (job) => {
logger.info(`JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has started!`);
});
worker.on('failed', (job, err) => {
triggerWorker.on('failed', (job, err) => {
const errorMessage = `
JOB ID: ${job.id} - FLOW ID: ${job.data.flowId} has failed to start with ${err.message}
\n ${err.stack}
@@ -60,6 +59,4 @@ worker.on('failed', (job, err) => {
});
});
process.on('SIGTERM', async () => {
await worker.close();
});
export default triggerWorker;